@atlanai/sdk 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -26
- package/dist/client.js +3 -1
- package/dist/evals.d.ts +46 -0
- package/dist/evals.js +230 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +8 -1
- package/dist/manifest.json +299 -3
- package/dist/raw/eval/apis/DatasetsApi.d.ts +304 -0
- package/dist/raw/eval/apis/DatasetsApi.js +706 -0
- package/dist/raw/eval/apis/ExperimentsApi.d.ts +371 -0
- package/dist/raw/eval/apis/ExperimentsApi.js +869 -0
- package/dist/raw/eval/apis/ScorersApi.d.ts +129 -0
- package/dist/raw/eval/apis/ScorersApi.js +308 -0
- package/dist/raw/eval/apis/index.d.ts +3 -0
- package/dist/raw/eval/apis/index.js +21 -0
- package/dist/raw/eval/index.d.ts +3 -0
- package/dist/raw/eval/index.js +21 -0
- package/dist/raw/eval/models/AccessView.d.ts +57 -0
- package/dist/raw/eval/models/AccessView.js +64 -0
- package/dist/raw/eval/models/ArtifactConflictProblem.d.ts +69 -0
- package/dist/raw/eval/models/ArtifactConflictProblem.js +66 -0
- package/dist/raw/eval/models/CostView.d.ts +50 -0
- package/dist/raw/eval/models/CostView.js +60 -0
- package/dist/raw/eval/models/CursorPage.d.ts +38 -0
- package/dist/raw/eval/models/CursorPage.js +52 -0
- package/dist/raw/eval/models/EvalArtifactEnvelope.d.ts +144 -0
- package/dist/raw/eval/models/EvalArtifactEnvelope.js +84 -0
- package/dist/raw/eval/models/EvalArtifactSearchRequest.d.ts +71 -0
- package/dist/raw/eval/models/EvalArtifactSearchRequest.js +60 -0
- package/dist/raw/eval/models/EvalBulkDatasetRecordResponse.d.ts +33 -0
- package/dist/raw/eval/models/EvalBulkDatasetRecordResponse.js +51 -0
- package/dist/raw/eval/models/EvalBulkDatasetRecordResult.d.ts +52 -0
- package/dist/raw/eval/models/EvalBulkDatasetRecordResult.js +60 -0
- package/dist/raw/eval/models/EvalBulkExperimentResultResponse.d.ts +33 -0
- package/dist/raw/eval/models/EvalBulkExperimentResultResponse.js +51 -0
- package/dist/raw/eval/models/EvalBulkExperimentResultSlot.d.ts +53 -0
- package/dist/raw/eval/models/EvalBulkExperimentResultSlot.js +60 -0
- package/dist/raw/eval/models/EvalCreateDataset400Response.d.ts +65 -0
- package/dist/raw/eval/models/EvalCreateDataset400Response.js +75 -0
- package/dist/raw/eval/models/EvalCreateDataset409Response.d.ts +63 -0
- package/dist/raw/eval/models/EvalCreateDataset409Response.js +75 -0
- package/dist/raw/eval/models/EvalCreateDataset413Response.d.ts +64 -0
- package/dist/raw/eval/models/EvalCreateDataset413Response.js +74 -0
- package/dist/raw/eval/models/EvalCreateDataset415Response.d.ts +64 -0
- package/dist/raw/eval/models/EvalCreateDataset415Response.js +74 -0
- package/dist/raw/eval/models/EvalCreateDataset422Response.d.ts +63 -0
- package/dist/raw/eval/models/EvalCreateDataset422Response.js +75 -0
- package/dist/raw/eval/models/EvalDatasetArtifactView.d.ts +277 -0
- package/dist/raw/eval/models/EvalDatasetArtifactView.js +146 -0
- package/dist/raw/eval/models/EvalDatasetCreate.d.ts +150 -0
- package/dist/raw/eval/models/EvalDatasetCreate.js +86 -0
- package/dist/raw/eval/models/EvalDatasetMeta.d.ts +46 -0
- package/dist/raw/eval/models/EvalDatasetMeta.js +48 -0
- package/dist/raw/eval/models/EvalDatasetPatch.d.ts +52 -0
- package/dist/raw/eval/models/EvalDatasetPatch.js +54 -0
- package/dist/raw/eval/models/EvalDatasetRecordArtifactView.d.ts +330 -0
- package/dist/raw/eval/models/EvalDatasetRecordArtifactView.js +161 -0
- package/dist/raw/eval/models/EvalDatasetRecordBulkCreate.d.ts +33 -0
- package/dist/raw/eval/models/EvalDatasetRecordBulkCreate.js +51 -0
- package/dist/raw/eval/models/EvalDatasetRecordCreate.d.ts +195 -0
- package/dist/raw/eval/models/EvalDatasetRecordCreate.js +101 -0
- package/dist/raw/eval/models/EvalDatasetRecordMeta.d.ts +96 -0
- package/dist/raw/eval/models/EvalDatasetRecordMeta.js +67 -0
- package/dist/raw/eval/models/EvalDatasetRecordPatch.d.ts +81 -0
- package/dist/raw/eval/models/EvalDatasetRecordPatch.js +62 -0
- package/dist/raw/eval/models/EvalExperimentArtifactView.d.ts +320 -0
- package/dist/raw/eval/models/EvalExperimentArtifactView.js +160 -0
- package/dist/raw/eval/models/EvalExperimentCreate.d.ts +193 -0
- package/dist/raw/eval/models/EvalExperimentCreate.js +100 -0
- package/dist/raw/eval/models/EvalExperimentMeta.d.ts +91 -0
- package/dist/raw/eval/models/EvalExperimentMeta.js +62 -0
- package/dist/raw/eval/models/EvalExperimentPatch.d.ts +63 -0
- package/dist/raw/eval/models/EvalExperimentPatch.js +57 -0
- package/dist/raw/eval/models/EvalExperimentResultArtifactView.d.ts +330 -0
- package/dist/raw/eval/models/EvalExperimentResultArtifactView.js +162 -0
- package/dist/raw/eval/models/EvalExperimentResultBulkCreate.d.ts +33 -0
- package/dist/raw/eval/models/EvalExperimentResultBulkCreate.js +51 -0
- package/dist/raw/eval/models/EvalExperimentResultCreate.d.ts +197 -0
- package/dist/raw/eval/models/EvalExperimentResultCreate.js +100 -0
- package/dist/raw/eval/models/EvalExperimentResultMeta.d.ts +102 -0
- package/dist/raw/eval/models/EvalExperimentResultMeta.js +66 -0
- package/dist/raw/eval/models/EvalExperimentStatus.d.ts +31 -0
- package/dist/raw/eval/models/EvalExperimentStatus.js +59 -0
- package/dist/raw/eval/models/EvalGetDataset404Response.d.ts +63 -0
- package/dist/raw/eval/models/EvalGetDataset404Response.js +75 -0
- package/dist/raw/eval/models/EvalGetExperimentTrace400Response.d.ts +63 -0
- package/dist/raw/eval/models/EvalGetExperimentTrace400Response.js +75 -0
- package/dist/raw/eval/models/EvalListDatasets400Response.d.ts +63 -0
- package/dist/raw/eval/models/EvalListDatasets400Response.js +75 -0
- package/dist/raw/eval/models/EvalListDatasets401Response.d.ts +79 -0
- package/dist/raw/eval/models/EvalListDatasets401Response.js +89 -0
- package/dist/raw/eval/models/EvalListDatasets403Response.d.ts +64 -0
- package/dist/raw/eval/models/EvalListDatasets403Response.js +74 -0
- package/dist/raw/eval/models/EvalListDatasets500Response.d.ts +63 -0
- package/dist/raw/eval/models/EvalListDatasets500Response.js +75 -0
- package/dist/raw/eval/models/EvalListDatasets503Response.d.ts +64 -0
- package/dist/raw/eval/models/EvalListDatasets503Response.js +74 -0
- package/dist/raw/eval/models/EvalListDatasets504Response.d.ts +63 -0
- package/dist/raw/eval/models/EvalListDatasets504Response.js +75 -0
- package/dist/raw/eval/models/EvalPageDatasetArtifactView.d.ts +44 -0
- package/dist/raw/eval/models/EvalPageDatasetArtifactView.js +56 -0
- package/dist/raw/eval/models/EvalPageDatasetRecordArtifactView.d.ts +44 -0
- package/dist/raw/eval/models/EvalPageDatasetRecordArtifactView.js +56 -0
- package/dist/raw/eval/models/EvalPageExperimentArtifactView.d.ts +44 -0
- package/dist/raw/eval/models/EvalPageExperimentArtifactView.js +56 -0
- package/dist/raw/eval/models/EvalPageExperimentResultArtifactView.d.ts +44 -0
- package/dist/raw/eval/models/EvalPageExperimentResultArtifactView.js +56 -0
- package/dist/raw/eval/models/EvalPageScorerArtifactView.d.ts +44 -0
- package/dist/raw/eval/models/EvalPageScorerArtifactView.js +56 -0
- package/dist/raw/eval/models/EvalPatchDataset412Response.d.ts +63 -0
- package/dist/raw/eval/models/EvalPatchDataset412Response.js +75 -0
- package/dist/raw/eval/models/EvalRecordSource.d.ts +27 -0
- package/dist/raw/eval/models/EvalRecordSource.js +55 -0
- package/dist/raw/eval/models/EvalScorerArtifactView.d.ts +301 -0
- package/dist/raw/eval/models/EvalScorerArtifactView.js +154 -0
- package/dist/raw/eval/models/EvalScorerCreate.d.ts +172 -0
- package/dist/raw/eval/models/EvalScorerCreate.js +98 -0
- package/dist/raw/eval/models/EvalScorerKind.d.ts +32 -0
- package/dist/raw/eval/models/EvalScorerKind.js +60 -0
- package/dist/raw/eval/models/EvalScorerMeta.d.ts +67 -0
- package/dist/raw/eval/models/EvalScorerMeta.js +60 -0
- package/dist/raw/eval/models/EvalScorerPatch.d.ts +75 -0
- package/dist/raw/eval/models/EvalScorerPatch.js +56 -0
- package/dist/raw/eval/models/EvalScorerScope.d.ts +32 -0
- package/dist/raw/eval/models/EvalScorerScope.js +60 -0
- package/dist/raw/eval/models/EvalSubjectKind.d.ts +33 -0
- package/dist/raw/eval/models/EvalSubjectKind.js +61 -0
- package/dist/raw/eval/models/GatewayArtifactBase.d.ts +250 -0
- package/dist/raw/eval/models/GatewayArtifactBase.js +139 -0
- package/dist/raw/eval/models/GatewayArtifactIdentityRevealView.d.ts +60 -0
- package/dist/raw/eval/models/GatewayArtifactIdentityRevealView.js +58 -0
- package/dist/raw/eval/models/GatewayPageMeta.d.ts +46 -0
- package/dist/raw/eval/models/GatewayPageMeta.js +58 -0
- package/dist/raw/eval/models/GatewaySourceEnvelope.d.ts +106 -0
- package/dist/raw/eval/models/GatewaySourceEnvelope.js +66 -0
- package/dist/raw/eval/models/GatewayTraceStatsBucket.d.ts +25 -0
- package/dist/raw/eval/models/GatewayTraceStatsBucket.js +53 -0
- package/dist/raw/eval/models/GatewayTraceStatsCompareMode.d.ts +25 -0
- package/dist/raw/eval/models/GatewayTraceStatsCompareMode.js +53 -0
- package/dist/raw/eval/models/GatewayTraceStatsComparison.d.ts +45 -0
- package/dist/raw/eval/models/GatewayTraceStatsComparison.js +59 -0
- package/dist/raw/eval/models/GatewayTraceStatsData.d.ts +24 -0
- package/dist/raw/eval/models/GatewayTraceStatsData.js +64 -0
- package/dist/raw/eval/models/GatewayTraceStatsDataOneOf.d.ts +46 -0
- package/dist/raw/eval/models/GatewayTraceStatsDataOneOf.js +66 -0
- package/dist/raw/eval/models/GatewayTraceStatsDataOneOf1.d.ts +53 -0
- package/dist/raw/eval/models/GatewayTraceStatsDataOneOf1.js +71 -0
- package/dist/raw/eval/models/GatewayTraceStatsDataOneOf2.d.ts +46 -0
- package/dist/raw/eval/models/GatewayTraceStatsDataOneOf2.js +66 -0
- package/dist/raw/eval/models/GatewayTraceStatsEnvelope.d.ts +50 -0
- package/dist/raw/eval/models/GatewayTraceStatsEnvelope.js +59 -0
- package/dist/raw/eval/models/GatewayTraceStatsFilter.d.ts +47 -0
- package/dist/raw/eval/models/GatewayTraceStatsFilter.js +59 -0
- package/dist/raw/eval/models/GatewayTraceStatsFilterOperator.d.ts +28 -0
- package/dist/raw/eval/models/GatewayTraceStatsFilterOperator.js +56 -0
- package/dist/raw/eval/models/GatewayTraceStatsFilterValue.d.ts +21 -0
- package/dist/raw/eval/models/GatewayTraceStatsFilterValue.js +85 -0
- package/dist/raw/eval/models/GatewayTraceStatsGroupBy.d.ts +35 -0
- package/dist/raw/eval/models/GatewayTraceStatsGroupBy.js +63 -0
- package/dist/raw/eval/models/GatewayTraceStatsMeasure.d.ts +39 -0
- package/dist/raw/eval/models/GatewayTraceStatsMeasure.js +67 -0
- package/dist/raw/eval/models/GatewayTraceStatsMeta.d.ts +49 -0
- package/dist/raw/eval/models/GatewayTraceStatsMeta.js +53 -0
- package/dist/raw/eval/models/GatewayTraceStatsOrder.d.ts +40 -0
- package/dist/raw/eval/models/GatewayTraceStatsOrder.js +56 -0
- package/dist/raw/eval/models/GatewayTraceStatsOrderBy.d.ts +25 -0
- package/dist/raw/eval/models/GatewayTraceStatsOrderBy.js +53 -0
- package/dist/raw/eval/models/GatewayTraceStatsOrderDirection.d.ts +25 -0
- package/dist/raw/eval/models/GatewayTraceStatsOrderDirection.js +53 -0
- package/dist/raw/eval/models/GatewayTraceStatsPanelQuery.d.ts +69 -0
- package/dist/raw/eval/models/GatewayTraceStatsPanelQuery.js +66 -0
- package/dist/raw/eval/models/GatewayTraceStatsRequest.d.ts +80 -0
- package/dist/raw/eval/models/GatewayTraceStatsRequest.js +72 -0
- package/dist/raw/eval/models/GatewayTraceStatsRequestType.d.ts +26 -0
- package/dist/raw/eval/models/GatewayTraceStatsRequestType.js +54 -0
- package/dist/raw/eval/models/GatewayTraceStatsScalarResult.d.ts +52 -0
- package/dist/raw/eval/models/GatewayTraceStatsScalarResult.js +59 -0
- package/dist/raw/eval/models/GatewayTraceStatsSeries.d.ts +52 -0
- package/dist/raw/eval/models/GatewayTraceStatsSeries.js +62 -0
- package/dist/raw/eval/models/GatewayTraceStatsSeriesPoint.d.ts +38 -0
- package/dist/raw/eval/models/GatewayTraceStatsSeriesPoint.js +52 -0
- package/dist/raw/eval/models/GatewayTraceStatsTableColumn.d.ts +45 -0
- package/dist/raw/eval/models/GatewayTraceStatsTableColumn.js +57 -0
- package/dist/raw/eval/models/GatewayTraceStatsTableColumnType.d.ts +25 -0
- package/dist/raw/eval/models/GatewayTraceStatsTableColumnType.js +53 -0
- package/dist/raw/eval/models/GatewayTraceStatsTableRow.d.ts +38 -0
- package/dist/raw/eval/models/GatewayTraceStatsTableRow.js +54 -0
- package/dist/raw/eval/models/GatewayTraceStatsWindow.d.ts +38 -0
- package/dist/raw/eval/models/GatewayTraceStatsWindow.js +54 -0
- package/dist/raw/eval/models/ProblemDetails.d.ts +76 -0
- package/dist/raw/eval/models/ProblemDetails.js +64 -0
- package/dist/raw/eval/models/RegistryEdgeTarget.d.ts +38 -0
- package/dist/raw/eval/models/RegistryEdgeTarget.js +54 -0
- package/dist/raw/eval/models/RegistryRelationshipEdge.d.ts +96 -0
- package/dist/raw/eval/models/RegistryRelationshipEdge.js +77 -0
- package/dist/raw/eval/models/RegistryRelationshipsView.d.ts +45 -0
- package/dist/raw/eval/models/RegistryRelationshipsView.js +59 -0
- package/dist/raw/eval/models/SpanListEnvelope.d.ts +40 -0
- package/dist/raw/eval/models/SpanListEnvelope.js +56 -0
- package/dist/raw/eval/models/SpanViewDto.d.ts +192 -0
- package/dist/raw/eval/models/SpanViewDto.js +117 -0
- package/dist/raw/eval/models/TraceDetailView.d.ts +177 -0
- package/dist/raw/eval/models/TraceDetailView.js +108 -0
- package/dist/raw/eval/models/TraceLinks.d.ts +45 -0
- package/dist/raw/eval/models/TraceLinks.js +58 -0
- package/dist/raw/eval/models/TraceListEnvelope.d.ts +47 -0
- package/dist/raw/eval/models/TraceListEnvelope.js +61 -0
- package/dist/raw/eval/models/TraceParentView.d.ts +68 -0
- package/dist/raw/eval/models/TraceParentView.js +64 -0
- package/dist/raw/eval/models/TraceStatus.d.ts +26 -0
- package/dist/raw/eval/models/TraceStatus.js +54 -0
- package/dist/raw/eval/models/TraceSummaryStats.d.ts +62 -0
- package/dist/raw/eval/models/TraceSummaryStats.js +62 -0
- package/dist/raw/eval/models/TraceSummaryView.d.ts +150 -0
- package/dist/raw/eval/models/TraceSummaryView.js +96 -0
- package/dist/raw/eval/models/UsageView.d.ts +56 -0
- package/dist/raw/eval/models/UsageView.js +66 -0
- package/dist/raw/eval/models/index.d.ts +100 -0
- package/dist/raw/eval/models/index.js +118 -0
- package/dist/raw/eval/runtime.d.ts +185 -0
- package/dist/raw/eval/runtime.js +369 -0
- package/dist/resources.json +307 -0
- package/dist/tracing/client.js +6 -4
- package/dist/tracing/propagation.d.ts +1 -0
- package/dist/tracing/propagation.js +7 -0
- package/dist/tracing/semconv.d.ts +8 -1
- package/dist/tracing/semconv.js +27 -13
- package/dist/tracing/spans.d.ts +8 -4
- package/dist/tracing/spans.js +28 -13
- package/dist/tracing/version.d.ts +1 -1
- package/dist/tracing/version.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/* tslint:disable */
|
|
3
|
+
/* eslint-disable */
|
|
4
|
+
/**
|
|
5
|
+
* Evaluation catalog
|
|
6
|
+
* The evaluation loop as registry artifacts: curated `dataset`s of examples pinned from production sessions and traces, `experiment` run snapshots with create-only per-case `experiment_result`s, and versioned `scorer` definitions. A verdict is a score span in the run\'s trace; `POST /experiments/{id}/summarize` rolls a run\'s score spans into its durable `summary`. The gateway executes and scores nothing — external harnesses, CI, and judges run the evals, emit score spans, and upload results here.
|
|
7
|
+
*
|
|
8
|
+
* The version of the OpenAPI document: 0.1.0
|
|
9
|
+
*
|
|
10
|
+
*
|
|
11
|
+
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
|
12
|
+
* https://openapi-generator.tech
|
|
13
|
+
* Do not edit the class manually.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.instanceOfEvalExperimentArtifactView = instanceOfEvalExperimentArtifactView;
|
|
17
|
+
exports.EvalExperimentArtifactViewFromJSON = EvalExperimentArtifactViewFromJSON;
|
|
18
|
+
exports.EvalExperimentArtifactViewFromJSONTyped = EvalExperimentArtifactViewFromJSONTyped;
|
|
19
|
+
exports.EvalExperimentArtifactViewToJSON = EvalExperimentArtifactViewToJSON;
|
|
20
|
+
exports.EvalExperimentArtifactViewToJSONTyped = EvalExperimentArtifactViewToJSONTyped;
|
|
21
|
+
const GatewayArtifactIdentityRevealView_1 = require("./GatewayArtifactIdentityRevealView");
|
|
22
|
+
const EvalSubjectKind_1 = require("./EvalSubjectKind");
|
|
23
|
+
const EvalExperimentStatus_1 = require("./EvalExperimentStatus");
|
|
24
|
+
const RegistryRelationshipsView_1 = require("./RegistryRelationshipsView");
|
|
25
|
+
/**
|
|
26
|
+
* Check if a given object implements the EvalExperimentArtifactView interface.
|
|
27
|
+
*/
|
|
28
|
+
function instanceOfEvalExperimentArtifactView(value) {
|
|
29
|
+
if ((!('artifactStatus' in value) && !('artifact_status' in value)) || (value['artifactStatus'] === undefined && value['artifact_status'] === undefined))
|
|
30
|
+
return false;
|
|
31
|
+
if ((!('commitTs' in value) && !('commit_ts' in value)) || (value['commitTs'] === undefined && value['commit_ts'] === undefined))
|
|
32
|
+
return false;
|
|
33
|
+
if ((!('createdAt' in value) && !('created_at' in value)) || (value['createdAt'] === undefined && value['created_at'] === undefined))
|
|
34
|
+
return false;
|
|
35
|
+
if ((!('createdBy' in value) && !('created_by' in value)) || (value['createdBy'] === undefined && value['created_by'] === undefined))
|
|
36
|
+
return false;
|
|
37
|
+
if (!('id' in value) || value['id'] === undefined)
|
|
38
|
+
return false;
|
|
39
|
+
if ((!('isLatest' in value) && !('is_latest' in value)) || (value['isLatest'] === undefined && value['is_latest'] === undefined))
|
|
40
|
+
return false;
|
|
41
|
+
if (!('kind' in value) || value['kind'] === undefined)
|
|
42
|
+
return false;
|
|
43
|
+
if (!('name' in value) || value['name'] === undefined)
|
|
44
|
+
return false;
|
|
45
|
+
if ((!('sourceType' in value) && !('source_type' in value)) || (value['sourceType'] === undefined && value['source_type'] === undefined))
|
|
46
|
+
return false;
|
|
47
|
+
if ((!('updatedAt' in value) && !('updated_at' in value)) || (value['updatedAt'] === undefined && value['updated_at'] === undefined))
|
|
48
|
+
return false;
|
|
49
|
+
if ((!('updatedBy' in value) && !('updated_by' in value)) || (value['updatedBy'] === undefined && value['updated_by'] === undefined))
|
|
50
|
+
return false;
|
|
51
|
+
if ((!('versionOrdinal' in value) && !('version_ordinal' in value)) || (value['versionOrdinal'] === undefined && value['version_ordinal'] === undefined))
|
|
52
|
+
return false;
|
|
53
|
+
if ((!('workspaceId' in value) && !('workspace_id' in value)) || (value['workspaceId'] === undefined && value['workspace_id'] === undefined))
|
|
54
|
+
return false;
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
function EvalExperimentArtifactViewFromJSON(json) {
|
|
58
|
+
return EvalExperimentArtifactViewFromJSONTyped(json, false);
|
|
59
|
+
}
|
|
60
|
+
function EvalExperimentArtifactViewFromJSONTyped(json, ignoreDiscriminator) {
|
|
61
|
+
if (json == null) {
|
|
62
|
+
return json;
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
'artifactStatus': json['artifact_status'],
|
|
66
|
+
'commitTs': json['commit_ts'],
|
|
67
|
+
'contentHashHex': json['content_hash_hex'] == null ? undefined : json['content_hash_hex'],
|
|
68
|
+
'createdAt': json['created_at'],
|
|
69
|
+
'createdBy': json['created_by'],
|
|
70
|
+
'description': json['description'] == null ? undefined : json['description'],
|
|
71
|
+
'displayName': json['display_name'] == null ? undefined : json['display_name'],
|
|
72
|
+
'extension': json['extension'] == null ? undefined : json['extension'],
|
|
73
|
+
'id': json['id'],
|
|
74
|
+
'identity': json['identity'] == null ? undefined : (0, GatewayArtifactIdentityRevealView_1.GatewayArtifactIdentityRevealViewFromJSON)(json['identity']),
|
|
75
|
+
'isLatest': json['is_latest'],
|
|
76
|
+
'kind': json['kind'],
|
|
77
|
+
'name': json['name'],
|
|
78
|
+
'oidcClientId': json['oidc_client_id'] == null ? undefined : json['oidc_client_id'],
|
|
79
|
+
'ownerId': json['owner_id'] == null ? undefined : json['owner_id'],
|
|
80
|
+
'projectId': json['project_id'] == null ? undefined : json['project_id'],
|
|
81
|
+
'repoId': json['repo_id'] == null ? undefined : json['repo_id'],
|
|
82
|
+
'sourceCreatedAt': json['source_created_at'] == null ? undefined : json['source_created_at'],
|
|
83
|
+
'sourceCreatedBy': json['source_created_by'] == null ? undefined : json['source_created_by'],
|
|
84
|
+
'sourceDigest': json['source_digest'] == null ? undefined : json['source_digest'],
|
|
85
|
+
'sourceId': json['source_id'] == null ? undefined : json['source_id'],
|
|
86
|
+
'sourceLastModifiedAt': json['source_last_modified_at'] == null ? undefined : json['source_last_modified_at'],
|
|
87
|
+
'sourceLastModifiedBy': json['source_last_modified_by'] == null ? undefined : json['source_last_modified_by'],
|
|
88
|
+
'sourceMetadata': json['source_metadata'] === undefined ? undefined : json['source_metadata'] === null ? null : json['source_metadata'],
|
|
89
|
+
'sourceSyncedAt': json['source_synced_at'] == null ? undefined : json['source_synced_at'],
|
|
90
|
+
'sourceType': json['source_type'],
|
|
91
|
+
'sourceUri': json['source_uri'] == null ? undefined : json['source_uri'],
|
|
92
|
+
'sourceVersion': json['source_version'] == null ? undefined : json['source_version'],
|
|
93
|
+
'updatedAt': json['updated_at'],
|
|
94
|
+
'updatedBy': json['updated_by'],
|
|
95
|
+
'versionLabel': json['version_label'] == null ? undefined : json['version_label'],
|
|
96
|
+
'versionOrdinal': json['version_ordinal'],
|
|
97
|
+
'workspaceId': json['workspace_id'],
|
|
98
|
+
'baselineExperimentId': json['baseline_experiment_id'] === undefined ? undefined : json['baseline_experiment_id'] === null ? null : json['baseline_experiment_id'],
|
|
99
|
+
'config': json['config'] === undefined ? undefined : json['config'] === null ? null : json['config'],
|
|
100
|
+
'datasetId': json['dataset_id'] === undefined ? undefined : json['dataset_id'] === null ? null : json['dataset_id'],
|
|
101
|
+
'experimentStatus': json['experiment_status'] == null ? undefined : (0, EvalExperimentStatus_1.EvalExperimentStatusFromJSON)(json['experiment_status']),
|
|
102
|
+
'subjectId': json['subject_id'] === undefined ? undefined : json['subject_id'] === null ? null : json['subject_id'],
|
|
103
|
+
'subjectKind': json['subject_kind'] === undefined ? undefined : json['subject_kind'] === null ? null : (0, EvalSubjectKind_1.EvalSubjectKindFromJSON)(json['subject_kind']),
|
|
104
|
+
'summary': json['summary'] === undefined ? undefined : json['summary'] === null ? null : json['summary'],
|
|
105
|
+
'permissions': json['permissions'] === undefined ? undefined : json['permissions'] === null ? null : json['permissions'],
|
|
106
|
+
'relationships': json['relationships'] === undefined ? undefined : json['relationships'] === null ? null : (0, RegistryRelationshipsView_1.RegistryRelationshipsViewFromJSON)(json['relationships']),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
function EvalExperimentArtifactViewToJSON(json) {
|
|
110
|
+
return EvalExperimentArtifactViewToJSONTyped(json, false);
|
|
111
|
+
}
|
|
112
|
+
function EvalExperimentArtifactViewToJSONTyped(value, ignoreDiscriminator = false) {
|
|
113
|
+
if (value == null) {
|
|
114
|
+
return value;
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
'artifact_status': value['artifactStatus'],
|
|
118
|
+
'commit_ts': value['commitTs'],
|
|
119
|
+
'content_hash_hex': value['contentHashHex'],
|
|
120
|
+
'created_at': value['createdAt'],
|
|
121
|
+
'created_by': value['createdBy'],
|
|
122
|
+
'description': value['description'],
|
|
123
|
+
'display_name': value['displayName'],
|
|
124
|
+
'extension': value['extension'],
|
|
125
|
+
'id': value['id'],
|
|
126
|
+
'identity': (0, GatewayArtifactIdentityRevealView_1.GatewayArtifactIdentityRevealViewToJSON)(value['identity']),
|
|
127
|
+
'is_latest': value['isLatest'],
|
|
128
|
+
'kind': value['kind'],
|
|
129
|
+
'name': value['name'],
|
|
130
|
+
'oidc_client_id': value['oidcClientId'],
|
|
131
|
+
'owner_id': value['ownerId'],
|
|
132
|
+
'project_id': value['projectId'],
|
|
133
|
+
'repo_id': value['repoId'],
|
|
134
|
+
'source_created_at': value['sourceCreatedAt'],
|
|
135
|
+
'source_created_by': value['sourceCreatedBy'],
|
|
136
|
+
'source_digest': value['sourceDigest'],
|
|
137
|
+
'source_id': value['sourceId'],
|
|
138
|
+
'source_last_modified_at': value['sourceLastModifiedAt'],
|
|
139
|
+
'source_last_modified_by': value['sourceLastModifiedBy'],
|
|
140
|
+
'source_metadata': value['sourceMetadata'],
|
|
141
|
+
'source_synced_at': value['sourceSyncedAt'],
|
|
142
|
+
'source_type': value['sourceType'],
|
|
143
|
+
'source_uri': value['sourceUri'],
|
|
144
|
+
'source_version': value['sourceVersion'],
|
|
145
|
+
'updated_at': value['updatedAt'],
|
|
146
|
+
'updated_by': value['updatedBy'],
|
|
147
|
+
'version_label': value['versionLabel'],
|
|
148
|
+
'version_ordinal': value['versionOrdinal'],
|
|
149
|
+
'workspace_id': value['workspaceId'],
|
|
150
|
+
'baseline_experiment_id': value['baselineExperimentId'],
|
|
151
|
+
'config': value['config'],
|
|
152
|
+
'dataset_id': value['datasetId'],
|
|
153
|
+
'experiment_status': (0, EvalExperimentStatus_1.EvalExperimentStatusToJSON)(value['experimentStatus']),
|
|
154
|
+
'subject_id': value['subjectId'],
|
|
155
|
+
'subject_kind': (0, EvalSubjectKind_1.EvalSubjectKindToJSON)(value['subjectKind']),
|
|
156
|
+
'summary': value['summary'],
|
|
157
|
+
'permissions': value['permissions'],
|
|
158
|
+
'relationships': (0, RegistryRelationshipsView_1.RegistryRelationshipsViewToJSON)(value['relationships']),
|
|
159
|
+
};
|
|
160
|
+
}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluation catalog
|
|
3
|
+
* The evaluation loop as registry artifacts: curated `dataset`s of examples pinned from production sessions and traces, `experiment` run snapshots with create-only per-case `experiment_result`s, and versioned `scorer` definitions. A verdict is a score span in the run\'s trace; `POST /experiments/{id}/summarize` rolls a run\'s score spans into its durable `summary`. The gateway executes and scores nothing — external harnesses, CI, and judges run the evals, emit score spans, and upload results here.
|
|
4
|
+
*
|
|
5
|
+
* The version of the OpenAPI document: 0.1.0
|
|
6
|
+
*
|
|
7
|
+
*
|
|
8
|
+
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
|
9
|
+
* https://openapi-generator.tech
|
|
10
|
+
* Do not edit the class manually.
|
|
11
|
+
*/
|
|
12
|
+
import type { EvalSubjectKind } from './EvalSubjectKind';
|
|
13
|
+
import type { EvalExperimentStatus } from './EvalExperimentStatus';
|
|
14
|
+
/**
|
|
15
|
+
* The caller-writable surface of a new experiment.
|
|
16
|
+
*
|
|
17
|
+
* Everything here describes *what is about to run* (or, for a one-shot
|
|
18
|
+
* import, what ran): the dataset, the subject pair, the configuration, the
|
|
19
|
+
* baseline. All of it is pinned after create.
|
|
20
|
+
* @export
|
|
21
|
+
* @interface EvalExperimentCreate
|
|
22
|
+
*/
|
|
23
|
+
export interface EvalExperimentCreate {
|
|
24
|
+
/**
|
|
25
|
+
* RFC 3339. The upstream's own creation time, never the commit time.
|
|
26
|
+
* @type {string}
|
|
27
|
+
* @memberof EvalExperimentCreate
|
|
28
|
+
*/
|
|
29
|
+
sourceCreatedAt?: string;
|
|
30
|
+
/**
|
|
31
|
+
* The upstream author, git `Name <email>` form (distinct from the registry
|
|
32
|
+
* principal `created_by`). The first author; see `source_last_modified_by`
|
|
33
|
+
* for who changed it last.
|
|
34
|
+
* @type {string}
|
|
35
|
+
* @memberof EvalExperimentCreate
|
|
36
|
+
*/
|
|
37
|
+
sourceCreatedBy?: string;
|
|
38
|
+
/**
|
|
39
|
+
* What the upstream calls this object. The reconciliation key for a re-sync,
|
|
40
|
+
* and the reason to prefer it over matching on `name`.
|
|
41
|
+
* @type {string}
|
|
42
|
+
* @memberof EvalExperimentCreate
|
|
43
|
+
*/
|
|
44
|
+
sourceId?: string;
|
|
45
|
+
/**
|
|
46
|
+
* RFC 3339.
|
|
47
|
+
* @type {string}
|
|
48
|
+
* @memberof EvalExperimentCreate
|
|
49
|
+
*/
|
|
50
|
+
sourceLastModifiedAt?: string;
|
|
51
|
+
/**
|
|
52
|
+
* The upstream author of the last change, git `Name <email>` form.
|
|
53
|
+
* @type {string}
|
|
54
|
+
* @memberof EvalExperimentCreate
|
|
55
|
+
*/
|
|
56
|
+
sourceLastModifiedBy?: string;
|
|
57
|
+
/**
|
|
58
|
+
*
|
|
59
|
+
* @type {any}
|
|
60
|
+
* @memberof EvalExperimentCreate
|
|
61
|
+
*/
|
|
62
|
+
sourceMetadata?: any | null;
|
|
63
|
+
/**
|
|
64
|
+
* RFC 3339. The time of the sync that produced this version.
|
|
65
|
+
* @type {string}
|
|
66
|
+
* @memberof EvalExperimentCreate
|
|
67
|
+
*/
|
|
68
|
+
sourceSyncedAt?: string;
|
|
69
|
+
/**
|
|
70
|
+
* The content authority: `git`, `atlan`, `local`, or `external` for a
|
|
71
|
+
* customer's own system. Omitted, the registry defaults it.
|
|
72
|
+
* @type {string}
|
|
73
|
+
* @memberof EvalExperimentCreate
|
|
74
|
+
*/
|
|
75
|
+
sourceType?: string;
|
|
76
|
+
/**
|
|
77
|
+
* Scheme-tagged origin — where the upstream object lives.
|
|
78
|
+
* @type {string}
|
|
79
|
+
* @memberof EvalExperimentCreate
|
|
80
|
+
*/
|
|
81
|
+
sourceUri?: string;
|
|
82
|
+
/**
|
|
83
|
+
* The upstream revision (a commit hash or tag), distinct from `version_label`.
|
|
84
|
+
* @type {string}
|
|
85
|
+
* @memberof EvalExperimentCreate
|
|
86
|
+
*/
|
|
87
|
+
sourceVersion?: string;
|
|
88
|
+
/**
|
|
89
|
+
* Human-readable description.
|
|
90
|
+
* @type {string}
|
|
91
|
+
* @memberof EvalExperimentCreate
|
|
92
|
+
*/
|
|
93
|
+
description?: string;
|
|
94
|
+
/**
|
|
95
|
+
* Human-readable label. Omitted, the gateway defaults it to `name`.
|
|
96
|
+
* @type {string}
|
|
97
|
+
* @memberof EvalExperimentCreate
|
|
98
|
+
*/
|
|
99
|
+
displayName?: string;
|
|
100
|
+
/**
|
|
101
|
+
* Stable machine handle ([AIP-122](https://google.aip.dev/122)) — short and
|
|
102
|
+
* identifier-shaped, kebab-case by convention. Unique per `(workspace,
|
|
103
|
+
* kind)` among live rows (`artifacts_name_idx`), but not identity: `id`
|
|
104
|
+
* is the only stable handle.
|
|
105
|
+
* @type {string}
|
|
106
|
+
* @memberof EvalExperimentCreate
|
|
107
|
+
*/
|
|
108
|
+
name: string;
|
|
109
|
+
/**
|
|
110
|
+
* Optional canonical owner (`users.user_id`). Omitted, the gateway
|
|
111
|
+
* attributes the caller.
|
|
112
|
+
* @type {string}
|
|
113
|
+
* @memberof EvalExperimentCreate
|
|
114
|
+
*/
|
|
115
|
+
ownerId?: string;
|
|
116
|
+
/**
|
|
117
|
+
* Optional `project_`-prefixed handle to group under. Must belong to
|
|
118
|
+
* `workspace_id` (else 422).
|
|
119
|
+
* @type {string}
|
|
120
|
+
* @memberof EvalExperimentCreate
|
|
121
|
+
*/
|
|
122
|
+
projectId?: string;
|
|
123
|
+
/**
|
|
124
|
+
* Optional `repo_`-prefixed source repository. No default — a repo is a
|
|
125
|
+
* factual claim about provenance, so it is never guessed.
|
|
126
|
+
* @type {string}
|
|
127
|
+
* @memberof EvalExperimentCreate
|
|
128
|
+
*/
|
|
129
|
+
repoId?: string;
|
|
130
|
+
/**
|
|
131
|
+
* `workspace_`-prefixed handle to create in. **Required** — there is
|
|
132
|
+
* deliberately no default, because guessing a tenant's namespace is
|
|
133
|
+
* invisible until someone else reads the data.
|
|
134
|
+
* @type {string}
|
|
135
|
+
* @memberof EvalExperimentCreate
|
|
136
|
+
*/
|
|
137
|
+
workspaceId: string;
|
|
138
|
+
/**
|
|
139
|
+
* The experiment this one is compared against.
|
|
140
|
+
* @type {string}
|
|
141
|
+
* @memberof EvalExperimentCreate
|
|
142
|
+
*/
|
|
143
|
+
baselineExperimentId?: string | null;
|
|
144
|
+
/**
|
|
145
|
+
* What runs: model, parameters, prompt hash, commit, retrieval strategy.
|
|
146
|
+
* @type {{ [key: string]: any; }}
|
|
147
|
+
* @memberof EvalExperimentCreate
|
|
148
|
+
*/
|
|
149
|
+
config?: {
|
|
150
|
+
[key: string]: any;
|
|
151
|
+
} | null;
|
|
152
|
+
/**
|
|
153
|
+
* The dataset this run evaluates over; omit for an ad-hoc run.
|
|
154
|
+
* @type {string}
|
|
155
|
+
* @memberof EvalExperimentCreate
|
|
156
|
+
*/
|
|
157
|
+
datasetId?: string | null;
|
|
158
|
+
/**
|
|
159
|
+
*
|
|
160
|
+
* @type {EvalExperimentStatus}
|
|
161
|
+
* @memberof EvalExperimentCreate
|
|
162
|
+
*/
|
|
163
|
+
experimentStatus?: EvalExperimentStatus;
|
|
164
|
+
/**
|
|
165
|
+
* The subject's artifact handle, resolved against `subject_kind`.
|
|
166
|
+
* @type {string}
|
|
167
|
+
* @memberof EvalExperimentCreate
|
|
168
|
+
*/
|
|
169
|
+
subjectId?: string | null;
|
|
170
|
+
/**
|
|
171
|
+
* Value of the `subject_kind` field.
|
|
172
|
+
* @type {EvalSubjectKind}
|
|
173
|
+
* @memberof EvalExperimentCreate
|
|
174
|
+
*/
|
|
175
|
+
subjectKind?: EvalSubjectKind | null;
|
|
176
|
+
/**
|
|
177
|
+
* Score aggregates, for a one-shot import. A live run leaves this absent
|
|
178
|
+
* and writes it at finalize.
|
|
179
|
+
* @type {{ [key: string]: any; }}
|
|
180
|
+
* @memberof EvalExperimentCreate
|
|
181
|
+
*/
|
|
182
|
+
summary?: {
|
|
183
|
+
[key: string]: any;
|
|
184
|
+
} | null;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Check if a given object implements the EvalExperimentCreate interface.
|
|
188
|
+
*/
|
|
189
|
+
export declare function instanceOfEvalExperimentCreate(value: object): value is EvalExperimentCreate;
|
|
190
|
+
export declare function EvalExperimentCreateFromJSON(json: any): EvalExperimentCreate;
|
|
191
|
+
export declare function EvalExperimentCreateFromJSONTyped(json: any, ignoreDiscriminator: boolean): EvalExperimentCreate;
|
|
192
|
+
export declare function EvalExperimentCreateToJSON(json: any): EvalExperimentCreate;
|
|
193
|
+
export declare function EvalExperimentCreateToJSONTyped(value?: EvalExperimentCreate | null, ignoreDiscriminator?: boolean): any;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/* tslint:disable */
|
|
3
|
+
/* eslint-disable */
|
|
4
|
+
/**
|
|
5
|
+
* Evaluation catalog
|
|
6
|
+
* The evaluation loop as registry artifacts: curated `dataset`s of examples pinned from production sessions and traces, `experiment` run snapshots with create-only per-case `experiment_result`s, and versioned `scorer` definitions. A verdict is a score span in the run\'s trace; `POST /experiments/{id}/summarize` rolls a run\'s score spans into its durable `summary`. The gateway executes and scores nothing — external harnesses, CI, and judges run the evals, emit score spans, and upload results here.
|
|
7
|
+
*
|
|
8
|
+
* The version of the OpenAPI document: 0.1.0
|
|
9
|
+
*
|
|
10
|
+
*
|
|
11
|
+
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
|
12
|
+
* https://openapi-generator.tech
|
|
13
|
+
* Do not edit the class manually.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.instanceOfEvalExperimentCreate = instanceOfEvalExperimentCreate;
|
|
17
|
+
exports.EvalExperimentCreateFromJSON = EvalExperimentCreateFromJSON;
|
|
18
|
+
exports.EvalExperimentCreateFromJSONTyped = EvalExperimentCreateFromJSONTyped;
|
|
19
|
+
exports.EvalExperimentCreateToJSON = EvalExperimentCreateToJSON;
|
|
20
|
+
exports.EvalExperimentCreateToJSONTyped = EvalExperimentCreateToJSONTyped;
|
|
21
|
+
const EvalSubjectKind_1 = require("./EvalSubjectKind");
|
|
22
|
+
const EvalExperimentStatus_1 = require("./EvalExperimentStatus");
|
|
23
|
+
/**
|
|
24
|
+
* Check if a given object implements the EvalExperimentCreate interface.
|
|
25
|
+
*/
|
|
26
|
+
function instanceOfEvalExperimentCreate(value) {
|
|
27
|
+
if (!('name' in value) || value['name'] === undefined)
|
|
28
|
+
return false;
|
|
29
|
+
if ((!('workspaceId' in value) && !('workspace_id' in value)) || (value['workspaceId'] === undefined && value['workspace_id'] === undefined))
|
|
30
|
+
return false;
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
function EvalExperimentCreateFromJSON(json) {
|
|
34
|
+
return EvalExperimentCreateFromJSONTyped(json, false);
|
|
35
|
+
}
|
|
36
|
+
function EvalExperimentCreateFromJSONTyped(json, ignoreDiscriminator) {
|
|
37
|
+
if (json == null) {
|
|
38
|
+
return json;
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
'sourceCreatedAt': json['source_created_at'] == null ? undefined : json['source_created_at'],
|
|
42
|
+
'sourceCreatedBy': json['source_created_by'] == null ? undefined : json['source_created_by'],
|
|
43
|
+
'sourceId': json['source_id'] == null ? undefined : json['source_id'],
|
|
44
|
+
'sourceLastModifiedAt': json['source_last_modified_at'] == null ? undefined : json['source_last_modified_at'],
|
|
45
|
+
'sourceLastModifiedBy': json['source_last_modified_by'] == null ? undefined : json['source_last_modified_by'],
|
|
46
|
+
'sourceMetadata': json['source_metadata'] === undefined ? undefined : json['source_metadata'] === null ? null : json['source_metadata'],
|
|
47
|
+
'sourceSyncedAt': json['source_synced_at'] == null ? undefined : json['source_synced_at'],
|
|
48
|
+
'sourceType': json['source_type'] == null ? undefined : json['source_type'],
|
|
49
|
+
'sourceUri': json['source_uri'] == null ? undefined : json['source_uri'],
|
|
50
|
+
'sourceVersion': json['source_version'] == null ? undefined : json['source_version'],
|
|
51
|
+
'description': json['description'] == null ? undefined : json['description'],
|
|
52
|
+
'displayName': json['display_name'] == null ? undefined : json['display_name'],
|
|
53
|
+
'name': json['name'],
|
|
54
|
+
'ownerId': json['owner_id'] == null ? undefined : json['owner_id'],
|
|
55
|
+
'projectId': json['project_id'] == null ? undefined : json['project_id'],
|
|
56
|
+
'repoId': json['repo_id'] == null ? undefined : json['repo_id'],
|
|
57
|
+
'workspaceId': json['workspace_id'],
|
|
58
|
+
'baselineExperimentId': json['baseline_experiment_id'] === undefined ? undefined : json['baseline_experiment_id'] === null ? null : json['baseline_experiment_id'],
|
|
59
|
+
'config': json['config'] === undefined ? undefined : json['config'] === null ? null : json['config'],
|
|
60
|
+
'datasetId': json['dataset_id'] === undefined ? undefined : json['dataset_id'] === null ? null : json['dataset_id'],
|
|
61
|
+
'experimentStatus': json['experiment_status'] == null ? undefined : (0, EvalExperimentStatus_1.EvalExperimentStatusFromJSON)(json['experiment_status']),
|
|
62
|
+
'subjectId': json['subject_id'] === undefined ? undefined : json['subject_id'] === null ? null : json['subject_id'],
|
|
63
|
+
'subjectKind': json['subject_kind'] === undefined ? undefined : json['subject_kind'] === null ? null : (0, EvalSubjectKind_1.EvalSubjectKindFromJSON)(json['subject_kind']),
|
|
64
|
+
'summary': json['summary'] === undefined ? undefined : json['summary'] === null ? null : json['summary'],
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
function EvalExperimentCreateToJSON(json) {
|
|
68
|
+
return EvalExperimentCreateToJSONTyped(json, false);
|
|
69
|
+
}
|
|
70
|
+
function EvalExperimentCreateToJSONTyped(value, ignoreDiscriminator = false) {
|
|
71
|
+
if (value == null) {
|
|
72
|
+
return value;
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
'source_created_at': value['sourceCreatedAt'],
|
|
76
|
+
'source_created_by': value['sourceCreatedBy'],
|
|
77
|
+
'source_id': value['sourceId'],
|
|
78
|
+
'source_last_modified_at': value['sourceLastModifiedAt'],
|
|
79
|
+
'source_last_modified_by': value['sourceLastModifiedBy'],
|
|
80
|
+
'source_metadata': value['sourceMetadata'],
|
|
81
|
+
'source_synced_at': value['sourceSyncedAt'],
|
|
82
|
+
'source_type': value['sourceType'],
|
|
83
|
+
'source_uri': value['sourceUri'],
|
|
84
|
+
'source_version': value['sourceVersion'],
|
|
85
|
+
'description': value['description'],
|
|
86
|
+
'display_name': value['displayName'],
|
|
87
|
+
'name': value['name'],
|
|
88
|
+
'owner_id': value['ownerId'],
|
|
89
|
+
'project_id': value['projectId'],
|
|
90
|
+
'repo_id': value['repoId'],
|
|
91
|
+
'workspace_id': value['workspaceId'],
|
|
92
|
+
'baseline_experiment_id': value['baselineExperimentId'],
|
|
93
|
+
'config': value['config'],
|
|
94
|
+
'dataset_id': value['datasetId'],
|
|
95
|
+
'experiment_status': (0, EvalExperimentStatus_1.EvalExperimentStatusToJSON)(value['experimentStatus']),
|
|
96
|
+
'subject_id': value['subjectId'],
|
|
97
|
+
'subject_kind': (0, EvalSubjectKind_1.EvalSubjectKindToJSON)(value['subjectKind']),
|
|
98
|
+
'summary': value['summary'],
|
|
99
|
+
};
|
|
100
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluation catalog
|
|
3
|
+
* The evaluation loop as registry artifacts: curated `dataset`s of examples pinned from production sessions and traces, `experiment` run snapshots with create-only per-case `experiment_result`s, and versioned `scorer` definitions. A verdict is a score span in the run\'s trace; `POST /experiments/{id}/summarize` rolls a run\'s score spans into its durable `summary`. The gateway executes and scores nothing — external harnesses, CI, and judges run the evals, emit score spans, and upload results here.
|
|
4
|
+
*
|
|
5
|
+
* The version of the OpenAPI document: 0.1.0
|
|
6
|
+
*
|
|
7
|
+
*
|
|
8
|
+
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
|
9
|
+
* https://openapi-generator.tech
|
|
10
|
+
* Do not edit the class manually.
|
|
11
|
+
*/
|
|
12
|
+
import type { EvalSubjectKind } from './EvalSubjectKind';
|
|
13
|
+
import type { EvalExperimentStatus } from './EvalExperimentStatus';
|
|
14
|
+
/**
|
|
15
|
+
* The `experiment` artifact kind: the recorded snapshot of one eval run.
|
|
16
|
+
*
|
|
17
|
+
* Created `running`, results uploaded as create-only children, then finalized
|
|
18
|
+
* by one narrow PATCH (`experiment_status` + `summary`). Everything that
|
|
19
|
+
* describes *what ran* — the dataset pointer, the subject pair, `config`, the
|
|
20
|
+
* baseline — is pinned after create, and a terminal experiment is frozen
|
|
21
|
+
* entirely: that is what makes a completed experiment comparable over time.
|
|
22
|
+
*
|
|
23
|
+
* `dataset_id` is a line pointer, deliberately **not** version-pinned: results
|
|
24
|
+
* snapshot their own `input`/`expected`, so the experiment stays an honest
|
|
25
|
+
* record even as the dataset evolves — and the dataset stays archivable.
|
|
26
|
+
* @export
|
|
27
|
+
* @interface EvalExperimentMeta
|
|
28
|
+
*/
|
|
29
|
+
export interface EvalExperimentMeta {
|
|
30
|
+
/**
|
|
31
|
+
* The experiment this one is compared against (same dataset, prior
|
|
32
|
+
* configuration). Resolved at write, so a dangling baseline cannot be
|
|
33
|
+
* recorded.
|
|
34
|
+
* @type {string}
|
|
35
|
+
* @memberof EvalExperimentMeta
|
|
36
|
+
*/
|
|
37
|
+
baselineExperimentId?: string | null;
|
|
38
|
+
/**
|
|
39
|
+
* What ran: model, parameters, prompt hash, commit, retrieval strategy —
|
|
40
|
+
* whatever identifies the configuration under test. Pinned after create,
|
|
41
|
+
* because it describes a run that already happened.
|
|
42
|
+
* @type {{ [key: string]: any; }}
|
|
43
|
+
* @memberof EvalExperimentMeta
|
|
44
|
+
*/
|
|
45
|
+
config?: {
|
|
46
|
+
[key: string]: any;
|
|
47
|
+
} | null;
|
|
48
|
+
/**
|
|
49
|
+
* The dataset this run evaluated over. Optional — an ad-hoc run has none —
|
|
50
|
+
* and immutable after create.
|
|
51
|
+
* @type {string}
|
|
52
|
+
* @memberof EvalExperimentMeta
|
|
53
|
+
*/
|
|
54
|
+
datasetId?: string | null;
|
|
55
|
+
/**
|
|
56
|
+
*
|
|
57
|
+
* @type {EvalExperimentStatus}
|
|
58
|
+
* @memberof EvalExperimentMeta
|
|
59
|
+
*/
|
|
60
|
+
experimentStatus?: EvalExperimentStatus;
|
|
61
|
+
/**
|
|
62
|
+
* The subject's artifact handle, resolved against `subject_kind`.
|
|
63
|
+
* @type {string}
|
|
64
|
+
* @memberof EvalExperimentMeta
|
|
65
|
+
*/
|
|
66
|
+
subjectId?: string | null;
|
|
67
|
+
/**
|
|
68
|
+
* Value of the `subject_kind` field.
|
|
69
|
+
* @type {EvalSubjectKind}
|
|
70
|
+
* @memberof EvalExperimentMeta
|
|
71
|
+
*/
|
|
72
|
+
subjectKind?: EvalSubjectKind | null;
|
|
73
|
+
/**
|
|
74
|
+
* Score aggregates, written at finalize — e.g.
|
|
75
|
+
* `{"scores": {"accuracy": {"mean": 0.82, "count": 50}}}`. The runner's
|
|
76
|
+
* vocabulary, round-tripped verbatim.
|
|
77
|
+
* @type {{ [key: string]: any; }}
|
|
78
|
+
* @memberof EvalExperimentMeta
|
|
79
|
+
*/
|
|
80
|
+
summary?: {
|
|
81
|
+
[key: string]: any;
|
|
82
|
+
} | null;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Check if a given object implements the EvalExperimentMeta interface.
|
|
86
|
+
*/
|
|
87
|
+
export declare function instanceOfEvalExperimentMeta(value: object): value is EvalExperimentMeta;
|
|
88
|
+
export declare function EvalExperimentMetaFromJSON(json: any): EvalExperimentMeta;
|
|
89
|
+
export declare function EvalExperimentMetaFromJSONTyped(json: any, ignoreDiscriminator: boolean): EvalExperimentMeta;
|
|
90
|
+
export declare function EvalExperimentMetaToJSON(json: any): EvalExperimentMeta;
|
|
91
|
+
export declare function EvalExperimentMetaToJSONTyped(value?: EvalExperimentMeta | null, ignoreDiscriminator?: boolean): any;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/* tslint:disable */
|
|
3
|
+
/* eslint-disable */
|
|
4
|
+
/**
|
|
5
|
+
* Evaluation catalog
|
|
6
|
+
* The evaluation loop as registry artifacts: curated `dataset`s of examples pinned from production sessions and traces, `experiment` run snapshots with create-only per-case `experiment_result`s, and versioned `scorer` definitions. A verdict is a score span in the run\'s trace; `POST /experiments/{id}/summarize` rolls a run\'s score spans into its durable `summary`. The gateway executes and scores nothing — external harnesses, CI, and judges run the evals, emit score spans, and upload results here.
|
|
7
|
+
*
|
|
8
|
+
* The version of the OpenAPI document: 0.1.0
|
|
9
|
+
*
|
|
10
|
+
*
|
|
11
|
+
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
|
|
12
|
+
* https://openapi-generator.tech
|
|
13
|
+
* Do not edit the class manually.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.instanceOfEvalExperimentMeta = instanceOfEvalExperimentMeta;
|
|
17
|
+
exports.EvalExperimentMetaFromJSON = EvalExperimentMetaFromJSON;
|
|
18
|
+
exports.EvalExperimentMetaFromJSONTyped = EvalExperimentMetaFromJSONTyped;
|
|
19
|
+
exports.EvalExperimentMetaToJSON = EvalExperimentMetaToJSON;
|
|
20
|
+
exports.EvalExperimentMetaToJSONTyped = EvalExperimentMetaToJSONTyped;
|
|
21
|
+
const EvalSubjectKind_1 = require("./EvalSubjectKind");
|
|
22
|
+
const EvalExperimentStatus_1 = require("./EvalExperimentStatus");
|
|
23
|
+
/**
|
|
24
|
+
* Check if a given object implements the EvalExperimentMeta interface.
|
|
25
|
+
*/
|
|
26
|
+
function instanceOfEvalExperimentMeta(value) {
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
function EvalExperimentMetaFromJSON(json) {
|
|
30
|
+
return EvalExperimentMetaFromJSONTyped(json, false);
|
|
31
|
+
}
|
|
32
|
+
function EvalExperimentMetaFromJSONTyped(json, ignoreDiscriminator) {
|
|
33
|
+
if (json == null) {
|
|
34
|
+
return json;
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
'baselineExperimentId': json['baseline_experiment_id'] === undefined ? undefined : json['baseline_experiment_id'] === null ? null : json['baseline_experiment_id'],
|
|
38
|
+
'config': json['config'] === undefined ? undefined : json['config'] === null ? null : json['config'],
|
|
39
|
+
'datasetId': json['dataset_id'] === undefined ? undefined : json['dataset_id'] === null ? null : json['dataset_id'],
|
|
40
|
+
'experimentStatus': json['experiment_status'] == null ? undefined : (0, EvalExperimentStatus_1.EvalExperimentStatusFromJSON)(json['experiment_status']),
|
|
41
|
+
'subjectId': json['subject_id'] === undefined ? undefined : json['subject_id'] === null ? null : json['subject_id'],
|
|
42
|
+
'subjectKind': json['subject_kind'] === undefined ? undefined : json['subject_kind'] === null ? null : (0, EvalSubjectKind_1.EvalSubjectKindFromJSON)(json['subject_kind']),
|
|
43
|
+
'summary': json['summary'] === undefined ? undefined : json['summary'] === null ? null : json['summary'],
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function EvalExperimentMetaToJSON(json) {
|
|
47
|
+
return EvalExperimentMetaToJSONTyped(json, false);
|
|
48
|
+
}
|
|
49
|
+
function EvalExperimentMetaToJSONTyped(value, ignoreDiscriminator = false) {
|
|
50
|
+
if (value == null) {
|
|
51
|
+
return value;
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
'baseline_experiment_id': value['baselineExperimentId'],
|
|
55
|
+
'config': value['config'],
|
|
56
|
+
'dataset_id': value['datasetId'],
|
|
57
|
+
'experiment_status': (0, EvalExperimentStatus_1.EvalExperimentStatusToJSON)(value['experimentStatus']),
|
|
58
|
+
'subject_id': value['subjectId'],
|
|
59
|
+
'subject_kind': (0, EvalSubjectKind_1.EvalSubjectKindToJSON)(value['subjectKind']),
|
|
60
|
+
'summary': value['summary'],
|
|
61
|
+
};
|
|
62
|
+
}
|