@sdeverywhere/check-core 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +2 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +256 -166
- package/dist/index.d.ts +256 -166
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -971,6 +971,171 @@ interface ComparisonViewGroup {
|
|
|
971
971
|
views: (ComparisonView | ComparisonUnresolvedView)[];
|
|
972
972
|
}
|
|
973
973
|
|
|
974
|
+
interface PerfReport {
|
|
975
|
+
readonly minTime: number;
|
|
976
|
+
readonly maxTime: number;
|
|
977
|
+
readonly avgTime: number;
|
|
978
|
+
readonly allTimes: number[];
|
|
979
|
+
}
|
|
980
|
+
declare class PerfStats {
|
|
981
|
+
private readonly times;
|
|
982
|
+
addRun(timeInMillis: number): void;
|
|
983
|
+
toReport(): PerfReport;
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
interface DiffPoint {
|
|
987
|
+
time: number;
|
|
988
|
+
valueL: number;
|
|
989
|
+
valueR: number;
|
|
990
|
+
}
|
|
991
|
+
type DiffValidity = 'neither' | 'left-only' | 'right-only' | 'both';
|
|
992
|
+
interface DiffReport {
|
|
993
|
+
validity: DiffValidity;
|
|
994
|
+
minValue: number;
|
|
995
|
+
maxValue: number;
|
|
996
|
+
avgDiff: number;
|
|
997
|
+
minDiff: number;
|
|
998
|
+
maxDiff: number;
|
|
999
|
+
maxDiffPoint: DiffPoint;
|
|
1000
|
+
}
|
|
1001
|
+
declare function diffDatasets(datasetL: Dataset | undefined, datasetR: Dataset | undefined): DiffReport;
|
|
1002
|
+
|
|
1003
|
+
/**
|
|
1004
|
+
* The report for a single comparison test (involving a dataset produced under
|
|
1005
|
+
* a specific input scenario). This includes the full `DiffReport`, whereas
|
|
1006
|
+
* a `ComparisonTestSummary` only includes the `maxDiff` value.
|
|
1007
|
+
*/
|
|
1008
|
+
interface ComparisonTestReport {
|
|
1009
|
+
scenarioKey: ComparisonScenarioKey;
|
|
1010
|
+
datasetKey: DatasetKey;
|
|
1011
|
+
diffReport: DiffReport;
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* A simplified/terse version of `ComparisonTestReport` that is used when writing
|
|
1015
|
+
* results to a JSON file. The object keys are terse and it only includes the
|
|
1016
|
+
* minimum set of fields (only the `maxDiff` value instead of the full `DiffReport`)
|
|
1017
|
+
* to keep the file smaller when there are many reported differences.
|
|
1018
|
+
*/
|
|
1019
|
+
interface ComparisonTestSummary {
|
|
1020
|
+
/** Short for `scenarioKey`. */
|
|
1021
|
+
s: ComparisonScenarioKey;
|
|
1022
|
+
/** Short for `datasetKey`. */
|
|
1023
|
+
d: DatasetKey;
|
|
1024
|
+
/** Short for `maxDiff`. */
|
|
1025
|
+
md: number;
|
|
1026
|
+
}
|
|
1027
|
+
/**
|
|
1028
|
+
* The roll-up report that contains the results of all individual comparison tests.
|
|
1029
|
+
*/
|
|
1030
|
+
interface ComparisonReport {
|
|
1031
|
+
/** The set of all comparison test reports. */
|
|
1032
|
+
testReports: ComparisonTestReport[];
|
|
1033
|
+
/** The perf report for the "left" model. */
|
|
1034
|
+
perfReportL: PerfReport;
|
|
1035
|
+
/** The perf report for the "right" model. */
|
|
1036
|
+
perfReportR: PerfReport;
|
|
1037
|
+
}
|
|
1038
|
+
/**
|
|
1039
|
+
* A simplified/terse version of `ComparisonReport` that only includes the minimum set
|
|
1040
|
+
* of fields needed by the reporting app (to keep the file smaller when there are many
|
|
1041
|
+
* reported differences). This only includes comparison results for which there is
|
|
1042
|
+
* a non-zero `maxDiff` value.
|
|
1043
|
+
*/
|
|
1044
|
+
interface ComparisonSummary {
|
|
1045
|
+
/** The simplified set of all terse comparison test summaries. */
|
|
1046
|
+
testSummaries: ComparisonTestSummary[];
|
|
1047
|
+
/** The perf report for the "left" model. */
|
|
1048
|
+
perfReportL: PerfReport;
|
|
1049
|
+
/** The perf report for the "right" model. */
|
|
1050
|
+
perfReportR: PerfReport;
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
type ComparisonGroupKind = 'by-dataset' | 'by-scenario';
|
|
1054
|
+
type ComparisonGroupKey = string;
|
|
1055
|
+
/**
|
|
1056
|
+
* A group of comparison test summaries associated with a particular scenario or dataset.
|
|
1057
|
+
*/
|
|
1058
|
+
interface ComparisonGroup {
|
|
1059
|
+
/** The kind of group, either 'by-dataset' or 'by-scenario'. */
|
|
1060
|
+
kind: ComparisonGroupKind;
|
|
1061
|
+
/**
|
|
1062
|
+
* The unique key for this group (a `DatasetKey` if grouped by dataset, or a
|
|
1063
|
+
* `ComparisonScenarioKey` if grouped by scenario).
|
|
1064
|
+
*/
|
|
1065
|
+
key: ComparisonGroupKey;
|
|
1066
|
+
/** The comparison test summaries for this group. */
|
|
1067
|
+
testSummaries: ComparisonTestSummary[];
|
|
1068
|
+
}
|
|
1069
|
+
/** Describes the "root" or primary item for a group of comparisons. */
|
|
1070
|
+
type ComparisonGroupRoot = ComparisonDataset | ComparisonScenario;
|
|
1071
|
+
/** A summary of scores for a group of comparisons. */
|
|
1072
|
+
interface ComparisonGroupScores {
|
|
1073
|
+
/** The total number of comparisons (sample size) for this group. */
|
|
1074
|
+
totalDiffCount: number;
|
|
1075
|
+
/** The sum of the `maxDiff` values for each threshold bucket. */
|
|
1076
|
+
totalMaxDiffByBucket: number[];
|
|
1077
|
+
/** The number of comparisons that fall into each threshold bucket. */
|
|
1078
|
+
diffCountByBucket: number[];
|
|
1079
|
+
/** The percentage of comparisons that fall into each threshold bucket. */
|
|
1080
|
+
diffPercentByBucket: number[];
|
|
1081
|
+
}
|
|
1082
|
+
/**
|
|
1083
|
+
* A summary of a group of comparisons that includes the resolved scenario/dataset metadata
|
|
1084
|
+
* and score information for the group.
|
|
1085
|
+
*/
|
|
1086
|
+
interface ComparisonGroupSummary {
|
|
1087
|
+
/** The metadata for the "root" or primary item for this group of comparisons. */
|
|
1088
|
+
root: ComparisonGroupRoot;
|
|
1089
|
+
/** The group containing the comparison summaries. */
|
|
1090
|
+
group: ComparisonGroup;
|
|
1091
|
+
/** The scores for this group, or undefined if comparisons were not performed for this group. */
|
|
1092
|
+
scores?: ComparisonGroupScores;
|
|
1093
|
+
}
|
|
1094
|
+
/**
|
|
1095
|
+
* Breaks down a set of by-scenario or by-dataset groupings into distinct categories.
|
|
1096
|
+
*/
|
|
1097
|
+
interface ComparisonGroupSummariesByCategory {
|
|
1098
|
+
/**
|
|
1099
|
+
* All groups in a map, keyed by "group key" (either a dataset key or scenario key).
|
|
1100
|
+
*/
|
|
1101
|
+
allGroupSummaries: Map<ComparisonGroupKey, ComparisonGroupSummary>;
|
|
1102
|
+
/**
|
|
1103
|
+
* Groups with items that have errors (are not valid) for both "left" and "right" models.
|
|
1104
|
+
*/
|
|
1105
|
+
withErrors: ComparisonGroupSummary[];
|
|
1106
|
+
/**
|
|
1107
|
+
* Groups with items that are only valid for the "left" model (for example, datasets that
|
|
1108
|
+
* were removed and no longer available in the "right" model).
|
|
1109
|
+
*/
|
|
1110
|
+
onlyInLeft: ComparisonGroupSummary[];
|
|
1111
|
+
/**
|
|
1112
|
+
* Groups with items that are only valid for the "right" model (for example, scenarios
|
|
1113
|
+
* for inputs that were added in the "right" model).
|
|
1114
|
+
*/
|
|
1115
|
+
onlyInRight: ComparisonGroupSummary[];
|
|
1116
|
+
/**
|
|
1117
|
+
* Groups with one or more comparisons that have non-zero `maxDiff` scores; the groups
|
|
1118
|
+
* will be sorted by `maxDiff`, with higher scores at the front of the array.
|
|
1119
|
+
*/
|
|
1120
|
+
withDiffs: ComparisonGroupSummary[];
|
|
1121
|
+
/**
|
|
1122
|
+
* Groups where all comparisons have `maxDiff` scores of zero (no differences between
|
|
1123
|
+
* "left" and "right").
|
|
1124
|
+
*/
|
|
1125
|
+
withoutDiffs: ComparisonGroupSummary[];
|
|
1126
|
+
}
|
|
1127
|
+
/**
|
|
1128
|
+
* Rolls up all by-scenario and by-dataset groupings.
|
|
1129
|
+
*/
|
|
1130
|
+
interface ComparisonCategorizedResults {
|
|
1131
|
+
/** All summaries for the comparison tests that were performed. */
|
|
1132
|
+
allTestSummaries: ComparisonTestSummary[];
|
|
1133
|
+
/** The full set of by-scenario groupings. */
|
|
1134
|
+
byScenario: ComparisonGroupSummariesByCategory;
|
|
1135
|
+
/** The full set of by-dataset groupings. */
|
|
1136
|
+
byDataset: ComparisonGroupSummariesByCategory;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
974
1139
|
/**
|
|
975
1140
|
* Provides access to the set of dataset definitions (`ComparisonDataset` instances) that are used
|
|
976
1141
|
* when comparing the two models.
|
|
@@ -1064,6 +1229,92 @@ interface ComparisonDatasetOptions {
|
|
|
1064
1229
|
*/
|
|
1065
1230
|
contextGraphIdsForDataset?: (dataset: ComparisonDataset, scenario: ComparisonScenario) => BundleGraphId[];
|
|
1066
1231
|
}
|
|
1232
|
+
/**
|
|
1233
|
+
* Describes a row in the comparison report summary view.
|
|
1234
|
+
*/
|
|
1235
|
+
interface ComparisonReportSummaryRow {
|
|
1236
|
+
/** The group summary represented by the row. */
|
|
1237
|
+
groupSummary: ComparisonGroupSummary;
|
|
1238
|
+
/** The custom title for the row (this overrides the default title derived from the summary). */
|
|
1239
|
+
title?: string;
|
|
1240
|
+
/** The custom subtitle for the row (this overrides the default subtitle derived from the summary). */
|
|
1241
|
+
subtitle?: string;
|
|
1242
|
+
}
|
|
1243
|
+
/**
|
|
1244
|
+
* Describes a section in the comparison report summary view.
|
|
1245
|
+
*/
|
|
1246
|
+
interface ComparisonReportSummarySection {
|
|
1247
|
+
/** The text to display for the section header. */
|
|
1248
|
+
headerText: string;
|
|
1249
|
+
/** The summary rows to display in the section. */
|
|
1250
|
+
rows: ComparisonReportSummaryRow[];
|
|
1251
|
+
/**
|
|
1252
|
+
* The initial expanded state of the section. If undefined, defaults to 'expanded-if-diffs',
|
|
1253
|
+
* meaning the section will be initially expanded only if any rows have differences, otherwise
|
|
1254
|
+
* it will be initially collapsed.
|
|
1255
|
+
*/
|
|
1256
|
+
initialState?: 'collapsed' | 'expanded' | 'expanded-if-diffs';
|
|
1257
|
+
}
|
|
1258
|
+
/**
|
|
1259
|
+
* Describes an item (box) in the comparison report detail view.
|
|
1260
|
+
*/
|
|
1261
|
+
interface ComparisonReportDetailItem {
|
|
1262
|
+
/** The title of the item. */
|
|
1263
|
+
title: string;
|
|
1264
|
+
/** The subtitle of the item (if any). */
|
|
1265
|
+
subtitle?: string;
|
|
1266
|
+
/** The scenario for the item. */
|
|
1267
|
+
scenario: ComparisonScenario;
|
|
1268
|
+
/** The test summary for the item. */
|
|
1269
|
+
testSummary: ComparisonTestSummary;
|
|
1270
|
+
}
|
|
1271
|
+
/**
|
|
1272
|
+
* Describes a row in the comparison report detail view.
|
|
1273
|
+
*/
|
|
1274
|
+
interface ComparisonReportDetailRow {
|
|
1275
|
+
/** The title of the row. */
|
|
1276
|
+
title: string;
|
|
1277
|
+
/** The subtitle of the row (if any). */
|
|
1278
|
+
subtitle?: string;
|
|
1279
|
+
/** The score for the row (the meaning of the value depends on the chosen statistical method). */
|
|
1280
|
+
score: number;
|
|
1281
|
+
/** The items in this row (one item per box). */
|
|
1282
|
+
items: ComparisonReportDetailItem[];
|
|
1283
|
+
}
|
|
1284
|
+
interface ComparisonReportOptions {
|
|
1285
|
+
/**
|
|
1286
|
+
* An optional function that allows for customizing the order and grouping of
|
|
1287
|
+
* sections and rows in the "comparisons by scenario" summary view.
|
|
1288
|
+
*
|
|
1289
|
+
* @param summaries The comparison summaries, one summary per scenario.
|
|
1290
|
+
* @returns The sections to display in the "comparisons by scenario" summary view.
|
|
1291
|
+
*/
|
|
1292
|
+
summarySectionsForComparisonsByScenario?: (summaries: ComparisonGroupSummariesByCategory) => ComparisonReportSummarySection[];
|
|
1293
|
+
/**
|
|
1294
|
+
* An optional function that allows for customizing the order and grouping of
|
|
1295
|
+
* sections and rows in the "comparisons by dataset" summary view.
|
|
1296
|
+
*
|
|
1297
|
+
* @param summaries The comparison summaries, one summary per dataset.
|
|
1298
|
+
* @returns The sections to display in the "comparisons by dataset" summary view.
|
|
1299
|
+
*/
|
|
1300
|
+
summarySectionsForComparisonsByDataset?: (summaries: ComparisonGroupSummariesByCategory) => ComparisonReportSummarySection[];
|
|
1301
|
+
/**
|
|
1302
|
+
* An optional function that allows for customizing the order of rows and boxes
|
|
1303
|
+
* in the detail view for a scenario.
|
|
1304
|
+
*
|
|
1305
|
+
* @param rows The original rows to be displayed in the detail view for a scenario.
|
|
1306
|
+
* @returns The customized rows to display in the detail view for a scenario.
|
|
1307
|
+
*/
|
|
1308
|
+
detailRowsForScenario?: (rows: ComparisonReportDetailRow[]) => ComparisonReportDetailRow[];
|
|
1309
|
+
/**
|
|
1310
|
+
* An optional function that allows for customizing the order of rows and boxes
|
|
1311
|
+
* in the detail view for a dataset.
|
|
1312
|
+
*
|
|
1313
|
+
* @param rows The original rows to be displayed in the detail view for a dataset.
|
|
1314
|
+
* @returns The customized rows to display in the detail view for a dataset.
|
|
1315
|
+
*/
|
|
1316
|
+
detailRowsForDataset?: (rows: ComparisonReportDetailRow[]) => ComparisonReportDetailRow[];
|
|
1317
|
+
}
|
|
1067
1318
|
interface ComparisonOptions {
|
|
1068
1319
|
/** The left-side ("baseline") bundle being compared. */
|
|
1069
1320
|
baseline: NamedBundle;
|
|
@@ -1079,6 +1330,8 @@ interface ComparisonOptions {
|
|
|
1079
1330
|
specs: (ComparisonSpecs | ComparisonSpecsSource)[];
|
|
1080
1331
|
/** Optional configuration for the datasets that are compared for different scenarios. */
|
|
1081
1332
|
datasets?: ComparisonDatasetOptions;
|
|
1333
|
+
/** Options for customizing the comparison report. */
|
|
1334
|
+
report?: ComparisonReportOptions;
|
|
1082
1335
|
}
|
|
1083
1336
|
interface ComparisonConfig {
|
|
1084
1337
|
/** The loaded left-side ("baseline") bundle being compared. */
|
|
@@ -1096,6 +1349,8 @@ interface ComparisonConfig {
|
|
|
1096
1349
|
datasets: ComparisonDatasets;
|
|
1097
1350
|
/** The set of resolved view groups. */
|
|
1098
1351
|
viewGroups: ComparisonViewGroup[];
|
|
1352
|
+
/** Options for customizing the comparison report. */
|
|
1353
|
+
reportOptions?: ComparisonReportOptions;
|
|
1099
1354
|
}
|
|
1100
1355
|
|
|
1101
1356
|
type ComparisonDataRequestKey = string;
|
|
@@ -1114,85 +1369,6 @@ declare class ComparisonDataCoordinator {
|
|
|
1114
1369
|
cancelRequest(key: ComparisonDataRequestKey): void;
|
|
1115
1370
|
}
|
|
1116
1371
|
|
|
1117
|
-
interface DiffPoint {
|
|
1118
|
-
time: number;
|
|
1119
|
-
valueL: number;
|
|
1120
|
-
valueR: number;
|
|
1121
|
-
}
|
|
1122
|
-
type DiffValidity = 'neither' | 'left-only' | 'right-only' | 'both';
|
|
1123
|
-
interface DiffReport {
|
|
1124
|
-
validity: DiffValidity;
|
|
1125
|
-
minValue: number;
|
|
1126
|
-
maxValue: number;
|
|
1127
|
-
avgDiff: number;
|
|
1128
|
-
minDiff: number;
|
|
1129
|
-
maxDiff: number;
|
|
1130
|
-
maxDiffPoint: DiffPoint;
|
|
1131
|
-
}
|
|
1132
|
-
declare function diffDatasets(datasetL: Dataset | undefined, datasetR: Dataset | undefined): DiffReport;
|
|
1133
|
-
|
|
1134
|
-
interface PerfReport {
|
|
1135
|
-
readonly minTime: number;
|
|
1136
|
-
readonly maxTime: number;
|
|
1137
|
-
readonly avgTime: number;
|
|
1138
|
-
readonly allTimes: number[];
|
|
1139
|
-
}
|
|
1140
|
-
declare class PerfStats {
|
|
1141
|
-
private readonly times;
|
|
1142
|
-
addRun(timeInMillis: number): void;
|
|
1143
|
-
toReport(): PerfReport;
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
/**
|
|
1147
|
-
* The report for a single comparison test (involving a dataset produced under
|
|
1148
|
-
* a specific input scenario). This includes the full `DiffReport`, whereas
|
|
1149
|
-
* a `ComparisonTestSummary` only includes the `maxDiff` value.
|
|
1150
|
-
*/
|
|
1151
|
-
interface ComparisonTestReport {
|
|
1152
|
-
scenarioKey: ComparisonScenarioKey;
|
|
1153
|
-
datasetKey: DatasetKey;
|
|
1154
|
-
diffReport: DiffReport;
|
|
1155
|
-
}
|
|
1156
|
-
/**
|
|
1157
|
-
* A simplified/terse version of `ComparisonTestReport` that is used when writing
|
|
1158
|
-
* results to a JSON file. The object keys are terse and it only includes the
|
|
1159
|
-
* minimum set of fields (only the `maxDiff` value instead of the full `DiffReport`)
|
|
1160
|
-
* to keep the file smaller when there are many reported differences.
|
|
1161
|
-
*/
|
|
1162
|
-
interface ComparisonTestSummary {
|
|
1163
|
-
/** Short for `scenarioKey`. */
|
|
1164
|
-
s: ComparisonScenarioKey;
|
|
1165
|
-
/** Short for `datasetKey`. */
|
|
1166
|
-
d: DatasetKey;
|
|
1167
|
-
/** Short for `maxDiff`. */
|
|
1168
|
-
md: number;
|
|
1169
|
-
}
|
|
1170
|
-
/**
|
|
1171
|
-
* The roll-up report that contains the results of all individual comparison tests.
|
|
1172
|
-
*/
|
|
1173
|
-
interface ComparisonReport {
|
|
1174
|
-
/** The set of all comparison test reports. */
|
|
1175
|
-
testReports: ComparisonTestReport[];
|
|
1176
|
-
/** The perf report for the "left" model. */
|
|
1177
|
-
perfReportL: PerfReport;
|
|
1178
|
-
/** The perf report for the "right" model. */
|
|
1179
|
-
perfReportR: PerfReport;
|
|
1180
|
-
}
|
|
1181
|
-
/**
|
|
1182
|
-
* A simplified/terse version of `ComparisonReport` that only includes the minimum set
|
|
1183
|
-
* of fields needed by the reporting app (to keep the file smaller when there are many
|
|
1184
|
-
* reported differences). This only includes comparison results for which there is
|
|
1185
|
-
* a non-zero `maxDiff` value.
|
|
1186
|
-
*/
|
|
1187
|
-
interface ComparisonSummary {
|
|
1188
|
-
/** The simplified set of all terse comparison test summaries. */
|
|
1189
|
-
testSummaries: ComparisonTestSummary[];
|
|
1190
|
-
/** The perf report for the "left" model. */
|
|
1191
|
-
perfReportL: PerfReport;
|
|
1192
|
-
/** The perf report for the "right" model. */
|
|
1193
|
-
perfReportR: PerfReport;
|
|
1194
|
-
}
|
|
1195
|
-
|
|
1196
1372
|
type GraphInclusion = 'neither' | 'left-only' | 'right-only' | 'both';
|
|
1197
1373
|
interface GraphComparisonMetadataReport {
|
|
1198
1374
|
/** The key for the metadata field. */
|
|
@@ -1237,92 +1413,6 @@ declare function diffGraphs(graphL: BundleGraphSpec | undefined, graphR: BundleG
|
|
|
1237
1413
|
*/
|
|
1238
1414
|
declare function comparisonSummaryFromReport(comparisonReport: ComparisonReport): ComparisonSummary;
|
|
1239
1415
|
|
|
1240
|
-
type ComparisonGroupKind = 'by-dataset' | 'by-scenario';
|
|
1241
|
-
type ComparisonGroupKey = string;
|
|
1242
|
-
/**
|
|
1243
|
-
* A group of comparison test summaries associated with a particular scenario or dataset.
|
|
1244
|
-
*/
|
|
1245
|
-
interface ComparisonGroup {
|
|
1246
|
-
/** The kind of group, either 'by-dataset' or 'by-scenario'. */
|
|
1247
|
-
kind: ComparisonGroupKind;
|
|
1248
|
-
/**
|
|
1249
|
-
* The unique key for this group (a `DatasetKey` if grouped by dataset, or a
|
|
1250
|
-
* `ComparisonScenarioKey` if grouped by scenario).
|
|
1251
|
-
*/
|
|
1252
|
-
key: ComparisonGroupKey;
|
|
1253
|
-
/** The comparison test summaries for this group. */
|
|
1254
|
-
testSummaries: ComparisonTestSummary[];
|
|
1255
|
-
}
|
|
1256
|
-
/** Describes the "root" or primary item for a group of comparisons. */
|
|
1257
|
-
type ComparisonGroupRoot = ComparisonDataset | ComparisonScenario;
|
|
1258
|
-
/** A summary of scores for a group of comparisons. */
|
|
1259
|
-
interface ComparisonGroupScores {
|
|
1260
|
-
/** The total number of comparisons (sample size) for this group. */
|
|
1261
|
-
totalDiffCount: number;
|
|
1262
|
-
/** The sum of the `maxDiff` values for each threshold bucket. */
|
|
1263
|
-
totalMaxDiffByBucket: number[];
|
|
1264
|
-
/** The number of comparisons that fall into each threshold bucket. */
|
|
1265
|
-
diffCountByBucket: number[];
|
|
1266
|
-
/** The percentage of comparisons that fall into each threshold bucket. */
|
|
1267
|
-
diffPercentByBucket: number[];
|
|
1268
|
-
}
|
|
1269
|
-
/**
|
|
1270
|
-
* A summary of a group of comparisons that includes the resolved scenario/dataset metadata
|
|
1271
|
-
* and score information for the group.
|
|
1272
|
-
*/
|
|
1273
|
-
interface ComparisonGroupSummary {
|
|
1274
|
-
/** The metadata for the "root" or primary item for this group of comparisons. */
|
|
1275
|
-
root: ComparisonGroupRoot;
|
|
1276
|
-
/** The group containing the comparison summaries. */
|
|
1277
|
-
group: ComparisonGroup;
|
|
1278
|
-
/** The scores for this group, or undefined if comparisons were not performed for this group. */
|
|
1279
|
-
scores?: ComparisonGroupScores;
|
|
1280
|
-
}
|
|
1281
|
-
/**
|
|
1282
|
-
* Breaks down a set of by-scenario or by-dataset groupings into distinct categories.
|
|
1283
|
-
*/
|
|
1284
|
-
interface ComparisonGroupSummariesByCategory {
|
|
1285
|
-
/**
|
|
1286
|
-
* All groups in a map, keyed by "group key" (either a dataset key or scenario key).
|
|
1287
|
-
*/
|
|
1288
|
-
allGroupSummaries: Map<ComparisonGroupKey, ComparisonGroupSummary>;
|
|
1289
|
-
/**
|
|
1290
|
-
* Groups with items that have errors (are not valid) for both "left" and "right" models.
|
|
1291
|
-
*/
|
|
1292
|
-
withErrors: ComparisonGroupSummary[];
|
|
1293
|
-
/**
|
|
1294
|
-
* Groups with items that are only valid for the "left" model (for example, datasets that
|
|
1295
|
-
* were removed and no longer available in the "right" model).
|
|
1296
|
-
*/
|
|
1297
|
-
onlyInLeft: ComparisonGroupSummary[];
|
|
1298
|
-
/**
|
|
1299
|
-
* Groups with items that are only valid for the "right" model (for example, scenarios
|
|
1300
|
-
* for inputs that were added in the "right" model).
|
|
1301
|
-
*/
|
|
1302
|
-
onlyInRight: ComparisonGroupSummary[];
|
|
1303
|
-
/**
|
|
1304
|
-
* Groups with one or more comparisons that have non-zero `maxDiff` scores; the groups
|
|
1305
|
-
* will be sorted by `maxDiff`, with higher scores at the front of the array.
|
|
1306
|
-
*/
|
|
1307
|
-
withDiffs: ComparisonGroupSummary[];
|
|
1308
|
-
/**
|
|
1309
|
-
* Groups where all comparisons have `maxDiff` scores of zero (no differences between
|
|
1310
|
-
* "left" and "right").
|
|
1311
|
-
*/
|
|
1312
|
-
withoutDiffs: ComparisonGroupSummary[];
|
|
1313
|
-
}
|
|
1314
|
-
/**
|
|
1315
|
-
* Rolls up all by-scenario and by-dataset groupings.
|
|
1316
|
-
*/
|
|
1317
|
-
interface ComparisonCategorizedResults {
|
|
1318
|
-
/** All summaries for the comparison tests that were performed. */
|
|
1319
|
-
allTestSummaries: ComparisonTestSummary[];
|
|
1320
|
-
/** The full set of by-scenario groupings. */
|
|
1321
|
-
byScenario: ComparisonGroupSummariesByCategory;
|
|
1322
|
-
/** The full set of by-dataset groupings. */
|
|
1323
|
-
byDataset: ComparisonGroupSummariesByCategory;
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
1416
|
/**
|
|
1327
1417
|
* Compute the overall scores for the given group of comparison test summaries.
|
|
1328
1418
|
*
|
|
@@ -1447,4 +1537,4 @@ declare function runSuite(config: Config, callbacks: RunSuiteCallbacks, options?
|
|
|
1447
1537
|
*/
|
|
1448
1538
|
declare function suiteSummaryFromReport(suiteReport: SuiteReport): SuiteSummary;
|
|
1449
1539
|
|
|
1450
|
-
export { type AllInputsSpec, type Bundle, type BundleGraphData, type BundleGraphDatasetSpec, type BundleGraphId, type BundleGraphSpec, type BundleGraphView, type BundleModel, CheckDataCoordinator, type CheckDataRequestKey, type CheckDatasetReport, type CheckGroupReport, type CheckKey, type CheckPredicateOp, type CheckPredicateOpConstantRef, type CheckPredicateOpDataRef, type CheckPredicateOpRef, type CheckPredicateReport, type CheckPredicateSummary, type CheckPredicateTimeOptions, type CheckPredicateTimeRange, type CheckPredicateTimeSingle, type CheckPredicateTimeSpec, type CheckReport, type CheckResult, type CheckResultErrorInfo, type CheckScenario, type CheckScenarioError, type CheckScenarioInputDesc, type CheckScenarioReport, type CheckStatus, type CheckSummary, type CheckTestReport, type ComparisonCategorizedResults, type ComparisonConfig, ComparisonDataCoordinator, type ComparisonDataRequestKey, type ComparisonDataset, type ComparisonDatasetName, type ComparisonDatasetOptions, type ComparisonDatasetSource, type ComparisonDatasetSpec, type ComparisonDatasets, type ComparisonGraphGroup, type ComparisonGraphGroupId, type ComparisonGraphGroupRefSpec, type ComparisonGraphGroupSpec, type ComparisonGraphId, type ComparisonGraphsArraySpec, type ComparisonGraphsPresetSpec, type ComparisonGroup, type ComparisonGroupKey, type ComparisonGroupKind, type ComparisonGroupRoot, type ComparisonGroupScores, type ComparisonGroupSummariesByCategory, type ComparisonGroupSummary, type ComparisonOptions, type ComparisonPlot, type ComparisonReport, type ComparisonResolverError, type ComparisonResolverInvalidValueError, type ComparisonResolverUnknownInputError, type ComparisonScenario, type ComparisonScenarioAllInputsSettings, type ComparisonScenarioGroup, type ComparisonScenarioGroupId, type ComparisonScenarioGroupRefSpec, type ComparisonScenarioGroupSpec, type ComparisonScenarioGroupTitle, type ComparisonScenarioId, type ComparisonScenarioInput, type ComparisonScenarioInputAtPositionSpec, type ComparisonScenarioInputAtValueSpec, type ComparisonScenarioInputName, type ComparisonScenarioInputPosition, type ComparisonScenarioInputSettings, type ComparisonScenarioInputSpec, type ComparisonScenarioInputState, type ComparisonScenarioKey, type ComparisonScenarioPresetMatrixSpec, type ComparisonScenarioRefSpec, type ComparisonScenarioSettings, type ComparisonScenarioSpec, type ComparisonScenarioSubtitle, type ComparisonScenarioTitle, type ComparisonScenarioWithAllInputsSpec, type ComparisonScenarioWithDistinctInputsSpec, type ComparisonScenarioWithInputsSpec, type ComparisonScenarios, type ComparisonSpecs, type ComparisonSpecsSource, type ComparisonSummary, type ComparisonTestReport, type ComparisonTestSummary, type ComparisonUnresolvedScenarioGroupRef, type ComparisonUnresolvedScenarioRef, type ComparisonUnresolvedView, type ComparisonView, type ComparisonViewBox, type ComparisonViewBoxSpec, type ComparisonViewGraphOrder, type ComparisonViewGraphsSpec, type ComparisonViewGroup, type ComparisonViewGroupSpec, type ComparisonViewGroupTitle, type ComparisonViewGroupWithScenariosSpec, type ComparisonViewGroupWithViewsSpec, type ComparisonViewItemSubtitle, type ComparisonViewItemTitle, type ComparisonViewRow, type ComparisonViewRowSpec, type ComparisonViewRowSubtitle, type ComparisonViewRowTitle, type ComparisonViewSpec, type ComparisonViewSubtitle, type ComparisonViewTitle, type Config, type ConfigInitOptions, type ConfigOptions, type DataSource, type Dataset, type DatasetGroupName, type DatasetKey, type DatasetMap, type DatasetsResult, type DiffPoint, type DiffReport, type DiffValidity, type Dimension, type GraphComparisonDatasetReport, type GraphComparisonMetadataReport, type GraphComparisonReport, type GraphInclusion, type ImplVar, type InputAliasName, type InputGroupName, type InputId, type InputPosition, type InputSetting, type InputSettingsSpec, type InputVar, type LegendItem, type LinkItem, type LoadedBundle, type ModelSpec, type NamedBundle, type OutputVar, type PerfReport, PerfRunner, PerfStats, type PositionSetting, type RelatedItem, type RunSuiteCallbacks, type RunSuiteOptions, type ScenarioSpec, type ScenarioSpecUid, type SourceName, type Subscript, type SuiteReport, type SuiteSummary, type ValueSetting, type VarId, categorizeComparisonTestSummaries, checkReportFromSummary, checkSummaryFromReport, comparisonSummaryFromReport, createConfig, datasetMessage, diffDatasets, diffGraphs, getScoresForTestSummaries, predicateMessage, runSuite, scenarioMessage, suiteSummaryFromReport };
|
|
1540
|
+
export { type AllInputsSpec, type Bundle, type BundleGraphData, type BundleGraphDatasetSpec, type BundleGraphId, type BundleGraphSpec, type BundleGraphView, type BundleModel, CheckDataCoordinator, type CheckDataRequestKey, type CheckDatasetReport, type CheckGroupReport, type CheckKey, type CheckPredicateOp, type CheckPredicateOpConstantRef, type CheckPredicateOpDataRef, type CheckPredicateOpRef, type CheckPredicateReport, type CheckPredicateSummary, type CheckPredicateTimeOptions, type CheckPredicateTimeRange, type CheckPredicateTimeSingle, type CheckPredicateTimeSpec, type CheckReport, type CheckResult, type CheckResultErrorInfo, type CheckScenario, type CheckScenarioError, type CheckScenarioInputDesc, type CheckScenarioReport, type CheckStatus, type CheckSummary, type CheckTestReport, type ComparisonCategorizedResults, type ComparisonConfig, ComparisonDataCoordinator, type ComparisonDataRequestKey, type ComparisonDataset, type ComparisonDatasetName, type ComparisonDatasetOptions, type ComparisonDatasetSource, type ComparisonDatasetSpec, type ComparisonDatasets, type ComparisonGraphGroup, type ComparisonGraphGroupId, type ComparisonGraphGroupRefSpec, type ComparisonGraphGroupSpec, type ComparisonGraphId, type ComparisonGraphsArraySpec, type ComparisonGraphsPresetSpec, type ComparisonGroup, type ComparisonGroupKey, type ComparisonGroupKind, type ComparisonGroupRoot, type ComparisonGroupScores, type ComparisonGroupSummariesByCategory, type ComparisonGroupSummary, type ComparisonOptions, type ComparisonPlot, type ComparisonReport, type ComparisonReportDetailItem, type ComparisonReportDetailRow, type ComparisonReportOptions, type ComparisonReportSummaryRow, type ComparisonReportSummarySection, type ComparisonResolverError, type ComparisonResolverInvalidValueError, type ComparisonResolverUnknownInputError, type ComparisonScenario, type ComparisonScenarioAllInputsSettings, type ComparisonScenarioGroup, type ComparisonScenarioGroupId, type ComparisonScenarioGroupRefSpec, type ComparisonScenarioGroupSpec, type ComparisonScenarioGroupTitle, type ComparisonScenarioId, type ComparisonScenarioInput, type ComparisonScenarioInputAtPositionSpec, type ComparisonScenarioInputAtValueSpec, type ComparisonScenarioInputName, type ComparisonScenarioInputPosition, type ComparisonScenarioInputSettings, type ComparisonScenarioInputSpec, type ComparisonScenarioInputState, type ComparisonScenarioKey, type ComparisonScenarioPresetMatrixSpec, type ComparisonScenarioRefSpec, type ComparisonScenarioSettings, type ComparisonScenarioSpec, type ComparisonScenarioSubtitle, type ComparisonScenarioTitle, type ComparisonScenarioWithAllInputsSpec, type ComparisonScenarioWithDistinctInputsSpec, type ComparisonScenarioWithInputsSpec, type ComparisonScenarios, type ComparisonSpecs, type ComparisonSpecsSource, type ComparisonSummary, type ComparisonTestReport, type ComparisonTestSummary, type ComparisonUnresolvedScenarioGroupRef, type ComparisonUnresolvedScenarioRef, type ComparisonUnresolvedView, type ComparisonView, type ComparisonViewBox, type ComparisonViewBoxSpec, type ComparisonViewGraphOrder, type ComparisonViewGraphsSpec, type ComparisonViewGroup, type ComparisonViewGroupSpec, type ComparisonViewGroupTitle, type ComparisonViewGroupWithScenariosSpec, type ComparisonViewGroupWithViewsSpec, type ComparisonViewItemSubtitle, type ComparisonViewItemTitle, type ComparisonViewRow, type ComparisonViewRowSpec, type ComparisonViewRowSubtitle, type ComparisonViewRowTitle, type ComparisonViewSpec, type ComparisonViewSubtitle, type ComparisonViewTitle, type Config, type ConfigInitOptions, type ConfigOptions, type DataSource, type Dataset, type DatasetGroupName, type DatasetKey, type DatasetMap, type DatasetsResult, type DiffPoint, type DiffReport, type DiffValidity, type Dimension, type GraphComparisonDatasetReport, type GraphComparisonMetadataReport, type GraphComparisonReport, type GraphInclusion, type ImplVar, type InputAliasName, type InputGroupName, type InputId, type InputPosition, type InputSetting, type InputSettingsSpec, type InputVar, type LegendItem, type LinkItem, type LoadedBundle, type ModelSpec, type NamedBundle, type OutputVar, type PerfReport, PerfRunner, PerfStats, type PositionSetting, type RelatedItem, type RunSuiteCallbacks, type RunSuiteOptions, type ScenarioSpec, type ScenarioSpecUid, type SourceName, type Subscript, type SuiteReport, type SuiteSummary, type ValueSetting, type VarId, categorizeComparisonTestSummaries, checkReportFromSummary, checkSummaryFromReport, comparisonSummaryFromReport, createConfig, datasetMessage, diffDatasets, diffGraphs, getScoresForTestSummaries, predicateMessage, runSuite, scenarioMessage, suiteSummaryFromReport };
|