@sdeverywhere/check-core 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +63 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +64 -5
- package/dist/index.d.ts +64 -5
- package/dist/index.js +63 -15
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -1203,16 +1203,34 @@ interface ComparisonDataset {
|
|
|
1203
1203
|
type ComparisonScenarioKey = string & {
|
|
1204
1204
|
_brand?: 'ComparisonScenarioKey';
|
|
1205
1205
|
};
|
|
1206
|
+
/** A fatal error indicating that no input variable matched the requested name. */
|
|
1206
1207
|
interface ComparisonResolverUnknownInputError {
|
|
1207
1208
|
kind: 'unknown-input';
|
|
1208
1209
|
}
|
|
1210
|
+
/** A fatal error indicating that no input setting group matched the requested ID. */
|
|
1209
1211
|
interface ComparisonResolverUnknownInputSettingGroupError {
|
|
1210
1212
|
kind: 'unknown-input-setting-group';
|
|
1211
1213
|
}
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
+
/**
|
|
1215
|
+
* A fatal resolution error. When this is set on an input state, the scenario
|
|
1216
|
+
* cannot be run for the affected side: spec construction for that side is
|
|
1217
|
+
* skipped and downstream UI annotations render it as an error.
|
|
1218
|
+
*/
|
|
1219
|
+
type ComparisonResolverError = ComparisonResolverUnknownInputError | ComparisonResolverUnknownInputSettingGroupError;
|
|
1220
|
+
/**
|
|
1221
|
+
* A non-fatal warning indicating that the resolved value falls outside the
|
|
1222
|
+
* declared `[minValue, maxValue]` range of a slider (or is not one of the
|
|
1223
|
+
* declared values of a switch). The scenario still runs with the requested
|
|
1224
|
+
* value; consumers (e.g. UI annotations) should flag it as a warning.
|
|
1225
|
+
*/
|
|
1226
|
+
interface ComparisonResolverValueOutOfRangeWarning {
|
|
1227
|
+
kind: 'value-out-of-range';
|
|
1214
1228
|
}
|
|
1215
|
-
|
|
1229
|
+
/**
|
|
1230
|
+
* A non-fatal resolution warning. Warnings do not prevent the scenario from
|
|
1231
|
+
* running; they are surfaced as annotations alongside the resolved value.
|
|
1232
|
+
*/
|
|
1233
|
+
type ComparisonResolverWarning = ComparisonResolverValueOutOfRangeWarning;
|
|
1216
1234
|
/** Describes the resolution state for a scenario input relative to a specific model. */
|
|
1217
1235
|
interface ComparisonScenarioInputState {
|
|
1218
1236
|
/** The matched input variable; can be undefined if no input matched. */
|
|
@@ -1221,8 +1239,14 @@ interface ComparisonScenarioInputState {
|
|
|
1221
1239
|
position?: InputPosition;
|
|
1222
1240
|
/** The value of the input, for the given position or explicit value. */
|
|
1223
1241
|
value?: number;
|
|
1224
|
-
/** The error info if the input could not be resolved. */
|
|
1242
|
+
/** The fatal error info if the input could not be resolved. */
|
|
1225
1243
|
error?: ComparisonResolverError;
|
|
1244
|
+
/**
|
|
1245
|
+
* Non-fatal advisory info about the resolved input. When set (without an
|
|
1246
|
+
* accompanying `error`), the input still resolved and the scenario can run;
|
|
1247
|
+
* consumers should surface the warning alongside the resolved value.
|
|
1248
|
+
*/
|
|
1249
|
+
warning?: ComparisonResolverWarning;
|
|
1226
1250
|
}
|
|
1227
1251
|
/** A scenario input that has been checked against both "left" and "right" model. */
|
|
1228
1252
|
interface ComparisonScenarioInput {
|
|
@@ -1377,15 +1401,50 @@ interface ComparisonViewGroup {
|
|
|
1377
1401
|
views: (ComparisonView | ComparisonUnresolvedView)[];
|
|
1378
1402
|
}
|
|
1379
1403
|
|
|
1404
|
+
/**
|
|
1405
|
+
* A summary of timing samples collected during a performance run.
|
|
1406
|
+
*/
|
|
1380
1407
|
interface PerfReport {
|
|
1408
|
+
/** Minimum sample time, in milliseconds. */
|
|
1381
1409
|
readonly minTime: number;
|
|
1410
|
+
/** Maximum sample time, in milliseconds. */
|
|
1382
1411
|
readonly maxTime: number;
|
|
1412
|
+
/**
|
|
1413
|
+
* Trimmed mean (interquartile mean) computed from the middle 50% of samples,
|
|
1414
|
+
* in milliseconds. This is more robust against outliers than a simple mean.
|
|
1415
|
+
*/
|
|
1383
1416
|
readonly avgTime: number;
|
|
1417
|
+
/** Median (50th percentile) sample time, in milliseconds. */
|
|
1418
|
+
readonly medianTime: number;
|
|
1419
|
+
/** 95th percentile sample time, in milliseconds. */
|
|
1420
|
+
readonly p95Time: number;
|
|
1421
|
+
/** Population standard deviation across all samples, in milliseconds. */
|
|
1422
|
+
readonly stdDev: number;
|
|
1423
|
+
/** All recorded sample times, sorted ascending, in milliseconds. */
|
|
1384
1424
|
readonly allTimes: number[];
|
|
1385
1425
|
}
|
|
1426
|
+
/**
|
|
1427
|
+
* Collect performance timing samples and produce a robust statistical summary.
|
|
1428
|
+
*/
|
|
1386
1429
|
declare class PerfStats {
|
|
1387
1430
|
private readonly times;
|
|
1431
|
+
/**
|
|
1432
|
+
* Record a single run time sample.
|
|
1433
|
+
*
|
|
1434
|
+
* @param timeInMillis The run time in milliseconds.
|
|
1435
|
+
*/
|
|
1388
1436
|
addRun(timeInMillis: number): void;
|
|
1437
|
+
/**
|
|
1438
|
+
* Get the raw run time samples that have been recorded.
|
|
1439
|
+
*
|
|
1440
|
+
* @returns A copy of the recorded run times, in insertion order.
|
|
1441
|
+
*/
|
|
1442
|
+
getTimes(): number[];
|
|
1443
|
+
/**
|
|
1444
|
+
* Produce a `PerfReport` summarizing the recorded samples.
|
|
1445
|
+
*
|
|
1446
|
+
* @returns The summary report.
|
|
1447
|
+
*/
|
|
1389
1448
|
toReport(): PerfReport;
|
|
1390
1449
|
}
|
|
1391
1450
|
|
|
@@ -2122,4 +2181,4 @@ declare function runSuite(config: Config, callbacks: RunSuiteCallbacks, options?
|
|
|
2122
2181
|
*/
|
|
2123
2182
|
declare function suiteSummaryFromReport(suiteReport: SuiteReport, elapsedMillis: number): SuiteSummary;
|
|
2124
2183
|
|
|
2125
|
-
export { type AllInputsSpec, type Bundle, type BundleGraphData, type BundleGraphDatasetSpec, type BundleGraphId, type BundleGraphSpec, type BundleGraphView, type BundleGraphViewOptions, type BundleModel, type CancelRunPerf, type CancelRunSuite, type CancelRunTrace as CancelTrace, type CheckConfig, CheckDataCoordinator, type CheckDataRef, type CheckDataRefKey, type CheckDataRequestKey, type CheckDataset, type CheckDatasetError, type CheckDatasetReport, type CheckGroupReport, type CheckKey, type CheckNameSpec, type CheckOptions, type CheckPredicateOp, type CheckPredicateOpConstantRef, type CheckPredicateOpDataRef, type CheckPredicateOpRef, type CheckPredicateReport, type CheckPredicateSummary, type CheckPredicateTimeOptions, type CheckPredicateTimeRange, type CheckPredicateTimeSingle, type CheckPredicateTimeSpec, type CheckReport, type CheckResult, type CheckResultErrorInfo, type CheckScenario, type CheckScenarioError, type CheckScenarioInputDesc, type CheckScenarioReport, type CheckStatus, type CheckSummary, type CheckTestReport, type ComparisonCategorizedResults, type ComparisonConfig, ComparisonDataCoordinator, type ComparisonDataRequestKey, type ComparisonDataset, type ComparisonDatasetName, type ComparisonDatasetOptions, type ComparisonDatasetSource, type ComparisonDatasetSpec, type ComparisonDatasets, type ComparisonGraphGroup, type ComparisonGraphGroupId, type ComparisonGraphGroupRefSpec, type ComparisonGraphGroupSpec, type ComparisonGraphId, type ComparisonGraphsArraySpec, type ComparisonGraphsPresetSpec, type ComparisonGroup, type ComparisonGroupKey, type ComparisonGroupKind, type ComparisonGroupRoot, type ComparisonGroupScores, type ComparisonGroupSummariesByCategory, type ComparisonGroupSummary, type ComparisonOptions, type ComparisonPlot, type ComparisonReport, type ComparisonReportDetailItem, type ComparisonReportDetailRow, type ComparisonReportOptions, type ComparisonReportSummaryRow, type ComparisonReportSummarySection, type ComparisonResolverError, type
|
|
2184
|
+
export { type AllInputsSpec, type Bundle, type BundleGraphData, type BundleGraphDatasetSpec, type BundleGraphId, type BundleGraphSpec, type BundleGraphView, type BundleGraphViewOptions, type BundleModel, type CancelRunPerf, type CancelRunSuite, type CancelRunTrace as CancelTrace, type CheckConfig, CheckDataCoordinator, type CheckDataRef, type CheckDataRefKey, type CheckDataRequestKey, type CheckDataset, type CheckDatasetError, type CheckDatasetReport, type CheckGroupReport, type CheckKey, type CheckNameSpec, type CheckOptions, type CheckPredicateOp, type CheckPredicateOpConstantRef, type CheckPredicateOpDataRef, type CheckPredicateOpRef, type CheckPredicateReport, type CheckPredicateSummary, type CheckPredicateTimeOptions, type CheckPredicateTimeRange, type CheckPredicateTimeSingle, type CheckPredicateTimeSpec, type CheckReport, type CheckResult, type CheckResultErrorInfo, type CheckScenario, type CheckScenarioError, type CheckScenarioInputDesc, type CheckScenarioReport, type CheckStatus, type CheckSummary, type CheckTestReport, type ComparisonCategorizedResults, type ComparisonConfig, ComparisonDataCoordinator, type ComparisonDataRequestKey, type ComparisonDataset, type ComparisonDatasetName, type ComparisonDatasetOptions, type ComparisonDatasetSource, type ComparisonDatasetSpec, type ComparisonDatasets, type ComparisonGraphGroup, type ComparisonGraphGroupId, type ComparisonGraphGroupRefSpec, type ComparisonGraphGroupSpec, type ComparisonGraphId, type ComparisonGraphsArraySpec, type ComparisonGraphsPresetSpec, type ComparisonGroup, type ComparisonGroupKey, type ComparisonGroupKind, type ComparisonGroupRoot, type ComparisonGroupScores, type ComparisonGroupSummariesByCategory, type ComparisonGroupSummary, type ComparisonOptions, type ComparisonPlot, type ComparisonReport, type ComparisonReportDetailItem, type ComparisonReportDetailRow, type ComparisonReportOptions, type ComparisonReportSummaryRow, type ComparisonReportSummarySection, type ComparisonResolverError, type ComparisonResolverUnknownInputError, type ComparisonResolverUnknownInputSettingGroupError, type ComparisonResolverValueOutOfRangeWarning, type ComparisonResolverWarning, type ComparisonScenario, type ComparisonScenarioAllInputsSettings, type ComparisonScenarioGroup, type ComparisonScenarioGroupId, type ComparisonScenarioGroupRefSpec, type ComparisonScenarioGroupSpec, type ComparisonScenarioGroupTitle, type ComparisonScenarioId, type ComparisonScenarioInput, type ComparisonScenarioInputAtPositionSpec, type ComparisonScenarioInputAtValueSpec, type ComparisonScenarioInputName, type ComparisonScenarioInputPosition, type ComparisonScenarioInputSettings, type ComparisonScenarioInputSpec, type ComparisonScenarioInputState, type ComparisonScenarioKey, type ComparisonScenarioPresetMatrixSpec, type ComparisonScenarioRefSpec, type ComparisonScenarioSettings, type ComparisonScenarioSpec, type ComparisonScenarioSubtitle, type ComparisonScenarioTitle, type ComparisonScenarioTitleSpec, type ComparisonScenarioWithAllInputsSpec, type ComparisonScenarioWithDistinctInputsSpec, type ComparisonScenarioWithInputsSpec, type ComparisonScenarioWithSettingGroupSpec, type ComparisonScenarios, type ComparisonSortMode, type ComparisonSpecs, type ComparisonSpecsSource, type ComparisonSummary, type ComparisonTestReport, type ComparisonTestSummary, type ComparisonUnresolvedScenarioGroupRef, type ComparisonUnresolvedScenarioRef, type ComparisonUnresolvedView, type ComparisonView, type ComparisonViewBox, type ComparisonViewBoxSpec, type ComparisonViewGraphOrder, type ComparisonViewGraphsSpec, type ComparisonViewGroup, type ComparisonViewGroupSpec, type ComparisonViewGroupTitle, type ComparisonViewGroupWithScenariosSpec, type ComparisonViewGroupWithViewsSpec, type ComparisonViewItemSubtitle, type ComparisonViewItemTitle, type ComparisonViewRow, type ComparisonViewRowSpec, type ComparisonViewRowSubtitle, type ComparisonViewRowTitle, type ComparisonViewSpec, type ComparisonViewSubtitle, type ComparisonViewTitle, type Config, type ConfigInitOptions, type ConfigOptions, type ConstantOverride, type DataSource, type Dataset, type DatasetGroupName, type DatasetKey, type DatasetMap, type DatasetsResult, type DiffPoint, type DiffReport, type DiffValidity, type EncodedImplVars, type EncodedSubscript, type EncodedVarInstance, type EncodedVarType, type EncodedVariable, type GetDatasetsOptions, type GraphComparisonDatasetReport, type GraphComparisonMetadataReport, type GraphComparisonReport, type GraphInclusion, type ImplVar, type ImplVarGroup, type InputAliasName, type InputGroupName, type InputId, type InputPosition, type InputSetting, type InputSettingGroupId, type InputSettingsSpec, type InputVar, type LegendItem, type LinkItem, type LoadedBundle, type LookupOverride, type ModelSpec, type NamedBundle, type OutputVar, type PerfReport, PerfStats, type PositionSetting, type RelatedItem, type RunPerfCallbacks, type RunPerfOptions, type RunSuiteCallbacks, type RunSuiteOptions, type ScenarioSpec, type ScenarioSpecUid, type SliderInputVar, type SourceName, type SuiteReport, type SuiteSummary, type SwitchInputVar, type RunTraceCallbacks as TraceCallbacks, type TraceCompareToBundleOptions, type TraceCompareToExtDataOptions, type TraceDatasetReport, type TraceOptions, type TraceReport, type ValueSetting, type VarId, categorizeComparisonTestSummaries, checkReportFromSummary, checkSummaryFromReport, comparisonSummaryFromReport, createCheckDataCoordinator, createCheckDataCoordinatorForTests, createComparisonDataCoordinator, createConfig, datasetMessage, decodeImplVars, diffDatasets, diffGraphs, encodeImplVars, getScoresForTestSummaries, predicateMessage, runPerf, runSuite, runTrace, scenarioMessage, suiteSummaryFromReport, testSummaryFromReport };
|
package/dist/index.d.ts
CHANGED
|
@@ -1203,16 +1203,34 @@ interface ComparisonDataset {
|
|
|
1203
1203
|
type ComparisonScenarioKey = string & {
|
|
1204
1204
|
_brand?: 'ComparisonScenarioKey';
|
|
1205
1205
|
};
|
|
1206
|
+
/** A fatal error indicating that no input variable matched the requested name. */
|
|
1206
1207
|
interface ComparisonResolverUnknownInputError {
|
|
1207
1208
|
kind: 'unknown-input';
|
|
1208
1209
|
}
|
|
1210
|
+
/** A fatal error indicating that no input setting group matched the requested ID. */
|
|
1209
1211
|
interface ComparisonResolverUnknownInputSettingGroupError {
|
|
1210
1212
|
kind: 'unknown-input-setting-group';
|
|
1211
1213
|
}
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
+
/**
|
|
1215
|
+
* A fatal resolution error. When this is set on an input state, the scenario
|
|
1216
|
+
* cannot be run for the affected side: spec construction for that side is
|
|
1217
|
+
* skipped and downstream UI annotations render it as an error.
|
|
1218
|
+
*/
|
|
1219
|
+
type ComparisonResolverError = ComparisonResolverUnknownInputError | ComparisonResolverUnknownInputSettingGroupError;
|
|
1220
|
+
/**
|
|
1221
|
+
* A non-fatal warning indicating that the resolved value falls outside the
|
|
1222
|
+
* declared `[minValue, maxValue]` range of a slider (or is not one of the
|
|
1223
|
+
* declared values of a switch). The scenario still runs with the requested
|
|
1224
|
+
* value; consumers (e.g. UI annotations) should flag it as a warning.
|
|
1225
|
+
*/
|
|
1226
|
+
interface ComparisonResolverValueOutOfRangeWarning {
|
|
1227
|
+
kind: 'value-out-of-range';
|
|
1214
1228
|
}
|
|
1215
|
-
|
|
1229
|
+
/**
|
|
1230
|
+
* A non-fatal resolution warning. Warnings do not prevent the scenario from
|
|
1231
|
+
* running; they are surfaced as annotations alongside the resolved value.
|
|
1232
|
+
*/
|
|
1233
|
+
type ComparisonResolverWarning = ComparisonResolverValueOutOfRangeWarning;
|
|
1216
1234
|
/** Describes the resolution state for a scenario input relative to a specific model. */
|
|
1217
1235
|
interface ComparisonScenarioInputState {
|
|
1218
1236
|
/** The matched input variable; can be undefined if no input matched. */
|
|
@@ -1221,8 +1239,14 @@ interface ComparisonScenarioInputState {
|
|
|
1221
1239
|
position?: InputPosition;
|
|
1222
1240
|
/** The value of the input, for the given position or explicit value. */
|
|
1223
1241
|
value?: number;
|
|
1224
|
-
/** The error info if the input could not be resolved. */
|
|
1242
|
+
/** The fatal error info if the input could not be resolved. */
|
|
1225
1243
|
error?: ComparisonResolverError;
|
|
1244
|
+
/**
|
|
1245
|
+
* Non-fatal advisory info about the resolved input. When set (without an
|
|
1246
|
+
* accompanying `error`), the input still resolved and the scenario can run;
|
|
1247
|
+
* consumers should surface the warning alongside the resolved value.
|
|
1248
|
+
*/
|
|
1249
|
+
warning?: ComparisonResolverWarning;
|
|
1226
1250
|
}
|
|
1227
1251
|
/** A scenario input that has been checked against both "left" and "right" model. */
|
|
1228
1252
|
interface ComparisonScenarioInput {
|
|
@@ -1377,15 +1401,50 @@ interface ComparisonViewGroup {
|
|
|
1377
1401
|
views: (ComparisonView | ComparisonUnresolvedView)[];
|
|
1378
1402
|
}
|
|
1379
1403
|
|
|
1404
|
+
/**
|
|
1405
|
+
* A summary of timing samples collected during a performance run.
|
|
1406
|
+
*/
|
|
1380
1407
|
interface PerfReport {
|
|
1408
|
+
/** Minimum sample time, in milliseconds. */
|
|
1381
1409
|
readonly minTime: number;
|
|
1410
|
+
/** Maximum sample time, in milliseconds. */
|
|
1382
1411
|
readonly maxTime: number;
|
|
1412
|
+
/**
|
|
1413
|
+
* Trimmed mean (interquartile mean) computed from the middle 50% of samples,
|
|
1414
|
+
* in milliseconds. This is more robust against outliers than a simple mean.
|
|
1415
|
+
*/
|
|
1383
1416
|
readonly avgTime: number;
|
|
1417
|
+
/** Median (50th percentile) sample time, in milliseconds. */
|
|
1418
|
+
readonly medianTime: number;
|
|
1419
|
+
/** 95th percentile sample time, in milliseconds. */
|
|
1420
|
+
readonly p95Time: number;
|
|
1421
|
+
/** Population standard deviation across all samples, in milliseconds. */
|
|
1422
|
+
readonly stdDev: number;
|
|
1423
|
+
/** All recorded sample times, sorted ascending, in milliseconds. */
|
|
1384
1424
|
readonly allTimes: number[];
|
|
1385
1425
|
}
|
|
1426
|
+
/**
|
|
1427
|
+
* Collect performance timing samples and produce a robust statistical summary.
|
|
1428
|
+
*/
|
|
1386
1429
|
declare class PerfStats {
|
|
1387
1430
|
private readonly times;
|
|
1431
|
+
/**
|
|
1432
|
+
* Record a single run time sample.
|
|
1433
|
+
*
|
|
1434
|
+
* @param timeInMillis The run time in milliseconds.
|
|
1435
|
+
*/
|
|
1388
1436
|
addRun(timeInMillis: number): void;
|
|
1437
|
+
/**
|
|
1438
|
+
* Get the raw run time samples that have been recorded.
|
|
1439
|
+
*
|
|
1440
|
+
* @returns A copy of the recorded run times, in insertion order.
|
|
1441
|
+
*/
|
|
1442
|
+
getTimes(): number[];
|
|
1443
|
+
/**
|
|
1444
|
+
* Produce a `PerfReport` summarizing the recorded samples.
|
|
1445
|
+
*
|
|
1446
|
+
* @returns The summary report.
|
|
1447
|
+
*/
|
|
1389
1448
|
toReport(): PerfReport;
|
|
1390
1449
|
}
|
|
1391
1450
|
|
|
@@ -2122,4 +2181,4 @@ declare function runSuite(config: Config, callbacks: RunSuiteCallbacks, options?
|
|
|
2122
2181
|
*/
|
|
2123
2182
|
declare function suiteSummaryFromReport(suiteReport: SuiteReport, elapsedMillis: number): SuiteSummary;
|
|
2124
2183
|
|
|
2125
|
-
export { type AllInputsSpec, type Bundle, type BundleGraphData, type BundleGraphDatasetSpec, type BundleGraphId, type BundleGraphSpec, type BundleGraphView, type BundleGraphViewOptions, type BundleModel, type CancelRunPerf, type CancelRunSuite, type CancelRunTrace as CancelTrace, type CheckConfig, CheckDataCoordinator, type CheckDataRef, type CheckDataRefKey, type CheckDataRequestKey, type CheckDataset, type CheckDatasetError, type CheckDatasetReport, type CheckGroupReport, type CheckKey, type CheckNameSpec, type CheckOptions, type CheckPredicateOp, type CheckPredicateOpConstantRef, type CheckPredicateOpDataRef, type CheckPredicateOpRef, type CheckPredicateReport, type CheckPredicateSummary, type CheckPredicateTimeOptions, type CheckPredicateTimeRange, type CheckPredicateTimeSingle, type CheckPredicateTimeSpec, type CheckReport, type CheckResult, type CheckResultErrorInfo, type CheckScenario, type CheckScenarioError, type CheckScenarioInputDesc, type CheckScenarioReport, type CheckStatus, type CheckSummary, type CheckTestReport, type ComparisonCategorizedResults, type ComparisonConfig, ComparisonDataCoordinator, type ComparisonDataRequestKey, type ComparisonDataset, type ComparisonDatasetName, type ComparisonDatasetOptions, type ComparisonDatasetSource, type ComparisonDatasetSpec, type ComparisonDatasets, type ComparisonGraphGroup, type ComparisonGraphGroupId, type ComparisonGraphGroupRefSpec, type ComparisonGraphGroupSpec, type ComparisonGraphId, type ComparisonGraphsArraySpec, type ComparisonGraphsPresetSpec, type ComparisonGroup, type ComparisonGroupKey, type ComparisonGroupKind, type ComparisonGroupRoot, type ComparisonGroupScores, type ComparisonGroupSummariesByCategory, type ComparisonGroupSummary, type ComparisonOptions, type ComparisonPlot, type ComparisonReport, type ComparisonReportDetailItem, type ComparisonReportDetailRow, type ComparisonReportOptions, type ComparisonReportSummaryRow, type ComparisonReportSummarySection, type ComparisonResolverError, type
|
|
2184
|
+
export { type AllInputsSpec, type Bundle, type BundleGraphData, type BundleGraphDatasetSpec, type BundleGraphId, type BundleGraphSpec, type BundleGraphView, type BundleGraphViewOptions, type BundleModel, type CancelRunPerf, type CancelRunSuite, type CancelRunTrace as CancelTrace, type CheckConfig, CheckDataCoordinator, type CheckDataRef, type CheckDataRefKey, type CheckDataRequestKey, type CheckDataset, type CheckDatasetError, type CheckDatasetReport, type CheckGroupReport, type CheckKey, type CheckNameSpec, type CheckOptions, type CheckPredicateOp, type CheckPredicateOpConstantRef, type CheckPredicateOpDataRef, type CheckPredicateOpRef, type CheckPredicateReport, type CheckPredicateSummary, type CheckPredicateTimeOptions, type CheckPredicateTimeRange, type CheckPredicateTimeSingle, type CheckPredicateTimeSpec, type CheckReport, type CheckResult, type CheckResultErrorInfo, type CheckScenario, type CheckScenarioError, type CheckScenarioInputDesc, type CheckScenarioReport, type CheckStatus, type CheckSummary, type CheckTestReport, type ComparisonCategorizedResults, type ComparisonConfig, ComparisonDataCoordinator, type ComparisonDataRequestKey, type ComparisonDataset, type ComparisonDatasetName, type ComparisonDatasetOptions, type ComparisonDatasetSource, type ComparisonDatasetSpec, type ComparisonDatasets, type ComparisonGraphGroup, type ComparisonGraphGroupId, type ComparisonGraphGroupRefSpec, type ComparisonGraphGroupSpec, type ComparisonGraphId, type ComparisonGraphsArraySpec, type ComparisonGraphsPresetSpec, type ComparisonGroup, type ComparisonGroupKey, type ComparisonGroupKind, type ComparisonGroupRoot, type ComparisonGroupScores, type ComparisonGroupSummariesByCategory, type ComparisonGroupSummary, type ComparisonOptions, type ComparisonPlot, type ComparisonReport, type ComparisonReportDetailItem, type ComparisonReportDetailRow, type ComparisonReportOptions, type ComparisonReportSummaryRow, type ComparisonReportSummarySection, type ComparisonResolverError, type ComparisonResolverUnknownInputError, type ComparisonResolverUnknownInputSettingGroupError, type ComparisonResolverValueOutOfRangeWarning, type ComparisonResolverWarning, type ComparisonScenario, type ComparisonScenarioAllInputsSettings, type ComparisonScenarioGroup, type ComparisonScenarioGroupId, type ComparisonScenarioGroupRefSpec, type ComparisonScenarioGroupSpec, type ComparisonScenarioGroupTitle, type ComparisonScenarioId, type ComparisonScenarioInput, type ComparisonScenarioInputAtPositionSpec, type ComparisonScenarioInputAtValueSpec, type ComparisonScenarioInputName, type ComparisonScenarioInputPosition, type ComparisonScenarioInputSettings, type ComparisonScenarioInputSpec, type ComparisonScenarioInputState, type ComparisonScenarioKey, type ComparisonScenarioPresetMatrixSpec, type ComparisonScenarioRefSpec, type ComparisonScenarioSettings, type ComparisonScenarioSpec, type ComparisonScenarioSubtitle, type ComparisonScenarioTitle, type ComparisonScenarioTitleSpec, type ComparisonScenarioWithAllInputsSpec, type ComparisonScenarioWithDistinctInputsSpec, type ComparisonScenarioWithInputsSpec, type ComparisonScenarioWithSettingGroupSpec, type ComparisonScenarios, type ComparisonSortMode, type ComparisonSpecs, type ComparisonSpecsSource, type ComparisonSummary, type ComparisonTestReport, type ComparisonTestSummary, type ComparisonUnresolvedScenarioGroupRef, type ComparisonUnresolvedScenarioRef, type ComparisonUnresolvedView, type ComparisonView, type ComparisonViewBox, type ComparisonViewBoxSpec, type ComparisonViewGraphOrder, type ComparisonViewGraphsSpec, type ComparisonViewGroup, type ComparisonViewGroupSpec, type ComparisonViewGroupTitle, type ComparisonViewGroupWithScenariosSpec, type ComparisonViewGroupWithViewsSpec, type ComparisonViewItemSubtitle, type ComparisonViewItemTitle, type ComparisonViewRow, type ComparisonViewRowSpec, type ComparisonViewRowSubtitle, type ComparisonViewRowTitle, type ComparisonViewSpec, type ComparisonViewSubtitle, type ComparisonViewTitle, type Config, type ConfigInitOptions, type ConfigOptions, type ConstantOverride, type DataSource, type Dataset, type DatasetGroupName, type DatasetKey, type DatasetMap, type DatasetsResult, type DiffPoint, type DiffReport, type DiffValidity, type EncodedImplVars, type EncodedSubscript, type EncodedVarInstance, type EncodedVarType, type EncodedVariable, type GetDatasetsOptions, type GraphComparisonDatasetReport, type GraphComparisonMetadataReport, type GraphComparisonReport, type GraphInclusion, type ImplVar, type ImplVarGroup, type InputAliasName, type InputGroupName, type InputId, type InputPosition, type InputSetting, type InputSettingGroupId, type InputSettingsSpec, type InputVar, type LegendItem, type LinkItem, type LoadedBundle, type LookupOverride, type ModelSpec, type NamedBundle, type OutputVar, type PerfReport, PerfStats, type PositionSetting, type RelatedItem, type RunPerfCallbacks, type RunPerfOptions, type RunSuiteCallbacks, type RunSuiteOptions, type ScenarioSpec, type ScenarioSpecUid, type SliderInputVar, type SourceName, type SuiteReport, type SuiteSummary, type SwitchInputVar, type RunTraceCallbacks as TraceCallbacks, type TraceCompareToBundleOptions, type TraceCompareToExtDataOptions, type TraceDatasetReport, type TraceOptions, type TraceReport, type ValueSetting, type VarId, categorizeComparisonTestSummaries, checkReportFromSummary, checkSummaryFromReport, comparisonSummaryFromReport, createCheckDataCoordinator, createCheckDataCoordinatorForTests, createComparisonDataCoordinator, createConfig, datasetMessage, decodeImplVars, diffDatasets, diffGraphs, encodeImplVars, getScoresForTestSummaries, predicateMessage, runPerf, runSuite, runTrace, scenarioMessage, suiteSummaryFromReport, testSummaryFromReport };
|
package/dist/index.js
CHANGED
|
@@ -3756,7 +3756,7 @@ function resolveScenarioForInputSpecs(modelInputsL, modelInputsR, key, id, title
|
|
|
3756
3756
|
};
|
|
3757
3757
|
}
|
|
3758
3758
|
function resolveScenarioForDistinctInputSpecs(modelInputsL, modelInputsR, key, id, title, subtitle, inputSpecsL, inputSpecsR) {
|
|
3759
|
-
const
|
|
3759
|
+
const inputsWithIssues = [];
|
|
3760
3760
|
const settingsL = [];
|
|
3761
3761
|
const settingsR = [];
|
|
3762
3762
|
function resolveInputSpec(side, modelInputs, inputSpec) {
|
|
@@ -3771,13 +3771,14 @@ function resolveScenarioForDistinctInputSpecs(modelInputsL, modelInputsR, key, i
|
|
|
3771
3771
|
default:
|
|
3772
3772
|
assertNever10(inputSpec);
|
|
3773
3773
|
}
|
|
3774
|
-
if (inputState.error !== void 0) {
|
|
3775
|
-
|
|
3774
|
+
if (inputState.error !== void 0 || inputState.warning !== void 0) {
|
|
3775
|
+
inputsWithIssues.push({
|
|
3776
3776
|
requestedName: inputSpec.inputName,
|
|
3777
3777
|
stateL: side === "left" ? inputState : {},
|
|
3778
3778
|
stateR: side === "right" ? inputState : {}
|
|
3779
3779
|
});
|
|
3780
|
-
}
|
|
3780
|
+
}
|
|
3781
|
+
if (inputState.error === void 0 && inputState.inputVar !== void 0) {
|
|
3781
3782
|
const inputSetting = inputSettingFromResolvedInputState(inputState);
|
|
3782
3783
|
if (side === "left") {
|
|
3783
3784
|
settingsL.push(inputSetting);
|
|
@@ -3788,9 +3789,12 @@ function resolveScenarioForDistinctInputSpecs(modelInputsL, modelInputsR, key, i
|
|
|
3788
3789
|
}
|
|
3789
3790
|
inputSpecsL.forEach((inputSpec) => resolveInputSpec("left", modelInputsL, inputSpec));
|
|
3790
3791
|
inputSpecsR.forEach((inputSpec) => resolveInputSpec("right", modelInputsR, inputSpec));
|
|
3792
|
+
const hasFatalError = inputsWithIssues.some(
|
|
3793
|
+
(input) => input.stateL.error !== void 0 || input.stateR.error !== void 0
|
|
3794
|
+
);
|
|
3791
3795
|
let specL;
|
|
3792
3796
|
let specR;
|
|
3793
|
-
if (
|
|
3797
|
+
if (!hasFatalError) {
|
|
3794
3798
|
specL = inputSettingsSpec(settingsL);
|
|
3795
3799
|
specR = inputSettingsSpec(settingsR);
|
|
3796
3800
|
}
|
|
@@ -3802,7 +3806,7 @@ function resolveScenarioForDistinctInputSpecs(modelInputsL, modelInputsR, key, i
|
|
|
3802
3806
|
subtitle,
|
|
3803
3807
|
settings: {
|
|
3804
3808
|
kind: "input-settings",
|
|
3805
|
-
inputs:
|
|
3809
|
+
inputs: inputsWithIssues
|
|
3806
3810
|
},
|
|
3807
3811
|
specL,
|
|
3808
3812
|
specR
|
|
@@ -3969,8 +3973,10 @@ function resolveInputVarAtValue(inputVar, value) {
|
|
|
3969
3973
|
};
|
|
3970
3974
|
} else {
|
|
3971
3975
|
return {
|
|
3972
|
-
|
|
3973
|
-
|
|
3976
|
+
inputVar,
|
|
3977
|
+
value,
|
|
3978
|
+
warning: {
|
|
3979
|
+
kind: "value-out-of-range"
|
|
3974
3980
|
}
|
|
3975
3981
|
};
|
|
3976
3982
|
}
|
|
@@ -4576,34 +4582,76 @@ function handleRenames(origCurrentBundle, renamedDatasetKeys) {
|
|
|
4576
4582
|
import { assertNever as assertNever11 } from "assert-never";
|
|
4577
4583
|
|
|
4578
4584
|
// src/perf/perf-stats.ts
|
|
4585
|
+
function percentile(sorted, p) {
|
|
4586
|
+
if (sorted.length === 1) {
|
|
4587
|
+
return sorted[0];
|
|
4588
|
+
}
|
|
4589
|
+
const rank = p * (sorted.length - 1);
|
|
4590
|
+
const lo = Math.floor(rank);
|
|
4591
|
+
const hi = Math.ceil(rank);
|
|
4592
|
+
if (lo === hi) {
|
|
4593
|
+
return sorted[lo];
|
|
4594
|
+
}
|
|
4595
|
+
const frac = rank - lo;
|
|
4596
|
+
return sorted[lo] + (sorted[hi] - sorted[lo]) * frac;
|
|
4597
|
+
}
|
|
4579
4598
|
var PerfStats = class {
|
|
4580
4599
|
constructor() {
|
|
4581
4600
|
this.times = [];
|
|
4582
4601
|
}
|
|
4602
|
+
/**
|
|
4603
|
+
* Record a single run time sample.
|
|
4604
|
+
*
|
|
4605
|
+
* @param timeInMillis The run time in milliseconds.
|
|
4606
|
+
*/
|
|
4583
4607
|
addRun(timeInMillis) {
|
|
4584
4608
|
this.times.push(timeInMillis);
|
|
4585
4609
|
}
|
|
4610
|
+
/**
|
|
4611
|
+
* Get the raw run time samples that have been recorded.
|
|
4612
|
+
*
|
|
4613
|
+
* @returns A copy of the recorded run times, in insertion order.
|
|
4614
|
+
*/
|
|
4615
|
+
getTimes() {
|
|
4616
|
+
return this.times.slice();
|
|
4617
|
+
}
|
|
4618
|
+
/**
|
|
4619
|
+
* Produce a `PerfReport` summarizing the recorded samples.
|
|
4620
|
+
*
|
|
4621
|
+
* @returns The summary report.
|
|
4622
|
+
*/
|
|
4586
4623
|
toReport() {
|
|
4587
4624
|
if (this.times.length === 0) {
|
|
4588
4625
|
return {
|
|
4589
4626
|
minTime: 0,
|
|
4590
4627
|
maxTime: 0,
|
|
4591
4628
|
avgTime: 0,
|
|
4629
|
+
medianTime: 0,
|
|
4630
|
+
p95Time: 0,
|
|
4631
|
+
stdDev: 0,
|
|
4592
4632
|
allTimes: []
|
|
4593
4633
|
};
|
|
4594
4634
|
}
|
|
4595
|
-
const
|
|
4596
|
-
const
|
|
4597
|
-
const
|
|
4598
|
-
const
|
|
4599
|
-
const
|
|
4635
|
+
const sortedTimes = this.times.slice().sort((a, b) => a - b);
|
|
4636
|
+
const n = sortedTimes.length;
|
|
4637
|
+
const minTime = sortedTimes[0];
|
|
4638
|
+
const maxTime = sortedTimes[n - 1];
|
|
4639
|
+
const minIndex = Math.floor(n / 4);
|
|
4640
|
+
const maxIndex = minIndex + Math.max(1, Math.ceil(n / 2));
|
|
4600
4641
|
const middleTimes = sortedTimes.slice(minIndex, maxIndex);
|
|
4601
|
-
const
|
|
4602
|
-
const
|
|
4642
|
+
const avgTime = middleTimes.reduce((a, b) => a + b, 0) / middleTimes.length;
|
|
4643
|
+
const medianTime = percentile(sortedTimes, 0.5);
|
|
4644
|
+
const p95Time = percentile(sortedTimes, 0.95);
|
|
4645
|
+
const mean = sortedTimes.reduce((a, b) => a + b, 0) / n;
|
|
4646
|
+
const variance = sortedTimes.reduce((acc, t) => acc + (t - mean) * (t - mean), 0) / n;
|
|
4647
|
+
const stdDev = Math.sqrt(variance);
|
|
4603
4648
|
return {
|
|
4604
4649
|
minTime,
|
|
4605
4650
|
maxTime,
|
|
4606
4651
|
avgTime,
|
|
4652
|
+
medianTime,
|
|
4653
|
+
p95Time,
|
|
4654
|
+
stdDev,
|
|
4607
4655
|
allTimes: sortedTimes
|
|
4608
4656
|
};
|
|
4609
4657
|
}
|