@sdeverywhere/check-core 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1790 -664
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +625 -284
- package/dist/index.js +1785 -656
- package/dist/index.js.map +1 -1
- package/package.json +5 -4
- package/schema/comparison.schema.json +456 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
declare type SourceName = string;
|
|
2
2
|
declare type VarId = string;
|
|
3
|
-
declare type ScenarioKey = string;
|
|
4
|
-
declare type ScenarioGroupKey = string;
|
|
5
3
|
declare type DatasetKey = string;
|
|
6
4
|
declare type Dataset = Map<number, number>;
|
|
7
5
|
declare type DatasetMap = Map<DatasetKey, Dataset>;
|
|
8
6
|
|
|
7
|
+
/** A unique identifier for the scenario, derived from its input settings. */
|
|
8
|
+
declare type ScenarioSpecUid = string;
|
|
9
9
|
declare type InputPosition = 'at-default' | 'at-minimum' | 'at-maximum';
|
|
10
10
|
interface PositionSetting {
|
|
11
11
|
kind: 'position';
|
|
@@ -18,38 +18,17 @@ interface ValueSetting {
|
|
|
18
18
|
value: number;
|
|
19
19
|
}
|
|
20
20
|
declare type InputSetting = PositionSetting | ValueSetting;
|
|
21
|
-
interface
|
|
22
|
-
kind: 'settings';
|
|
23
|
-
|
|
24
|
-
groupKey: ScenarioGroupKey;
|
|
21
|
+
interface InputSettingsSpec {
|
|
22
|
+
kind: 'input-settings';
|
|
23
|
+
uid: ScenarioSpecUid;
|
|
25
24
|
settings: InputSetting[];
|
|
26
25
|
}
|
|
27
|
-
interface
|
|
26
|
+
interface AllInputsSpec {
|
|
28
27
|
kind: 'all-inputs';
|
|
29
|
-
|
|
30
|
-
groupKey: ScenarioGroupKey;
|
|
28
|
+
uid: ScenarioSpecUid;
|
|
31
29
|
position: InputPosition;
|
|
32
30
|
}
|
|
33
|
-
declare type
|
|
34
|
-
declare function positionSetting(inputVarId: VarId, position: InputPosition): InputSetting;
|
|
35
|
-
declare function valueSetting(inputVarId: VarId, value: number): InputSetting;
|
|
36
|
-
declare function settingsScenario(key: ScenarioKey, groupKey: ScenarioGroupKey, settings: InputSetting[]): Scenario;
|
|
37
|
-
declare function inputAtPositionScenario(inputVarId: VarId, groupKey: ScenarioGroupKey, position: InputPosition): Scenario;
|
|
38
|
-
declare function inputAtValueScenario(inputVarId: VarId, groupKey: ScenarioGroupKey, value: number): Scenario;
|
|
39
|
-
declare function allInputsAtPositionScenario(position: InputPosition): Scenario;
|
|
40
|
-
/**
|
|
41
|
-
* Return an array of scenarios that can be used to run the model
|
|
42
|
-
* with a matrix of output/input scenarios.
|
|
43
|
-
*
|
|
44
|
-
* For each output variable, run the model:
|
|
45
|
-
* - once with all inputs at their default
|
|
46
|
-
* - once with all inputs at their minimum
|
|
47
|
-
* - once with all inputs at their maximum
|
|
48
|
-
* - twice for each input
|
|
49
|
-
* - once with single input at its minimum
|
|
50
|
-
* - once with single input at its maximum
|
|
51
|
-
*/
|
|
52
|
-
declare function matrixScenarios(inputVarIds: VarId[]): Scenario[];
|
|
31
|
+
declare type ScenarioSpec = InputSettingsSpec | AllInputsSpec;
|
|
53
32
|
|
|
54
33
|
interface DatasetsResult {
|
|
55
34
|
/**
|
|
@@ -64,7 +43,7 @@ interface DatasetsResult {
|
|
|
64
43
|
}
|
|
65
44
|
interface DataSource {
|
|
66
45
|
/** Return the datasets that result from running the given scenario. */
|
|
67
|
-
getDatasetsForScenario(
|
|
46
|
+
getDatasetsForScenario(scenarioSpec: ScenarioSpec, datasetKeys: DatasetKey[]): Promise<DatasetsResult>;
|
|
68
47
|
}
|
|
69
48
|
|
|
70
49
|
/**
|
|
@@ -76,24 +55,58 @@ interface RelatedItem {
|
|
|
76
55
|
id: string;
|
|
77
56
|
locationPath: string[];
|
|
78
57
|
}
|
|
58
|
+
/** A unique, stable input identifier. */
|
|
59
|
+
declare type InputId = string;
|
|
79
60
|
/**
|
|
80
61
|
* Holds information about an input variable used in the model.
|
|
81
62
|
*/
|
|
82
63
|
interface InputVar {
|
|
64
|
+
/**
|
|
65
|
+
* Whether this input is controlled by a continuous range/slider or a discrete on/off switch.
|
|
66
|
+
* If undefined, 'slider' will be assumed.
|
|
67
|
+
*/
|
|
68
|
+
kind?: 'slider' | 'switch';
|
|
69
|
+
/**
|
|
70
|
+
* A unique, stable identifier string for this input.
|
|
71
|
+
*
|
|
72
|
+
* This can be used to identify an input variable in a way that is resilient
|
|
73
|
+
* to the variable's name being changed between two versions of the model.
|
|
74
|
+
*
|
|
75
|
+
* For example, if both the "left" and "right" versions of the model have an
|
|
76
|
+
* input with an `inputId` of 2, but the variable is called "Variable 2" in
|
|
77
|
+
* the left and "Variable Two" in the right, the inputs can be correlated and
|
|
78
|
+
* compared despite the different variable names.
|
|
79
|
+
*/
|
|
80
|
+
inputId: InputId;
|
|
81
|
+
/** The variable identifier (typically a simplified/canonical ID, like the form used in SDE). */
|
|
83
82
|
varId: VarId;
|
|
83
|
+
/** The full variable name as used in the modeling tool. */
|
|
84
84
|
varName: string;
|
|
85
|
+
/** The default value of the input. */
|
|
85
86
|
defaultValue: number;
|
|
87
|
+
/** The minimum value of the input. */
|
|
86
88
|
minValue: number;
|
|
89
|
+
/** The maximum value of the input. */
|
|
87
90
|
maxValue: number;
|
|
91
|
+
/** The metadata for the related input control. */
|
|
88
92
|
relatedItem?: RelatedItem;
|
|
89
93
|
}
|
|
90
94
|
/**
|
|
91
95
|
* Holds information about an output variable used in the model.
|
|
92
96
|
*/
|
|
93
97
|
interface OutputVar {
|
|
98
|
+
/** The unique dataset key for this variable (it should include `sourceName` and `varId`). */
|
|
99
|
+
datasetKey: DatasetKey;
|
|
100
|
+
/**
|
|
101
|
+
* The source for the variable (e.g., undefined for a normal model output, "Data" for a variable
|
|
102
|
+
* that is defined in an external data file).
|
|
103
|
+
*/
|
|
94
104
|
sourceName?: SourceName;
|
|
105
|
+
/** The variable identifier (typically a simplified/canonical ID, like the form used in SDE). */
|
|
95
106
|
varId: VarId;
|
|
107
|
+
/** The full variable name as used in the modeling tool. */
|
|
96
108
|
varName: string;
|
|
109
|
+
/** The metadata for the related visuals/graphs in which this variable is used. */
|
|
97
110
|
relatedItems?: RelatedItem[];
|
|
98
111
|
}
|
|
99
112
|
/**
|
|
@@ -122,7 +135,7 @@ interface Dimension {
|
|
|
122
135
|
interface ImplVar {
|
|
123
136
|
/** The variable identifier, as used in SDE. */
|
|
124
137
|
varId: VarId;
|
|
125
|
-
/** The variable name, as used in
|
|
138
|
+
/** The variable name, as used in the modeling tool. */
|
|
126
139
|
varName: string;
|
|
127
140
|
/** The variable index, used by SDE to reference the value in the generated model. */
|
|
128
141
|
varIndex: number;
|
|
@@ -211,10 +224,12 @@ interface ModelSpec {
|
|
|
211
224
|
outputVars: Map<DatasetKey, OutputVar>;
|
|
212
225
|
/** The map of all variables (both internal and exported) in this version of the model. */
|
|
213
226
|
implVars: Map<DatasetKey, ImplVar>;
|
|
227
|
+
/** The custom input variable aliases defined for this model. */
|
|
228
|
+
inputAliases?: Map<string, VarId>;
|
|
214
229
|
/** The custom input variable groups defined for this model. */
|
|
215
|
-
inputGroups
|
|
230
|
+
inputGroups?: Map<string, InputVar[]>;
|
|
216
231
|
/** The custom dataset (output variable) groups defined for this model. */
|
|
217
|
-
datasetGroups
|
|
232
|
+
datasetGroups?: Map<string, DatasetKey[]>;
|
|
218
233
|
/** The start time (year) for the model. */
|
|
219
234
|
startTime?: number;
|
|
220
235
|
/** The end time (year) for the model. */
|
|
@@ -229,18 +244,13 @@ interface ModelSpec {
|
|
|
229
244
|
interface BundleModel extends DataSource {
|
|
230
245
|
/** The spec for the bundled model. */
|
|
231
246
|
modelSpec: ModelSpec;
|
|
232
|
-
/**
|
|
233
|
-
* Return the set of context graphs to be displayed for the given dataset. If left
|
|
234
|
-
* undefined, the set of graphs will be determined using the advertised graph specs.
|
|
235
|
-
*/
|
|
236
|
-
getGraphsForDataset?(datasetKey: DatasetKey): BundleGraphId[];
|
|
237
247
|
/**
|
|
238
248
|
* Load the data used to display the graph by running the model with inputs
|
|
239
249
|
* configured for the given scenario.
|
|
240
250
|
*/
|
|
241
|
-
getGraphDataForScenario(
|
|
251
|
+
getGraphDataForScenario(scenarioSpec: ScenarioSpec, graphId: BundleGraphId): Promise<BundleGraphData>;
|
|
242
252
|
/** Return the links to be displayed for the graph in the given scenario. */
|
|
243
|
-
getGraphLinksForScenario(
|
|
253
|
+
getGraphLinksForScenario(scenarioSpec: ScenarioSpec, graphId: BundleGraphId): LinkItem[];
|
|
244
254
|
}
|
|
245
255
|
/**
|
|
246
256
|
* Provides access to the model that is contained in this bundle for use in
|
|
@@ -288,7 +298,7 @@ declare class CheckDataCoordinator {
|
|
|
288
298
|
readonly bundleModel: BundleModel;
|
|
289
299
|
private readonly taskQueue;
|
|
290
300
|
constructor(bundleModel: BundleModel);
|
|
291
|
-
requestDataset(requestKey: CheckDataRequestKey,
|
|
301
|
+
requestDataset(requestKey: CheckDataRequestKey, scenarioSpec: ScenarioSpec, datasetKey: DatasetKey, onResponse: (dataset: Dataset) => void): void;
|
|
292
302
|
cancelRequest(key: CheckDataRequestKey): void;
|
|
293
303
|
}
|
|
294
304
|
|
|
@@ -344,8 +354,8 @@ interface CheckScenarioInputDesc {
|
|
|
344
354
|
value?: number;
|
|
345
355
|
}
|
|
346
356
|
interface CheckScenario {
|
|
347
|
-
/** The
|
|
348
|
-
|
|
357
|
+
/** The spec used to configure the model with the matched input(s); can be undefined if input(s) failed to match. */
|
|
358
|
+
spec?: ScenarioSpec;
|
|
349
359
|
/** The name of the associated input group, if any. */
|
|
350
360
|
inputGroupName?: string;
|
|
351
361
|
/** The descriptions of the inputs; if empty, it is an "all inputs" scenario. */
|
|
@@ -355,7 +365,7 @@ interface CheckScenario {
|
|
|
355
365
|
}
|
|
356
366
|
|
|
357
367
|
/**
|
|
358
|
-
* The key type for data references (in the form `<
|
|
368
|
+
* The key type for data references (in the form `<ScenarioUid::DatasetKey>`).
|
|
359
369
|
*/
|
|
360
370
|
declare type CheckDataRefKey = string;
|
|
361
371
|
/**
|
|
@@ -478,176 +488,439 @@ declare function checkSummaryFromReport(checkReport: CheckReport): CheckSummary;
|
|
|
478
488
|
*
|
|
479
489
|
* @param checkConfig The config used to reconstruct the check test structure.
|
|
480
490
|
* @param checkSummary The simplified check summary.
|
|
481
|
-
* @param simplifyScenarios If true, reduce the number of scenarios generated for a `matrix`.
|
|
482
491
|
* @return The converted check report.
|
|
483
492
|
*/
|
|
484
|
-
declare function checkReportFromSummary(checkConfig: CheckConfig, checkSummary: CheckSummary
|
|
493
|
+
declare function checkReportFromSummary(checkConfig: CheckConfig, checkSummary: CheckSummary): CheckReport | undefined;
|
|
485
494
|
|
|
495
|
+
declare type ComparisonScenarioId = string;
|
|
496
|
+
declare type ComparisonScenarioTitle = string;
|
|
497
|
+
declare type ComparisonScenarioSubtitle = string;
|
|
498
|
+
declare type ComparisonScenarioInputName = string;
|
|
499
|
+
declare type ComparisonScenarioInputPosition = 'default' | 'min' | 'max';
|
|
486
500
|
/**
|
|
487
|
-
*
|
|
501
|
+
* Specifies an input that is set to a specific position (default / min / max).
|
|
488
502
|
*/
|
|
489
|
-
interface
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
scenario: Scenario;
|
|
496
|
-
/** The key for the datasets being compared. */
|
|
497
|
-
datasetKey: DatasetKey;
|
|
503
|
+
interface ComparisonScenarioInputAtPositionSpec {
|
|
504
|
+
kind: 'input-at-position';
|
|
505
|
+
/** The requested input name or alias. */
|
|
506
|
+
inputName: ComparisonScenarioInputName;
|
|
507
|
+
/** The requested position of the input. */
|
|
508
|
+
position: ComparisonScenarioInputPosition;
|
|
498
509
|
}
|
|
499
510
|
/**
|
|
500
|
-
*
|
|
511
|
+
* Specifies an input that is set to a specific number value.
|
|
501
512
|
*/
|
|
502
|
-
interface
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
relatedItems: RelatedItem[];
|
|
513
|
+
interface ComparisonScenarioInputAtValueSpec {
|
|
514
|
+
kind: 'input-at-value';
|
|
515
|
+
/** The requested input name or alias. */
|
|
516
|
+
inputName: ComparisonScenarioInputName;
|
|
517
|
+
/** The number value of the input. */
|
|
518
|
+
value: number;
|
|
509
519
|
}
|
|
510
520
|
/**
|
|
511
|
-
*
|
|
512
|
-
*
|
|
521
|
+
* A single input setting for a scenario. An input can be set to a specific number value,
|
|
522
|
+
* or it can be set to a "position" (default / min / max).
|
|
513
523
|
*/
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
524
|
+
declare type ComparisonScenarioInputSpec = ComparisonScenarioInputAtPositionSpec | ComparisonScenarioInputAtValueSpec;
|
|
525
|
+
/**
|
|
526
|
+
* Specifies a single scenario that sets one or more inputs to a value/position.
|
|
527
|
+
*/
|
|
528
|
+
interface ComparisonScenarioWithInputsSpec {
|
|
529
|
+
kind: 'scenario-with-inputs';
|
|
530
|
+
/** The unique identifier for the scenario. */
|
|
531
|
+
id?: ComparisonScenarioId;
|
|
532
|
+
/** The title of the scenario. */
|
|
533
|
+
title?: ComparisonScenarioTitle;
|
|
534
|
+
/** The subtitle of the scenario. */
|
|
535
|
+
subtitle?: ComparisonScenarioSubtitle;
|
|
536
|
+
/** The input settings for this scenario. */
|
|
537
|
+
inputs: ComparisonScenarioInputSpec[];
|
|
519
538
|
}
|
|
520
|
-
|
|
521
539
|
/**
|
|
522
|
-
*
|
|
540
|
+
* Specifies a single scenario that sets all available inputs to position.
|
|
523
541
|
*/
|
|
524
|
-
interface
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
relatedItems: RelatedItem[];
|
|
542
|
+
interface ComparisonScenarioWithAllInputsSpec {
|
|
543
|
+
kind: 'scenario-with-all-inputs';
|
|
544
|
+
/** The unique identifier for the scenario. */
|
|
545
|
+
id?: ComparisonScenarioId;
|
|
546
|
+
/** The title of the scenario. */
|
|
547
|
+
title?: ComparisonScenarioTitle;
|
|
548
|
+
/** The subtitle of the scenario. */
|
|
549
|
+
subtitle?: ComparisonScenarioSubtitle;
|
|
550
|
+
/** The position that will be used for all available inputs. */
|
|
551
|
+
position: ComparisonScenarioInputPosition;
|
|
535
552
|
}
|
|
536
553
|
/**
|
|
537
|
-
*
|
|
554
|
+
* Special preset that expands to many scenarios:
|
|
555
|
+
* - one scenario with all inputs at their default
|
|
556
|
+
* - two scenarios for each available input:
|
|
557
|
+
* - one scenario with the input at its minimum
|
|
558
|
+
* - one scenario with the input at its maximum
|
|
538
559
|
*/
|
|
539
|
-
interface
|
|
560
|
+
interface ComparisonScenarioPresetMatrixSpec {
|
|
561
|
+
kind: 'scenario-matrix';
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* A definition of input scenario(s). A scenario can set one input to a value/position, or it
|
|
565
|
+
* can set multiple inputs to particular values/positions.
|
|
566
|
+
*/
|
|
567
|
+
declare type ComparisonScenarioSpec = ComparisonScenarioWithInputsSpec | ComparisonScenarioWithAllInputsSpec | ComparisonScenarioPresetMatrixSpec;
|
|
568
|
+
/** A reference to a scenario definition. */
|
|
569
|
+
interface ComparisonScenarioRefSpec {
|
|
570
|
+
kind: 'scenario-ref';
|
|
571
|
+
/** The ID of the scenario that is referenced. */
|
|
572
|
+
scenarioId: ComparisonScenarioId;
|
|
573
|
+
/** The optional title that is used instead of the referenced scenario's title. */
|
|
574
|
+
title?: ComparisonScenarioTitle;
|
|
575
|
+
/** The optional subtitle that is used instead of the referenced scenario's subtitle. */
|
|
576
|
+
subtitle?: ComparisonScenarioSubtitle;
|
|
577
|
+
}
|
|
578
|
+
declare type ComparisonScenarioGroupId = string;
|
|
579
|
+
declare type ComparisonScenarioGroupTitle = string;
|
|
580
|
+
/**
|
|
581
|
+
* A definition of a group of input scenarios. Multiple scenarios can be grouped together under a single name, and
|
|
582
|
+
* can later be referenced by group ID in a view definition.
|
|
583
|
+
*/
|
|
584
|
+
interface ComparisonScenarioGroupSpec {
|
|
585
|
+
kind: 'scenario-group';
|
|
586
|
+
/** The unique identifier for the group. */
|
|
587
|
+
id?: ComparisonScenarioGroupId;
|
|
588
|
+
/** The title of the group. */
|
|
589
|
+
title: ComparisonScenarioGroupTitle;
|
|
590
|
+
/** The scenarios that are included in this group. */
|
|
591
|
+
scenarios: (ComparisonScenarioSpec | ComparisonScenarioRefSpec)[];
|
|
592
|
+
}
|
|
593
|
+
/** A reference to a scenario group definition. */
|
|
594
|
+
interface ComparisonScenarioGroupRefSpec {
|
|
595
|
+
kind: 'scenario-group-ref';
|
|
596
|
+
/** The ID of the scenario group that is referenced. */
|
|
597
|
+
groupId: ComparisonScenarioGroupId;
|
|
598
|
+
}
|
|
599
|
+
declare type ComparisonViewTitle = string;
|
|
600
|
+
declare type ComparisonViewSubtitle = string;
|
|
601
|
+
declare type ComparisonViewGraphId = string;
|
|
602
|
+
/**
|
|
603
|
+
* Specifies a list of graphs to be shown in a view.
|
|
604
|
+
*/
|
|
605
|
+
interface ComparisonViewGraphsArraySpec {
|
|
606
|
+
kind: 'graphs-array';
|
|
607
|
+
/** The array of IDs for graphs to show. */
|
|
608
|
+
graphIds: ComparisonViewGraphId[];
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Specifies a preset list of graphs to be shown in a view.
|
|
612
|
+
*/
|
|
613
|
+
interface ComparisonViewGraphsPresetSpec {
|
|
614
|
+
kind: 'graphs-preset';
|
|
615
|
+
/** The preset (currently only "all" is supported, which shows all available graphs). */
|
|
616
|
+
preset: 'all';
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Specifies a set of graphs to be shown in a view.
|
|
620
|
+
*/
|
|
621
|
+
declare type ComparisonViewGraphsSpec = ComparisonViewGraphsArraySpec | ComparisonViewGraphsPresetSpec;
|
|
622
|
+
/**
|
|
623
|
+
* A definition of a view. A view presents a set of graphs for a single input scenario.
|
|
624
|
+
*/
|
|
625
|
+
interface ComparisonViewSpec {
|
|
626
|
+
kind: 'view';
|
|
627
|
+
/** The title of the view. If undefined, the title will be inferred from the scenario. */
|
|
628
|
+
title?: ComparisonViewTitle;
|
|
629
|
+
/** The subtitle of the view. If undefined, the subtitle will be inferred from the scenario. */
|
|
630
|
+
subtitle?: ComparisonViewGroupTitle;
|
|
631
|
+
/** The scenario to be shown in the view. */
|
|
632
|
+
scenarioId: ComparisonScenarioId;
|
|
633
|
+
/** The graphs to be shown for each scenario view. */
|
|
634
|
+
graphs: ComparisonViewGraphsSpec;
|
|
635
|
+
}
|
|
636
|
+
declare type ComparisonViewGroupTitle = string;
|
|
637
|
+
/**
|
|
638
|
+
* Specifies a view group with an explicit array of view definitions.
|
|
639
|
+
*/
|
|
640
|
+
interface ComparisonViewGroupWithViewsSpec {
|
|
641
|
+
kind: 'view-group-with-views';
|
|
642
|
+
/** The title of the group of views. */
|
|
643
|
+
title: ComparisonViewGroupTitle;
|
|
644
|
+
/** The views that are included in this group. */
|
|
645
|
+
views: ComparisonViewSpec[];
|
|
646
|
+
}
|
|
647
|
+
/**
|
|
648
|
+
* Specifies a view group by declaring the scenarios included in the group (one view per scenario), along
|
|
649
|
+
* with a set of graphs that will shown in each view.
|
|
650
|
+
*/
|
|
651
|
+
interface ComparisonViewGroupWithScenariosSpec {
|
|
652
|
+
kind: 'view-group-with-scenarios';
|
|
653
|
+
/** The title of the group of views. */
|
|
654
|
+
title: ComparisonViewGroupTitle;
|
|
655
|
+
/** The scenarios to be included (one view will be created for each scenario). */
|
|
656
|
+
scenarios: (ComparisonScenarioRefSpec | ComparisonScenarioGroupRefSpec)[];
|
|
657
|
+
/** The graphs to be shown for each scenario view. */
|
|
658
|
+
graphs: ComparisonViewGraphsSpec;
|
|
659
|
+
}
|
|
660
|
+
/**
|
|
661
|
+
* A definition of a group of views. Multiple related views can be grouped together under a single title
|
|
662
|
+
* to make them easy to distinguish in a report.
|
|
663
|
+
*/
|
|
664
|
+
declare type ComparisonViewGroupSpec = ComparisonViewGroupWithViewsSpec | ComparisonViewGroupWithScenariosSpec;
|
|
665
|
+
/**
|
|
666
|
+
* Contains the scenario and view definitions from one or more sources (JSON/YAML files or manually
|
|
667
|
+
* defined specs).
|
|
668
|
+
*/
|
|
669
|
+
interface ComparisonSpecs {
|
|
670
|
+
/** The requested scenarios. */
|
|
671
|
+
scenarios: ComparisonScenarioSpec[];
|
|
672
|
+
/** The requested scenario groups. */
|
|
673
|
+
scenarioGroups: ComparisonScenarioGroupSpec[];
|
|
674
|
+
/** The requested view groups. */
|
|
675
|
+
viewGroups: ComparisonViewGroupSpec[];
|
|
676
|
+
}
|
|
677
|
+
/** A source of comparison scenario and specifications. */
|
|
678
|
+
interface ComparisonSpecsSource {
|
|
679
|
+
kind: 'yaml' | 'json';
|
|
680
|
+
/** The source filename, if known. */
|
|
681
|
+
filename?: string;
|
|
682
|
+
/** A string containing YAML or JSON content. */
|
|
683
|
+
content: string;
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
/** A resolved dataset that is being compared. */
|
|
687
|
+
interface ComparisonDataset {
|
|
688
|
+
kind: 'dataset';
|
|
689
|
+
/** The unique key for the dataset (i.e., output variable or static data). */
|
|
690
|
+
key: DatasetKey;
|
|
691
|
+
/**
|
|
692
|
+
* The resolved output variable from the "left" model that corresponds to this dataset,
|
|
693
|
+
* or undefined if the variable is not defined in the left model.
|
|
694
|
+
*/
|
|
695
|
+
outputVarL?: OutputVar;
|
|
696
|
+
/**
|
|
697
|
+
* The resolved output variable from the "right" model that corresponds to this dataset,
|
|
698
|
+
* or undefined if the variable is not defined in the right model.
|
|
699
|
+
*/
|
|
700
|
+
outputVarR?: OutputVar;
|
|
701
|
+
}
|
|
702
|
+
/** A unique key for a `ComparisonScenario`, generated internally for use by the library. */
|
|
703
|
+
declare type ComparisonScenarioKey = string & {
|
|
704
|
+
_brand?: 'ComparisonScenarioKey';
|
|
705
|
+
};
|
|
706
|
+
interface ComparisonResolverUnknownInputError {
|
|
707
|
+
kind: 'unknown-input';
|
|
708
|
+
}
|
|
709
|
+
interface ComparisonResolverInvalidValueError {
|
|
710
|
+
kind: 'invalid-value';
|
|
711
|
+
}
|
|
712
|
+
declare type ComparisonResolverError = ComparisonResolverUnknownInputError | ComparisonResolverInvalidValueError;
|
|
713
|
+
/** Describes the resolution state for a scenario input relative to a specific model. */
|
|
714
|
+
interface ComparisonScenarioInputState {
|
|
715
|
+
/** The matched input variable; can be undefined if no input matched. */
|
|
716
|
+
inputVar?: InputVar;
|
|
717
|
+
/** The position of the input, if this is a position scenario. */
|
|
718
|
+
position?: InputPosition;
|
|
719
|
+
/** The value of the input, for the given position or explicit value. */
|
|
720
|
+
value?: number;
|
|
721
|
+
/** The error info if the input could not be resolved. */
|
|
722
|
+
error?: ComparisonResolverError;
|
|
723
|
+
}
|
|
724
|
+
/** A scenario input that has been checked against both "left" and "right" model. */
|
|
725
|
+
interface ComparisonScenarioInput {
|
|
726
|
+
/** The requested name of the input. */
|
|
727
|
+
requestedName: string;
|
|
728
|
+
/** The resolved state of the input for the "left" model. */
|
|
729
|
+
stateL: ComparisonScenarioInputState;
|
|
730
|
+
/** The resolved state of the input for the "right" model. */
|
|
731
|
+
stateR: ComparisonScenarioInputState;
|
|
732
|
+
}
|
|
733
|
+
/** A configuration that sets model inputs to specific values. */
|
|
734
|
+
interface ComparisonScenarioInputSettings {
|
|
735
|
+
kind: 'input-settings';
|
|
736
|
+
/** The resolutions for the specified inputs in the scenario. */
|
|
737
|
+
inputs: ComparisonScenarioInput[];
|
|
738
|
+
}
|
|
739
|
+
/** A configuration that sets all inputs in the model to a certain position. */
|
|
740
|
+
interface ComparisonScenarioAllInputsSettings {
|
|
741
|
+
kind: 'all-inputs-settings';
|
|
742
|
+
/** The input position that will be applied to all available inputs. */
|
|
743
|
+
position: InputPosition;
|
|
744
|
+
}
|
|
745
|
+
/**
|
|
746
|
+
* The configuration for an input scenario, either a set of individual input settings, or one
|
|
747
|
+
* that sets all inputs in the model to a certain position.
|
|
748
|
+
*/
|
|
749
|
+
declare type ComparisonScenarioSettings = ComparisonScenarioInputSettings | ComparisonScenarioAllInputsSettings;
|
|
750
|
+
/** A single resolved input scenario. */
|
|
751
|
+
interface ComparisonScenario {
|
|
752
|
+
kind: 'scenario';
|
|
753
|
+
/** The unique key for the scenario, generated internally for use by the library. */
|
|
754
|
+
key: ComparisonScenarioKey;
|
|
755
|
+
/** The unique user-defined identifier for the scenario. */
|
|
756
|
+
id?: ComparisonScenarioId;
|
|
540
757
|
/** The scenario title. */
|
|
541
758
|
title: string;
|
|
542
759
|
/** The scenario subtitle. */
|
|
543
760
|
subtitle?: string;
|
|
761
|
+
/** The resolved settings for the model inputs in this scenario. */
|
|
762
|
+
settings: ComparisonScenarioSettings;
|
|
763
|
+
/** The input scenario used to configure the "left" model, or undefined if data not available. */
|
|
764
|
+
specL?: ScenarioSpec;
|
|
765
|
+
/** The input scenario used to configure the "right" model, or undefined if data not available. */
|
|
766
|
+
specR?: ScenarioSpec;
|
|
767
|
+
}
|
|
768
|
+
/** An unresolved input scenario reference. */
|
|
769
|
+
interface ComparisonUnresolvedScenarioRef {
|
|
770
|
+
kind: 'unresolved-scenario-ref';
|
|
771
|
+
/** The ID of the referenced scenario that could not be resolved. */
|
|
772
|
+
scenarioId: ComparisonScenarioId;
|
|
773
|
+
}
|
|
774
|
+
/** A resolved group of input scenarios. */
|
|
775
|
+
interface ComparisonScenarioGroup {
|
|
776
|
+
kind: 'scenario-group';
|
|
777
|
+
/** The unique identifier for the group. */
|
|
778
|
+
id?: ComparisonScenarioGroupId;
|
|
779
|
+
/** The title of the group. */
|
|
780
|
+
title: ComparisonScenarioGroupTitle;
|
|
544
781
|
/**
|
|
545
|
-
* The
|
|
546
|
-
*
|
|
782
|
+
* The scenarios that are included in this group. This includes scenario that were successfully
|
|
783
|
+
* resolved as well as scenario references that could not be resolved.
|
|
547
784
|
*/
|
|
548
|
-
|
|
785
|
+
scenarios: (ComparisonScenario | ComparisonUnresolvedScenarioRef)[];
|
|
786
|
+
}
|
|
787
|
+
/** An unresolved scenario group reference. */
|
|
788
|
+
interface ComparisonUnresolvedScenarioGroupRef {
|
|
789
|
+
kind: 'unresolved-scenario-group-ref';
|
|
790
|
+
/** The ID of the referenced scenario group that could not be resolved. */
|
|
791
|
+
scenarioGroupId: ComparisonScenarioGroupId;
|
|
792
|
+
}
|
|
793
|
+
/** A resolved view definition. A view presents a set of graphs for a single input scenario. */
|
|
794
|
+
interface ComparisonView {
|
|
795
|
+
kind: 'view';
|
|
796
|
+
/** The title of the view. */
|
|
797
|
+
title: ComparisonViewTitle;
|
|
798
|
+
/** The subtitle of the view. */
|
|
799
|
+
subtitle?: ComparisonViewSubtitle;
|
|
800
|
+
/** The resolved scenario to be shown in the view. */
|
|
801
|
+
scenario: ComparisonScenario;
|
|
802
|
+
/** The graphs to be shown for each scenario view. */
|
|
803
|
+
graphs: 'all' | ComparisonViewGraphId[];
|
|
804
|
+
}
|
|
805
|
+
/** An unresolved view. */
|
|
806
|
+
interface ComparisonUnresolvedView {
|
|
807
|
+
kind: 'unresolved-view';
|
|
808
|
+
/** The requested title of the view, if provided. */
|
|
809
|
+
title?: ComparisonViewTitle;
|
|
810
|
+
/** The requested subtitle of the view, if provided. */
|
|
811
|
+
subtitle?: ComparisonViewSubtitle;
|
|
812
|
+
/** The ID of the referenced scenario that could not be resolved. */
|
|
813
|
+
scenarioId?: ComparisonScenarioId;
|
|
814
|
+
/** The ID of the referenced scenario group that could not be resolved. */
|
|
815
|
+
scenarioGroupId?: ComparisonScenarioGroupId;
|
|
816
|
+
}
|
|
817
|
+
/** A resolved group of compared scenario/graph views. */
|
|
818
|
+
interface ComparisonViewGroup {
|
|
819
|
+
kind: 'view-group';
|
|
820
|
+
/** The title of the group of views. */
|
|
821
|
+
title: ComparisonViewGroupTitle;
|
|
822
|
+
/** The array of resolved (and unresolved) views that are included in this group. */
|
|
823
|
+
views: (ComparisonView | ComparisonUnresolvedView)[];
|
|
549
824
|
}
|
|
550
825
|
|
|
551
826
|
/**
|
|
552
|
-
* Provides access to the set of
|
|
827
|
+
* Provides access to the set of dataset definitions (`ComparisonDataset` instances) that are used
|
|
828
|
+
* when comparing the two models.
|
|
553
829
|
*/
|
|
554
|
-
interface
|
|
830
|
+
interface ComparisonDatasets {
|
|
555
831
|
/**
|
|
556
|
-
* Return
|
|
832
|
+
* Return all `ComparisonDataset` instances that are available for comparisons.
|
|
557
833
|
*/
|
|
558
|
-
|
|
834
|
+
getAllDatasets(): IterableIterator<ComparisonDataset>;
|
|
559
835
|
/**
|
|
560
|
-
* Return the
|
|
836
|
+
* Return the dataset metadata for the given key.
|
|
561
837
|
*
|
|
562
|
-
* @param
|
|
838
|
+
* @param datasetKey The key for the dataset.
|
|
563
839
|
*/
|
|
564
|
-
|
|
840
|
+
getDataset(datasetKey: DatasetKey): ComparisonDataset | undefined;
|
|
565
841
|
/**
|
|
566
|
-
* Return the
|
|
842
|
+
* Return the keys for the datasets that should be compared for the given scenario.
|
|
567
843
|
*
|
|
568
|
-
* @param
|
|
844
|
+
* @param scenario The scenario definition.
|
|
569
845
|
*/
|
|
570
|
-
|
|
846
|
+
getDatasetKeysForScenario(scenario: ComparisonScenario): DatasetKey[];
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
interface ComparisonScenarios {
|
|
571
850
|
/**
|
|
572
|
-
* Return
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
*
|
|
577
|
-
* used to customize the title and subtitle for that item (for example, it will show
|
|
578
|
-
* the default value of the input associated with the row).
|
|
851
|
+
* Return all `ComparisonScenario` instances that are available for comparisons.
|
|
852
|
+
*/
|
|
853
|
+
getAllScenarios(): IterableIterator<ComparisonScenario>;
|
|
854
|
+
/**
|
|
855
|
+
* Return the scenario definition for the given key.
|
|
579
856
|
*
|
|
580
|
-
* @param
|
|
581
|
-
* @param groupKey The key for the group in which the scenario will be displayed.
|
|
857
|
+
* @param key The key for the scenario.
|
|
582
858
|
*/
|
|
583
|
-
|
|
859
|
+
getScenario(key: ComparisonScenarioKey): ComparisonScenario | undefined;
|
|
584
860
|
}
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
*/
|
|
588
|
-
interface CompareDatasets {
|
|
589
|
-
/** The mapping of renamed dataset keys. */
|
|
590
|
-
renamedDatasetKeys?: Map<DatasetKey, DatasetKey>;
|
|
861
|
+
|
|
862
|
+
interface ComparisonDatasetOptions {
|
|
591
863
|
/**
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
* @param scenario The scenario.
|
|
864
|
+
* The mapping of renamed dataset keys (old or "left" name as the map key,
|
|
865
|
+
* new or "right" name as the value).
|
|
595
866
|
*/
|
|
596
|
-
|
|
867
|
+
renamedDatasetKeys?: Map<DatasetKey, DatasetKey>;
|
|
597
868
|
/**
|
|
598
|
-
*
|
|
869
|
+
* An optional function that allows for limiting the datasets that are compared
|
|
870
|
+
* for a given scenario. By default, all datasets are compared for a given
|
|
871
|
+
* scenario, but if a custom function is provided, it can return a subset of
|
|
872
|
+
* datasets (for example, to omit datasets that are not relevant).
|
|
599
873
|
*/
|
|
600
|
-
|
|
874
|
+
datasetKeysForScenario?: (allDatasetKeys: DatasetKey[], scenario: ComparisonScenario) => DatasetKey[];
|
|
601
875
|
}
|
|
602
|
-
interface
|
|
876
|
+
interface ComparisonOptions {
|
|
877
|
+
/** The left-side ("baseline") bundle being compared. */
|
|
603
878
|
baseline: NamedBundle;
|
|
879
|
+
/**
|
|
880
|
+
* The array of thresholds used to color differences, e.g., [1, 5, 10] will use
|
|
881
|
+
* buckets of 0%, 0-1%, 1-5%, 5-10%, and >10%.
|
|
882
|
+
*/
|
|
604
883
|
thresholds: number[];
|
|
605
|
-
|
|
606
|
-
|
|
884
|
+
/**
|
|
885
|
+
* The requested comparison scenario and view specifications. These can be
|
|
886
|
+
* specified in YAML or JSON files, or using `Spec` objects.
|
|
887
|
+
*/
|
|
888
|
+
specs: (ComparisonSpecs | ComparisonSpecsSource)[];
|
|
889
|
+
/** Optional configuration for the datasets that are compared for different scenarios. */
|
|
890
|
+
datasets?: ComparisonDatasetOptions;
|
|
607
891
|
}
|
|
608
|
-
interface
|
|
892
|
+
interface ComparisonConfig {
|
|
893
|
+
/** The loaded left-side ("baseline") bundle being compared. */
|
|
609
894
|
bundleL: LoadedBundle;
|
|
895
|
+
/** The loaded right-side ("current") bundle being compared. */
|
|
610
896
|
bundleR: LoadedBundle;
|
|
897
|
+
/**
|
|
898
|
+
* The array of thresholds used to color differences, e.g., [1, 5, 10] will use
|
|
899
|
+
* buckets of 0%, 0-1%, 1-5%, 5-10%, and >10%.
|
|
900
|
+
*/
|
|
611
901
|
thresholds: number[];
|
|
612
|
-
scenarios
|
|
613
|
-
|
|
902
|
+
/** The set of resolved scenarios that will be compared. */
|
|
903
|
+
scenarios: ComparisonScenarios;
|
|
904
|
+
/** The set of resolved datasets that will be compared. */
|
|
905
|
+
datasets: ComparisonDatasets;
|
|
906
|
+
/** The set of resolved view groups. */
|
|
907
|
+
viewGroups: ComparisonViewGroup[];
|
|
614
908
|
}
|
|
615
909
|
|
|
616
|
-
declare type
|
|
910
|
+
declare type ComparisonDataRequestKey = string;
|
|
617
911
|
/**
|
|
618
912
|
* Coordinates loading of data in parallel from two models.
|
|
619
913
|
*/
|
|
620
|
-
declare class
|
|
914
|
+
declare class ComparisonDataCoordinator {
|
|
621
915
|
readonly bundleModelL: BundleModel;
|
|
622
916
|
readonly bundleModelR: BundleModel;
|
|
623
917
|
private readonly taskQueue;
|
|
624
918
|
constructor(bundleModelL: BundleModel, bundleModelR: BundleModel);
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
interface PerfReport {
|
|
631
|
-
readonly minTime: number;
|
|
632
|
-
readonly maxTime: number;
|
|
633
|
-
readonly avgTime: number;
|
|
634
|
-
readonly allTimes: number[];
|
|
635
|
-
}
|
|
636
|
-
declare class PerfStats {
|
|
637
|
-
private readonly times;
|
|
638
|
-
addRun(timeInMillis: number): void;
|
|
639
|
-
toReport(): PerfReport;
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
interface CompareDatasetReport {
|
|
643
|
-
scenarioKey: ScenarioKey;
|
|
644
|
-
datasetKey: DatasetKey;
|
|
645
|
-
diffReport: DiffReport;
|
|
646
|
-
}
|
|
647
|
-
interface CompareReport {
|
|
648
|
-
datasetReports: CompareDatasetReport[];
|
|
649
|
-
perfReportL: PerfReport;
|
|
650
|
-
perfReportR: PerfReport;
|
|
919
|
+
private processDatasetRequest;
|
|
920
|
+
private processGraphDataRequest;
|
|
921
|
+
requestDatasetMaps(requestKey: ComparisonDataRequestKey, scenarioSpecL: ScenarioSpec, scenarioSpecR: ScenarioSpec, datasetKeys: DatasetKey[], onResponse: (datasetMapL?: DatasetMap, datasetMapR?: DatasetMap) => void): void;
|
|
922
|
+
requestGraphData(requestKey: ComparisonDataRequestKey, scenarioSpecL: ScenarioSpec, scenarioSpecR: ScenarioSpec, graphId: BundleGraphId, onResponse: (graphDataL?: BundleGraphData, graphDataR?: BundleGraphData) => void): void;
|
|
923
|
+
cancelRequest(key: ComparisonDataRequestKey): void;
|
|
651
924
|
}
|
|
652
925
|
|
|
653
926
|
interface DiffPoint {
|
|
@@ -666,43 +939,71 @@ interface DiffReport {
|
|
|
666
939
|
maxDiffPoint: DiffPoint;
|
|
667
940
|
}
|
|
668
941
|
declare function diffDatasets(datasetL: Dataset | undefined, datasetR: Dataset | undefined): DiffReport;
|
|
669
|
-
|
|
942
|
+
|
|
943
|
+
interface PerfReport {
|
|
944
|
+
readonly minTime: number;
|
|
945
|
+
readonly maxTime: number;
|
|
946
|
+
readonly avgTime: number;
|
|
947
|
+
readonly allTimes: number[];
|
|
948
|
+
}
|
|
949
|
+
declare class PerfStats {
|
|
950
|
+
private readonly times;
|
|
951
|
+
addRun(timeInMillis: number): void;
|
|
952
|
+
toReport(): PerfReport;
|
|
953
|
+
}
|
|
670
954
|
|
|
671
955
|
/**
|
|
672
|
-
*
|
|
673
|
-
*
|
|
674
|
-
*
|
|
956
|
+
* The report for a single comparison test (involving a dataset produced under
|
|
957
|
+
* a specific input scenario). This includes the full `DiffReport`, whereas
|
|
958
|
+
* a `ComparisonTestSummary` only includes the `maxDiff` value.
|
|
959
|
+
*/
|
|
960
|
+
interface ComparisonTestReport {
|
|
961
|
+
scenarioKey: ComparisonScenarioKey;
|
|
962
|
+
datasetKey: DatasetKey;
|
|
963
|
+
diffReport: DiffReport;
|
|
964
|
+
}
|
|
965
|
+
/**
|
|
966
|
+
* A simplified/terse version of `ComparisonTestReport` that is used when writing
|
|
967
|
+
* results to a JSON file. The object keys are terse and it only includes the
|
|
968
|
+
* minimum set of fields (only the `maxDiff` value instead of the full `DiffReport`)
|
|
675
969
|
* to keep the file smaller when there are many reported differences.
|
|
676
970
|
*/
|
|
677
|
-
interface
|
|
971
|
+
interface ComparisonTestSummary {
|
|
678
972
|
/** Short for `scenarioKey`. */
|
|
679
|
-
s:
|
|
973
|
+
s: ComparisonScenarioKey;
|
|
680
974
|
/** Short for `datasetKey`. */
|
|
681
975
|
d: DatasetKey;
|
|
682
976
|
/** Short for `maxDiff`. */
|
|
683
977
|
md: number;
|
|
684
978
|
}
|
|
685
979
|
/**
|
|
686
|
-
*
|
|
687
|
-
* format of the JSON objects emitted by the CLI in terse mode.
|
|
980
|
+
* The roll-up report that contains the results of all individual comparison tests.
|
|
688
981
|
*/
|
|
689
|
-
interface
|
|
690
|
-
|
|
982
|
+
interface ComparisonReport {
|
|
983
|
+
/** The set of all comparison test reports. */
|
|
984
|
+
testReports: ComparisonTestReport[];
|
|
985
|
+
/** The perf report for the "left" model. */
|
|
691
986
|
perfReportL: PerfReport;
|
|
987
|
+
/** The perf report for the "right" model. */
|
|
692
988
|
perfReportR: PerfReport;
|
|
693
989
|
}
|
|
694
990
|
/**
|
|
695
|
-
*
|
|
696
|
-
*
|
|
697
|
-
* reported differences.
|
|
698
|
-
*
|
|
699
|
-
* @param compareReport The full compare report.
|
|
700
|
-
* @return The converted compare summary.
|
|
991
|
+
* A simplified/terse version of `ComparisonReport` that only includes the minimum set
|
|
992
|
+
* of fields needed by the reporting app (to keep the file smaller when there are many
|
|
993
|
+
* reported differences). This only includes comparison results for which there is
|
|
994
|
+
* a non-zero `maxDiff` value.
|
|
701
995
|
*/
|
|
702
|
-
|
|
996
|
+
interface ComparisonSummary {
|
|
997
|
+
/** The simplified set of all terse comparison test summaries. */
|
|
998
|
+
testSummaries: ComparisonTestSummary[];
|
|
999
|
+
/** The perf report for the "left" model. */
|
|
1000
|
+
perfReportL: PerfReport;
|
|
1001
|
+
/** The perf report for the "right" model. */
|
|
1002
|
+
perfReportR: PerfReport;
|
|
1003
|
+
}
|
|
703
1004
|
|
|
704
1005
|
declare type GraphInclusion = 'neither' | 'left-only' | 'right-only' | 'both';
|
|
705
|
-
interface
|
|
1006
|
+
interface GraphComparisonMetadataReport {
|
|
706
1007
|
/** The key for the metadata field. */
|
|
707
1008
|
key: string;
|
|
708
1009
|
/** The value of the metadata field in the left bundle. */
|
|
@@ -710,147 +1011,181 @@ interface GraphMetadataReport {
|
|
|
710
1011
|
/** The value of the metadata field in the right bundle. */
|
|
711
1012
|
valueR?: string;
|
|
712
1013
|
}
|
|
713
|
-
interface
|
|
1014
|
+
interface GraphComparisonDatasetReport {
|
|
714
1015
|
/** The dataset key. */
|
|
715
1016
|
datasetKey: DatasetKey;
|
|
716
1017
|
/** The max diff for this dataset. */
|
|
717
1018
|
maxDiff?: number;
|
|
718
1019
|
}
|
|
719
|
-
interface
|
|
1020
|
+
interface GraphComparisonReport {
|
|
720
1021
|
/** Indicates which bundles the graph is defined in. */
|
|
721
1022
|
inclusion: GraphInclusion;
|
|
722
1023
|
/** The metadata fields with differences. */
|
|
723
|
-
metadataReports:
|
|
1024
|
+
metadataReports: GraphComparisonMetadataReport[];
|
|
724
1025
|
/** The datasets with differences. */
|
|
725
|
-
datasetReports:
|
|
1026
|
+
datasetReports: GraphComparisonDatasetReport[];
|
|
726
1027
|
}
|
|
727
1028
|
/**
|
|
728
|
-
*
|
|
1029
|
+
* Comparison the metadata and datasets for the given graphs.
|
|
729
1030
|
*
|
|
730
1031
|
* @param graphL The graph defined in the left bundle.
|
|
731
1032
|
* @param graphR The graph defined in the right bundle.
|
|
732
|
-
* @param scenarioKey The scenario used for comparing datasets.
|
|
733
|
-
* @param
|
|
1033
|
+
* @param scenarioKey The key of the scenario used for comparing datasets.
|
|
1034
|
+
* @param testSummaries The set of test summaries from a previous comparison run.
|
|
734
1035
|
*/
|
|
735
|
-
declare function diffGraphs(graphL: BundleGraphSpec | undefined, graphR: BundleGraphSpec | undefined, scenarioKey:
|
|
1036
|
+
declare function diffGraphs(graphL: BundleGraphSpec | undefined, graphR: BundleGraphSpec | undefined, scenarioKey: ComparisonScenarioKey, testSummaries: ComparisonTestSummary[]): GraphComparisonReport;
|
|
736
1037
|
|
|
737
|
-
|
|
1038
|
+
/**
|
|
1039
|
+
* Convert a full `ComparisonReport` to a simplified `ComparisonSummary` that includes
|
|
1040
|
+
* the minimum set of fields needed to keep the file smaller when there are many
|
|
1041
|
+
* reported differences. This only includes comparison results for which there
|
|
1042
|
+
* is a non-zero `maxDiff` value.
|
|
1043
|
+
*
|
|
1044
|
+
* @param comparisonReport The full comparison report.
|
|
1045
|
+
* @return The terse summary.
|
|
1046
|
+
*/
|
|
1047
|
+
declare function comparisonSummaryFromReport(comparisonReport: ComparisonReport): ComparisonSummary;
|
|
1048
|
+
|
|
1049
|
+
declare type ComparisonGroupKind = 'by-dataset' | 'by-scenario';
|
|
1050
|
+
declare type ComparisonGroupKey = string;
|
|
1051
|
+
/**
|
|
1052
|
+
* A group of comparison test summaries associated with a particular scenario or dataset.
|
|
1053
|
+
*/
|
|
1054
|
+
interface ComparisonGroup {
|
|
1055
|
+
/** The kind of group, either 'by-dataset' or 'by-scenario'. */
|
|
1056
|
+
kind: ComparisonGroupKind;
|
|
738
1057
|
/**
|
|
739
|
-
* The
|
|
740
|
-
*
|
|
1058
|
+
* The unique key for this group (a `DatasetKey` if grouped by dataset, or a
|
|
1059
|
+
* `ComparisonScenarioKey` if grouped by scenario).
|
|
741
1060
|
*/
|
|
742
|
-
|
|
1061
|
+
key: ComparisonGroupKey;
|
|
1062
|
+
/** The comparison test summaries for this group. */
|
|
1063
|
+
testSummaries: ComparisonTestSummary[];
|
|
1064
|
+
}
|
|
1065
|
+
/** Describes the "root" or primary item for a group of comparisons. */
|
|
1066
|
+
declare type ComparisonGroupRoot = ComparisonDataset | ComparisonScenario;
|
|
1067
|
+
/** A summary of scores for a group of comparisons. */
|
|
1068
|
+
interface ComparisonGroupScores {
|
|
1069
|
+
/** The total number of comparisons (sample size) for this group. */
|
|
1070
|
+
totalDiffCount: number;
|
|
1071
|
+
/** The sum of the `maxDiff` values for each threshold bucket. */
|
|
1072
|
+
totalMaxDiffByBucket: number[];
|
|
1073
|
+
/** The number of comparisons that fall into each threshold bucket. */
|
|
1074
|
+
diffCountByBucket: number[];
|
|
1075
|
+
/** The percentage of comparisons that fall into each threshold bucket. */
|
|
1076
|
+
diffPercentByBucket: number[];
|
|
1077
|
+
}
|
|
1078
|
+
/**
|
|
1079
|
+
* A summary of a group of comparisons that includes the resolved scenario/dataset metadata
|
|
1080
|
+
* and score information for the group.
|
|
1081
|
+
*/
|
|
1082
|
+
interface ComparisonGroupSummary {
|
|
1083
|
+
/** The metadata for the "root" or primary item for this group of comparisons. */
|
|
1084
|
+
root: ComparisonGroupRoot;
|
|
1085
|
+
/** The group containing the comparison summaries. */
|
|
1086
|
+
group: ComparisonGroup;
|
|
1087
|
+
/** The scores for this group, or undefined if comparisons were not performed for this group. */
|
|
1088
|
+
scores?: ComparisonGroupScores;
|
|
1089
|
+
}
|
|
1090
|
+
/**
|
|
1091
|
+
* Breaks down a set of by-scenario or by-dataset groupings into distinct categories.
|
|
1092
|
+
*/
|
|
1093
|
+
interface ComparisonGroupSummariesByCategory {
|
|
743
1094
|
/**
|
|
744
|
-
*
|
|
1095
|
+
* All groups in a map, keyed by "group key" (either a dataset key or scenario key).
|
|
745
1096
|
*/
|
|
746
|
-
|
|
1097
|
+
allGroupSummaries: Map<ComparisonGroupKey, ComparisonGroupSummary>;
|
|
747
1098
|
/**
|
|
748
|
-
*
|
|
1099
|
+
* Groups with items that have errors (are not valid) for both "left" and "right" models.
|
|
1100
|
+
*/
|
|
1101
|
+
withErrors: ComparisonGroupSummary[];
|
|
1102
|
+
/**
|
|
1103
|
+
* Groups with items that are only valid for the "left" model (for example, datasets that
|
|
1104
|
+
* were removed and no longer available in the "right" model).
|
|
749
1105
|
*/
|
|
750
|
-
|
|
1106
|
+
onlyInLeft: ComparisonGroupSummary[];
|
|
1107
|
+
/**
|
|
1108
|
+
* Groups with items that are only valid for the "right" model (for example, scenarios
|
|
1109
|
+
* for inputs that were added in the "right" model).
|
|
1110
|
+
*/
|
|
1111
|
+
onlyInRight: ComparisonGroupSummary[];
|
|
1112
|
+
/**
|
|
1113
|
+
* Groups with one or more comparisons that have non-zero `maxDiff` scores; the groups
|
|
1114
|
+
* will be sorted by `maxDiff`, with higher scores at the front of the array.
|
|
1115
|
+
*/
|
|
1116
|
+
withDiffs: ComparisonGroupSummary[];
|
|
1117
|
+
/**
|
|
1118
|
+
* Groups where all comparisons have `maxDiff` scores of zero (no differences between
|
|
1119
|
+
* "left" and "right").
|
|
1120
|
+
*/
|
|
1121
|
+
withoutDiffs: ComparisonGroupSummary[];
|
|
751
1122
|
}
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
1123
|
+
/**
|
|
1124
|
+
* Rolls up all by-scenario and by-dataset groupings.
|
|
1125
|
+
*/
|
|
1126
|
+
interface ComparisonCategorizedResults {
|
|
1127
|
+
/** The full set of by-scenario groupings. */
|
|
1128
|
+
byScenario: ComparisonGroupSummariesByCategory;
|
|
1129
|
+
/** The full set of by-dataset groupings. */
|
|
1130
|
+
byDataset: ComparisonGroupSummariesByCategory;
|
|
755
1131
|
}
|
|
756
1132
|
|
|
757
|
-
declare function createConfig(options: ConfigOptions): Promise<Config>;
|
|
758
|
-
|
|
759
1133
|
/**
|
|
760
|
-
*
|
|
761
|
-
*
|
|
762
|
-
*
|
|
763
|
-
* This class computes the union of the available dataset keys and handles
|
|
764
|
-
* renames so that if any variables were renamed in the "right" bundle, the
|
|
765
|
-
* old key will be used so that the variable can still be compared.
|
|
1134
|
+
* Given a set of terse test summaries (which only includes summaries for tests with non-zero `maxDiff`
|
|
1135
|
+
* scores), restore the full set of summaries and then categorize them.
|
|
766
1136
|
*
|
|
767
|
-
*
|
|
768
|
-
*
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
*/
|
|
772
|
-
declare class DatasetManager implements CompareDatasets {
|
|
773
|
-
private readonly bundleL;
|
|
774
|
-
private readonly bundleR;
|
|
775
|
-
readonly renamedDatasetKeys?: Map<DatasetKey, DatasetKey>;
|
|
776
|
-
readonly allOutputVarKeys: DatasetKey[];
|
|
777
|
-
readonly modelOutputVarKeys: DatasetKey[];
|
|
778
|
-
/**
|
|
779
|
-
* @param bundleL The "left" bundle being compared.
|
|
780
|
-
* @param bundleR The "right" bundle being compared.
|
|
781
|
-
* @param renamedDatasetKeys The mapping of renamed dataset keys.
|
|
782
|
-
*/
|
|
783
|
-
constructor(bundleL: Bundle, bundleR: Bundle, renamedDatasetKeys?: Map<DatasetKey, DatasetKey>);
|
|
784
|
-
getDatasetKeysForScenario(scenario: Scenario): DatasetKey[];
|
|
785
|
-
getDatasetInfo(datasetKey: DatasetKey): DatasetInfo | undefined;
|
|
786
|
-
}
|
|
1137
|
+
* @param comparisonConfig The comparison configuration.
|
|
1138
|
+
* @param terseSummaries The set of terse test summaries.
|
|
1139
|
+
*/
|
|
1140
|
+
declare function categorizeComparisonTestSummaries(comparisonConfig: ComparisonConfig, terseSummaries: ComparisonTestSummary[]): ComparisonCategorizedResults;
|
|
787
1141
|
|
|
788
1142
|
/**
|
|
789
|
-
*
|
|
790
|
-
*
|
|
1143
|
+
* Additional options that are passed to `getConfigOptions`. These can be used to customize
|
|
1144
|
+
* the `ConfigOptions`, for example, if the `simplifyScenarios` flag is true, a reduced set
|
|
1145
|
+
* of tests can be provided in the `ConfigOptions` so that the tests run faster in a local
|
|
1146
|
+
* development situation.
|
|
791
1147
|
*/
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
private readonly defaultInfoForGroup;
|
|
798
|
-
private readonly groupInfo;
|
|
1148
|
+
interface ConfigInitOptions {
|
|
1149
|
+
/** If defined, overrides the displayed name of the baseline ("left") bundle. */
|
|
1150
|
+
bundleNameL?: string;
|
|
1151
|
+
/** If defined, overrides the displayed name of the current ("right") bundle. */
|
|
1152
|
+
bundleNameR?: string;
|
|
799
1153
|
/**
|
|
800
|
-
*
|
|
801
|
-
*
|
|
1154
|
+
* A hint that the user wants tests to run faster. If true, you can return a
|
|
1155
|
+
* configuration that runs a smaller subset of tests than normal.
|
|
802
1156
|
*/
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
1157
|
+
simplifyScenarios?: boolean;
|
|
1158
|
+
}
|
|
1159
|
+
/**
|
|
1160
|
+
* The user-specified options used by the library to resolve and initialize a `Config` instance.
|
|
1161
|
+
*/
|
|
1162
|
+
interface ConfigOptions {
|
|
808
1163
|
/**
|
|
809
|
-
*
|
|
810
|
-
*
|
|
811
|
-
* text instead of showing the default message ("...at default").
|
|
812
|
-
*
|
|
813
|
-
* @param groupKey The scenario group key.
|
|
814
|
-
* @param scenarioInfo The custom info to be displayed.
|
|
1164
|
+
* The bundle being checked. This bundle will also be compared against the
|
|
1165
|
+
* "baseline" bundle, if `comparison` options are defined.
|
|
815
1166
|
*/
|
|
816
|
-
|
|
1167
|
+
current: NamedBundle;
|
|
817
1168
|
/**
|
|
818
|
-
*
|
|
819
|
-
*
|
|
820
|
-
* @param scenario The scenario to be added.
|
|
821
|
-
* @param scenarioInfo The custom title/subtitle for the scenario. If left undefined, the
|
|
822
|
-
* default (possibly generic) info will be used.
|
|
823
|
-
* @param groupInfo The custom title/subtitle for the group. If left undefined, the
|
|
824
|
-
* default (possibly generic) info will be used.
|
|
1169
|
+
* The model check options.
|
|
825
1170
|
*/
|
|
826
|
-
|
|
1171
|
+
check: CheckOptions;
|
|
827
1172
|
/**
|
|
828
|
-
*
|
|
829
|
-
* the given model.
|
|
830
|
-
*
|
|
831
|
-
* This function computes a matrix of input scenarios using the input variables
|
|
832
|
-
* advertised by the given bundles. It will generate scenarios such that for
|
|
833
|
-
* any output variable, the model will be run:
|
|
834
|
-
* - once with all inputs at their default
|
|
835
|
-
* - once with all inputs at their minimum
|
|
836
|
-
* - once with all inputs at their maximum
|
|
837
|
-
* - twice for each input
|
|
838
|
-
* - once with single input at its minimum
|
|
839
|
-
* - once with single input at its maximum
|
|
840
|
-
*
|
|
841
|
-
* This is intended to be a simple, general purpose way to create a set of
|
|
842
|
-
* scenarios, but every model is different, so you can replace this with a
|
|
843
|
-
* function to generate a different set of scenarios that is better suited
|
|
844
|
-
* for the model you are testing.
|
|
1173
|
+
* The model comparison options.
|
|
845
1174
|
*/
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
1175
|
+
comparison?: ComparisonOptions;
|
|
1176
|
+
}
|
|
1177
|
+
/**
|
|
1178
|
+
* The resolved configuration for check and comparison tests.
|
|
1179
|
+
*/
|
|
1180
|
+
interface Config {
|
|
1181
|
+
/** The resolved check test configuration. */
|
|
1182
|
+
check: CheckConfig;
|
|
1183
|
+
/** The resolved comparison test configuration. */
|
|
1184
|
+
comparison?: ComparisonConfig;
|
|
852
1185
|
}
|
|
853
1186
|
|
|
1187
|
+
declare function createConfig(options: ConfigOptions): Promise<Config>;
|
|
1188
|
+
|
|
854
1189
|
declare class PerfRunner {
|
|
855
1190
|
readonly bundleModelL: BundleModel;
|
|
856
1191
|
readonly bundleModelR: BundleModel;
|
|
@@ -862,9 +1197,23 @@ declare class PerfRunner {
|
|
|
862
1197
|
start(): void;
|
|
863
1198
|
}
|
|
864
1199
|
|
|
1200
|
+
/**
|
|
1201
|
+
* The report for a single run of the full check+comparison test suite.
|
|
1202
|
+
*/
|
|
865
1203
|
interface SuiteReport {
|
|
866
1204
|
checkReport: CheckReport;
|
|
867
|
-
|
|
1205
|
+
comparisonReport?: ComparisonReport;
|
|
1206
|
+
}
|
|
1207
|
+
/**
|
|
1208
|
+
* A simplified/terse version of `SuiteReport` that is used when writing
|
|
1209
|
+
* results to a JSON file. The object keys are terse and it only includes
|
|
1210
|
+
* the minimum set of fields (e.g., only the `maxDiff` value instead of the
|
|
1211
|
+
* full `DiffReport` for each comparison test) to keep the file smaller
|
|
1212
|
+
* when there are many reported differences.
|
|
1213
|
+
*/
|
|
1214
|
+
interface SuiteSummary {
|
|
1215
|
+
checkSummary: CheckSummary;
|
|
1216
|
+
comparisonSummary?: ComparisonSummary;
|
|
868
1217
|
}
|
|
869
1218
|
|
|
870
1219
|
declare type CancelRunSuite = () => void;
|
|
@@ -887,14 +1236,6 @@ interface RunSuiteOptions {
|
|
|
887
1236
|
*/
|
|
888
1237
|
declare function runSuite(config: Config, callbacks: RunSuiteCallbacks, options?: RunSuiteOptions): CancelRunSuite;
|
|
889
1238
|
|
|
890
|
-
/**
|
|
891
|
-
* A simplified/terse version of `SuiteReport` that matches the
|
|
892
|
-
* format of the JSON objects emitted by the CLI in terse mode.
|
|
893
|
-
*/
|
|
894
|
-
interface SuiteSummary {
|
|
895
|
-
checkSummary: CheckSummary;
|
|
896
|
-
compareSummary?: CompareSummary;
|
|
897
|
-
}
|
|
898
1239
|
/**
|
|
899
1240
|
* Convert a full `SuiteReport` to a simplified `SuiteSummary` that only includes
|
|
900
1241
|
* failed/errored checks or comparisons with differences.
|
|
@@ -904,4 +1245,4 @@ interface SuiteSummary {
|
|
|
904
1245
|
*/
|
|
905
1246
|
declare function suiteSummaryFromReport(suiteReport: SuiteReport): SuiteSummary;
|
|
906
1247
|
|
|
907
|
-
export {
|
|
1248
|
+
export { AllInputsSpec, Bundle, BundleGraphData, BundleGraphDatasetSpec, BundleGraphId, BundleGraphSpec, BundleGraphView, BundleModel, CheckDataCoordinator, CheckDataRequestKey, CheckDatasetReport, CheckGroupReport, CheckKey, CheckPredicateOp, CheckPredicateOpConstantRef, CheckPredicateOpDataRef, CheckPredicateOpRef, CheckPredicateReport, CheckPredicateSummary, CheckPredicateTimeOptions, CheckPredicateTimeRange, CheckPredicateTimeSingle, CheckPredicateTimeSpec, CheckReport, CheckResult, CheckResultErrorInfo, CheckScenario, CheckScenarioError, CheckScenarioInputDesc, CheckScenarioReport, CheckStatus, CheckSummary, CheckTestReport, ComparisonCategorizedResults, ComparisonConfig, ComparisonDataCoordinator, ComparisonDataRequestKey, ComparisonDataset, ComparisonDatasetOptions, ComparisonDatasets, ComparisonGroup, ComparisonGroupKey, ComparisonGroupKind, ComparisonGroupRoot, ComparisonGroupScores, ComparisonGroupSummariesByCategory, ComparisonGroupSummary, ComparisonOptions, ComparisonReport, ComparisonResolverError, ComparisonResolverInvalidValueError, ComparisonResolverUnknownInputError, ComparisonScenario, ComparisonScenarioAllInputsSettings, ComparisonScenarioGroup, ComparisonScenarioGroupId, ComparisonScenarioGroupRefSpec, ComparisonScenarioGroupSpec, ComparisonScenarioGroupTitle, ComparisonScenarioId, ComparisonScenarioInput, ComparisonScenarioInputAtPositionSpec, ComparisonScenarioInputAtValueSpec, ComparisonScenarioInputName, ComparisonScenarioInputPosition, ComparisonScenarioInputSettings, ComparisonScenarioInputSpec, ComparisonScenarioInputState, ComparisonScenarioKey, ComparisonScenarioPresetMatrixSpec, ComparisonScenarioRefSpec, ComparisonScenarioSettings, ComparisonScenarioSpec, ComparisonScenarioSubtitle, ComparisonScenarioTitle, ComparisonScenarioWithAllInputsSpec, ComparisonScenarioWithInputsSpec, ComparisonScenarios, ComparisonSpecs, ComparisonSpecsSource, ComparisonSummary, ComparisonTestReport, ComparisonTestSummary, ComparisonUnresolvedScenarioGroupRef, ComparisonUnresolvedScenarioRef, ComparisonUnresolvedView, ComparisonView, ComparisonViewGraphId, ComparisonViewGraphsArraySpec, ComparisonViewGraphsPresetSpec, ComparisonViewGraphsSpec, ComparisonViewGroup, ComparisonViewGroupSpec, ComparisonViewGroupTitle, ComparisonViewGroupWithScenariosSpec, ComparisonViewGroupWithViewsSpec, ComparisonViewSpec, ComparisonViewSubtitle, ComparisonViewTitle, Config, ConfigInitOptions, ConfigOptions, DataSource, Dataset, DatasetKey, DatasetMap, DatasetsResult, DiffPoint, DiffReport, DiffValidity, Dimension, GraphComparisonDatasetReport, GraphComparisonMetadataReport, GraphComparisonReport, GraphInclusion, ImplVar, InputId, InputPosition, InputSetting, InputSettingsSpec, InputVar, LegendItem, LinkItem, LoadedBundle, ModelSpec, NamedBundle, OutputVar, PerfReport, PerfRunner, PerfStats, PositionSetting, RelatedItem, RunSuiteCallbacks, RunSuiteOptions, ScenarioSpec, ScenarioSpecUid, SourceName, Subscript, SuiteReport, SuiteSummary, ValueSetting, VarId, categorizeComparisonTestSummaries, checkReportFromSummary, checkSummaryFromReport, comparisonSummaryFromReport, createConfig, datasetMessage, diffDatasets, diffGraphs, predicateMessage, runSuite, scenarioMessage, suiteSummaryFromReport };
|