@sdeverywhere/check-core 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -1,11 +1,11 @@
1
1
  declare type SourceName = string;
2
2
  declare type VarId = string;
3
- declare type ScenarioKey = string;
4
- declare type ScenarioGroupKey = string;
5
3
  declare type DatasetKey = string;
6
4
  declare type Dataset = Map<number, number>;
7
5
  declare type DatasetMap = Map<DatasetKey, Dataset>;
8
6
 
7
+ /** A unique identifier for the scenario, derived from its input settings. */
8
+ declare type ScenarioSpecUid = string;
9
9
  declare type InputPosition = 'at-default' | 'at-minimum' | 'at-maximum';
10
10
  interface PositionSetting {
11
11
  kind: 'position';
@@ -18,38 +18,17 @@ interface ValueSetting {
18
18
  value: number;
19
19
  }
20
20
  declare type InputSetting = PositionSetting | ValueSetting;
21
- interface SettingsScenario {
22
- kind: 'settings';
23
- key: ScenarioKey;
24
- groupKey: ScenarioGroupKey;
21
+ interface InputSettingsSpec {
22
+ kind: 'input-settings';
23
+ uid: ScenarioSpecUid;
25
24
  settings: InputSetting[];
26
25
  }
27
- interface AllInputsScenario {
26
+ interface AllInputsSpec {
28
27
  kind: 'all-inputs';
29
- key: ScenarioKey;
30
- groupKey: ScenarioGroupKey;
28
+ uid: ScenarioSpecUid;
31
29
  position: InputPosition;
32
30
  }
33
- declare type Scenario = SettingsScenario | AllInputsScenario;
34
- declare function positionSetting(inputVarId: VarId, position: InputPosition): InputSetting;
35
- declare function valueSetting(inputVarId: VarId, value: number): InputSetting;
36
- declare function settingsScenario(key: ScenarioKey, groupKey: ScenarioGroupKey, settings: InputSetting[]): Scenario;
37
- declare function inputAtPositionScenario(inputVarId: VarId, groupKey: ScenarioGroupKey, position: InputPosition): Scenario;
38
- declare function inputAtValueScenario(inputVarId: VarId, groupKey: ScenarioGroupKey, value: number): Scenario;
39
- declare function allInputsAtPositionScenario(position: InputPosition): Scenario;
40
- /**
41
- * Return an array of scenarios that can be used to run the model
42
- * with a matrix of output/input scenarios.
43
- *
44
- * For each output variable, run the model:
45
- * - once with all inputs at their default
46
- * - once with all inputs at their minimum
47
- * - once with all inputs at their maximum
48
- * - twice for each input
49
- * - once with single input at its minimum
50
- * - once with single input at its maximum
51
- */
52
- declare function matrixScenarios(inputVarIds: VarId[]): Scenario[];
31
+ declare type ScenarioSpec = InputSettingsSpec | AllInputsSpec;
53
32
 
54
33
  interface DatasetsResult {
55
34
  /**
@@ -64,7 +43,7 @@ interface DatasetsResult {
64
43
  }
65
44
  interface DataSource {
66
45
  /** Return the datasets that result from running the given scenario. */
67
- getDatasetsForScenario(scenario: Scenario, datasetKeys: DatasetKey[]): Promise<DatasetsResult>;
46
+ getDatasetsForScenario(scenarioSpec: ScenarioSpec, datasetKeys: DatasetKey[]): Promise<DatasetsResult>;
68
47
  }
69
48
 
70
49
  /**
@@ -76,24 +55,58 @@ interface RelatedItem {
76
55
  id: string;
77
56
  locationPath: string[];
78
57
  }
58
+ /** A unique, stable input identifier. */
59
+ declare type InputId = string;
79
60
  /**
80
61
  * Holds information about an input variable used in the model.
81
62
  */
82
63
  interface InputVar {
64
+ /**
65
+ * Whether this input is controlled by a continuous range/slider or a discrete on/off switch.
66
+ * If undefined, 'slider' will be assumed.
67
+ */
68
+ kind?: 'slider' | 'switch';
69
+ /**
70
+ * A unique, stable identifier string for this input.
71
+ *
72
+ * This can be used to identify an input variable in a way that is resilient
73
+ * to the variable's name being changed between two versions of the model.
74
+ *
75
+ * For example, if both the "left" and "right" versions of the model have an
76
+ * input with an `inputId` of 2, but the variable is called "Variable 2" in
77
+ * the left and "Variable Two" in the right, the inputs can be correlated and
78
+ * compared despite the different variable names.
79
+ */
80
+ inputId: InputId;
81
+ /** The variable identifier (typically a simplified/canonical ID, like the form used in SDE). */
83
82
  varId: VarId;
83
+ /** The full variable name as used in the modeling tool. */
84
84
  varName: string;
85
+ /** The default value of the input. */
85
86
  defaultValue: number;
87
+ /** The minimum value of the input. */
86
88
  minValue: number;
89
+ /** The maximum value of the input. */
87
90
  maxValue: number;
91
+ /** The metadata for the related input control. */
88
92
  relatedItem?: RelatedItem;
89
93
  }
90
94
  /**
91
95
  * Holds information about an output variable used in the model.
92
96
  */
93
97
  interface OutputVar {
98
+ /** The unique dataset key for this variable (it should include `sourceName` and `varId`). */
99
+ datasetKey: DatasetKey;
100
+ /**
101
+ * The source for the variable (e.g., undefined for a normal model output, "Data" for a variable
102
+ * that is defined in an external data file).
103
+ */
94
104
  sourceName?: SourceName;
105
+ /** The variable identifier (typically a simplified/canonical ID, like the form used in SDE). */
95
106
  varId: VarId;
107
+ /** The full variable name as used in the modeling tool. */
96
108
  varName: string;
109
+ /** The metadata for the related visuals/graphs in which this variable is used. */
97
110
  relatedItems?: RelatedItem[];
98
111
  }
99
112
  /**
@@ -122,7 +135,7 @@ interface Dimension {
122
135
  interface ImplVar {
123
136
  /** The variable identifier, as used in SDE. */
124
137
  varId: VarId;
125
- /** The variable name, as used in Vensim. */
138
+ /** The variable name, as used in the modeling tool. */
126
139
  varName: string;
127
140
  /** The variable index, used by SDE to reference the value in the generated model. */
128
141
  varIndex: number;
@@ -211,10 +224,12 @@ interface ModelSpec {
211
224
  outputVars: Map<DatasetKey, OutputVar>;
212
225
  /** The map of all variables (both internal and exported) in this version of the model. */
213
226
  implVars: Map<DatasetKey, ImplVar>;
227
+ /** The custom input variable aliases defined for this model. */
228
+ inputAliases?: Map<string, VarId>;
214
229
  /** The custom input variable groups defined for this model. */
215
- inputGroups: Map<string, InputVar[]>;
230
+ inputGroups?: Map<string, InputVar[]>;
216
231
  /** The custom dataset (output variable) groups defined for this model. */
217
- datasetGroups: Map<string, DatasetKey[]>;
232
+ datasetGroups?: Map<string, DatasetKey[]>;
218
233
  /** The start time (year) for the model. */
219
234
  startTime?: number;
220
235
  /** The end time (year) for the model. */
@@ -229,18 +244,13 @@ interface ModelSpec {
229
244
  interface BundleModel extends DataSource {
230
245
  /** The spec for the bundled model. */
231
246
  modelSpec: ModelSpec;
232
- /**
233
- * Return the set of context graphs to be displayed for the given dataset. If left
234
- * undefined, the set of graphs will be determined using the advertised graph specs.
235
- */
236
- getGraphsForDataset?(datasetKey: DatasetKey): BundleGraphId[];
237
247
  /**
238
248
  * Load the data used to display the graph by running the model with inputs
239
249
  * configured for the given scenario.
240
250
  */
241
- getGraphDataForScenario(scenario: Scenario, graphId: BundleGraphId): Promise<BundleGraphData>;
251
+ getGraphDataForScenario(scenarioSpec: ScenarioSpec, graphId: BundleGraphId): Promise<BundleGraphData>;
242
252
  /** Return the links to be displayed for the graph in the given scenario. */
243
- getGraphLinksForScenario(scenario: Scenario, graphId: BundleGraphId): LinkItem[];
253
+ getGraphLinksForScenario(scenarioSpec: ScenarioSpec, graphId: BundleGraphId): LinkItem[];
244
254
  }
245
255
  /**
246
256
  * Provides access to the model that is contained in this bundle for use in
@@ -288,7 +298,7 @@ declare class CheckDataCoordinator {
288
298
  readonly bundleModel: BundleModel;
289
299
  private readonly taskQueue;
290
300
  constructor(bundleModel: BundleModel);
291
- requestDataset(requestKey: CheckDataRequestKey, scenario: Scenario, datasetKey: DatasetKey, onResponse: (dataset: Dataset) => void): void;
301
+ requestDataset(requestKey: CheckDataRequestKey, scenarioSpec: ScenarioSpec, datasetKey: DatasetKey, onResponse: (dataset: Dataset) => void): void;
292
302
  cancelRequest(key: CheckDataRequestKey): void;
293
303
  }
294
304
 
@@ -344,8 +354,8 @@ interface CheckScenarioInputDesc {
344
354
  value?: number;
345
355
  }
346
356
  interface CheckScenario {
347
- /** The scenario for the matched input(s); can be undefined if input(s) failed to match. */
348
- scenario?: Scenario;
357
+ /** The spec used to configure the model with the matched input(s); can be undefined if input(s) failed to match. */
358
+ spec?: ScenarioSpec;
349
359
  /** The name of the associated input group, if any. */
350
360
  inputGroupName?: string;
351
361
  /** The descriptions of the inputs; if empty, it is an "all inputs" scenario. */
@@ -355,7 +365,7 @@ interface CheckScenario {
355
365
  }
356
366
 
357
367
  /**
358
- * The key type for data references (in the form `<ScenarioKey::DatasetKey>`).
368
+ * The key type for data references (in the form `<ScenarioUid::DatasetKey>`).
359
369
  */
360
370
  declare type CheckDataRefKey = string;
361
371
  /**
@@ -478,176 +488,439 @@ declare function checkSummaryFromReport(checkReport: CheckReport): CheckSummary;
478
488
  *
479
489
  * @param checkConfig The config used to reconstruct the check test structure.
480
490
  * @param checkSummary The simplified check summary.
481
- * @param simplifyScenarios If true, reduce the number of scenarios generated for a `matrix`.
482
491
  * @return The converted check report.
483
492
  */
484
- declare function checkReportFromSummary(checkConfig: CheckConfig, checkSummary: CheckSummary, simplifyScenarios: boolean): CheckReport | undefined;
493
+ declare function checkReportFromSummary(checkConfig: CheckConfig, checkSummary: CheckSummary): CheckReport | undefined;
485
494
 
495
+ declare type ComparisonScenarioId = string;
496
+ declare type ComparisonScenarioTitle = string;
497
+ declare type ComparisonScenarioSubtitle = string;
498
+ declare type ComparisonScenarioInputName = string;
499
+ declare type ComparisonScenarioInputPosition = 'default' | 'min' | 'max';
486
500
  /**
487
- * Describes a scenario/dataset comparison.
501
+ * Specifies an input that is set to a specific position (default / min / max).
488
502
  */
489
- interface CompareItem {
490
- /** The item title. */
491
- title: string;
492
- /** The item subtitle. */
493
- subtitle?: string;
494
- /** The scenario used for comparison. */
495
- scenario: Scenario;
496
- /** The key for the datasets being compared. */
497
- datasetKey: DatasetKey;
503
+ interface ComparisonScenarioInputAtPositionSpec {
504
+ kind: 'input-at-position';
505
+ /** The requested input name or alias. */
506
+ inputName: ComparisonScenarioInputName;
507
+ /** The requested position of the input. */
508
+ position: ComparisonScenarioInputPosition;
498
509
  }
499
510
  /**
500
- * The title and subtitle info for a group of comparisons.
511
+ * Specifies an input that is set to a specific number value.
501
512
  */
502
- interface CompareGroupInfo {
503
- /** The group title. */
504
- title: string;
505
- /** The group subtitle. */
506
- subtitle?: string;
507
- /** The related items (e.g. graphs or sliders) for the group. */
508
- relatedItems: RelatedItem[];
513
+ interface ComparisonScenarioInputAtValueSpec {
514
+ kind: 'input-at-value';
515
+ /** The requested input name or alias. */
516
+ inputName: ComparisonScenarioInputName;
517
+ /** The number value of the input. */
518
+ value: number;
509
519
  }
510
520
  /**
511
- * Describes a group of comparisons. The results can be grouped either
512
- * by dataset or by scenario.
521
+ * A single input setting for a scenario. An input can be set to a specific number value,
522
+ * or it can be set to a "position" (default / min / max).
513
523
  */
514
- interface CompareGroup {
515
- /** The group title/subtitle info. */
516
- info: CompareGroupInfo;
517
- /** The items in the group. */
518
- items: CompareItem[];
524
+ declare type ComparisonScenarioInputSpec = ComparisonScenarioInputAtPositionSpec | ComparisonScenarioInputAtValueSpec;
525
+ /**
526
+ * Specifies a single scenario that sets one or more inputs to a value/position.
527
+ */
528
+ interface ComparisonScenarioWithInputsSpec {
529
+ kind: 'scenario-with-inputs';
530
+ /** The unique identifier for the scenario. */
531
+ id?: ComparisonScenarioId;
532
+ /** The title of the scenario. */
533
+ title?: ComparisonScenarioTitle;
534
+ /** The subtitle of the scenario. */
535
+ subtitle?: ComparisonScenarioSubtitle;
536
+ /** The input settings for this scenario. */
537
+ inputs: ComparisonScenarioInputSpec[];
519
538
  }
520
-
521
539
  /**
522
- * The variable/source name info associated with a dataset.
540
+ * Specifies a single scenario that sets all available inputs to position.
523
541
  */
524
- interface DatasetInfo {
525
- /** The variable name. */
526
- varName: string;
527
- /** The new variable name, if it was renamed. */
528
- newVarName?: string;
529
- /** The source name. */
530
- sourceName?: string;
531
- /** The new source name, if it was renamed. */
532
- newSourceName?: string;
533
- /** The related items (e.g. graphs) for the dataset. */
534
- relatedItems: RelatedItem[];
542
+ interface ComparisonScenarioWithAllInputsSpec {
543
+ kind: 'scenario-with-all-inputs';
544
+ /** The unique identifier for the scenario. */
545
+ id?: ComparisonScenarioId;
546
+ /** The title of the scenario. */
547
+ title?: ComparisonScenarioTitle;
548
+ /** The subtitle of the scenario. */
549
+ subtitle?: ComparisonScenarioSubtitle;
550
+ /** The position that will be used for all available inputs. */
551
+ position: ComparisonScenarioInputPosition;
535
552
  }
536
553
  /**
537
- * The human-readable title and subtitle for a scenario.
554
+ * Special preset that expands to many scenarios:
555
+ * - one scenario with all inputs at their default
556
+ * - two scenarios for each available input:
557
+ * - one scenario with the input at its minimum
558
+ * - one scenario with the input at its maximum
538
559
  */
539
- interface ScenarioInfo {
560
+ interface ComparisonScenarioPresetMatrixSpec {
561
+ kind: 'scenario-matrix';
562
+ }
563
+ /**
564
+ * A definition of input scenario(s). A scenario can set one input to a value/position, or it
565
+ * can set multiple inputs to particular values/positions.
566
+ */
567
+ declare type ComparisonScenarioSpec = ComparisonScenarioWithInputsSpec | ComparisonScenarioWithAllInputsSpec | ComparisonScenarioPresetMatrixSpec;
568
+ /** A reference to a scenario definition. */
569
+ interface ComparisonScenarioRefSpec {
570
+ kind: 'scenario-ref';
571
+ /** The ID of the scenario that is referenced. */
572
+ scenarioId: ComparisonScenarioId;
573
+ /** The optional title that is used instead of the referenced scenario's title. */
574
+ title?: ComparisonScenarioTitle;
575
+ /** The optional subtitle that is used instead of the referenced scenario's subtitle. */
576
+ subtitle?: ComparisonScenarioSubtitle;
577
+ }
578
+ declare type ComparisonScenarioGroupId = string;
579
+ declare type ComparisonScenarioGroupTitle = string;
580
+ /**
581
+ * A definition of a group of input scenarios. Multiple scenarios can be grouped together under a single name, and
582
+ * can later be referenced by group ID in a view definition.
583
+ */
584
+ interface ComparisonScenarioGroupSpec {
585
+ kind: 'scenario-group';
586
+ /** The unique identifier for the group. */
587
+ id?: ComparisonScenarioGroupId;
588
+ /** The title of the group. */
589
+ title: ComparisonScenarioGroupTitle;
590
+ /** The scenarios that are included in this group. */
591
+ scenarios: (ComparisonScenarioSpec | ComparisonScenarioRefSpec)[];
592
+ }
593
+ /** A reference to a scenario group definition. */
594
+ interface ComparisonScenarioGroupRefSpec {
595
+ kind: 'scenario-group-ref';
596
+ /** The ID of the scenario group that is referenced. */
597
+ groupId: ComparisonScenarioGroupId;
598
+ }
599
+ declare type ComparisonViewTitle = string;
600
+ declare type ComparisonViewSubtitle = string;
601
+ declare type ComparisonViewGraphId = string;
602
+ /**
603
+ * Specifies a list of graphs to be shown in a view.
604
+ */
605
+ interface ComparisonViewGraphsArraySpec {
606
+ kind: 'graphs-array';
607
+ /** The array of IDs for graphs to show. */
608
+ graphIds: ComparisonViewGraphId[];
609
+ }
610
+ /**
611
+ * Specifies a preset list of graphs to be shown in a view.
612
+ */
613
+ interface ComparisonViewGraphsPresetSpec {
614
+ kind: 'graphs-preset';
615
+ /** The preset (currently only "all" is supported, which shows all available graphs). */
616
+ preset: 'all';
617
+ }
618
+ /**
619
+ * Specifies a set of graphs to be shown in a view.
620
+ */
621
+ declare type ComparisonViewGraphsSpec = ComparisonViewGraphsArraySpec | ComparisonViewGraphsPresetSpec;
622
+ /**
623
+ * A definition of a view. A view presents a set of graphs for a single input scenario.
624
+ */
625
+ interface ComparisonViewSpec {
626
+ kind: 'view';
627
+ /** The title of the view. If undefined, the title will be inferred from the scenario. */
628
+ title?: ComparisonViewTitle;
629
+ /** The subtitle of the view. If undefined, the subtitle will be inferred from the scenario. */
630
+ subtitle?: ComparisonViewGroupTitle;
631
+ /** The scenario to be shown in the view. */
632
+ scenarioId: ComparisonScenarioId;
633
+ /** The graphs to be shown for each scenario view. */
634
+ graphs: ComparisonViewGraphsSpec;
635
+ }
636
+ declare type ComparisonViewGroupTitle = string;
637
+ /**
638
+ * Specifies a view group with an explicit array of view definitions.
639
+ */
640
+ interface ComparisonViewGroupWithViewsSpec {
641
+ kind: 'view-group-with-views';
642
+ /** The title of the group of views. */
643
+ title: ComparisonViewGroupTitle;
644
+ /** The views that are included in this group. */
645
+ views: ComparisonViewSpec[];
646
+ }
647
+ /**
648
+ * Specifies a view group by declaring the scenarios included in the group (one view per scenario), along
649
+ * with a set of graphs that will shown in each view.
650
+ */
651
+ interface ComparisonViewGroupWithScenariosSpec {
652
+ kind: 'view-group-with-scenarios';
653
+ /** The title of the group of views. */
654
+ title: ComparisonViewGroupTitle;
655
+ /** The scenarios to be included (one view will be created for each scenario). */
656
+ scenarios: (ComparisonScenarioRefSpec | ComparisonScenarioGroupRefSpec)[];
657
+ /** The graphs to be shown for each scenario view. */
658
+ graphs: ComparisonViewGraphsSpec;
659
+ }
660
+ /**
661
+ * A definition of a group of views. Multiple related views can be grouped together under a single title
662
+ * to make them easy to distinguish in a report.
663
+ */
664
+ declare type ComparisonViewGroupSpec = ComparisonViewGroupWithViewsSpec | ComparisonViewGroupWithScenariosSpec;
665
+ /**
666
+ * Contains the scenario and view definitions from one or more sources (JSON/YAML files or manually
667
+ * defined specs).
668
+ */
669
+ interface ComparisonSpecs {
670
+ /** The requested scenarios. */
671
+ scenarios: ComparisonScenarioSpec[];
672
+ /** The requested scenario groups. */
673
+ scenarioGroups: ComparisonScenarioGroupSpec[];
674
+ /** The requested view groups. */
675
+ viewGroups: ComparisonViewGroupSpec[];
676
+ }
677
+ /** A source of comparison scenario and specifications. */
678
+ interface ComparisonSpecsSource {
679
+ kind: 'yaml' | 'json';
680
+ /** The source filename, if known. */
681
+ filename?: string;
682
+ /** A string containing YAML or JSON content. */
683
+ content: string;
684
+ }
685
+
686
+ /** A resolved dataset that is being compared. */
687
+ interface ComparisonDataset {
688
+ kind: 'dataset';
689
+ /** The unique key for the dataset (i.e., output variable or static data). */
690
+ key: DatasetKey;
691
+ /**
692
+ * The resolved output variable from the "left" model that corresponds to this dataset,
693
+ * or undefined if the variable is not defined in the left model.
694
+ */
695
+ outputVarL?: OutputVar;
696
+ /**
697
+ * The resolved output variable from the "right" model that corresponds to this dataset,
698
+ * or undefined if the variable is not defined in the right model.
699
+ */
700
+ outputVarR?: OutputVar;
701
+ }
702
+ /** A unique key for a `ComparisonScenario`, generated internally for use by the library. */
703
+ declare type ComparisonScenarioKey = string & {
704
+ _brand?: 'ComparisonScenarioKey';
705
+ };
706
+ interface ComparisonResolverUnknownInputError {
707
+ kind: 'unknown-input';
708
+ }
709
+ interface ComparisonResolverInvalidValueError {
710
+ kind: 'invalid-value';
711
+ }
712
+ declare type ComparisonResolverError = ComparisonResolverUnknownInputError | ComparisonResolverInvalidValueError;
713
+ /** Describes the resolution state for a scenario input relative to a specific model. */
714
+ interface ComparisonScenarioInputState {
715
+ /** The matched input variable; can be undefined if no input matched. */
716
+ inputVar?: InputVar;
717
+ /** The position of the input, if this is a position scenario. */
718
+ position?: InputPosition;
719
+ /** The value of the input, for the given position or explicit value. */
720
+ value?: number;
721
+ /** The error info if the input could not be resolved. */
722
+ error?: ComparisonResolverError;
723
+ }
724
+ /** A scenario input that has been checked against both "left" and "right" model. */
725
+ interface ComparisonScenarioInput {
726
+ /** The requested name of the input. */
727
+ requestedName: string;
728
+ /** The resolved state of the input for the "left" model. */
729
+ stateL: ComparisonScenarioInputState;
730
+ /** The resolved state of the input for the "right" model. */
731
+ stateR: ComparisonScenarioInputState;
732
+ }
733
+ /** A configuration that sets model inputs to specific values. */
734
+ interface ComparisonScenarioInputSettings {
735
+ kind: 'input-settings';
736
+ /** The resolutions for the specified inputs in the scenario. */
737
+ inputs: ComparisonScenarioInput[];
738
+ }
739
+ /** A configuration that sets all inputs in the model to a certain position. */
740
+ interface ComparisonScenarioAllInputsSettings {
741
+ kind: 'all-inputs-settings';
742
+ /** The input position that will be applied to all available inputs. */
743
+ position: InputPosition;
744
+ }
745
+ /**
746
+ * The configuration for an input scenario, either a set of individual input settings, or one
747
+ * that sets all inputs in the model to a certain position.
748
+ */
749
+ declare type ComparisonScenarioSettings = ComparisonScenarioInputSettings | ComparisonScenarioAllInputsSettings;
750
+ /** A single resolved input scenario. */
751
+ interface ComparisonScenario {
752
+ kind: 'scenario';
753
+ /** The unique key for the scenario, generated internally for use by the library. */
754
+ key: ComparisonScenarioKey;
755
+ /** The unique user-defined identifier for the scenario. */
756
+ id?: ComparisonScenarioId;
540
757
  /** The scenario title. */
541
758
  title: string;
542
759
  /** The scenario subtitle. */
543
760
  subtitle?: string;
761
+ /** The resolved settings for the model inputs in this scenario. */
762
+ settings: ComparisonScenarioSettings;
763
+ /** The input scenario used to configure the "left" model, or undefined if data not available. */
764
+ specL?: ScenarioSpec;
765
+ /** The input scenario used to configure the "right" model, or undefined if data not available. */
766
+ specR?: ScenarioSpec;
767
+ }
768
+ /** An unresolved input scenario reference. */
769
+ interface ComparisonUnresolvedScenarioRef {
770
+ kind: 'unresolved-scenario-ref';
771
+ /** The ID of the referenced scenario that could not be resolved. */
772
+ scenarioId: ComparisonScenarioId;
773
+ }
774
+ /** A resolved group of input scenarios. */
775
+ interface ComparisonScenarioGroup {
776
+ kind: 'scenario-group';
777
+ /** The unique identifier for the group. */
778
+ id?: ComparisonScenarioGroupId;
779
+ /** The title of the group. */
780
+ title: ComparisonScenarioGroupTitle;
544
781
  /**
545
- * The position of the scenario when displayed in a row. Typically, 0 means
546
- * "left", 1 means "middle", and so on.
782
+ * The scenarios that are included in this group. This includes scenario that were successfully
783
+ * resolved as well as scenario references that could not be resolved.
547
784
  */
548
- position: number;
785
+ scenarios: (ComparisonScenario | ComparisonUnresolvedScenarioRef)[];
786
+ }
787
+ /** An unresolved scenario group reference. */
788
+ interface ComparisonUnresolvedScenarioGroupRef {
789
+ kind: 'unresolved-scenario-group-ref';
790
+ /** The ID of the referenced scenario group that could not be resolved. */
791
+ scenarioGroupId: ComparisonScenarioGroupId;
792
+ }
793
+ /** A resolved view definition. A view presents a set of graphs for a single input scenario. */
794
+ interface ComparisonView {
795
+ kind: 'view';
796
+ /** The title of the view. */
797
+ title: ComparisonViewTitle;
798
+ /** The subtitle of the view. */
799
+ subtitle?: ComparisonViewSubtitle;
800
+ /** The resolved scenario to be shown in the view. */
801
+ scenario: ComparisonScenario;
802
+ /** The graphs to be shown for each scenario view. */
803
+ graphs: 'all' | ComparisonViewGraphId[];
804
+ }
805
+ /** An unresolved view. */
806
+ interface ComparisonUnresolvedView {
807
+ kind: 'unresolved-view';
808
+ /** The requested title of the view, if provided. */
809
+ title?: ComparisonViewTitle;
810
+ /** The requested subtitle of the view, if provided. */
811
+ subtitle?: ComparisonViewSubtitle;
812
+ /** The ID of the referenced scenario that could not be resolved. */
813
+ scenarioId?: ComparisonScenarioId;
814
+ /** The ID of the referenced scenario group that could not be resolved. */
815
+ scenarioGroupId?: ComparisonScenarioGroupId;
816
+ }
817
+ /** A resolved group of compared scenario/graph views. */
818
+ interface ComparisonViewGroup {
819
+ kind: 'view-group';
820
+ /** The title of the group of views. */
821
+ title: ComparisonViewGroupTitle;
822
+ /** The array of resolved (and unresolved) views that are included in this group. */
823
+ views: (ComparisonView | ComparisonUnresolvedView)[];
549
824
  }
550
825
 
551
826
  /**
552
- * Provides access to the set of scenarios that are used when comparing the two models.
827
+ * Provides access to the set of dataset definitions (`ComparisonDataset` instances) that are used
828
+ * when comparing the two models.
553
829
  */
554
- interface CompareScenarios {
830
+ interface ComparisonDatasets {
555
831
  /**
556
- * Return an array containing all configured scenarios.
832
+ * Return all `ComparisonDataset` instances that are available for comparisons.
557
833
  */
558
- getScenarios(): Scenario[];
834
+ getAllDatasets(): IterableIterator<ComparisonDataset>;
559
835
  /**
560
- * Return the scenario for the given key.
836
+ * Return the dataset metadata for the given key.
561
837
  *
562
- * @param scenarioKey The key for the scenario.
838
+ * @param datasetKey The key for the dataset.
563
839
  */
564
- getScenario(scenarioKey: ScenarioKey): Scenario | undefined;
840
+ getDataset(datasetKey: DatasetKey): ComparisonDataset | undefined;
565
841
  /**
566
- * Return the group info for the given group key.
842
+ * Return the keys for the datasets that should be compared for the given scenario.
567
843
  *
568
- * @param groupKey The scenario group key.
844
+ * @param scenario The scenario definition.
569
845
  */
570
- getScenarioGroupInfo(groupKey: ScenarioGroupKey): CompareGroupInfo | undefined;
846
+ getDatasetKeysForScenario(scenario: ComparisonScenario): DatasetKey[];
847
+ }
848
+
849
+ interface ComparisonScenarios {
571
850
  /**
572
- * Return the title/subtitle info for the given scenario.
573
- *
574
- * Note that the given `groupKey` could be different than `scenario.groupKey`. This
575
- * will be the case for the "all inputs at default" item that is included in each
576
- * row in the detail view for reference purposes. The provided `groupKey` will be
577
- * used to customize the title and subtitle for that item (for example, it will show
578
- * the default value of the input associated with the row).
851
+ * Return all `ComparisonScenario` instances that are available for comparisons.
852
+ */
853
+ getAllScenarios(): IterableIterator<ComparisonScenario>;
854
+ /**
855
+ * Return the scenario definition for the given key.
579
856
  *
580
- * @param scenario The scenario to be displayed.
581
- * @param groupKey The key for the group in which the scenario will be displayed.
857
+ * @param key The key for the scenario.
582
858
  */
583
- getScenarioInfo(scenario: Scenario, groupKey: ScenarioGroupKey): ScenarioInfo | undefined;
859
+ getScenario(key: ComparisonScenarioKey): ComparisonScenario | undefined;
584
860
  }
585
- /**
586
- * Provides access to the set of datasets that are configured for comparison.
587
- */
588
- interface CompareDatasets {
589
- /** The mapping of renamed dataset keys. */
590
- renamedDatasetKeys?: Map<DatasetKey, DatasetKey>;
861
+
862
+ interface ComparisonDatasetOptions {
591
863
  /**
592
- * Return the keys for the datasets that should be compared for the given scenario.
593
- *
594
- * @param scenario The scenario.
864
+ * The mapping of renamed dataset keys (old or "left" name as the map key,
865
+ * new or "right" name as the value).
595
866
  */
596
- getDatasetKeysForScenario(scenario: Scenario): DatasetKey[];
867
+ renamedDatasetKeys?: Map<DatasetKey, DatasetKey>;
597
868
  /**
598
- * Return the dataset info for the given key.
869
+ * An optional function that allows for limiting the datasets that are compared
870
+ * for a given scenario. By default, all datasets are compared for a given
871
+ * scenario, but if a custom function is provided, it can return a subset of
872
+ * datasets (for example, to omit datasets that are not relevant).
599
873
  */
600
- getDatasetInfo(datasetKey: DatasetKey): DatasetInfo | undefined;
874
+ datasetKeysForScenario?: (allDatasetKeys: DatasetKey[], scenario: ComparisonScenario) => DatasetKey[];
601
875
  }
602
- interface CompareOptions {
876
+ interface ComparisonOptions {
877
+ /** The left-side ("baseline") bundle being compared. */
603
878
  baseline: NamedBundle;
879
+ /**
880
+ * The array of thresholds used to color differences, e.g., [1, 5, 10] will use
881
+ * buckets of 0%, 0-1%, 1-5%, 5-10%, and >10%.
882
+ */
604
883
  thresholds: number[];
605
- scenarios: CompareScenarios;
606
- datasets: CompareDatasets;
884
+ /**
885
+ * The requested comparison scenario and view specifications. These can be
886
+ * specified in YAML or JSON files, or using `Spec` objects.
887
+ */
888
+ specs: (ComparisonSpecs | ComparisonSpecsSource)[];
889
+ /** Optional configuration for the datasets that are compared for different scenarios. */
890
+ datasets?: ComparisonDatasetOptions;
607
891
  }
608
- interface CompareConfig {
892
+ interface ComparisonConfig {
893
+ /** The loaded left-side ("baseline") bundle being compared. */
609
894
  bundleL: LoadedBundle;
895
+ /** The loaded right-side ("current") bundle being compared. */
610
896
  bundleR: LoadedBundle;
897
+ /**
898
+ * The array of thresholds used to color differences, e.g., [1, 5, 10] will use
899
+ * buckets of 0%, 0-1%, 1-5%, 5-10%, and >10%.
900
+ */
611
901
  thresholds: number[];
612
- scenarios: CompareScenarios;
613
- datasets: CompareDatasets;
902
+ /** The set of resolved scenarios that will be compared. */
903
+ scenarios: ComparisonScenarios;
904
+ /** The set of resolved datasets that will be compared. */
905
+ datasets: ComparisonDatasets;
906
+ /** The set of resolved view groups. */
907
+ viewGroups: ComparisonViewGroup[];
614
908
  }
615
909
 
616
- declare type CompareDataRequestKey = string;
910
+ declare type ComparisonDataRequestKey = string;
617
911
  /**
618
912
  * Coordinates loading of data in parallel from two models.
619
913
  */
620
- declare class CompareDataCoordinator {
914
+ declare class ComparisonDataCoordinator {
621
915
  readonly bundleModelL: BundleModel;
622
916
  readonly bundleModelR: BundleModel;
623
917
  private readonly taskQueue;
624
918
  constructor(bundleModelL: BundleModel, bundleModelR: BundleModel);
625
- requestDatasetMaps(requestKey: CompareDataRequestKey, scenario: Scenario, datasetKeys: DatasetKey[], onResponse: (datasetMapL: DatasetMap, datasetMapR: DatasetMap) => void): void;
626
- requestGraphData(requestKey: CompareDataRequestKey, bundle: 'left' | 'right', scenario: Scenario, graphId: BundleGraphId, onResponse: (graphData: BundleGraphData) => void): void;
627
- cancelRequest(key: CompareDataRequestKey): void;
628
- }
629
-
630
- interface PerfReport {
631
- readonly minTime: number;
632
- readonly maxTime: number;
633
- readonly avgTime: number;
634
- readonly allTimes: number[];
635
- }
636
- declare class PerfStats {
637
- private readonly times;
638
- addRun(timeInMillis: number): void;
639
- toReport(): PerfReport;
640
- }
641
-
642
- interface CompareDatasetReport {
643
- scenarioKey: ScenarioKey;
644
- datasetKey: DatasetKey;
645
- diffReport: DiffReport;
646
- }
647
- interface CompareReport {
648
- datasetReports: CompareDatasetReport[];
649
- perfReportL: PerfReport;
650
- perfReportR: PerfReport;
919
+ private processDatasetRequest;
920
+ private processGraphDataRequest;
921
+ requestDatasetMaps(requestKey: ComparisonDataRequestKey, scenarioSpecL: ScenarioSpec, scenarioSpecR: ScenarioSpec, datasetKeys: DatasetKey[], onResponse: (datasetMapL?: DatasetMap, datasetMapR?: DatasetMap) => void): void;
922
+ requestGraphData(requestKey: ComparisonDataRequestKey, scenarioSpecL: ScenarioSpec, scenarioSpecR: ScenarioSpec, graphId: BundleGraphId, onResponse: (graphDataL?: BundleGraphData, graphDataR?: BundleGraphData) => void): void;
923
+ cancelRequest(key: ComparisonDataRequestKey): void;
651
924
  }
652
925
 
653
926
  interface DiffPoint {
@@ -666,43 +939,71 @@ interface DiffReport {
666
939
  maxDiffPoint: DiffPoint;
667
940
  }
668
941
  declare function diffDatasets(datasetL: Dataset | undefined, datasetR: Dataset | undefined): DiffReport;
669
- declare function compareDatasets(scenarioKey: ScenarioKey, datasetKey: DatasetKey, datasetMapL: DatasetMap, datasetMapR: DatasetMap): CompareDatasetReport;
942
+
943
+ interface PerfReport {
944
+ readonly minTime: number;
945
+ readonly maxTime: number;
946
+ readonly avgTime: number;
947
+ readonly allTimes: number[];
948
+ }
949
+ declare class PerfStats {
950
+ private readonly times;
951
+ addRun(timeInMillis: number): void;
952
+ toReport(): PerfReport;
953
+ }
670
954
 
671
955
  /**
672
- * A simplified/terse version of `CompareDatasetReport` that matches the
673
- * format of the JSON objects emitted by the CLI in terse mode.
674
- * The object keys are terse and it only includes the minimum set of fields
956
+ * The report for a single comparison test (involving a dataset produced under
957
+ * a specific input scenario). This includes the full `DiffReport`, whereas
958
+ * a `ComparisonTestSummary` only includes the `maxDiff` value.
959
+ */
960
+ interface ComparisonTestReport {
961
+ scenarioKey: ComparisonScenarioKey;
962
+ datasetKey: DatasetKey;
963
+ diffReport: DiffReport;
964
+ }
965
+ /**
966
+ * A simplified/terse version of `ComparisonTestReport` that is used when writing
967
+ * results to a JSON file. The object keys are terse and it only includes the
968
+ * minimum set of fields (only the `maxDiff` value instead of the full `DiffReport`)
675
969
  * to keep the file smaller when there are many reported differences.
676
970
  */
677
- interface CompareDatasetSummary {
971
+ interface ComparisonTestSummary {
678
972
  /** Short for `scenarioKey`. */
679
- s: ScenarioKey;
973
+ s: ComparisonScenarioKey;
680
974
  /** Short for `datasetKey`. */
681
975
  d: DatasetKey;
682
976
  /** Short for `maxDiff`. */
683
977
  md: number;
684
978
  }
685
979
  /**
686
- * A simplified/terse version of `CompareReport` that matches the
687
- * format of the JSON objects emitted by the CLI in terse mode.
980
+ * The roll-up report that contains the results of all individual comparison tests.
688
981
  */
689
- interface CompareSummary {
690
- datasetSummaries: CompareDatasetSummary[];
982
+ interface ComparisonReport {
983
+ /** The set of all comparison test reports. */
984
+ testReports: ComparisonTestReport[];
985
+ /** The perf report for the "left" model. */
691
986
  perfReportL: PerfReport;
987
+ /** The perf report for the "right" model. */
692
988
  perfReportR: PerfReport;
693
989
  }
694
990
  /**
695
- * Convert a full `CompareReport` to a simplified `CompareSummary` that includes
696
- * the minimum set of fields needed to keep the file smaller when there are many
697
- * reported differences.
698
- *
699
- * @param compareReport The full compare report.
700
- * @return The converted compare summary.
991
+ * A simplified/terse version of `ComparisonReport` that only includes the minimum set
992
+ * of fields needed by the reporting app (to keep the file smaller when there are many
993
+ * reported differences). This only includes comparison results for which there is
994
+ * a non-zero `maxDiff` value.
701
995
  */
702
- declare function compareSummaryFromReport(compareReport: CompareReport): CompareSummary;
996
+ interface ComparisonSummary {
997
+ /** The simplified set of all terse comparison test summaries. */
998
+ testSummaries: ComparisonTestSummary[];
999
+ /** The perf report for the "left" model. */
1000
+ perfReportL: PerfReport;
1001
+ /** The perf report for the "right" model. */
1002
+ perfReportR: PerfReport;
1003
+ }
703
1004
 
704
1005
  declare type GraphInclusion = 'neither' | 'left-only' | 'right-only' | 'both';
705
- interface GraphMetadataReport {
1006
+ interface GraphComparisonMetadataReport {
706
1007
  /** The key for the metadata field. */
707
1008
  key: string;
708
1009
  /** The value of the metadata field in the left bundle. */
@@ -710,147 +1011,181 @@ interface GraphMetadataReport {
710
1011
  /** The value of the metadata field in the right bundle. */
711
1012
  valueR?: string;
712
1013
  }
713
- interface GraphDatasetReport {
1014
+ interface GraphComparisonDatasetReport {
714
1015
  /** The dataset key. */
715
1016
  datasetKey: DatasetKey;
716
1017
  /** The max diff for this dataset. */
717
1018
  maxDiff?: number;
718
1019
  }
719
- interface GraphReport {
1020
+ interface GraphComparisonReport {
720
1021
  /** Indicates which bundles the graph is defined in. */
721
1022
  inclusion: GraphInclusion;
722
1023
  /** The metadata fields with differences. */
723
- metadataReports: GraphMetadataReport[];
1024
+ metadataReports: GraphComparisonMetadataReport[];
724
1025
  /** The datasets with differences. */
725
- datasetReports: GraphDatasetReport[];
1026
+ datasetReports: GraphComparisonDatasetReport[];
726
1027
  }
727
1028
  /**
728
- * Compare the metadata and datasets for the given graphs.
1029
+ * Comparison the metadata and datasets for the given graphs.
729
1030
  *
730
1031
  * @param graphL The graph defined in the left bundle.
731
1032
  * @param graphR The graph defined in the right bundle.
732
- * @param scenarioKey The scenario used for comparing datasets.
733
- * @param datasetSummaries The set of summaries from a previous comparison run.
1033
+ * @param scenarioKey The key of the scenario used for comparing datasets.
1034
+ * @param testSummaries The set of test summaries from a previous comparison run.
734
1035
  */
735
- declare function diffGraphs(graphL: BundleGraphSpec | undefined, graphR: BundleGraphSpec | undefined, scenarioKey: ScenarioKey, datasetSummaries: CompareDatasetSummary[]): GraphReport;
1036
+ declare function diffGraphs(graphL: BundleGraphSpec | undefined, graphR: BundleGraphSpec | undefined, scenarioKey: ComparisonScenarioKey, testSummaries: ComparisonTestSummary[]): GraphComparisonReport;
736
1037
 
737
- interface ConfigOptions {
1038
+ /**
1039
+ * Convert a full `ComparisonReport` to a simplified `ComparisonSummary` that includes
1040
+ * the minimum set of fields needed to keep the file smaller when there are many
1041
+ * reported differences. This only includes comparison results for which there
1042
+ * is a non-zero `maxDiff` value.
1043
+ *
1044
+ * @param comparisonReport The full comparison report.
1045
+ * @return The terse summary.
1046
+ */
1047
+ declare function comparisonSummaryFromReport(comparisonReport: ComparisonReport): ComparisonSummary;
1048
+
1049
+ declare type ComparisonGroupKind = 'by-dataset' | 'by-scenario';
1050
+ declare type ComparisonGroupKey = string;
1051
+ /**
1052
+ * A group of comparison test summaries associated with a particular scenario or dataset.
1053
+ */
1054
+ interface ComparisonGroup {
1055
+ /** The kind of group, either 'by-dataset' or 'by-scenario'. */
1056
+ kind: ComparisonGroupKind;
738
1057
  /**
739
- * The bundle being checked. This bundle will also be compared against the
740
- * "baseline" bundle, if `compare` is defined.
1058
+ * The unique key for this group (a `DatasetKey` if grouped by dataset, or a
1059
+ * `ComparisonScenarioKey` if grouped by scenario).
741
1060
  */
742
- current: NamedBundle;
1061
+ key: ComparisonGroupKey;
1062
+ /** The comparison test summaries for this group. */
1063
+ testSummaries: ComparisonTestSummary[];
1064
+ }
1065
+ /** Describes the "root" or primary item for a group of comparisons. */
1066
+ declare type ComparisonGroupRoot = ComparisonDataset | ComparisonScenario;
1067
+ /** A summary of scores for a group of comparisons. */
1068
+ interface ComparisonGroupScores {
1069
+ /** The total number of comparisons (sample size) for this group. */
1070
+ totalDiffCount: number;
1071
+ /** The sum of the `maxDiff` values for each threshold bucket. */
1072
+ totalMaxDiffByBucket: number[];
1073
+ /** The number of comparisons that fall into each threshold bucket. */
1074
+ diffCountByBucket: number[];
1075
+ /** The percentage of comparisons that fall into each threshold bucket. */
1076
+ diffPercentByBucket: number[];
1077
+ }
1078
+ /**
1079
+ * A summary of a group of comparisons that includes the resolved scenario/dataset metadata
1080
+ * and score information for the group.
1081
+ */
1082
+ interface ComparisonGroupSummary {
1083
+ /** The metadata for the "root" or primary item for this group of comparisons. */
1084
+ root: ComparisonGroupRoot;
1085
+ /** The group containing the comparison summaries. */
1086
+ group: ComparisonGroup;
1087
+ /** The scores for this group, or undefined if comparisons were not performed for this group. */
1088
+ scores?: ComparisonGroupScores;
1089
+ }
1090
+ /**
1091
+ * Breaks down a set of by-scenario or by-dataset groupings into distinct categories.
1092
+ */
1093
+ interface ComparisonGroupSummariesByCategory {
743
1094
  /**
744
- * The model check options.
1095
+ * All groups in a map, keyed by "group key" (either a dataset key or scenario key).
745
1096
  */
746
- check: CheckOptions;
1097
+ allGroupSummaries: Map<ComparisonGroupKey, ComparisonGroupSummary>;
747
1098
  /**
748
- * The model comparison options.
1099
+ * Groups with items that have errors (are not valid) for both "left" and "right" models.
1100
+ */
1101
+ withErrors: ComparisonGroupSummary[];
1102
+ /**
1103
+ * Groups with items that are only valid for the "left" model (for example, datasets that
1104
+ * were removed and no longer available in the "right" model).
749
1105
  */
750
- compare?: CompareOptions;
1106
+ onlyInLeft: ComparisonGroupSummary[];
1107
+ /**
1108
+ * Groups with items that are only valid for the "right" model (for example, scenarios
1109
+ * for inputs that were added in the "right" model).
1110
+ */
1111
+ onlyInRight: ComparisonGroupSummary[];
1112
+ /**
1113
+ * Groups with one or more comparisons that have non-zero `maxDiff` scores; the groups
1114
+ * will be sorted by `maxDiff`, with higher scores at the front of the array.
1115
+ */
1116
+ withDiffs: ComparisonGroupSummary[];
1117
+ /**
1118
+ * Groups where all comparisons have `maxDiff` scores of zero (no differences between
1119
+ * "left" and "right").
1120
+ */
1121
+ withoutDiffs: ComparisonGroupSummary[];
751
1122
  }
752
- interface Config {
753
- check: CheckConfig;
754
- compare?: CompareConfig;
1123
+ /**
1124
+ * Rolls up all by-scenario and by-dataset groupings.
1125
+ */
1126
+ interface ComparisonCategorizedResults {
1127
+ /** The full set of by-scenario groupings. */
1128
+ byScenario: ComparisonGroupSummariesByCategory;
1129
+ /** The full set of by-dataset groupings. */
1130
+ byDataset: ComparisonGroupSummariesByCategory;
755
1131
  }
756
1132
 
757
- declare function createConfig(options: ConfigOptions): Promise<Config>;
758
-
759
1133
  /**
760
- * Manages a set of dataset keys (corresponding to the available model outputs
761
- * in the given bundles) that can be used to compare two versions of the model.
762
- *
763
- * This class computes the union of the available dataset keys and handles
764
- * renames so that if any variables were renamed in the "right" bundle, the
765
- * old key will be used so that the variable can still be compared.
1134
+ * Given a set of terse test summaries (which only includes summaries for tests with non-zero `maxDiff`
1135
+ * scores), restore the full set of summaries and then categorize them.
766
1136
  *
767
- * This is intended to be a simple, general purpose way to create a set of
768
- * dataset keys, but every model is different, so you can replace this with
769
- * a different set of dataset keys that is better suited for the model you
770
- * are testing.
771
- */
772
- declare class DatasetManager implements CompareDatasets {
773
- private readonly bundleL;
774
- private readonly bundleR;
775
- readonly renamedDatasetKeys?: Map<DatasetKey, DatasetKey>;
776
- readonly allOutputVarKeys: DatasetKey[];
777
- readonly modelOutputVarKeys: DatasetKey[];
778
- /**
779
- * @param bundleL The "left" bundle being compared.
780
- * @param bundleR The "right" bundle being compared.
781
- * @param renamedDatasetKeys The mapping of renamed dataset keys.
782
- */
783
- constructor(bundleL: Bundle, bundleR: Bundle, renamedDatasetKeys?: Map<DatasetKey, DatasetKey>);
784
- getDatasetKeysForScenario(scenario: Scenario): DatasetKey[];
785
- getDatasetInfo(datasetKey: DatasetKey): DatasetInfo | undefined;
786
- }
1137
+ * @param comparisonConfig The comparison configuration.
1138
+ * @param terseSummaries The set of terse test summaries.
1139
+ */
1140
+ declare function categorizeComparisonTestSummaries(comparisonConfig: ComparisonConfig, terseSummaries: ComparisonTestSummary[]): ComparisonCategorizedResults;
787
1141
 
788
1142
  /**
789
- * Manages a set of scenarios (corresponding to the available model inputs
790
- * in the given bundles) that can be used to compare two versions of the model.
1143
+ * Additional options that are passed to `getConfigOptions`. These can be used to customize
1144
+ * the `ConfigOptions`, for example, if the `simplifyScenarios` flag is true, a reduced set
1145
+ * of tests can be provided in the `ConfigOptions` so that the tests run faster in a local
1146
+ * development situation.
791
1147
  */
792
- declare class ScenarioManager implements CompareScenarios {
793
- private readonly bundleL;
794
- private readonly bundleR;
795
- private readonly scenarios;
796
- private readonly scenarioInfo;
797
- private readonly defaultInfoForGroup;
798
- private readonly groupInfo;
1148
+ interface ConfigInitOptions {
1149
+ /** If defined, overrides the displayed name of the baseline ("left") bundle. */
1150
+ bundleNameL?: string;
1151
+ /** If defined, overrides the displayed name of the current ("right") bundle. */
1152
+ bundleNameR?: string;
799
1153
  /**
800
- * @param bundleL The "left" bundle being compared.
801
- * @param bundleR The "right" bundle being compared.
1154
+ * A hint that the user wants tests to run faster. If true, you can return a
1155
+ * configuration that runs a smaller subset of tests than normal.
802
1156
  */
803
- constructor(bundleL: Bundle, bundleR: Bundle);
804
- getScenarios(): Scenario[];
805
- getScenario(scenarioKey: ScenarioKey): Scenario | undefined;
806
- getScenarioGroupInfo(groupKey: ScenarioGroupKey): CompareGroupInfo | undefined;
807
- getScenarioInfo(scenario: Scenario, groupKey: ScenarioGroupKey): ScenarioInfo | undefined;
1157
+ simplifyScenarios?: boolean;
1158
+ }
1159
+ /**
1160
+ * The user-specified options used by the library to resolve and initialize a `Config` instance.
1161
+ */
1162
+ interface ConfigOptions {
808
1163
  /**
809
- * Override the title and subtitle that are displayed when the "all inputs at default"
810
- * scenario is included for a particular group. This can be used to customize the
811
- * text instead of showing the default message ("...at default").
812
- *
813
- * @param groupKey The scenario group key.
814
- * @param scenarioInfo The custom info to be displayed.
1164
+ * The bundle being checked. This bundle will also be compared against the
1165
+ * "baseline" bundle, if `comparison` options are defined.
815
1166
  */
816
- setDefaultScenarioInfoForGroup(groupKey: ScenarioGroupKey, scenarioInfo: ScenarioInfo): void;
1167
+ current: NamedBundle;
817
1168
  /**
818
- * Add a scenario to the set.
819
- *
820
- * @param scenario The scenario to be added.
821
- * @param scenarioInfo The custom title/subtitle for the scenario. If left undefined, the
822
- * default (possibly generic) info will be used.
823
- * @param groupInfo The custom title/subtitle for the group. If left undefined, the
824
- * default (possibly generic) info will be used.
1169
+ * The model check options.
825
1170
  */
826
- addScenario(scenario: Scenario, scenarioInfo?: ScenarioInfo, groupInfo?: CompareGroupInfo): void;
1171
+ check: CheckOptions;
827
1172
  /**
828
- * Adds a set of scenarios that can be used to compare the two versions of
829
- * the given model.
830
- *
831
- * This function computes a matrix of input scenarios using the input variables
832
- * advertised by the given bundles. It will generate scenarios such that for
833
- * any output variable, the model will be run:
834
- * - once with all inputs at their default
835
- * - once with all inputs at their minimum
836
- * - once with all inputs at their maximum
837
- * - twice for each input
838
- * - once with single input at its minimum
839
- * - once with single input at its maximum
840
- *
841
- * This is intended to be a simple, general purpose way to create a set of
842
- * scenarios, but every model is different, so you can replace this with a
843
- * function to generate a different set of scenarios that is better suited
844
- * for the model you are testing.
1173
+ * The model comparison options.
845
1174
  */
846
- addScenarioMatrix(): void;
847
- private getGroupInfoForScenario;
848
- private getInfoForScenario;
849
- private getInfoForPositionSetting;
850
- private getRelatedItemsForSettings;
851
- private getInputVarForSetting;
1175
+ comparison?: ComparisonOptions;
1176
+ }
1177
+ /**
1178
+ * The resolved configuration for check and comparison tests.
1179
+ */
1180
+ interface Config {
1181
+ /** The resolved check test configuration. */
1182
+ check: CheckConfig;
1183
+ /** The resolved comparison test configuration. */
1184
+ comparison?: ComparisonConfig;
852
1185
  }
853
1186
 
1187
+ declare function createConfig(options: ConfigOptions): Promise<Config>;
1188
+
854
1189
  declare class PerfRunner {
855
1190
  readonly bundleModelL: BundleModel;
856
1191
  readonly bundleModelR: BundleModel;
@@ -862,9 +1197,23 @@ declare class PerfRunner {
862
1197
  start(): void;
863
1198
  }
864
1199
 
1200
+ /**
1201
+ * The report for a single run of the full check+comparison test suite.
1202
+ */
865
1203
  interface SuiteReport {
866
1204
  checkReport: CheckReport;
867
- compareReport?: CompareReport;
1205
+ comparisonReport?: ComparisonReport;
1206
+ }
1207
+ /**
1208
+ * A simplified/terse version of `SuiteReport` that is used when writing
1209
+ * results to a JSON file. The object keys are terse and it only includes
1210
+ * the minimum set of fields (e.g., only the `maxDiff` value instead of the
1211
+ * full `DiffReport` for each comparison test) to keep the file smaller
1212
+ * when there are many reported differences.
1213
+ */
1214
+ interface SuiteSummary {
1215
+ checkSummary: CheckSummary;
1216
+ comparisonSummary?: ComparisonSummary;
868
1217
  }
869
1218
 
870
1219
  declare type CancelRunSuite = () => void;
@@ -887,14 +1236,6 @@ interface RunSuiteOptions {
887
1236
  */
888
1237
  declare function runSuite(config: Config, callbacks: RunSuiteCallbacks, options?: RunSuiteOptions): CancelRunSuite;
889
1238
 
890
- /**
891
- * A simplified/terse version of `SuiteReport` that matches the
892
- * format of the JSON objects emitted by the CLI in terse mode.
893
- */
894
- interface SuiteSummary {
895
- checkSummary: CheckSummary;
896
- compareSummary?: CompareSummary;
897
- }
898
1239
  /**
899
1240
  * Convert a full `SuiteReport` to a simplified `SuiteSummary` that only includes
900
1241
  * failed/errored checks or comparisons with differences.
@@ -904,4 +1245,4 @@ interface SuiteSummary {
904
1245
  */
905
1246
  declare function suiteSummaryFromReport(suiteReport: SuiteReport): SuiteSummary;
906
1247
 
907
- export { AllInputsScenario, Bundle, BundleGraphData, BundleGraphDatasetSpec, BundleGraphId, BundleGraphSpec, BundleGraphView, BundleModel, CheckDataCoordinator, CheckDataRequestKey, CheckDatasetReport, CheckGroupReport, CheckKey, CheckPredicateOp, CheckPredicateOpConstantRef, CheckPredicateOpDataRef, CheckPredicateOpRef, CheckPredicateReport, CheckPredicateSummary, CheckPredicateTimeOptions, CheckPredicateTimeRange, CheckPredicateTimeSingle, CheckPredicateTimeSpec, CheckReport, CheckResult, CheckResultErrorInfo, CheckScenarioReport, CheckStatus, CheckSummary, CheckTestReport, CompareConfig, CompareDataCoordinator, CompareDataRequestKey, CompareDatasetReport, CompareDatasetSummary, CompareDatasets, CompareGroup, CompareGroupInfo, CompareItem, CompareOptions, CompareReport, CompareScenarios, CompareSummary, Config, ConfigOptions, DataSource, Dataset, DatasetInfo, DatasetKey, DatasetManager, DatasetMap, DatasetsResult, DiffPoint, DiffReport, DiffValidity, Dimension, GraphDatasetReport, GraphInclusion, GraphMetadataReport, GraphReport, ImplVar, InputPosition, InputSetting, InputVar, LegendItem, LinkItem, LoadedBundle, ModelSpec, NamedBundle, OutputVar, PerfReport, PerfRunner, PerfStats, PositionSetting, RelatedItem, RunSuiteCallbacks, RunSuiteOptions, Scenario, ScenarioGroupKey, ScenarioInfo, ScenarioKey, ScenarioManager, SettingsScenario, SourceName, Subscript, SuiteReport, SuiteSummary, ValueSetting, VarId, allInputsAtPositionScenario, checkReportFromSummary, checkSummaryFromReport, compareDatasets, compareSummaryFromReport, createConfig, datasetMessage, diffDatasets, diffGraphs, inputAtPositionScenario, inputAtValueScenario, matrixScenarios, positionSetting, predicateMessage, runSuite, scenarioMessage, settingsScenario, suiteSummaryFromReport, valueSetting };
1248
+ export { AllInputsSpec, Bundle, BundleGraphData, BundleGraphDatasetSpec, BundleGraphId, BundleGraphSpec, BundleGraphView, BundleModel, CheckDataCoordinator, CheckDataRequestKey, CheckDatasetReport, CheckGroupReport, CheckKey, CheckPredicateOp, CheckPredicateOpConstantRef, CheckPredicateOpDataRef, CheckPredicateOpRef, CheckPredicateReport, CheckPredicateSummary, CheckPredicateTimeOptions, CheckPredicateTimeRange, CheckPredicateTimeSingle, CheckPredicateTimeSpec, CheckReport, CheckResult, CheckResultErrorInfo, CheckScenario, CheckScenarioError, CheckScenarioInputDesc, CheckScenarioReport, CheckStatus, CheckSummary, CheckTestReport, ComparisonCategorizedResults, ComparisonConfig, ComparisonDataCoordinator, ComparisonDataRequestKey, ComparisonDataset, ComparisonDatasetOptions, ComparisonDatasets, ComparisonGroup, ComparisonGroupKey, ComparisonGroupKind, ComparisonGroupRoot, ComparisonGroupScores, ComparisonGroupSummariesByCategory, ComparisonGroupSummary, ComparisonOptions, ComparisonReport, ComparisonResolverError, ComparisonResolverInvalidValueError, ComparisonResolverUnknownInputError, ComparisonScenario, ComparisonScenarioAllInputsSettings, ComparisonScenarioGroup, ComparisonScenarioGroupId, ComparisonScenarioGroupRefSpec, ComparisonScenarioGroupSpec, ComparisonScenarioGroupTitle, ComparisonScenarioId, ComparisonScenarioInput, ComparisonScenarioInputAtPositionSpec, ComparisonScenarioInputAtValueSpec, ComparisonScenarioInputName, ComparisonScenarioInputPosition, ComparisonScenarioInputSettings, ComparisonScenarioInputSpec, ComparisonScenarioInputState, ComparisonScenarioKey, ComparisonScenarioPresetMatrixSpec, ComparisonScenarioRefSpec, ComparisonScenarioSettings, ComparisonScenarioSpec, ComparisonScenarioSubtitle, ComparisonScenarioTitle, ComparisonScenarioWithAllInputsSpec, ComparisonScenarioWithInputsSpec, ComparisonScenarios, ComparisonSpecs, ComparisonSpecsSource, ComparisonSummary, ComparisonTestReport, ComparisonTestSummary, ComparisonUnresolvedScenarioGroupRef, ComparisonUnresolvedScenarioRef, ComparisonUnresolvedView, ComparisonView, ComparisonViewGraphId, ComparisonViewGraphsArraySpec, ComparisonViewGraphsPresetSpec, ComparisonViewGraphsSpec, ComparisonViewGroup, ComparisonViewGroupSpec, ComparisonViewGroupTitle, ComparisonViewGroupWithScenariosSpec, ComparisonViewGroupWithViewsSpec, ComparisonViewSpec, ComparisonViewSubtitle, ComparisonViewTitle, Config, ConfigInitOptions, ConfigOptions, DataSource, Dataset, DatasetKey, DatasetMap, DatasetsResult, DiffPoint, DiffReport, DiffValidity, Dimension, GraphComparisonDatasetReport, GraphComparisonMetadataReport, GraphComparisonReport, GraphInclusion, ImplVar, InputId, InputPosition, InputSetting, InputSettingsSpec, InputVar, LegendItem, LinkItem, LoadedBundle, ModelSpec, NamedBundle, OutputVar, PerfReport, PerfRunner, PerfStats, PositionSetting, RelatedItem, RunSuiteCallbacks, RunSuiteOptions, ScenarioSpec, ScenarioSpecUid, SourceName, Subscript, SuiteReport, SuiteSummary, ValueSetting, VarId, categorizeComparisonTestSummaries, checkReportFromSummary, checkSummaryFromReport, comparisonSummaryFromReport, createConfig, datasetMessage, diffDatasets, diffGraphs, predicateMessage, runSuite, scenarioMessage, suiteSummaryFromReport };