@enwithai/product-check 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ATTRIBUTION.md +6 -6
- package/CHANGELOG.md +66 -0
- package/NOTICE.md +5 -3
- package/README.md +34 -19
- package/SECURITY.md +5 -1
- package/dashboard/assurance.css +71 -0
- package/dashboard/assurance.js +21 -0
- package/dashboard/dashboard.css +6 -0
- package/dist/assurance/adapters.d.ts +32 -0
- package/dist/assurance/adapters.js +76 -0
- package/dist/assurance/approvals.d.ts +34 -0
- package/dist/assurance/approvals.js +133 -0
- package/dist/assurance/cli.d.ts +13 -0
- package/dist/assurance/cli.js +863 -0
- package/dist/assurance/dashboard.d.ts +18 -0
- package/dist/assurance/dashboard.js +267 -0
- package/dist/assurance/diff.d.ts +20 -0
- package/dist/assurance/diff.js +99 -0
- package/dist/assurance/evidence-log.d.ts +20 -0
- package/dist/assurance/evidence-log.js +100 -0
- package/dist/assurance/fix-packets.d.ts +12 -0
- package/dist/assurance/fix-packets.js +178 -0
- package/dist/assurance/gate.d.ts +37 -0
- package/dist/assurance/gate.js +324 -0
- package/dist/assurance/lanes.d.ts +61 -0
- package/dist/assurance/lanes.js +380 -0
- package/dist/assurance/policy.d.ts +10 -0
- package/dist/assurance/policy.js +85 -0
- package/dist/assurance/proof.d.ts +35 -0
- package/dist/assurance/proof.js +146 -0
- package/dist/assurance/register.d.ts +41 -0
- package/dist/assurance/register.js +480 -0
- package/dist/assurance/render-md.d.ts +17 -0
- package/dist/assurance/render-md.js +212 -0
- package/dist/assurance/repo-state.d.ts +15 -0
- package/dist/assurance/repo-state.js +79 -0
- package/dist/assurance/review.d.ts +52 -0
- package/dist/assurance/review.js +408 -0
- package/dist/assurance/risk.d.ts +21 -0
- package/dist/assurance/risk.js +184 -0
- package/dist/assurance/types.d.ts +573 -0
- package/dist/assurance/types.js +6 -0
- package/dist/assurance/util.d.ts +24 -0
- package/dist/assurance/util.js +89 -0
- package/dist/assurance/verification.d.ts +30 -0
- package/dist/assurance/verification.js +212 -0
- package/dist/assurance/verify.d.ts +26 -0
- package/dist/assurance/verify.js +304 -0
- package/dist/blueprint/build.d.ts +3 -1
- package/dist/blueprint/build.js +41 -5
- package/dist/cli/run.js +103 -5
- package/dist/contract/contract.d.ts +4 -0
- package/dist/contract/contract.js +16 -0
- package/dist/dashboard/render.d.ts +5 -0
- package/dist/dashboard/render.js +107 -3
- package/dist/discovery/pops.d.ts +2 -0
- package/dist/discovery/pops.js +7 -5
- package/dist/exec/process.d.ts +27 -0
- package/dist/exec/process.js +175 -0
- package/dist/importers/operational.d.ts +56 -0
- package/dist/importers/operational.js +217 -0
- package/dist/importers/reports.d.ts +24 -0
- package/dist/importers/reports.js +37 -1
- package/dist/inspectors/source/surface.d.ts +30 -0
- package/dist/inspectors/source/surface.js +529 -0
- package/dist/parse/yaml.d.ts +4 -0
- package/dist/parse/yaml.js +39 -0
- package/dist/reporting/markdown.d.ts +1 -0
- package/dist/reporting/markdown.js +18 -1
- package/dist/reporting/spec-markdown.d.ts +9 -0
- package/dist/reporting/spec-markdown.js +144 -0
- package/dist/run.d.ts +2 -0
- package/dist/run.js +52 -2
- package/dist/schema/registry.js +10 -0
- package/dist/scoring/score.js +1 -1
- package/dist/spec/build.d.ts +28 -0
- package/dist/spec/build.js +278 -0
- package/dist/types.d.ts +78 -2
- package/dist/types.js +6 -1
- package/dist/validation/catalog.d.ts +1 -1
- package/dist/validation/catalog.js +125 -2
- package/dist/validation/context.d.ts +6 -1
- package/dist/validation/import-evaluators.js +37 -3
- package/dist/validation/integrity.js +4 -0
- package/dist/validation/operational-evaluators.d.ts +3 -0
- package/dist/validation/operational-evaluators.js +153 -0
- package/dist/validation/source-evaluators.js +1 -1
- package/package.json +4 -1
- package/schemas/approval.schema.json +98 -0
- package/schemas/as-built-spec.schema.json +356 -0
- package/schemas/assurance-common.schema.json +291 -0
- package/schemas/assurance-policy.schema.json +185 -0
- package/schemas/blueprint.schema.json +3 -0
- package/schemas/change-review.schema.json +711 -0
- package/schemas/closure-register.schema.json +499 -0
- package/schemas/evidence-record.schema.json +94 -0
- package/schemas/fix-packet.schema.json +272 -0
- package/schemas/release-gate.schema.json +308 -0
- package/schemas/reviewer-findings.schema.json +49 -0
- package/schemas/scan-contract.schema.json +20 -0
- package/schemas/scorecard.schema.json +1 -1
- package/schemas/verification-run.schema.json +199 -0
package/dist/types.d.ts
CHANGED
|
@@ -3,7 +3,13 @@
|
|
|
3
3
|
* schemas with both the built-in validator and Ajv.
|
|
4
4
|
*/
|
|
5
5
|
export declare const SCHEMA_VERSION: "0.1.0";
|
|
6
|
-
export declare const RUBRIC_VERSION: "pc-rubric-0.
|
|
6
|
+
export declare const RUBRIC_VERSION: "pc-rubric-0.2";
|
|
7
|
+
/**
|
|
8
|
+
* Rubric versions a stored scorecard may carry. pc-rubric-0.2 only adds control PC-T-17; the scoring rules are those
|
|
9
|
+
* of pc-rubric-0.1, so earlier results still recompute and load as baselines.
|
|
10
|
+
*/
|
|
11
|
+
export declare const RUBRIC_VERSIONS: readonly ["pc-rubric-0.1", "pc-rubric-0.2"];
|
|
12
|
+
export type RubricVersion = (typeof RUBRIC_VERSIONS)[number];
|
|
7
13
|
/** The three independent assessment pillars (CT3: Customer × Technology × Trust). */
|
|
8
14
|
export type Pillar = "customer" | "technology" | "trust";
|
|
9
15
|
export declare const PILLARS: readonly Pillar[];
|
|
@@ -101,6 +107,10 @@ export interface ScanContract {
|
|
|
101
107
|
spec_check: string | null;
|
|
102
108
|
impact_check: string | null;
|
|
103
109
|
outcome_check: string | null;
|
|
110
|
+
architecture_check: string | null;
|
|
111
|
+
openslo: string[];
|
|
112
|
+
alert_rules: string[];
|
|
113
|
+
service_graph: string | null;
|
|
104
114
|
};
|
|
105
115
|
control_mappings: ControlMapping[];
|
|
106
116
|
risk_acceptances: RiskAcceptance[];
|
|
@@ -250,7 +260,7 @@ export interface GateResult {
|
|
|
250
260
|
export interface Scorecard {
|
|
251
261
|
schema_version: typeof SCHEMA_VERSION;
|
|
252
262
|
kind: "scorecard";
|
|
253
|
-
rubric_version:
|
|
263
|
+
rubric_version: RubricVersion;
|
|
254
264
|
pillars: Record<Pillar, PillarScore>;
|
|
255
265
|
overall_coverage: number;
|
|
256
266
|
gates: GateResult[];
|
|
@@ -346,6 +356,8 @@ export interface Blueprint {
|
|
|
346
356
|
as_is: {
|
|
347
357
|
components: ArchitectureComponent[];
|
|
348
358
|
uncertainties: string[];
|
|
359
|
+
/** The as-built spec derived from the repository (absent when source was not inspected). */
|
|
360
|
+
as_built?: AsBuiltBody;
|
|
349
361
|
};
|
|
350
362
|
to_be: {
|
|
351
363
|
changes: TargetChange[];
|
|
@@ -484,3 +496,67 @@ export interface Results {
|
|
|
484
496
|
history: RunSummary[];
|
|
485
497
|
limitations: Limitation[];
|
|
486
498
|
}
|
|
499
|
+
/** Certainty of a statement in an as-built specification. */
|
|
500
|
+
export type SpecCertainty = "observed" | "inferred" | "hypothesis";
|
|
501
|
+
/** One element of the discovered product surface, with its evidence. */
|
|
502
|
+
export interface SurfaceEntry {
|
|
503
|
+
id: string;
|
|
504
|
+
kind: "page" | "endpoint" | "entity" | "config" | "integration" | "auth-boundary" | "test";
|
|
505
|
+
name: string;
|
|
506
|
+
method: string | null;
|
|
507
|
+
area: string;
|
|
508
|
+
detail: string;
|
|
509
|
+
location: {
|
|
510
|
+
path: string;
|
|
511
|
+
line: number | null;
|
|
512
|
+
};
|
|
513
|
+
certainty: "observed" | "inferred";
|
|
514
|
+
auth_hint: boolean;
|
|
515
|
+
evidence_ids: string[];
|
|
516
|
+
}
|
|
517
|
+
/** A requirement statement derived from the surface. */
|
|
518
|
+
export interface SpecRequirement {
|
|
519
|
+
id: string;
|
|
520
|
+
statement: string;
|
|
521
|
+
certainty: SpecCertainty;
|
|
522
|
+
surface_ids: string[];
|
|
523
|
+
evidence_ids: string[];
|
|
524
|
+
}
|
|
525
|
+
/** A user story grouping one area of the surface. */
|
|
526
|
+
export interface SpecStory {
|
|
527
|
+
id: string;
|
|
528
|
+
area: string;
|
|
529
|
+
title: string;
|
|
530
|
+
critical_journey: string | null;
|
|
531
|
+
surface_ids: string[];
|
|
532
|
+
requirement_ids: string[];
|
|
533
|
+
test_behaviours: string[];
|
|
534
|
+
}
|
|
535
|
+
/** The as-built specification document. */
|
|
536
|
+
export interface AsBuiltSpec {
|
|
537
|
+
schema_version: typeof SCHEMA_VERSION;
|
|
538
|
+
kind: "as-built-spec";
|
|
539
|
+
generated_at: string;
|
|
540
|
+
tool: {
|
|
541
|
+
name: "product-check";
|
|
542
|
+
version: string;
|
|
543
|
+
};
|
|
544
|
+
target: {
|
|
545
|
+
name: string;
|
|
546
|
+
repository: string;
|
|
547
|
+
commit: string | null;
|
|
548
|
+
ref: string | null;
|
|
549
|
+
};
|
|
550
|
+
surface: SurfaceEntry[];
|
|
551
|
+
stories: SpecStory[];
|
|
552
|
+
requirements: SpecRequirement[];
|
|
553
|
+
non_functional: SpecRequirement[];
|
|
554
|
+
success_criteria: SpecRequirement[];
|
|
555
|
+
edge_cases: SpecRequirement[];
|
|
556
|
+
clarifications: string[];
|
|
557
|
+
uncertainties: string[];
|
|
558
|
+
evidence: Evidence[];
|
|
559
|
+
limitations: Limitation[];
|
|
560
|
+
}
|
|
561
|
+
/** The derived content of an as-built spec, as embedded in a blueprint. */
|
|
562
|
+
export type AsBuiltBody = Pick<AsBuiltSpec, "surface" | "stories" | "requirements" | "non_functional" | "success_criteria" | "edge_cases" | "clarifications" | "uncertainties">;
|
package/dist/types.js
CHANGED
|
@@ -5,6 +5,11 @@
|
|
|
5
5
|
* schemas with both the built-in validator and Ajv.
|
|
6
6
|
*/
|
|
7
7
|
export const SCHEMA_VERSION = "0.1.0";
|
|
8
|
-
export const RUBRIC_VERSION = "pc-rubric-0.
|
|
8
|
+
export const RUBRIC_VERSION = "pc-rubric-0.2";
|
|
9
|
+
/**
|
|
10
|
+
* Rubric versions a stored scorecard may carry. pc-rubric-0.2 only adds control PC-T-17; the scoring rules are those
|
|
11
|
+
* of pc-rubric-0.1, so earlier results still recompute and load as baselines.
|
|
12
|
+
*/
|
|
13
|
+
export const RUBRIC_VERSIONS = ["pc-rubric-0.1", RUBRIC_VERSION];
|
|
9
14
|
export const PILLARS = ["customer", "technology", "trust"];
|
|
10
15
|
export const TIERS = ["CT-T0", "CT-T1", "CT-T2", "CT-T3", "CT-T4", "CT-T5", "CT-T6"];
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The control catalog (rubric pc-rubric-0.
|
|
2
|
+
* The control catalog (rubric pc-rubric-0.2). Each control states what is expected, which pillar and lifecycle
|
|
3
3
|
* tier it belongs to, its weight, whether it backs a hard gate, and the remediation knowledge used by the
|
|
4
4
|
* blueprint. Evaluation logic lives in evaluators; this file is data only.
|
|
5
5
|
*/
|
|
@@ -739,10 +739,13 @@ export const CONTROLS = [
|
|
|
739
739
|
},
|
|
740
740
|
root_cause: "No production telemetry is connected to the assessment.",
|
|
741
741
|
remediation: rem({
|
|
742
|
-
change: "Connect production metrics, logs and alerts
|
|
742
|
+
change: "Connect production metrics, logs and alerts, then export the alert rules (or OpenSLO alert policies) and a service graph and import them (--alert-rules, --openslo, --service-graph).",
|
|
743
743
|
why: "Launch watch depends on detection speed.",
|
|
744
744
|
to_be: "Alerts on error rate, latency and journey success.",
|
|
745
|
-
acceptance: [
|
|
745
|
+
acceptance: [
|
|
746
|
+
"Alerting is configured for critical signals.",
|
|
747
|
+
"Control PC-T-14 evaluates PASS on retest with alert configuration and a service graph imported.",
|
|
748
|
+
],
|
|
746
749
|
regression: "Alert test on deployment.",
|
|
747
750
|
effort: "M",
|
|
748
751
|
}),
|
|
@@ -805,6 +808,126 @@ export const CONTROLS = [
|
|
|
805
808
|
}),
|
|
806
809
|
},
|
|
807
810
|
// ------------------------------------------------------------------------------------------------- Trust
|
|
811
|
+
{
|
|
812
|
+
id: "PC-T-17",
|
|
813
|
+
pillar: "technology",
|
|
814
|
+
tier: "CT-T3",
|
|
815
|
+
title: "Architecture model matches the code",
|
|
816
|
+
expected: "An imported Architecture Check report shows no verified drift of medium severity or above between the declared architecture model and the code.",
|
|
817
|
+
category: "architecture",
|
|
818
|
+
weight: 1,
|
|
819
|
+
gate: null,
|
|
820
|
+
severity: "P2",
|
|
821
|
+
ux: null,
|
|
822
|
+
pops: ["repository"],
|
|
823
|
+
impact: {
|
|
824
|
+
customer: null,
|
|
825
|
+
technology: "Reviews, onboarding and risk decisions rely on an architecture model that no longer describes the system.",
|
|
826
|
+
trust: null,
|
|
827
|
+
},
|
|
828
|
+
root_cause: "The code changed (or the model was edited) without the other being updated.",
|
|
829
|
+
remediation: rem({
|
|
830
|
+
change: "Resolve each verified Architecture Check finding: update the model where the code is right, or change the code where it breaks a declared boundary.",
|
|
831
|
+
why: "A launch decision about the architecture is only as good as the model it is made against.",
|
|
832
|
+
to_be: "The declared model and the code agree, and Architecture Check runs on every pull request.",
|
|
833
|
+
acceptance: [
|
|
834
|
+
"The Architecture Check report has no verified finding of medium severity or above.",
|
|
835
|
+
"Control PC-T-17 evaluates PASS on retest.",
|
|
836
|
+
],
|
|
837
|
+
regression: "Architecture Check in pull-request CI.",
|
|
838
|
+
effort: "S",
|
|
839
|
+
}),
|
|
840
|
+
},
|
|
841
|
+
{
|
|
842
|
+
id: "PC-T-18",
|
|
843
|
+
pillar: "technology",
|
|
844
|
+
tier: "CT-T4",
|
|
845
|
+
title: "SLOs define an objective, a time window and an indicator",
|
|
846
|
+
expected: "Imported OpenSLO v1 files define at least one SLO, and every SLO has a service, one time window, an objective target and an indicator.",
|
|
847
|
+
category: "launch-watch",
|
|
848
|
+
weight: 1,
|
|
849
|
+
gate: null,
|
|
850
|
+
severity: "P2",
|
|
851
|
+
ux: null,
|
|
852
|
+
pops: ["observability"],
|
|
853
|
+
impact: {
|
|
854
|
+
customer: null,
|
|
855
|
+
technology: "Without complete SLOs nobody can tell whether the launched service meets its reliability target.",
|
|
856
|
+
trust: null,
|
|
857
|
+
},
|
|
858
|
+
root_cause: "SLOs are missing or were written without a measurable objective, window or indicator.",
|
|
859
|
+
remediation: rem({
|
|
860
|
+
change: "Define an OpenSLO v1 SLO per critical service with a target, one time window and an indicator (inline or an SLI reference).",
|
|
861
|
+
why: "A launch watch needs an agreed, measurable target to watch against.",
|
|
862
|
+
to_be: "Every critical service has a complete SLO under version control.",
|
|
863
|
+
acceptance: [
|
|
864
|
+
"Every imported SLO lists a service, a time window, an objective target and an indicator.",
|
|
865
|
+
"Control PC-T-18 evaluates PASS on retest.",
|
|
866
|
+
],
|
|
867
|
+
regression: "OpenSLO files validated in CI (oslo validate) and imported on every assessment.",
|
|
868
|
+
effort: "S",
|
|
869
|
+
}),
|
|
870
|
+
},
|
|
871
|
+
{
|
|
872
|
+
id: "PC-T-19",
|
|
873
|
+
pillar: "technology",
|
|
874
|
+
tier: "CT-T4",
|
|
875
|
+
title: "Every SLO is alerted on",
|
|
876
|
+
expected: "Every imported SLO has an OpenSLO alert policy or an imported alerting rule that names it.",
|
|
877
|
+
category: "launch-watch",
|
|
878
|
+
weight: 1,
|
|
879
|
+
gate: null,
|
|
880
|
+
severity: "P2",
|
|
881
|
+
ux: null,
|
|
882
|
+
pops: ["observability"],
|
|
883
|
+
impact: {
|
|
884
|
+
customer: null,
|
|
885
|
+
technology: "An SLO without an alert is breached silently; users notice before the team does.",
|
|
886
|
+
trust: null,
|
|
887
|
+
},
|
|
888
|
+
root_cause: "SLOs were defined without alert policies or burn-rate alert rules.",
|
|
889
|
+
remediation: rem({
|
|
890
|
+
change: "Attach an alert policy to each SLO, or generate burn-rate alerts (Sloth, Pyrra) that carry the SLO name as a label.",
|
|
891
|
+
why: "Detection speed decides how much error budget an incident consumes.",
|
|
892
|
+
to_be: "Every SLO pages through a burn-rate alert.",
|
|
893
|
+
acceptance: [
|
|
894
|
+
"Every SLO has an alert policy or an alerting rule labelled with its name.",
|
|
895
|
+
"Control PC-T-19 evaluates PASS on retest.",
|
|
896
|
+
],
|
|
897
|
+
regression: "Alert rules generated from the SLO files in CI.",
|
|
898
|
+
effort: "S",
|
|
899
|
+
}),
|
|
900
|
+
},
|
|
901
|
+
{
|
|
902
|
+
id: "PC-T-20",
|
|
903
|
+
pillar: "technology",
|
|
904
|
+
tier: "CT-T4",
|
|
905
|
+
title: "Declared dependencies are observed at runtime",
|
|
906
|
+
expected: "Each declared relationship that Architecture Check could not confirm statically appears as an edge in an imported service graph.",
|
|
907
|
+
category: "launch-watch",
|
|
908
|
+
weight: 1,
|
|
909
|
+
gate: null,
|
|
910
|
+
severity: "P2",
|
|
911
|
+
ux: null,
|
|
912
|
+
pops: ["observability", "repository"],
|
|
913
|
+
impact: {
|
|
914
|
+
customer: null,
|
|
915
|
+
technology: "The architecture model claims dependencies that may not exist at runtime, or that run through paths nobody traced.",
|
|
916
|
+
trust: null,
|
|
917
|
+
},
|
|
918
|
+
root_cause: "The model lists relationships that neither the code nor runtime traces show.",
|
|
919
|
+
remediation: rem({
|
|
920
|
+
change: "Check each listed relationship: remove it from the model if it no longer exists, or instrument the call so traces show it.",
|
|
921
|
+
why: "Runtime evidence is the only evidence for calls that static analysis cannot see.",
|
|
922
|
+
to_be: "Every declared relationship has static or runtime evidence.",
|
|
923
|
+
acceptance: [
|
|
924
|
+
"No declared relationship lacks both static evidence and a runtime edge.",
|
|
925
|
+
"Control PC-T-20 evaluates PASS on retest.",
|
|
926
|
+
],
|
|
927
|
+
regression: "Architecture Check and a service-graph export on every release.",
|
|
928
|
+
effort: "S",
|
|
929
|
+
}),
|
|
930
|
+
},
|
|
808
931
|
{
|
|
809
932
|
id: "PC-S-01",
|
|
810
933
|
pillar: "trust",
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
import type { EvidenceCollector } from "../evidence/collector.js";
|
|
5
5
|
import type { ObservationSet } from "../inspectors/http/observe.js";
|
|
6
6
|
import type { RepoFacts } from "../inspectors/source/facts.js";
|
|
7
|
-
import type { AuditSummary, ImpactCheckSummary, OutcomeCheckSummary, SpecCheckSummary } from "../importers/reports.js";
|
|
7
|
+
import type { ArchitectureCheckSummary, AuditSummary, ImpactCheckSummary, OutcomeCheckSummary, SpecCheckSummary } from "../importers/reports.js";
|
|
8
|
+
import type { AlertRulesSummary, OpenSloSummary, ServiceGraphSummary } from "../importers/operational.js";
|
|
8
9
|
import type { TestRun } from "../importers/tests.js";
|
|
9
10
|
import type { Confidence, Epistemic, EvidenceStrength, Results, ScanContract, Severity, UxSeverity, Validation, ValidationStatus } from "../types.js";
|
|
10
11
|
/** An imported artifact with its source path and checksum. */
|
|
@@ -27,6 +28,10 @@ export interface EvalContext {
|
|
|
27
28
|
spec: Imported<SpecCheckSummary> | null;
|
|
28
29
|
impact: Imported<ImpactCheckSummary> | null;
|
|
29
30
|
outcome: Imported<OutcomeCheckSummary> | null;
|
|
31
|
+
architecture: Imported<ArchitectureCheckSummary> | null;
|
|
32
|
+
openslo: Imported<OpenSloSummary> | null;
|
|
33
|
+
alertRules: Imported<AlertRulesSummary> | null;
|
|
34
|
+
serviceGraph: Imported<ServiceGraphSummary> | null;
|
|
30
35
|
baseline: Results | null;
|
|
31
36
|
aiInScope: boolean;
|
|
32
37
|
}
|
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
// SPDX-FileCopyrightText: 2026 EnWithAI Technologies
|
|
3
3
|
/**
|
|
4
4
|
* Evaluators backed by imported artifacts: executed test results (including contract control_mappings that link
|
|
5
|
-
* named tests to any control), npm audit,
|
|
5
|
+
* named tests to any control), npm audit, Spec/Impact/Outcome/Architecture Check reports, and the operational
|
|
6
|
+
* files of CT-T4 (operational-evaluators.ts). Imported results are evidence
|
|
6
7
|
* of executions that happened elsewhere; Product Check records where they came from and their checksum.
|
|
7
8
|
*/
|
|
8
9
|
import { isEndToEnd, tally } from "../importers/tests.js";
|
|
9
10
|
import { checked, unchecked, } from "./context.js";
|
|
11
|
+
import { OPERATIONAL_EVALUATORS } from "./operational-evaluators.js";
|
|
10
12
|
/** Evidence for a set of test cases from one results file. */
|
|
11
13
|
function testEv(ctx, run, cases, what) {
|
|
12
14
|
const t = tally(cases);
|
|
@@ -270,6 +272,35 @@ const PC_T_12 = (ctx) => {
|
|
|
270
272
|
},
|
|
271
273
|
]);
|
|
272
274
|
};
|
|
275
|
+
const PC_T_17 = (ctx) => {
|
|
276
|
+
if (!ctx.architecture)
|
|
277
|
+
return unchecked("NOT_TESTED", "No Architecture Check report was imported (optional; pass --architecture-check <report.json>).");
|
|
278
|
+
const s = ctx.architecture.data;
|
|
279
|
+
const rules = s.verified_rules.length > 0 ? ` (${s.verified_rules.join(", ")})` : "";
|
|
280
|
+
const id = ctx.ev.add({
|
|
281
|
+
kind: "imported-report",
|
|
282
|
+
pop: "repository",
|
|
283
|
+
strength: "imported-report",
|
|
284
|
+
provenance: "imported",
|
|
285
|
+
locator: ctx.architecture.source,
|
|
286
|
+
revision: null,
|
|
287
|
+
summary: `Architecture Check ${s.schema_version}: ${s.model_path ?? "model"} (${s.model_format ?? "unknown format"}), ${s.findings} finding(s); ${s.verified_drift} verified drift${rules}, ${s.hypotheses} hypotheses, ${s.unknown} unknown`,
|
|
288
|
+
checksum: ctx.architecture.sha256,
|
|
289
|
+
});
|
|
290
|
+
if (s.verified_drift === 0)
|
|
291
|
+
return checked("PASS", "No verified architecture drift of medium severity or above.", "imported-report", "imported-report", [id]);
|
|
292
|
+
return checked("PARTIAL", `${s.verified_drift} verified architecture drift finding(s)${rules}.`, "imported-report", "imported-report", [id], [
|
|
293
|
+
{
|
|
294
|
+
fingerprint: "architecture-drift",
|
|
295
|
+
location: s.model_path ?? ctx.architecture.source,
|
|
296
|
+
actual: `${s.verified_drift} verified drift finding(s) of medium severity or above${rules}.`,
|
|
297
|
+
evidence: [id],
|
|
298
|
+
epistemic: "fact",
|
|
299
|
+
confidence: "HIGH",
|
|
300
|
+
title: "The architecture model and the code have drifted",
|
|
301
|
+
},
|
|
302
|
+
]);
|
|
303
|
+
};
|
|
273
304
|
const PC_T_15 = (ctx) => {
|
|
274
305
|
if (!ctx.outcome)
|
|
275
306
|
return unchecked("NOT_TESTED", "No Outcome Check report was imported (optional; pass --outcome-check <report.json>).");
|
|
@@ -327,9 +358,12 @@ export const IMPORT_EVALUATORS = {
|
|
|
327
358
|
"PC-T-03": PC_T_03,
|
|
328
359
|
"PC-T-12": PC_T_12,
|
|
329
360
|
"PC-T-13": mappedOnly("PC-T-13", "Rollback needs an executed rehearsal."),
|
|
330
|
-
"PC-T-14": (ctx) => mappedOutcome(ctx, "PC-T-14") ??
|
|
331
|
-
unchecked("NOT_TESTED", "No production monitoring adapter exists in this release (read-only telemetry import is planned)."),
|
|
361
|
+
"PC-T-14": (ctx) => mappedOutcome(ctx, "PC-T-14") ?? OPERATIONAL_EVALUATORS["PC-T-14"](ctx),
|
|
332
362
|
"PC-T-15": PC_T_15,
|
|
363
|
+
"PC-T-17": PC_T_17,
|
|
364
|
+
"PC-T-18": OPERATIONAL_EVALUATORS["PC-T-18"],
|
|
365
|
+
"PC-T-19": OPERATIONAL_EVALUATORS["PC-T-19"],
|
|
366
|
+
"PC-T-20": OPERATIONAL_EVALUATORS["PC-T-20"],
|
|
333
367
|
"PC-S-13": (ctx) => ctx.contract.product.has_user_accounts === false
|
|
334
368
|
? unchecked("NOT_APPLICABLE", "The contract declares that the product has no user accounts or privileged paths.")
|
|
335
369
|
: mappedOnly("PC-S-13", "Authorization bypass testing needs credentialed negative tests, which Product Check does not run itself.")(ctx),
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { formatErrors, validateAgainst } from "../schema/registry.js";
|
|
9
9
|
import { computeScorecard } from "../scoring/score.js";
|
|
10
10
|
import { computeVerdict } from "../scoring/verdict.js";
|
|
11
|
+
import { RUBRIC_VERSIONS } from "../types.js";
|
|
11
12
|
import { CONTROL_BY_ID } from "./catalog.js";
|
|
12
13
|
/** Stable JSON for comparisons. */
|
|
13
14
|
const canon = (v) => JSON.stringify(v);
|
|
@@ -90,6 +91,9 @@ export function checkResults(doc) {
|
|
|
90
91
|
errors.push("comparison.baseline_run_id does not match run.baseline.");
|
|
91
92
|
const previous = r.history.length >= 2 ? (r.history.at(-2) ?? null) : null;
|
|
92
93
|
const scorecard = computeScorecard(r.validations, r.findings, r.scorecard.parameters, previous);
|
|
94
|
+
// Earlier rubrics share the scoring rules; keep the stored version so their results still recompute.
|
|
95
|
+
if (RUBRIC_VERSIONS.includes(r.scorecard.rubric_version))
|
|
96
|
+
scorecard.rubric_version = r.scorecard.rubric_version;
|
|
93
97
|
if (canon(scorecard) !== canon(r.scorecard))
|
|
94
98
|
errors.push("The scorecard does not recompute from the validations and findings (edited after generation?).");
|
|
95
99
|
else {
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
// SPDX-FileCopyrightText: 2026 EnWithAI Technologies
|
|
3
|
+
/**
|
|
4
|
+
* CT-T4 evaluators backed by exported operational files (docs/operational-imports.md): PC-T-14 (monitoring and
|
|
5
|
+
* alerting), PC-T-18 (SLOs complete), PC-T-19 (SLOs alerted) and PC-T-20 (declared dependencies observed at
|
|
6
|
+
* runtime). Configuration files are facts about configuration; a service graph is a snapshot of one time window.
|
|
7
|
+
*/
|
|
8
|
+
import { sloMatchesRule } from "../importers/operational.js";
|
|
9
|
+
import { checked, unchecked, } from "./context.js";
|
|
10
|
+
/** Evidence for the imported OpenSLO files. */
|
|
11
|
+
function sloEvidence(ctx, s, summary) {
|
|
12
|
+
return ctx.ev.add({
|
|
13
|
+
kind: "imported-report",
|
|
14
|
+
pop: "observability",
|
|
15
|
+
strength: "imported-report",
|
|
16
|
+
provenance: "imported",
|
|
17
|
+
locator: s.source,
|
|
18
|
+
revision: null,
|
|
19
|
+
summary,
|
|
20
|
+
checksum: s.sha256,
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
const PC_T_18 = (ctx) => {
|
|
24
|
+
if (!ctx.openslo)
|
|
25
|
+
return unchecked("NOT_TESTED", "No OpenSLO files were imported (optional; pass --openslo <file>).");
|
|
26
|
+
const s = ctx.openslo.data;
|
|
27
|
+
const incomplete = s.slos.filter((o) => o.missing.length > 0);
|
|
28
|
+
const services = [...new Set(s.slos.map((o) => o.service).filter((x) => x !== null))].sort();
|
|
29
|
+
const id = sloEvidence(ctx, ctx.openslo, `OpenSLO: ${s.slos.length} SLO(s) for ${services.length} service(s)${services.length ? ` (${services.slice(0, 10).join(", ")})` : ""}; ${incomplete.length} incomplete; ${s.slis} SLI(s), ${s.alert_policies} alert polic(ies); ${s.unsupported} document(s) of another OpenSLO version`);
|
|
30
|
+
if (s.slos.length === 0)
|
|
31
|
+
return checked("PARTIAL", "The imported OpenSLO files define no SLO.", "imported-report", "imported-report", [id], [
|
|
32
|
+
{
|
|
33
|
+
fingerprint: "no-slo",
|
|
34
|
+
location: ctx.openslo.source,
|
|
35
|
+
actual: "No SLO is defined in the imported OpenSLO files.",
|
|
36
|
+
evidence: [id],
|
|
37
|
+
epistemic: "fact",
|
|
38
|
+
confidence: "HIGH",
|
|
39
|
+
title: "No SLO is defined",
|
|
40
|
+
},
|
|
41
|
+
]);
|
|
42
|
+
if (incomplete.length === 0)
|
|
43
|
+
return checked("PASS", `All ${s.slos.length} SLO(s) define a service, a time window, an objective target and an indicator.`, "imported-report", "imported-report", [id]);
|
|
44
|
+
return checked("PARTIAL", `${incomplete.length} of ${s.slos.length} SLO(s) are incomplete.`, "imported-report", "imported-report", [id], incomplete.map((o) => ({
|
|
45
|
+
fingerprint: `slo-incomplete:${o.name}`,
|
|
46
|
+
location: o.name,
|
|
47
|
+
actual: `SLO ${o.name} is missing: ${o.missing.join(", ")}.`,
|
|
48
|
+
evidence: [id],
|
|
49
|
+
epistemic: "fact",
|
|
50
|
+
confidence: "HIGH",
|
|
51
|
+
title: "SLO is incomplete",
|
|
52
|
+
})));
|
|
53
|
+
};
|
|
54
|
+
const PC_T_19 = (ctx) => {
|
|
55
|
+
if (!ctx.openslo)
|
|
56
|
+
return unchecked("NOT_TESTED", "No OpenSLO files were imported (optional; pass --openslo <file>).");
|
|
57
|
+
const s = ctx.openslo.data;
|
|
58
|
+
if (s.slos.length === 0)
|
|
59
|
+
return unchecked("NOT_TESTED", "The imported OpenSLO files define no SLO.");
|
|
60
|
+
const rules = ctx.alertRules?.data.alerting ?? [];
|
|
61
|
+
const byPolicy = s.slos.filter((o) => o.has_alert_policy);
|
|
62
|
+
const byRule = s.slos.filter((o) => !o.has_alert_policy && rules.some((r) => sloMatchesRule(o.name, r)));
|
|
63
|
+
const uncovered = s.slos.filter((o) => !byPolicy.includes(o) && !byRule.includes(o));
|
|
64
|
+
const id = sloEvidence(ctx, ctx.openslo, `OpenSLO alerting: ${s.slos.length} SLO(s); ${byPolicy.length} by alert policy, ${byRule.length} by alert-rule name match (inference)${ctx.alertRules ? ` against ${rules.length} alerting rule(s) in ${ctx.alertRules.source}` : "; no alert rules imported"}; ${uncovered.length} without an alert`);
|
|
65
|
+
if (uncovered.length === 0)
|
|
66
|
+
return checked("PASS", "Every SLO has an alert policy or a matching alerting rule.", "imported-report", "imported-report", [id]);
|
|
67
|
+
return checked("PARTIAL", `${uncovered.length} of ${s.slos.length} SLO(s) have no alert policy and no matching alerting rule.`, "imported-report", "imported-report", [id], uncovered.map((o) => ({
|
|
68
|
+
fingerprint: `slo-unalerted:${o.name}`,
|
|
69
|
+
location: o.name,
|
|
70
|
+
actual: `SLO ${o.name} has no alert policy and no alerting rule names it${o.dangling_policy_refs.length ? ` (unresolved alertPolicyRef: ${o.dangling_policy_refs.join(", ")})` : ""}.`,
|
|
71
|
+
evidence: [id],
|
|
72
|
+
epistemic: "fact",
|
|
73
|
+
confidence: "HIGH",
|
|
74
|
+
title: "SLO is not alerted on",
|
|
75
|
+
})));
|
|
76
|
+
};
|
|
77
|
+
/** Lower case, letters and digits only. */
|
|
78
|
+
const norm = (s) => s.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
79
|
+
const lastSegment = (id) => id.split(".").at(-1) ?? id;
|
|
80
|
+
const PC_T_20 = (ctx) => {
|
|
81
|
+
if (!ctx.serviceGraph)
|
|
82
|
+
return unchecked("NOT_TESTED", "No service graph was imported (optional; pass --service-graph <file>).");
|
|
83
|
+
if (!ctx.architecture)
|
|
84
|
+
return unchecked("NOT_TESTED", "A service graph was imported, but the declared relationships come from an Architecture Check report (--architecture-check).");
|
|
85
|
+
const g = ctx.serviceGraph.data;
|
|
86
|
+
const candidates = ctx.architecture.data.unobserved;
|
|
87
|
+
const confirmed = candidates.filter((c) => g.edges.some((e) => norm(e.client) === norm(lastSegment(c.from)) && norm(e.server) === norm(lastSegment(c.to))));
|
|
88
|
+
const missing = candidates.filter((c) => !confirmed.includes(c));
|
|
89
|
+
const id = ctx.ev.add({
|
|
90
|
+
kind: "imported-report",
|
|
91
|
+
pop: "observability",
|
|
92
|
+
strength: "imported-report",
|
|
93
|
+
provenance: "imported",
|
|
94
|
+
locator: ctx.serviceGraph.source,
|
|
95
|
+
revision: null,
|
|
96
|
+
summary: `Service graph (${g.format}): ${g.edges.length} edge(s), ${g.external_peers} external peer(s); ${candidates.length} declared relationship(s) without static evidence, ${confirmed.length} observed at runtime (name match). A snapshot of one time window that the export does not record.`,
|
|
97
|
+
checksum: ctx.serviceGraph.sha256,
|
|
98
|
+
});
|
|
99
|
+
if (missing.length === 0)
|
|
100
|
+
return checked("PASS", candidates.length === 0
|
|
101
|
+
? "Every declared relationship already has static evidence; nothing needed runtime confirmation."
|
|
102
|
+
: `All ${candidates.length} declared relationship(s) without static evidence were observed at runtime.`, "imported-report", "imported-report", [id]);
|
|
103
|
+
return checked("PARTIAL", `${missing.length} declared relationship(s) have neither static evidence nor a runtime edge in this snapshot.`, "imported-report", "imported-report", [id], missing.map((c) => ({
|
|
104
|
+
fingerprint: `relationship-not-observed:${c.from}->${c.to}`,
|
|
105
|
+
location: `${c.from} -> ${c.to}`,
|
|
106
|
+
actual: `The declared relationship ${c.from} -> ${c.to} has no static evidence and no runtime edge in the imported service graph.`,
|
|
107
|
+
evidence: [id],
|
|
108
|
+
epistemic: "inference",
|
|
109
|
+
confidence: "MEDIUM",
|
|
110
|
+
title: "Declared dependency not observed at runtime",
|
|
111
|
+
})));
|
|
112
|
+
};
|
|
113
|
+
const PC_T_14 = (ctx) => {
|
|
114
|
+
const alerting = (ctx.alertRules?.data.alerting.length ?? 0) > 0 ||
|
|
115
|
+
(ctx.openslo?.data.slos.some((o) => o.has_alert_policy) ?? false);
|
|
116
|
+
const telemetry = (ctx.serviceGraph?.data.edges.length ?? 0) > 0;
|
|
117
|
+
if (!ctx.alertRules && !ctx.openslo && !ctx.serviceGraph)
|
|
118
|
+
return unchecked("NOT_TESTED", "No operational files were imported (optional; pass --openslo, --alert-rules and/or --service-graph).");
|
|
119
|
+
const sources = [ctx.openslo, ctx.alertRules, ctx.serviceGraph].filter((x) => x !== null);
|
|
120
|
+
const id = ctx.ev.add({
|
|
121
|
+
kind: "imported-report",
|
|
122
|
+
pop: "observability",
|
|
123
|
+
strength: "imported-report",
|
|
124
|
+
provenance: "imported",
|
|
125
|
+
locator: sources.map((s) => s.source).join(", "),
|
|
126
|
+
revision: null,
|
|
127
|
+
summary: `Monitoring: alert configuration ${alerting ? "present" : "absent"} (${ctx.alertRules?.data.alerting.length ?? 0} alerting rule(s), ${ctx.openslo?.data.slos.filter((o) => o.has_alert_policy).length ?? 0} SLO(s) with an alert policy); runtime telemetry ${telemetry ? `present (${ctx.serviceGraph?.data.edges.length ?? 0} service-graph edge(s))` : "absent"}`,
|
|
128
|
+
checksum: sources.length === 1 ? (sources[0]?.sha256 ?? null) : null,
|
|
129
|
+
});
|
|
130
|
+
if (alerting && telemetry)
|
|
131
|
+
return checked("PASS", "Alert configuration and runtime telemetry were imported.", "imported-report", "imported-report", [id]);
|
|
132
|
+
const missingHalf = alerting
|
|
133
|
+
? "runtime telemetry (a service graph)"
|
|
134
|
+
: "alert configuration (alert rules or SLO alert policies)";
|
|
135
|
+
return checked("PARTIAL", `Missing ${missingHalf}.`, "imported-report", "imported-report", [id], [
|
|
136
|
+
{
|
|
137
|
+
fingerprint: "monitoring-incomplete",
|
|
138
|
+
location: sources.map((s) => s.source).join(", "),
|
|
139
|
+
actual: `The imported files show no ${missingHalf}.`,
|
|
140
|
+
evidence: [id],
|
|
141
|
+
epistemic: "inference",
|
|
142
|
+
confidence: "MEDIUM",
|
|
143
|
+
title: "Monitoring or alerting evidence is incomplete",
|
|
144
|
+
},
|
|
145
|
+
]);
|
|
146
|
+
};
|
|
147
|
+
/** Evaluators for the operational controls, keyed by control ID. */
|
|
148
|
+
export const OPERATIONAL_EVALUATORS = {
|
|
149
|
+
"PC-T-14": PC_T_14,
|
|
150
|
+
"PC-T-18": PC_T_18,
|
|
151
|
+
"PC-T-19": PC_T_19,
|
|
152
|
+
"PC-T-20": PC_T_20,
|
|
153
|
+
};
|
|
@@ -36,7 +36,7 @@ function scopeEv(ctx, files, what) {
|
|
|
36
36
|
provenance: "observed",
|
|
37
37
|
locator: `repository (${files.length} text files)`,
|
|
38
38
|
revision: ctx.revision,
|
|
39
|
-
summary: `Searched ${files.length} readable, non-ignored text files for ${what} (rubric pc-rubric-0.
|
|
39
|
+
summary: `Searched ${files.length} readable, non-ignored text files for ${what} (rubric pc-rubric-0.2).`,
|
|
40
40
|
checksum: fingerprint,
|
|
41
41
|
});
|
|
42
42
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@enwithai/product-check",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Evidence-led product assurance and software blueprinting: scan a repository and/or an authorized deployed app, validate Customer, Technology and Trust controls with evidence, and get findings, a remediation blueprint, a readiness verdict and a local dashboard. Local-first; JSON is the source of truth.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "EnWithAI Technologies",
|
|
@@ -73,5 +73,8 @@
|
|
|
73
73
|
},
|
|
74
74
|
"publishConfig": {
|
|
75
75
|
"access": "public"
|
|
76
|
+
},
|
|
77
|
+
"dependencies": {
|
|
78
|
+
"yaml": "2.9.1"
|
|
76
79
|
}
|
|
77
80
|
}
|