monora-ai 2.0.0 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +441 -150
- package/dist/aims_governance.d.ts +238 -0
- package/dist/aims_governance.d.ts.map +1 -0
- package/dist/aims_governance.js +922 -0
- package/dist/alerts.d.ts +16 -0
- package/dist/alerts.d.ts.map +1 -1
- package/dist/alerts.js +16 -0
- package/dist/api.d.ts +6 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +6 -0
- package/dist/assessment.d.ts +269 -0
- package/dist/assessment.d.ts.map +1 -0
- package/dist/assessment.js +1232 -0
- package/dist/attestation.js +23 -1
- package/dist/attribution.d.ts +349 -0
- package/dist/attribution.d.ts.map +1 -0
- package/dist/attribution.js +987 -0
- package/dist/autodetect.d.ts +69 -1
- package/dist/autodetect.d.ts.map +1 -1
- package/dist/autodetect.js +644 -1
- package/dist/bias.d.ts +130 -0
- package/dist/bias.d.ts.map +1 -0
- package/dist/bias.js +223 -0
- package/dist/circuit_breaker.js +3 -3
- package/dist/cli/diagnostics.d.ts +5 -1
- package/dist/cli/diagnostics.d.ts.map +1 -1
- package/dist/cli/diagnostics.js +31 -8
- package/dist/cli/doctor.d.ts +25 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +381 -0
- package/dist/cli/fix.d.ts +16 -0
- package/dist/cli/fix.d.ts.map +1 -0
- package/dist/cli/fix.js +284 -0
- package/dist/cli/init.d.ts +57 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +205 -0
- package/dist/cli.js +1611 -126
- package/dist/complianceTargets.d.ts +111 -0
- package/dist/complianceTargets.d.ts.map +1 -0
- package/dist/complianceTargets.js +521 -0
- package/dist/config.d.ts +301 -17
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +428 -36
- package/dist/config_migrations.d.ts +41 -0
- package/dist/config_migrations.d.ts.map +1 -1
- package/dist/config_migrations.js +205 -0
- package/dist/config_schema.d.ts +2900 -731
- package/dist/config_schema.d.ts.map +1 -1
- package/dist/config_schema.js +257 -55
- package/dist/context.d.ts +34 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +118 -7
- package/dist/control_backbone.d.ts +122 -0
- package/dist/control_backbone.d.ts.map +1 -0
- package/dist/control_backbone.js +698 -0
- package/dist/data-governance.d.ts +187 -0
- package/dist/data-governance.d.ts.map +1 -0
- package/dist/data-governance.js +424 -0
- package/dist/dataResidency.d.ts +44 -0
- package/dist/dataResidency.d.ts.map +1 -0
- package/dist/dataResidency.js +203 -0
- package/dist/dispatcher.d.ts +32 -0
- package/dist/dispatcher.d.ts.map +1 -1
- package/dist/dispatcher.js +91 -4
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js +38 -0
- package/dist/evidence_store.d.ts +103 -0
- package/dist/evidence_store.d.ts.map +1 -0
- package/dist/evidence_store.js +459 -0
- package/dist/executiveSummary.d.ts +65 -8
- package/dist/executiveSummary.d.ts.map +1 -1
- package/dist/executiveSummary.js +289 -26
- package/dist/identity.d.ts +143 -0
- package/dist/identity.d.ts.map +1 -0
- package/dist/identity.js +231 -0
- package/dist/impact-assessment.d.ts +350 -0
- package/dist/impact-assessment.d.ts.map +1 -0
- package/dist/impact-assessment.js +580 -0
- package/dist/index.d.ts +25 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +300 -4
- package/dist/instrumentation.d.ts +1 -1
- package/dist/instrumentation.d.ts.map +1 -1
- package/dist/instrumentation.js +243 -27
- package/dist/integrations/anthropic.d.ts +3 -0
- package/dist/integrations/anthropic.d.ts.map +1 -1
- package/dist/integrations/anthropic.js +284 -79
- package/dist/integrations/governance.d.ts +33 -0
- package/dist/integrations/governance.d.ts.map +1 -0
- package/dist/integrations/governance.js +208 -0
- package/dist/integrations/langchain.d.ts +7 -0
- package/dist/integrations/langchain.d.ts.map +1 -1
- package/dist/integrations/langchain.js +387 -143
- package/dist/integrations/openai.d.ts +9 -0
- package/dist/integrations/openai.d.ts.map +1 -1
- package/dist/integrations/openai.js +673 -73
- package/dist/iso42001_consolidation.d.ts +16 -0
- package/dist/iso42001_consolidation.d.ts.map +1 -0
- package/dist/iso42001_consolidation.js +413 -0
- package/dist/iso42001_workflows.d.ts +263 -0
- package/dist/iso42001_workflows.d.ts.map +1 -0
- package/dist/iso42001_workflows.js +781 -0
- package/dist/lifecycle.d.ts +299 -0
- package/dist/lifecycle.d.ts.map +1 -0
- package/dist/lifecycle.js +624 -0
- package/dist/lineage.d.ts +2 -2
- package/dist/lineage.d.ts.map +1 -1
- package/dist/lineage.js +12 -17
- package/dist/middleware/express.d.ts.map +1 -1
- package/dist/middleware/express.js +33 -3
- package/dist/middleware/nextjs.d.ts.map +1 -1
- package/dist/middleware/nextjs.js +42 -68
- package/dist/model.d.ts +143 -0
- package/dist/model.d.ts.map +1 -0
- package/dist/model.js +371 -0
- package/dist/onboarding.d.ts +42 -0
- package/dist/onboarding.d.ts.map +1 -0
- package/dist/onboarding.js +1022 -0
- package/dist/oversight.d.ts +264 -0
- package/dist/oversight.d.ts.map +1 -0
- package/dist/oversight.js +497 -0
- package/dist/pdf_report.d.ts.map +1 -1
- package/dist/pdf_report.js +42 -21
- package/dist/presets.d.ts +88 -0
- package/dist/presets.d.ts.map +1 -0
- package/dist/presets.js +520 -0
- package/dist/propagation.d.ts.map +1 -1
- package/dist/propagation.js +34 -2
- package/dist/quotas.d.ts +171 -0
- package/dist/quotas.d.ts.map +1 -0
- package/dist/quotas.js +259 -0
- package/dist/register.d.ts +13 -0
- package/dist/register.d.ts.map +1 -0
- package/dist/register.js +99 -0
- package/dist/registry.d.ts +1 -0
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +7 -0
- package/dist/registryData.json +43 -6
- package/dist/report.d.ts +2 -1
- package/dist/report.d.ts.map +1 -1
- package/dist/report.js +189 -2
- package/dist/reporting.d.ts +125 -0
- package/dist/reporting.d.ts.map +1 -1
- package/dist/reporting.js +196 -5
- package/dist/resources.d.ts +285 -0
- package/dist/resources.d.ts.map +1 -0
- package/dist/resources.js +643 -0
- package/dist/risk.d.ts +120 -0
- package/dist/risk.d.ts.map +1 -0
- package/dist/risk.js +220 -0
- package/dist/runtime.d.ts +74 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +598 -22
- package/dist/schemaInference.d.ts +92 -0
- package/dist/schemaInference.d.ts.map +1 -0
- package/dist/schemaInference.js +466 -0
- package/dist/schema_validation.js +2 -2
- package/dist/schemas/config.schema.json +169 -6
- package/dist/schemas/event.schema.json +4 -0
- package/dist/security_report.js +4 -4
- package/dist/signing.d.ts +1 -1
- package/dist/signing.d.ts.map +1 -1
- package/dist/signing.js +4 -0
- package/dist/sinks/file.d.ts +19 -1
- package/dist/sinks/file.d.ts.map +1 -1
- package/dist/sinks/file.js +82 -13
- package/dist/sinks/https.d.ts +10 -0
- package/dist/sinks/https.d.ts.map +1 -1
- package/dist/sinks/https.js +76 -16
- package/dist/sinks/stdout.d.ts +1 -0
- package/dist/sinks/stdout.d.ts.map +1 -1
- package/dist/sinks/stdout.js +12 -1
- package/dist/spec.d.ts +159 -0
- package/dist/spec.d.ts.map +1 -0
- package/dist/spec.js +391 -0
- package/dist/stakeholders.d.ts +199 -0
- package/dist/stakeholders.d.ts.map +1 -0
- package/dist/stakeholders.js +398 -0
- package/dist/standards.d.ts.map +1 -1
- package/dist/standards.js +160 -2
- package/dist/standards_ingest.d.ts +2 -2
- package/dist/standards_ingest.d.ts.map +1 -1
- package/dist/standards_ingest.js +105 -23
- package/dist/streaming.d.ts.map +1 -1
- package/dist/streaming.js +7 -2
- package/dist/telemetry.d.ts +16 -2
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +79 -14
- package/dist/templates/controls/iso42001_control_catalog.json +1443 -0
- package/dist/traced_emitter.d.ts +3 -0
- package/dist/traced_emitter.d.ts.map +1 -1
- package/dist/traced_emitter.js +142 -25
- package/dist/trust_package.d.ts +21 -1
- package/dist/trust_package.d.ts.map +1 -1
- package/dist/trust_package.js +101 -4
- package/dist/verify.d.ts.map +1 -1
- package/dist/verify.js +9 -2
- package/dist/wal.d.ts.map +1 -1
- package/dist/wal.js +2 -1
- package/package.json +14 -1
- package/scripts/postinstall.js +119 -97
- package/templates/controls/iso42001_control_catalog.json +1443 -0
package/dist/bias.d.ts
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bias metrics capture for ISO 42001 fairness compliance.
|
|
3
|
+
*
|
|
4
|
+
* This module provides bias measurement tracking for AI outputs, supporting
|
|
5
|
+
* ISO 42001 fairness controls and EU AI Act Article 10 requirements.
|
|
6
|
+
*
|
|
7
|
+
* Cross-SDK Parity:
|
|
8
|
+
* Both Python and Node.js SDKs provide identical bias metrics APIs:
|
|
9
|
+
* - recordBiasMeasurement() / record_bias_measurement()
|
|
10
|
+
* - checkThresholdViolations() / check_threshold_violations()
|
|
11
|
+
* - BiasMetrics interface / BiasMetrics dataclass
|
|
12
|
+
*/
|
|
13
|
+
import { MonoraConfig } from './config';
|
|
14
|
+
export type MeasurementType = 'demographic_parity' | 'equalized_odds' | 'disparate_impact' | 'custom';
|
|
15
|
+
/**
|
|
16
|
+
* Bias measurement metrics.
|
|
17
|
+
*/
|
|
18
|
+
export interface BiasMetrics {
|
|
19
|
+
/** Ratio comparing outcomes across groups (0-1, 1=parity) */
|
|
20
|
+
demographicParityRatio?: number;
|
|
21
|
+
/** Difference in true positive rates across groups (0=equal) */
|
|
22
|
+
equalizedOddsDiff?: number;
|
|
23
|
+
/** 4/5ths rule ratio (>=0.8 typically required) */
|
|
24
|
+
disparateImpact?: number;
|
|
25
|
+
/** Additional custom metric values */
|
|
26
|
+
customMetrics?: Record<string, number>;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Convert BiasMetrics to dictionary for event inclusion.
|
|
30
|
+
*/
|
|
31
|
+
export declare function metricsToDict(metrics: BiasMetrics): Record<string, any>;
|
|
32
|
+
/**
|
|
33
|
+
* A complete bias measurement.
|
|
34
|
+
*/
|
|
35
|
+
export interface BiasMeasurement {
|
|
36
|
+
/** The model being measured */
|
|
37
|
+
model: string;
|
|
38
|
+
/** Type of measurement performed */
|
|
39
|
+
measurementType: MeasurementType;
|
|
40
|
+
/** Demographic dimensions measured */
|
|
41
|
+
dimensions: string[];
|
|
42
|
+
/** The computed bias metrics */
|
|
43
|
+
metrics: BiasMetrics;
|
|
44
|
+
/** Number of samples in the measurement */
|
|
45
|
+
sampleSize: number;
|
|
46
|
+
/** Name of the evaluation dataset used */
|
|
47
|
+
evaluationDataset?: string;
|
|
48
|
+
/** List of thresholds violated */
|
|
49
|
+
thresholdViolations: string[];
|
|
50
|
+
/** When the measurement was taken */
|
|
51
|
+
timestamp: string;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Convert BiasMeasurement to event body dictionary.
|
|
55
|
+
*/
|
|
56
|
+
export declare function measurementToEventBody(measurement: BiasMeasurement): Record<string, any>;
|
|
57
|
+
export type BiasCallback = (model: string, inputs: any[], outputs: any[]) => BiasMetrics;
|
|
58
|
+
/**
|
|
59
|
+
* Options for recording a bias measurement.
|
|
60
|
+
*/
|
|
61
|
+
export interface RecordBiasMeasurementOptions {
|
|
62
|
+
/** Type of measurement (demographic_parity, equalized_odds, etc.) */
|
|
63
|
+
measurementType?: MeasurementType;
|
|
64
|
+
/** Demographic dimensions measured (gender, age_group, etc.) */
|
|
65
|
+
dimensions?: string[];
|
|
66
|
+
/** Dict of metric name to value */
|
|
67
|
+
metrics?: Record<string, number>;
|
|
68
|
+
/** Number of samples in measurement */
|
|
69
|
+
sampleSize?: number;
|
|
70
|
+
/** Name of evaluation dataset */
|
|
71
|
+
evaluationDataset?: string;
|
|
72
|
+
/** Optional config (uses runtime state if not provided) */
|
|
73
|
+
config?: MonoraConfig;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Record a bias measurement for a model.
|
|
77
|
+
*
|
|
78
|
+
* This stores the measurement and checks for threshold violations.
|
|
79
|
+
*
|
|
80
|
+
* @param model - The model being measured.
|
|
81
|
+
* @param options - Measurement options.
|
|
82
|
+
* @returns The created BiasMeasurement.
|
|
83
|
+
*
|
|
84
|
+
* @example
|
|
85
|
+
* ```typescript
|
|
86
|
+
* recordBiasMeasurement('gpt-4o', {
|
|
87
|
+
* measurementType: 'demographic_parity',
|
|
88
|
+
* dimensions: ['gender'],
|
|
89
|
+
* metrics: { demographic_parity_ratio: 0.85, sample_size: 500 }
|
|
90
|
+
* });
|
|
91
|
+
* ```
|
|
92
|
+
*/
|
|
93
|
+
export declare function recordBiasMeasurement(model: string, options?: RecordBiasMeasurementOptions): BiasMeasurement;
|
|
94
|
+
/**
|
|
95
|
+
* Check if any bias metrics violate configured thresholds.
|
|
96
|
+
*
|
|
97
|
+
* @param metrics - The bias metrics to check.
|
|
98
|
+
* @param config - The Monora configuration.
|
|
99
|
+
* @returns List of violated threshold names.
|
|
100
|
+
*/
|
|
101
|
+
export declare function checkThresholdViolations(metrics: BiasMetrics, config: MonoraConfig): string[];
|
|
102
|
+
/**
|
|
103
|
+
* Set a custom callback for automatic bias measurement.
|
|
104
|
+
*
|
|
105
|
+
* The callback is invoked after LLM calls to compute bias metrics
|
|
106
|
+
* based on inputs and outputs.
|
|
107
|
+
*
|
|
108
|
+
* @param callback - Function taking (model, inputs, outputs) => BiasMetrics,
|
|
109
|
+
* or null to clear the callback.
|
|
110
|
+
*/
|
|
111
|
+
export declare function setMeasurementCallback(callback: BiasCallback | null): void;
|
|
112
|
+
/**
|
|
113
|
+
* Get the current measurement callback.
|
|
114
|
+
*/
|
|
115
|
+
export declare function getMeasurementCallback(): BiasCallback | null;
|
|
116
|
+
/**
|
|
117
|
+
* Get all recorded bias measurements.
|
|
118
|
+
*/
|
|
119
|
+
export declare function getAllMeasurements(): BiasMeasurement[];
|
|
120
|
+
/**
|
|
121
|
+
* Clear all stored measurements.
|
|
122
|
+
*/
|
|
123
|
+
export declare function clearMeasurements(): void;
|
|
124
|
+
/**
|
|
125
|
+
* Get summary of bias measurements for reporting.
|
|
126
|
+
*
|
|
127
|
+
* @returns Summary dict for compliance reports.
|
|
128
|
+
*/
|
|
129
|
+
export declare function getBiasSummary(): Record<string, any>;
|
|
130
|
+
//# sourceMappingURL=bias.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bias.d.ts","sourceRoot":"","sources":["../src/bias.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAGxC,MAAM,MAAM,eAAe,GAAG,oBAAoB,GAAG,gBAAgB,GAAG,kBAAkB,GAAG,QAAQ,CAAC;AAEtG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,6DAA6D;IAC7D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,gEAAgE;IAChE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mDAAmD;IACnD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,sCAAsC;IACtC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAoBvE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,eAAe,EAAE,eAAe,CAAC;IACjC,sCAAsC;IACtC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,gCAAgC;IAChC,OAAO,EAAE,WAAW,CAAC;IACrB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kCAAkC;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAWxF;AAMD,MAAM,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,WAAW,CAAC;AAGzF;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,qEAAqE;IACrE,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,gEAAgE;IAChE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,mCAAmC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,2DAA2D;IAC3D,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,4BAAiC,GACzC,eAAe,CA8CjB;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,WAAW,EACpB,MAAM,EAAE,YAAY,GACnB,MAAM,EAAE,CA8CV;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,GAAG,IAAI,CAE1E;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,YAAY,GAAG,IAAI,CAE5D;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,eAAe,EAAE,CAEtD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED;;;;GAIG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAuBpD"}
|
package/dist/bias.js
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Bias metrics capture for ISO 42001 fairness compliance.
|
|
4
|
+
*
|
|
5
|
+
* This module provides bias measurement tracking for AI outputs, supporting
|
|
6
|
+
* ISO 42001 fairness controls and EU AI Act Article 10 requirements.
|
|
7
|
+
*
|
|
8
|
+
* Cross-SDK Parity:
|
|
9
|
+
* Both Python and Node.js SDKs provide identical bias metrics APIs:
|
|
10
|
+
* - recordBiasMeasurement() / record_bias_measurement()
|
|
11
|
+
* - checkThresholdViolations() / check_threshold_violations()
|
|
12
|
+
* - BiasMetrics interface / BiasMetrics dataclass
|
|
13
|
+
*/
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.metricsToDict = metricsToDict;
|
|
16
|
+
exports.measurementToEventBody = measurementToEventBody;
|
|
17
|
+
exports.recordBiasMeasurement = recordBiasMeasurement;
|
|
18
|
+
exports.checkThresholdViolations = checkThresholdViolations;
|
|
19
|
+
exports.setMeasurementCallback = setMeasurementCallback;
|
|
20
|
+
exports.getMeasurementCallback = getMeasurementCallback;
|
|
21
|
+
exports.getAllMeasurements = getAllMeasurements;
|
|
22
|
+
exports.clearMeasurements = clearMeasurements;
|
|
23
|
+
exports.getBiasSummary = getBiasSummary;
|
|
24
|
+
const logger_1 = require("./logger");
|
|
25
|
+
/**
|
|
26
|
+
* Convert BiasMetrics to dictionary for event inclusion.
|
|
27
|
+
*/
|
|
28
|
+
function metricsToDict(metrics) {
|
|
29
|
+
const result = {};
|
|
30
|
+
if (metrics.demographicParityRatio !== undefined) {
|
|
31
|
+
result.demographic_parity_ratio = metrics.demographicParityRatio;
|
|
32
|
+
}
|
|
33
|
+
if (metrics.equalizedOddsDiff !== undefined) {
|
|
34
|
+
result.equalized_odds_diff = metrics.equalizedOddsDiff;
|
|
35
|
+
}
|
|
36
|
+
if (metrics.disparateImpact !== undefined) {
|
|
37
|
+
result.disparate_impact = metrics.disparateImpact;
|
|
38
|
+
}
|
|
39
|
+
if (metrics.customMetrics) {
|
|
40
|
+
Object.assign(result, metrics.customMetrics);
|
|
41
|
+
}
|
|
42
|
+
return result;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Convert BiasMeasurement to event body dictionary.
|
|
46
|
+
*/
|
|
47
|
+
function measurementToEventBody(measurement) {
|
|
48
|
+
return {
|
|
49
|
+
model: measurement.model,
|
|
50
|
+
measurement_type: measurement.measurementType,
|
|
51
|
+
dimensions: measurement.dimensions,
|
|
52
|
+
metrics: metricsToDict(measurement.metrics),
|
|
53
|
+
sample_size: measurement.sampleSize,
|
|
54
|
+
evaluation_dataset: measurement.evaluationDataset,
|
|
55
|
+
threshold_violations: measurement.thresholdViolations,
|
|
56
|
+
timestamp: measurement.timestamp,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
// Stored measurements for reporting
|
|
60
|
+
const measurements = [];
|
|
61
|
+
let measurementCallback = null;
|
|
62
|
+
/**
|
|
63
|
+
* Record a bias measurement for a model.
|
|
64
|
+
*
|
|
65
|
+
* This stores the measurement and checks for threshold violations.
|
|
66
|
+
*
|
|
67
|
+
* @param model - The model being measured.
|
|
68
|
+
* @param options - Measurement options.
|
|
69
|
+
* @returns The created BiasMeasurement.
|
|
70
|
+
*
|
|
71
|
+
* @example
|
|
72
|
+
* ```typescript
|
|
73
|
+
* recordBiasMeasurement('gpt-4o', {
|
|
74
|
+
* measurementType: 'demographic_parity',
|
|
75
|
+
* dimensions: ['gender'],
|
|
76
|
+
* metrics: { demographic_parity_ratio: 0.85, sample_size: 500 }
|
|
77
|
+
* });
|
|
78
|
+
* ```
|
|
79
|
+
*/
|
|
80
|
+
function recordBiasMeasurement(model, options = {}) {
|
|
81
|
+
const { measurementType = 'custom', dimensions = [], metrics = {}, sampleSize = 0, evaluationDataset, config = {}, } = options;
|
|
82
|
+
const biasConfig = config.bias || {};
|
|
83
|
+
// Parse metrics into BiasMetrics
|
|
84
|
+
const biasMetrics = {
|
|
85
|
+
demographicParityRatio: metrics.demographic_parity_ratio,
|
|
86
|
+
equalizedOddsDiff: metrics.equalized_odds_diff,
|
|
87
|
+
disparateImpact: metrics.disparate_impact,
|
|
88
|
+
customMetrics: Object.fromEntries(Object.entries(metrics).filter(([k]) => !['demographic_parity_ratio', 'equalized_odds_diff', 'disparate_impact'].includes(k))),
|
|
89
|
+
};
|
|
90
|
+
// Check threshold violations
|
|
91
|
+
const violations = checkThresholdViolations(biasMetrics, config);
|
|
92
|
+
const measurement = {
|
|
93
|
+
model,
|
|
94
|
+
measurementType,
|
|
95
|
+
dimensions: dimensions.length > 0 ? dimensions : (biasConfig.dimensions || []),
|
|
96
|
+
metrics: biasMetrics,
|
|
97
|
+
sampleSize,
|
|
98
|
+
evaluationDataset,
|
|
99
|
+
thresholdViolations: violations,
|
|
100
|
+
timestamp: new Date().toISOString(),
|
|
101
|
+
};
|
|
102
|
+
measurements.push(measurement);
|
|
103
|
+
// Send alert if violations and configured
|
|
104
|
+
if (violations.length > 0 && (biasConfig.alert_on_violation ?? true)) {
|
|
105
|
+
sendViolationAlert(measurement, config);
|
|
106
|
+
}
|
|
107
|
+
return measurement;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Check if any bias metrics violate configured thresholds.
|
|
111
|
+
*
|
|
112
|
+
* @param metrics - The bias metrics to check.
|
|
113
|
+
* @param config - The Monora configuration.
|
|
114
|
+
* @returns List of violated threshold names.
|
|
115
|
+
*/
|
|
116
|
+
function checkThresholdViolations(metrics, config) {
|
|
117
|
+
const violations = [];
|
|
118
|
+
const biasConfig = config.bias || {};
|
|
119
|
+
const thresholds = biasConfig.thresholds || {};
|
|
120
|
+
// Check demographic parity ratio (must be >= threshold)
|
|
121
|
+
if (metrics.demographicParityRatio !== undefined) {
|
|
122
|
+
const threshold = thresholds.demographic_parity_ratio ?? 0.8;
|
|
123
|
+
if (metrics.demographicParityRatio < threshold) {
|
|
124
|
+
violations.push(`demographic_parity_ratio (${metrics.demographicParityRatio.toFixed(3)} < ${threshold})`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
// Check disparate impact (must be >= threshold, typically 0.8 for 4/5ths rule)
|
|
128
|
+
if (metrics.disparateImpact !== undefined) {
|
|
129
|
+
const threshold = thresholds.disparate_impact ?? 0.8;
|
|
130
|
+
if (metrics.disparateImpact < threshold) {
|
|
131
|
+
violations.push(`disparate_impact (${metrics.disparateImpact.toFixed(3)} < ${threshold})`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
// Check equalized odds diff (must be <= threshold, lower is better)
|
|
135
|
+
if (metrics.equalizedOddsDiff !== undefined) {
|
|
136
|
+
const threshold = thresholds.equalized_odds_diff ?? 0.1;
|
|
137
|
+
if (metrics.equalizedOddsDiff > threshold) {
|
|
138
|
+
violations.push(`equalized_odds_diff (${metrics.equalizedOddsDiff.toFixed(3)} > ${threshold})`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// Check custom thresholds
|
|
142
|
+
if (metrics.customMetrics) {
|
|
143
|
+
for (const [metricName, value] of Object.entries(metrics.customMetrics)) {
|
|
144
|
+
const threshold = thresholds[metricName];
|
|
145
|
+
if (threshold !== undefined && value < threshold) {
|
|
146
|
+
violations.push(`${metricName} (${value.toFixed(3)} < ${threshold})`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return violations;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Set a custom callback for automatic bias measurement.
|
|
154
|
+
*
|
|
155
|
+
* The callback is invoked after LLM calls to compute bias metrics
|
|
156
|
+
* based on inputs and outputs.
|
|
157
|
+
*
|
|
158
|
+
* @param callback - Function taking (model, inputs, outputs) => BiasMetrics,
|
|
159
|
+
* or null to clear the callback.
|
|
160
|
+
*/
|
|
161
|
+
function setMeasurementCallback(callback) {
|
|
162
|
+
measurementCallback = callback;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Get the current measurement callback.
|
|
166
|
+
*/
|
|
167
|
+
function getMeasurementCallback() {
|
|
168
|
+
return measurementCallback;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Get all recorded bias measurements.
|
|
172
|
+
*/
|
|
173
|
+
function getAllMeasurements() {
|
|
174
|
+
return [...measurements];
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Clear all stored measurements.
|
|
178
|
+
*/
|
|
179
|
+
function clearMeasurements() {
|
|
180
|
+
measurements.length = 0;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Get summary of bias measurements for reporting.
|
|
184
|
+
*
|
|
185
|
+
* @returns Summary dict for compliance reports.
|
|
186
|
+
*/
|
|
187
|
+
function getBiasSummary() {
|
|
188
|
+
if (measurements.length === 0) {
|
|
189
|
+
return {
|
|
190
|
+
measurements_recorded: 0,
|
|
191
|
+
threshold_violations: 0,
|
|
192
|
+
dimensions_measured: [],
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
const allDimensions = new Set();
|
|
196
|
+
let totalViolations = 0;
|
|
197
|
+
for (const m of measurements) {
|
|
198
|
+
m.dimensions.forEach((d) => allDimensions.add(d));
|
|
199
|
+
totalViolations += m.thresholdViolations.length;
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
measurements_recorded: measurements.length,
|
|
203
|
+
threshold_violations: totalViolations,
|
|
204
|
+
dimensions_measured: Array.from(allDimensions).sort(),
|
|
205
|
+
models_measured: Array.from(new Set(measurements.map((m) => m.model))),
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Send alert for threshold violations.
|
|
210
|
+
*/
|
|
211
|
+
function sendViolationAlert(measurement, config) {
|
|
212
|
+
try {
|
|
213
|
+
const alertConfig = config.alerts;
|
|
214
|
+
if (!alertConfig?.violation_webhook) {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
// Log the violation - actual alert sending would use the alerts module
|
|
218
|
+
logger_1.logger.warning('Bias threshold violations detected for model %s: %s', measurement.model, measurement.thresholdViolations.join(', '));
|
|
219
|
+
}
|
|
220
|
+
catch (error) {
|
|
221
|
+
logger_1.logger.warning('Failed to send bias violation alert: %s', error);
|
|
222
|
+
}
|
|
223
|
+
}
|
package/dist/circuit_breaker.js
CHANGED
|
@@ -110,7 +110,7 @@ class CircuitBreaker {
|
|
|
110
110
|
* In closed state, resets failure count.
|
|
111
111
|
*/
|
|
112
112
|
recordSuccess() {
|
|
113
|
-
if (this.
|
|
113
|
+
if (this.state === CircuitState.HALF_OPEN) {
|
|
114
114
|
this._successCount++;
|
|
115
115
|
logger_1.logger.debug('[%s] Success recorded in HALF_OPEN (%d/%d)', this.name, this._successCount, this.successThreshold);
|
|
116
116
|
if (this._successCount >= this.successThreshold) {
|
|
@@ -119,7 +119,7 @@ class CircuitBreaker {
|
|
|
119
119
|
this._failureCount = 0;
|
|
120
120
|
}
|
|
121
121
|
}
|
|
122
|
-
else if (this.
|
|
122
|
+
else if (this.state === CircuitState.CLOSED) {
|
|
123
123
|
// Reset failure count on success
|
|
124
124
|
this._failureCount = 0;
|
|
125
125
|
}
|
|
@@ -133,7 +133,7 @@ class CircuitBreaker {
|
|
|
133
133
|
recordFailure() {
|
|
134
134
|
this._failureCount++;
|
|
135
135
|
this._lastFailureTime = Date.now();
|
|
136
|
-
if (this.
|
|
136
|
+
if (this.state === CircuitState.HALF_OPEN) {
|
|
137
137
|
logger_1.logger.warning('[%s] Circuit reopening HALF_OPEN -> OPEN after failure', this.name);
|
|
138
138
|
this._state = CircuitState.OPEN;
|
|
139
139
|
}
|
|
@@ -2,13 +2,17 @@
|
|
|
2
2
|
* Configuration diagnostics and validation.
|
|
3
3
|
*/
|
|
4
4
|
import { MonoraConfig } from '../config';
|
|
5
|
+
import { ValidationMode } from '../config_schema';
|
|
5
6
|
interface ValidationResult {
|
|
6
7
|
errors: string[];
|
|
7
8
|
warnings: string[];
|
|
8
9
|
}
|
|
9
|
-
export declare function validateConfig(config: MonoraConfig
|
|
10
|
+
export declare function validateConfig(config: MonoraConfig, options?: {
|
|
11
|
+
mode?: ValidationMode;
|
|
12
|
+
}): ValidationResult;
|
|
10
13
|
export declare function doctorChecks(config: MonoraConfig, options?: {
|
|
11
14
|
checkNetwork?: boolean;
|
|
15
|
+
mode?: ValidationMode;
|
|
12
16
|
}): ValidationResult;
|
|
13
17
|
export declare function emitResults(results: ValidationResult, options?: {
|
|
14
18
|
json?: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diagnostics.d.ts","sourceRoot":"","sources":["../../src/cli/diagnostics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,
|
|
1
|
+
{"version":3,"file":"diagnostics.d.ts","sourceRoot":"","sources":["../../src/cli/diagnostics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAyB,cAAc,EAA4C,MAAM,kBAAkB,CAAC;AAQnH,UAAU,gBAAgB;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,cAAc,CAAA;CAAE,GAAG,gBAAgB,CAmK1G;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,YAAY,EACpB,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAC;IAAC,IAAI,CAAC,EAAE,cAAc,CAAA;CAAE,GAC1D,gBAAgB,CAqDlB;AA+CD,wBAAgB,WAAW,CACzB,OAAO,EAAE,gBAAgB,EACzB,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,GAC3B,MAAM,CAqCR"}
|
package/dist/cli/diagnostics.js
CHANGED
|
@@ -42,17 +42,35 @@ exports.emitResults = emitResults;
|
|
|
42
42
|
const fs = __importStar(require("fs"));
|
|
43
43
|
const net = __importStar(require("net"));
|
|
44
44
|
const url_1 = require("url");
|
|
45
|
-
const
|
|
45
|
+
const config_schema_1 = require("../config_schema");
|
|
46
|
+
const config_migrations_1 = require("../config_migrations");
|
|
46
47
|
const ALLOWED_QUEUE_MODES = new Set(['warn', 'raise', 'block']);
|
|
47
48
|
const ALLOWED_SINK_FAILURE_MODES = new Set(['warn', 'raise', 'silent']);
|
|
48
49
|
const ALLOWED_ROTATION = new Set(['none', 'daily', 'size']);
|
|
49
50
|
const ALLOWED_DATA_HANDLING = new Set(['redact', 'block', 'allow']);
|
|
50
|
-
function validateConfig(config) {
|
|
51
|
+
function validateConfig(config, options) {
|
|
51
52
|
const errors = [];
|
|
52
53
|
const warnings = [];
|
|
53
|
-
const
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
const mode = options?.mode || 'strict';
|
|
55
|
+
const schemaWarnings = [];
|
|
56
|
+
try {
|
|
57
|
+
(0, config_schema_1.validateConfig)(config, {
|
|
58
|
+
mode,
|
|
59
|
+
logger: { warn: (message) => schemaWarnings.push(message) },
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
if (error instanceof config_schema_1.ConfigValidationError) {
|
|
64
|
+
for (const issue of error.errors) {
|
|
65
|
+
errors.push(`schema: ${issue.path.join('.')}: ${issue.message}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
errors.push(`schema: ${error?.message || error}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (schemaWarnings.length > 0) {
|
|
73
|
+
warnings.push(...schemaWarnings);
|
|
56
74
|
}
|
|
57
75
|
// Validate sinks
|
|
58
76
|
const sinks = config.sinks || [];
|
|
@@ -80,7 +98,7 @@ function validateConfig(config) {
|
|
|
80
98
|
}
|
|
81
99
|
continue;
|
|
82
100
|
}
|
|
83
|
-
if (sinkType === 'https') {
|
|
101
|
+
if (sinkType === 'https' || sinkType === 'http') {
|
|
84
102
|
const endpoint = sink.endpoint;
|
|
85
103
|
if (!endpoint) {
|
|
86
104
|
errors.push(`${prefix}: https sink requires 'endpoint'.`);
|
|
@@ -167,11 +185,16 @@ function validateConfig(config) {
|
|
|
167
185
|
if (instrumentation.enabled && !instrumentation.targets?.length) {
|
|
168
186
|
warnings.push('instrumentation.enabled true but no targets configured.');
|
|
169
187
|
}
|
|
188
|
+
// Production readiness warnings
|
|
189
|
+
const readinessWarnings = (0, config_migrations_1.validateProductionReadiness)(config);
|
|
190
|
+
for (const warning of readinessWarnings) {
|
|
191
|
+
warnings.push(`production: [${warning.category}] ${warning.message} (recommendation: ${warning.recommendation})`);
|
|
192
|
+
}
|
|
170
193
|
return { errors, warnings };
|
|
171
194
|
}
|
|
172
195
|
function doctorChecks(config, options) {
|
|
173
196
|
const checkNetwork = options?.checkNetwork ?? true;
|
|
174
|
-
const result = validateConfig(config);
|
|
197
|
+
const result = validateConfig(config, { mode: options?.mode });
|
|
175
198
|
const errors = [...result.errors];
|
|
176
199
|
const warnings = [...result.warnings];
|
|
177
200
|
// Check axios for HTTPS sink
|
|
@@ -187,7 +210,7 @@ function doctorChecks(config, options) {
|
|
|
187
210
|
errors.push(`${prefix}: cannot write to ${path}.`);
|
|
188
211
|
}
|
|
189
212
|
}
|
|
190
|
-
if (sinkType === 'https') {
|
|
213
|
+
if (sinkType === 'https' || sinkType === 'http') {
|
|
191
214
|
if (!axiosAvailable) {
|
|
192
215
|
errors.push(`${prefix}: axios not installed (required for HTTPS sink).`);
|
|
193
216
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Config doctor checks for Monora CLI.
|
|
3
|
+
*/
|
|
4
|
+
import { ValidationMode } from '../config_schema';
|
|
5
|
+
export interface DoctorCheck {
|
|
6
|
+
name: string;
|
|
7
|
+
ok: boolean;
|
|
8
|
+
level: 'error' | 'warning';
|
|
9
|
+
message: string;
|
|
10
|
+
recommendation?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface DoctorResult {
|
|
13
|
+
ok: boolean;
|
|
14
|
+
errors: string[];
|
|
15
|
+
warnings: string[];
|
|
16
|
+
checks: DoctorCheck[];
|
|
17
|
+
}
|
|
18
|
+
export declare function doctorChecks(configPath: string, options?: {
|
|
19
|
+
checkNetwork?: boolean;
|
|
20
|
+
mode?: ValidationMode;
|
|
21
|
+
}): Promise<DoctorResult>;
|
|
22
|
+
export declare function emitDoctorResults(result: DoctorResult, options?: {
|
|
23
|
+
json?: boolean;
|
|
24
|
+
}): number;
|
|
25
|
+
//# sourceMappingURL=doctor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../src/cli/doctor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,OAAO,EAAyB,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAEzF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAID,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAC;IAAC,IAAI,CAAC,EAAE,cAAc,CAAA;CAAE,GAC1D,OAAO,CAAC,YAAY,CAAC,CAgHvB;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,MAAM,CAkC5F"}
|