monora-ai 2.1.0 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +333 -159
- package/dist/aims_governance.d.ts +238 -0
- package/dist/aims_governance.d.ts.map +1 -0
- package/dist/aims_governance.js +922 -0
- package/dist/alerts.d.ts +16 -0
- package/dist/alerts.d.ts.map +1 -1
- package/dist/alerts.js +16 -0
- package/dist/api.d.ts +6 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +6 -0
- package/dist/assessment.d.ts +85 -0
- package/dist/assessment.d.ts.map +1 -1
- package/dist/assessment.js +506 -13
- package/dist/attribution.d.ts +44 -3
- package/dist/attribution.d.ts.map +1 -1
- package/dist/attribution.js +197 -10
- package/dist/autodetect.d.ts +68 -0
- package/dist/autodetect.d.ts.map +1 -1
- package/dist/autodetect.js +639 -0
- package/dist/bias.d.ts +130 -0
- package/dist/bias.d.ts.map +1 -0
- package/dist/bias.js +223 -0
- package/dist/cli/diagnostics.d.ts +5 -1
- package/dist/cli/diagnostics.d.ts.map +1 -1
- package/dist/cli/diagnostics.js +23 -6
- package/dist/cli/doctor.d.ts +25 -0
- package/dist/cli/doctor.d.ts.map +1 -0
- package/dist/cli/doctor.js +381 -0
- package/dist/cli/fix.d.ts +16 -0
- package/dist/cli/fix.d.ts.map +1 -0
- package/dist/cli/fix.js +284 -0
- package/dist/cli/init.d.ts +57 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +205 -0
- package/dist/cli.js +1550 -176
- package/dist/complianceTargets.d.ts +111 -0
- package/dist/complianceTargets.d.ts.map +1 -0
- package/dist/complianceTargets.js +521 -0
- package/dist/config.d.ts +261 -16
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +381 -32
- package/dist/config_migrations.d.ts.map +1 -1
- package/dist/config_migrations.js +38 -1
- package/dist/config_schema.d.ts +2490 -1035
- package/dist/config_schema.d.ts.map +1 -1
- package/dist/config_schema.js +233 -64
- package/dist/context.d.ts +34 -0
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +118 -7
- package/dist/control_backbone.d.ts +122 -0
- package/dist/control_backbone.d.ts.map +1 -0
- package/dist/control_backbone.js +698 -0
- package/dist/data-governance.d.ts +187 -0
- package/dist/data-governance.d.ts.map +1 -0
- package/dist/data-governance.js +424 -0
- package/dist/dataResidency.d.ts +44 -0
- package/dist/dataResidency.d.ts.map +1 -0
- package/dist/dataResidency.js +203 -0
- package/dist/dispatcher.d.ts.map +1 -1
- package/dist/dispatcher.js +17 -5
- package/dist/evidence_store.d.ts +103 -0
- package/dist/evidence_store.d.ts.map +1 -0
- package/dist/evidence_store.js +459 -0
- package/dist/executiveSummary.d.ts +15 -0
- package/dist/executiveSummary.d.ts.map +1 -1
- package/dist/executiveSummary.js +135 -22
- package/dist/identity.d.ts +143 -0
- package/dist/identity.d.ts.map +1 -0
- package/dist/identity.js +231 -0
- package/dist/impact-assessment.d.ts +350 -0
- package/dist/impact-assessment.d.ts.map +1 -0
- package/dist/impact-assessment.js +580 -0
- package/dist/index.d.ts +20 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +247 -5
- package/dist/instrumentation.d.ts +1 -1
- package/dist/instrumentation.d.ts.map +1 -1
- package/dist/instrumentation.js +123 -22
- package/dist/integrations/anthropic.d.ts +3 -0
- package/dist/integrations/anthropic.d.ts.map +1 -1
- package/dist/integrations/anthropic.js +282 -80
- package/dist/integrations/governance.d.ts +33 -0
- package/dist/integrations/governance.d.ts.map +1 -0
- package/dist/integrations/governance.js +208 -0
- package/dist/integrations/langchain.d.ts +4 -0
- package/dist/integrations/langchain.d.ts.map +1 -1
- package/dist/integrations/langchain.js +362 -142
- package/dist/integrations/openai.d.ts +9 -0
- package/dist/integrations/openai.d.ts.map +1 -1
- package/dist/integrations/openai.js +673 -73
- package/dist/iso42001_consolidation.d.ts +16 -0
- package/dist/iso42001_consolidation.d.ts.map +1 -0
- package/dist/iso42001_consolidation.js +413 -0
- package/dist/iso42001_workflows.d.ts +263 -0
- package/dist/iso42001_workflows.d.ts.map +1 -0
- package/dist/iso42001_workflows.js +781 -0
- package/dist/lifecycle.d.ts +299 -0
- package/dist/lifecycle.d.ts.map +1 -0
- package/dist/lifecycle.js +624 -0
- package/dist/lineage.d.ts +2 -2
- package/dist/lineage.d.ts.map +1 -1
- package/dist/lineage.js +9 -16
- package/dist/middleware/express.d.ts.map +1 -1
- package/dist/middleware/express.js +18 -3
- package/dist/middleware/nextjs.js +2 -2
- package/dist/model.d.ts +143 -0
- package/dist/model.d.ts.map +1 -0
- package/dist/model.js +371 -0
- package/dist/onboarding.d.ts +42 -0
- package/dist/onboarding.d.ts.map +1 -0
- package/dist/onboarding.js +1022 -0
- package/dist/oversight.d.ts +264 -0
- package/dist/oversight.d.ts.map +1 -0
- package/dist/oversight.js +497 -0
- package/dist/presets.js +7 -7
- package/dist/quotas.d.ts +171 -0
- package/dist/quotas.d.ts.map +1 -0
- package/dist/quotas.js +259 -0
- package/dist/register.d.ts +13 -0
- package/dist/register.d.ts.map +1 -0
- package/dist/register.js +99 -0
- package/dist/registry.d.ts +1 -0
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +7 -0
- package/dist/registryData.json +43 -6
- package/dist/report.d.ts +2 -1
- package/dist/report.d.ts.map +1 -1
- package/dist/report.js +189 -2
- package/dist/reporting.d.ts +125 -0
- package/dist/reporting.d.ts.map +1 -1
- package/dist/reporting.js +192 -2
- package/dist/resources.d.ts +285 -0
- package/dist/resources.d.ts.map +1 -0
- package/dist/resources.js +643 -0
- package/dist/risk.d.ts +120 -0
- package/dist/risk.d.ts.map +1 -0
- package/dist/risk.js +220 -0
- package/dist/runtime.d.ts +73 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +415 -18
- package/dist/schemaInference.d.ts +92 -0
- package/dist/schemaInference.d.ts.map +1 -0
- package/dist/schemaInference.js +466 -0
- package/dist/schema_validation.js +2 -2
- package/dist/schemas/config.schema.json +118 -4
- package/dist/security_report.js +4 -4
- package/dist/signing.d.ts +1 -1
- package/dist/signing.d.ts.map +1 -1
- package/dist/signing.js +4 -0
- package/dist/sinks/file.d.ts +19 -1
- package/dist/sinks/file.d.ts.map +1 -1
- package/dist/sinks/file.js +82 -13
- package/dist/sinks/https.d.ts +10 -0
- package/dist/sinks/https.d.ts.map +1 -1
- package/dist/sinks/https.js +76 -16
- package/dist/sinks/stdout.d.ts +1 -0
- package/dist/sinks/stdout.d.ts.map +1 -1
- package/dist/sinks/stdout.js +12 -1
- package/dist/spec.d.ts +159 -0
- package/dist/spec.d.ts.map +1 -0
- package/dist/spec.js +391 -0
- package/dist/stakeholders.d.ts +199 -0
- package/dist/stakeholders.d.ts.map +1 -0
- package/dist/stakeholders.js +398 -0
- package/dist/standards.d.ts.map +1 -1
- package/dist/standards.js +160 -2
- package/dist/standards_ingest.d.ts.map +1 -1
- package/dist/standards_ingest.js +1 -4
- package/dist/telemetry.d.ts +16 -2
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +77 -14
- package/dist/templates/controls/iso42001_control_catalog.json +1443 -0
- package/dist/traced_emitter.d.ts.map +1 -1
- package/dist/traced_emitter.js +19 -9
- package/dist/trust_package.d.ts +19 -1
- package/dist/trust_package.d.ts.map +1 -1
- package/dist/trust_package.js +89 -2
- package/dist/verify.d.ts.map +1 -1
- package/dist/verify.js +9 -2
- package/dist/wal.d.ts.map +1 -1
- package/dist/wal.js +2 -1
- package/package.json +14 -1
- package/scripts/postinstall.js +105 -210
- package/templates/controls/iso42001_control_catalog.json +1443 -0
package/dist/bias.d.ts
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bias metrics capture for ISO 42001 fairness compliance.
|
|
3
|
+
*
|
|
4
|
+
* This module provides bias measurement tracking for AI outputs, supporting
|
|
5
|
+
* ISO 42001 fairness controls and EU AI Act Article 10 requirements.
|
|
6
|
+
*
|
|
7
|
+
* Cross-SDK Parity:
|
|
8
|
+
* Both Python and Node.js SDKs provide identical bias metrics APIs:
|
|
9
|
+
* - recordBiasMeasurement() / record_bias_measurement()
|
|
10
|
+
* - checkThresholdViolations() / check_threshold_violations()
|
|
11
|
+
* - BiasMetrics interface / BiasMetrics dataclass
|
|
12
|
+
*/
|
|
13
|
+
import { MonoraConfig } from './config';
|
|
14
|
+
export type MeasurementType = 'demographic_parity' | 'equalized_odds' | 'disparate_impact' | 'custom';
|
|
15
|
+
/**
|
|
16
|
+
* Bias measurement metrics.
|
|
17
|
+
*/
|
|
18
|
+
export interface BiasMetrics {
|
|
19
|
+
/** Ratio comparing outcomes across groups (0-1, 1=parity) */
|
|
20
|
+
demographicParityRatio?: number;
|
|
21
|
+
/** Difference in true positive rates across groups (0=equal) */
|
|
22
|
+
equalizedOddsDiff?: number;
|
|
23
|
+
/** 4/5ths rule ratio (>=0.8 typically required) */
|
|
24
|
+
disparateImpact?: number;
|
|
25
|
+
/** Additional custom metric values */
|
|
26
|
+
customMetrics?: Record<string, number>;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Convert BiasMetrics to dictionary for event inclusion.
|
|
30
|
+
*/
|
|
31
|
+
export declare function metricsToDict(metrics: BiasMetrics): Record<string, any>;
|
|
32
|
+
/**
|
|
33
|
+
* A complete bias measurement.
|
|
34
|
+
*/
|
|
35
|
+
export interface BiasMeasurement {
|
|
36
|
+
/** The model being measured */
|
|
37
|
+
model: string;
|
|
38
|
+
/** Type of measurement performed */
|
|
39
|
+
measurementType: MeasurementType;
|
|
40
|
+
/** Demographic dimensions measured */
|
|
41
|
+
dimensions: string[];
|
|
42
|
+
/** The computed bias metrics */
|
|
43
|
+
metrics: BiasMetrics;
|
|
44
|
+
/** Number of samples in the measurement */
|
|
45
|
+
sampleSize: number;
|
|
46
|
+
/** Name of the evaluation dataset used */
|
|
47
|
+
evaluationDataset?: string;
|
|
48
|
+
/** List of thresholds violated */
|
|
49
|
+
thresholdViolations: string[];
|
|
50
|
+
/** When the measurement was taken */
|
|
51
|
+
timestamp: string;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Convert BiasMeasurement to event body dictionary.
|
|
55
|
+
*/
|
|
56
|
+
export declare function measurementToEventBody(measurement: BiasMeasurement): Record<string, any>;
|
|
57
|
+
export type BiasCallback = (model: string, inputs: any[], outputs: any[]) => BiasMetrics;
|
|
58
|
+
/**
|
|
59
|
+
* Options for recording a bias measurement.
|
|
60
|
+
*/
|
|
61
|
+
export interface RecordBiasMeasurementOptions {
|
|
62
|
+
/** Type of measurement (demographic_parity, equalized_odds, etc.) */
|
|
63
|
+
measurementType?: MeasurementType;
|
|
64
|
+
/** Demographic dimensions measured (gender, age_group, etc.) */
|
|
65
|
+
dimensions?: string[];
|
|
66
|
+
/** Dict of metric name to value */
|
|
67
|
+
metrics?: Record<string, number>;
|
|
68
|
+
/** Number of samples in measurement */
|
|
69
|
+
sampleSize?: number;
|
|
70
|
+
/** Name of evaluation dataset */
|
|
71
|
+
evaluationDataset?: string;
|
|
72
|
+
/** Optional config (uses runtime state if not provided) */
|
|
73
|
+
config?: MonoraConfig;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Record a bias measurement for a model.
|
|
77
|
+
*
|
|
78
|
+
* This stores the measurement and checks for threshold violations.
|
|
79
|
+
*
|
|
80
|
+
* @param model - The model being measured.
|
|
81
|
+
* @param options - Measurement options.
|
|
82
|
+
* @returns The created BiasMeasurement.
|
|
83
|
+
*
|
|
84
|
+
* @example
|
|
85
|
+
* ```typescript
|
|
86
|
+
* recordBiasMeasurement('gpt-4o', {
|
|
87
|
+
* measurementType: 'demographic_parity',
|
|
88
|
+
* dimensions: ['gender'],
|
|
89
|
+
* metrics: { demographic_parity_ratio: 0.85, sample_size: 500 }
|
|
90
|
+
* });
|
|
91
|
+
* ```
|
|
92
|
+
*/
|
|
93
|
+
export declare function recordBiasMeasurement(model: string, options?: RecordBiasMeasurementOptions): BiasMeasurement;
|
|
94
|
+
/**
|
|
95
|
+
* Check if any bias metrics violate configured thresholds.
|
|
96
|
+
*
|
|
97
|
+
* @param metrics - The bias metrics to check.
|
|
98
|
+
* @param config - The Monora configuration.
|
|
99
|
+
* @returns List of violated threshold names.
|
|
100
|
+
*/
|
|
101
|
+
export declare function checkThresholdViolations(metrics: BiasMetrics, config: MonoraConfig): string[];
|
|
102
|
+
/**
|
|
103
|
+
* Set a custom callback for automatic bias measurement.
|
|
104
|
+
*
|
|
105
|
+
* The callback is invoked after LLM calls to compute bias metrics
|
|
106
|
+
* based on inputs and outputs.
|
|
107
|
+
*
|
|
108
|
+
* @param callback - Function taking (model, inputs, outputs) => BiasMetrics,
|
|
109
|
+
* or null to clear the callback.
|
|
110
|
+
*/
|
|
111
|
+
export declare function setMeasurementCallback(callback: BiasCallback | null): void;
|
|
112
|
+
/**
|
|
113
|
+
* Get the current measurement callback.
|
|
114
|
+
*/
|
|
115
|
+
export declare function getMeasurementCallback(): BiasCallback | null;
|
|
116
|
+
/**
|
|
117
|
+
* Get all recorded bias measurements.
|
|
118
|
+
*/
|
|
119
|
+
export declare function getAllMeasurements(): BiasMeasurement[];
|
|
120
|
+
/**
|
|
121
|
+
* Clear all stored measurements.
|
|
122
|
+
*/
|
|
123
|
+
export declare function clearMeasurements(): void;
|
|
124
|
+
/**
|
|
125
|
+
* Get summary of bias measurements for reporting.
|
|
126
|
+
*
|
|
127
|
+
* @returns Summary dict for compliance reports.
|
|
128
|
+
*/
|
|
129
|
+
export declare function getBiasSummary(): Record<string, any>;
|
|
130
|
+
//# sourceMappingURL=bias.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bias.d.ts","sourceRoot":"","sources":["../src/bias.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAGxC,MAAM,MAAM,eAAe,GAAG,oBAAoB,GAAG,gBAAgB,GAAG,kBAAkB,GAAG,QAAQ,CAAC;AAEtG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,6DAA6D;IAC7D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,gEAAgE;IAChE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mDAAmD;IACnD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,sCAAsC;IACtC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAoBvE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,oCAAoC;IACpC,eAAe,EAAE,eAAe,CAAC;IACjC,sCAAsC;IACtC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,gCAAgC;IAChC,OAAO,EAAE,WAAW,CAAC;IACrB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kCAAkC;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAWxF;AAMD,MAAM,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,WAAW,CAAC;AAGzF;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,qEAAqE;IACrE,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,gEAAgE;IAChE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,mCAAmC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,2DAA2D;IAC3D,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,4BAAiC,GACzC,eAAe,CA8CjB;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,WAAW,EACpB,MAAM,EAAE,YAAY,GACnB,MAAM,EAAE,CA8CV;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,GAAG,IAAI,CAE1E;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,YAAY,GAAG,IAAI,CAE5D;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,eAAe,EAAE,CAEtD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED;;;;GAIG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAuBpD"}
|
package/dist/bias.js
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Bias metrics capture for ISO 42001 fairness compliance.
|
|
4
|
+
*
|
|
5
|
+
* This module provides bias measurement tracking for AI outputs, supporting
|
|
6
|
+
* ISO 42001 fairness controls and EU AI Act Article 10 requirements.
|
|
7
|
+
*
|
|
8
|
+
* Cross-SDK Parity:
|
|
9
|
+
* Both Python and Node.js SDKs provide identical bias metrics APIs:
|
|
10
|
+
* - recordBiasMeasurement() / record_bias_measurement()
|
|
11
|
+
* - checkThresholdViolations() / check_threshold_violations()
|
|
12
|
+
* - BiasMetrics interface / BiasMetrics dataclass
|
|
13
|
+
*/
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.metricsToDict = metricsToDict;
|
|
16
|
+
exports.measurementToEventBody = measurementToEventBody;
|
|
17
|
+
exports.recordBiasMeasurement = recordBiasMeasurement;
|
|
18
|
+
exports.checkThresholdViolations = checkThresholdViolations;
|
|
19
|
+
exports.setMeasurementCallback = setMeasurementCallback;
|
|
20
|
+
exports.getMeasurementCallback = getMeasurementCallback;
|
|
21
|
+
exports.getAllMeasurements = getAllMeasurements;
|
|
22
|
+
exports.clearMeasurements = clearMeasurements;
|
|
23
|
+
exports.getBiasSummary = getBiasSummary;
|
|
24
|
+
const logger_1 = require("./logger");
|
|
25
|
+
/**
|
|
26
|
+
* Convert BiasMetrics to dictionary for event inclusion.
|
|
27
|
+
*/
|
|
28
|
+
function metricsToDict(metrics) {
|
|
29
|
+
const result = {};
|
|
30
|
+
if (metrics.demographicParityRatio !== undefined) {
|
|
31
|
+
result.demographic_parity_ratio = metrics.demographicParityRatio;
|
|
32
|
+
}
|
|
33
|
+
if (metrics.equalizedOddsDiff !== undefined) {
|
|
34
|
+
result.equalized_odds_diff = metrics.equalizedOddsDiff;
|
|
35
|
+
}
|
|
36
|
+
if (metrics.disparateImpact !== undefined) {
|
|
37
|
+
result.disparate_impact = metrics.disparateImpact;
|
|
38
|
+
}
|
|
39
|
+
if (metrics.customMetrics) {
|
|
40
|
+
Object.assign(result, metrics.customMetrics);
|
|
41
|
+
}
|
|
42
|
+
return result;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Convert BiasMeasurement to event body dictionary.
|
|
46
|
+
*/
|
|
47
|
+
function measurementToEventBody(measurement) {
|
|
48
|
+
return {
|
|
49
|
+
model: measurement.model,
|
|
50
|
+
measurement_type: measurement.measurementType,
|
|
51
|
+
dimensions: measurement.dimensions,
|
|
52
|
+
metrics: metricsToDict(measurement.metrics),
|
|
53
|
+
sample_size: measurement.sampleSize,
|
|
54
|
+
evaluation_dataset: measurement.evaluationDataset,
|
|
55
|
+
threshold_violations: measurement.thresholdViolations,
|
|
56
|
+
timestamp: measurement.timestamp,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
// Stored measurements for reporting
|
|
60
|
+
const measurements = [];
|
|
61
|
+
let measurementCallback = null;
|
|
62
|
+
/**
|
|
63
|
+
* Record a bias measurement for a model.
|
|
64
|
+
*
|
|
65
|
+
* This stores the measurement and checks for threshold violations.
|
|
66
|
+
*
|
|
67
|
+
* @param model - The model being measured.
|
|
68
|
+
* @param options - Measurement options.
|
|
69
|
+
* @returns The created BiasMeasurement.
|
|
70
|
+
*
|
|
71
|
+
* @example
|
|
72
|
+
* ```typescript
|
|
73
|
+
* recordBiasMeasurement('gpt-4o', {
|
|
74
|
+
* measurementType: 'demographic_parity',
|
|
75
|
+
* dimensions: ['gender'],
|
|
76
|
+
* metrics: { demographic_parity_ratio: 0.85, sample_size: 500 }
|
|
77
|
+
* });
|
|
78
|
+
* ```
|
|
79
|
+
*/
|
|
80
|
+
function recordBiasMeasurement(model, options = {}) {
|
|
81
|
+
const { measurementType = 'custom', dimensions = [], metrics = {}, sampleSize = 0, evaluationDataset, config = {}, } = options;
|
|
82
|
+
const biasConfig = config.bias || {};
|
|
83
|
+
// Parse metrics into BiasMetrics
|
|
84
|
+
const biasMetrics = {
|
|
85
|
+
demographicParityRatio: metrics.demographic_parity_ratio,
|
|
86
|
+
equalizedOddsDiff: metrics.equalized_odds_diff,
|
|
87
|
+
disparateImpact: metrics.disparate_impact,
|
|
88
|
+
customMetrics: Object.fromEntries(Object.entries(metrics).filter(([k]) => !['demographic_parity_ratio', 'equalized_odds_diff', 'disparate_impact'].includes(k))),
|
|
89
|
+
};
|
|
90
|
+
// Check threshold violations
|
|
91
|
+
const violations = checkThresholdViolations(biasMetrics, config);
|
|
92
|
+
const measurement = {
|
|
93
|
+
model,
|
|
94
|
+
measurementType,
|
|
95
|
+
dimensions: dimensions.length > 0 ? dimensions : (biasConfig.dimensions || []),
|
|
96
|
+
metrics: biasMetrics,
|
|
97
|
+
sampleSize,
|
|
98
|
+
evaluationDataset,
|
|
99
|
+
thresholdViolations: violations,
|
|
100
|
+
timestamp: new Date().toISOString(),
|
|
101
|
+
};
|
|
102
|
+
measurements.push(measurement);
|
|
103
|
+
// Send alert if violations and configured
|
|
104
|
+
if (violations.length > 0 && (biasConfig.alert_on_violation ?? true)) {
|
|
105
|
+
sendViolationAlert(measurement, config);
|
|
106
|
+
}
|
|
107
|
+
return measurement;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Check if any bias metrics violate configured thresholds.
|
|
111
|
+
*
|
|
112
|
+
* @param metrics - The bias metrics to check.
|
|
113
|
+
* @param config - The Monora configuration.
|
|
114
|
+
* @returns List of violated threshold names.
|
|
115
|
+
*/
|
|
116
|
+
function checkThresholdViolations(metrics, config) {
|
|
117
|
+
const violations = [];
|
|
118
|
+
const biasConfig = config.bias || {};
|
|
119
|
+
const thresholds = biasConfig.thresholds || {};
|
|
120
|
+
// Check demographic parity ratio (must be >= threshold)
|
|
121
|
+
if (metrics.demographicParityRatio !== undefined) {
|
|
122
|
+
const threshold = thresholds.demographic_parity_ratio ?? 0.8;
|
|
123
|
+
if (metrics.demographicParityRatio < threshold) {
|
|
124
|
+
violations.push(`demographic_parity_ratio (${metrics.demographicParityRatio.toFixed(3)} < ${threshold})`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
// Check disparate impact (must be >= threshold, typically 0.8 for 4/5ths rule)
|
|
128
|
+
if (metrics.disparateImpact !== undefined) {
|
|
129
|
+
const threshold = thresholds.disparate_impact ?? 0.8;
|
|
130
|
+
if (metrics.disparateImpact < threshold) {
|
|
131
|
+
violations.push(`disparate_impact (${metrics.disparateImpact.toFixed(3)} < ${threshold})`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
// Check equalized odds diff (must be <= threshold, lower is better)
|
|
135
|
+
if (metrics.equalizedOddsDiff !== undefined) {
|
|
136
|
+
const threshold = thresholds.equalized_odds_diff ?? 0.1;
|
|
137
|
+
if (metrics.equalizedOddsDiff > threshold) {
|
|
138
|
+
violations.push(`equalized_odds_diff (${metrics.equalizedOddsDiff.toFixed(3)} > ${threshold})`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// Check custom thresholds
|
|
142
|
+
if (metrics.customMetrics) {
|
|
143
|
+
for (const [metricName, value] of Object.entries(metrics.customMetrics)) {
|
|
144
|
+
const threshold = thresholds[metricName];
|
|
145
|
+
if (threshold !== undefined && value < threshold) {
|
|
146
|
+
violations.push(`${metricName} (${value.toFixed(3)} < ${threshold})`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return violations;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Set a custom callback for automatic bias measurement.
|
|
154
|
+
*
|
|
155
|
+
* The callback is invoked after LLM calls to compute bias metrics
|
|
156
|
+
* based on inputs and outputs.
|
|
157
|
+
*
|
|
158
|
+
* @param callback - Function taking (model, inputs, outputs) => BiasMetrics,
|
|
159
|
+
* or null to clear the callback.
|
|
160
|
+
*/
|
|
161
|
+
function setMeasurementCallback(callback) {
|
|
162
|
+
measurementCallback = callback;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Get the current measurement callback.
|
|
166
|
+
*/
|
|
167
|
+
function getMeasurementCallback() {
|
|
168
|
+
return measurementCallback;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Get all recorded bias measurements.
|
|
172
|
+
*/
|
|
173
|
+
function getAllMeasurements() {
|
|
174
|
+
return [...measurements];
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Clear all stored measurements.
|
|
178
|
+
*/
|
|
179
|
+
function clearMeasurements() {
|
|
180
|
+
measurements.length = 0;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Get summary of bias measurements for reporting.
|
|
184
|
+
*
|
|
185
|
+
* @returns Summary dict for compliance reports.
|
|
186
|
+
*/
|
|
187
|
+
function getBiasSummary() {
|
|
188
|
+
if (measurements.length === 0) {
|
|
189
|
+
return {
|
|
190
|
+
measurements_recorded: 0,
|
|
191
|
+
threshold_violations: 0,
|
|
192
|
+
dimensions_measured: [],
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
const allDimensions = new Set();
|
|
196
|
+
let totalViolations = 0;
|
|
197
|
+
for (const m of measurements) {
|
|
198
|
+
m.dimensions.forEach((d) => allDimensions.add(d));
|
|
199
|
+
totalViolations += m.thresholdViolations.length;
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
measurements_recorded: measurements.length,
|
|
203
|
+
threshold_violations: totalViolations,
|
|
204
|
+
dimensions_measured: Array.from(allDimensions).sort(),
|
|
205
|
+
models_measured: Array.from(new Set(measurements.map((m) => m.model))),
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Send alert for threshold violations.
|
|
210
|
+
*/
|
|
211
|
+
function sendViolationAlert(measurement, config) {
|
|
212
|
+
try {
|
|
213
|
+
const alertConfig = config.alerts;
|
|
214
|
+
if (!alertConfig?.violation_webhook) {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
// Log the violation - actual alert sending would use the alerts module
|
|
218
|
+
logger_1.logger.warning('Bias threshold violations detected for model %s: %s', measurement.model, measurement.thresholdViolations.join(', '));
|
|
219
|
+
}
|
|
220
|
+
catch (error) {
|
|
221
|
+
logger_1.logger.warning('Failed to send bias violation alert: %s', error);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
@@ -2,13 +2,17 @@
|
|
|
2
2
|
* Configuration diagnostics and validation.
|
|
3
3
|
*/
|
|
4
4
|
import { MonoraConfig } from '../config';
|
|
5
|
+
import { ValidationMode } from '../config_schema';
|
|
5
6
|
interface ValidationResult {
|
|
6
7
|
errors: string[];
|
|
7
8
|
warnings: string[];
|
|
8
9
|
}
|
|
9
|
-
export declare function validateConfig(config: MonoraConfig
|
|
10
|
+
export declare function validateConfig(config: MonoraConfig, options?: {
|
|
11
|
+
mode?: ValidationMode;
|
|
12
|
+
}): ValidationResult;
|
|
10
13
|
export declare function doctorChecks(config: MonoraConfig, options?: {
|
|
11
14
|
checkNetwork?: boolean;
|
|
15
|
+
mode?: ValidationMode;
|
|
12
16
|
}): ValidationResult;
|
|
13
17
|
export declare function emitResults(results: ValidationResult, options?: {
|
|
14
18
|
json?: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diagnostics.d.ts","sourceRoot":"","sources":["../../src/cli/diagnostics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,
|
|
1
|
+
{"version":3,"file":"diagnostics.d.ts","sourceRoot":"","sources":["../../src/cli/diagnostics.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAyB,cAAc,EAA4C,MAAM,kBAAkB,CAAC;AAQnH,UAAU,gBAAgB;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,cAAc,CAAA;CAAE,GAAG,gBAAgB,CAmK1G;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,YAAY,EACpB,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAC;IAAC,IAAI,CAAC,EAAE,cAAc,CAAA;CAAE,GAC1D,gBAAgB,CAqDlB;AA+CD,wBAAgB,WAAW,CACzB,OAAO,EAAE,gBAAgB,EACzB,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,GAC3B,MAAM,CAqCR"}
|
package/dist/cli/diagnostics.js
CHANGED
|
@@ -42,18 +42,35 @@ exports.emitResults = emitResults;
|
|
|
42
42
|
const fs = __importStar(require("fs"));
|
|
43
43
|
const net = __importStar(require("net"));
|
|
44
44
|
const url_1 = require("url");
|
|
45
|
-
const
|
|
45
|
+
const config_schema_1 = require("../config_schema");
|
|
46
46
|
const config_migrations_1 = require("../config_migrations");
|
|
47
47
|
const ALLOWED_QUEUE_MODES = new Set(['warn', 'raise', 'block']);
|
|
48
48
|
const ALLOWED_SINK_FAILURE_MODES = new Set(['warn', 'raise', 'silent']);
|
|
49
49
|
const ALLOWED_ROTATION = new Set(['none', 'daily', 'size']);
|
|
50
50
|
const ALLOWED_DATA_HANDLING = new Set(['redact', 'block', 'allow']);
|
|
51
|
-
function validateConfig(config) {
|
|
51
|
+
function validateConfig(config, options) {
|
|
52
52
|
const errors = [];
|
|
53
53
|
const warnings = [];
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
|
|
54
|
+
const mode = options?.mode || 'strict';
|
|
55
|
+
const schemaWarnings = [];
|
|
56
|
+
try {
|
|
57
|
+
(0, config_schema_1.validateConfig)(config, {
|
|
58
|
+
mode,
|
|
59
|
+
logger: { warn: (message) => schemaWarnings.push(message) },
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
if (error instanceof config_schema_1.ConfigValidationError) {
|
|
64
|
+
for (const issue of error.errors) {
|
|
65
|
+
errors.push(`schema: ${issue.path.join('.')}: ${issue.message}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
errors.push(`schema: ${error?.message || error}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (schemaWarnings.length > 0) {
|
|
73
|
+
warnings.push(...schemaWarnings);
|
|
57
74
|
}
|
|
58
75
|
// Validate sinks
|
|
59
76
|
const sinks = config.sinks || [];
|
|
@@ -177,7 +194,7 @@ function validateConfig(config) {
|
|
|
177
194
|
}
|
|
178
195
|
function doctorChecks(config, options) {
|
|
179
196
|
const checkNetwork = options?.checkNetwork ?? true;
|
|
180
|
-
const result = validateConfig(config);
|
|
197
|
+
const result = validateConfig(config, { mode: options?.mode });
|
|
181
198
|
const errors = [...result.errors];
|
|
182
199
|
const warnings = [...result.warnings];
|
|
183
200
|
// Check axios for HTTPS sink
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Config doctor checks for Monora CLI.
|
|
3
|
+
*/
|
|
4
|
+
import { ValidationMode } from '../config_schema';
|
|
5
|
+
export interface DoctorCheck {
|
|
6
|
+
name: string;
|
|
7
|
+
ok: boolean;
|
|
8
|
+
level: 'error' | 'warning';
|
|
9
|
+
message: string;
|
|
10
|
+
recommendation?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface DoctorResult {
|
|
13
|
+
ok: boolean;
|
|
14
|
+
errors: string[];
|
|
15
|
+
warnings: string[];
|
|
16
|
+
checks: DoctorCheck[];
|
|
17
|
+
}
|
|
18
|
+
export declare function doctorChecks(configPath: string, options?: {
|
|
19
|
+
checkNetwork?: boolean;
|
|
20
|
+
mode?: ValidationMode;
|
|
21
|
+
}): Promise<DoctorResult>;
|
|
22
|
+
export declare function emitDoctorResults(result: DoctorResult, options?: {
|
|
23
|
+
json?: boolean;
|
|
24
|
+
}): number;
|
|
25
|
+
//# sourceMappingURL=doctor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../src/cli/doctor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,OAAO,EAAyB,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAEzF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAID,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAC;IAAC,IAAI,CAAC,EAAE,cAAc,CAAA;CAAE,GAC1D,OAAO,CAAC,YAAY,CAAC,CAgHvB;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,MAAM,CAkC5F"}
|