observability-toolkit 1.8.4 → 2.0.0

package/dist/tools/query-verifications.test.js

@@ -0,0 +1,156 @@
+/**
+ * Tests for query-verifications tool
+ */
+import { describe, it, mock } from 'node:test';
+import assert from 'node:assert';
+import { queryVerifications, queryVerificationsSchema, queryVerificationsTool, buildSummary, } from './query-verifications.js';
+import { validateLimitSchema, validateDateRangeSchema } from '../test-helpers/schema-validators.js';
+describe('queryVerifications', () => {
+    // Use shared schema validators
+    validateLimitSchema(queryVerificationsSchema, 'queryVerificationsSchema', 50);
+    validateDateRangeSchema(queryVerificationsSchema, 'queryVerificationsSchema');
+    describe('schema validation', () => {
+        it('should accept empty input with defaults', () => {
+            const result = queryVerificationsSchema.parse({});
+            assert.strictEqual(result.limit, 50);
+        });
+        it('should accept all optional parameters', () => {
+            const input = {
+                sessionId: 'sess-123',
+                verificationType: 'approval',
+                startDate: '2026-01-28',
+                endDate: '2026-01-28',
+                limit: 100,
+            };
+            const result = queryVerificationsSchema.parse(input);
+            assert.strictEqual(result.sessionId, 'sess-123');
+            assert.strictEqual(result.verificationType, 'approval');
+        });
+        it('should enforce max limit of 1000', () => {
+            assert.throws(() => {
+                queryVerificationsSchema.parse({ limit: 1001 });
+            });
+        });
+        it('should accept all verification types', () => {
+            const types = ['approval', 'rejection', 'override', 'review'];
+            for (const type of types) {
+                const result = queryVerificationsSchema.parse({ verificationType: type });
+                assert.strictEqual(result.verificationType, type);
+            }
+        });
+        it('should reject invalid verification type', () => {
+            assert.throws(() => {
+                queryVerificationsSchema.parse({ verificationType: 'invalid' });
+            });
+        });
+    });
+    describe('buildSummary', () => {
+        it('should count by verification type', () => {
+            const events = [
+                { timestamp: '2026-01-31T12:00:00Z', sessionId: 's1', verificationType: 'approval' },
+                { timestamp: '2026-01-31T12:01:00Z', sessionId: 's2', verificationType: 'approval' },
+                { timestamp: '2026-01-31T12:02:00Z', sessionId: 's3', verificationType: 'rejection' },
+            ];
+            const summary = buildSummary(events);
+            assert.strictEqual(summary.byVerificationType['approval'], 2);
+            assert.strictEqual(summary.byVerificationType['rejection'], 1);
+        });
+        it('should count by verifier ID', () => {
+            const events = [
+                { timestamp: '2026-01-31T12:00:00Z', sessionId: 's1', verificationType: 'approval', verifierId: 'user-001' },
+                { timestamp: '2026-01-31T12:01:00Z', sessionId: 's2', verificationType: 'approval', verifierId: 'user-001' },
+                { timestamp: '2026-01-31T12:02:00Z', sessionId: 's3', verificationType: 'rejection', verifierId: 'user-002' },
+            ];
+            const summary = buildSummary(events);
+            assert.strictEqual(summary.byVerifierId['user-001'], 2);
+            assert.strictEqual(summary.byVerifierId['user-002'], 1);
+        });
+        it('should handle empty array', () => {
+            const summary = buildSummary([]);
+            assert.deepStrictEqual(summary.byVerificationType, {});
+            assert.deepStrictEqual(summary.byVerifierId, {});
+        });
+    });
+    describe('handler with mocked backend', () => {
+        it('should return empty results when no verifications found', async () => {
+            const mockQueryFn = mock.fn(async () => []);
+            const options = { queryFn: mockQueryFn };
+            const result = await queryVerifications({}, options);
+            assert.strictEqual(result.count, 0);
+            assert.deepStrictEqual(result.verifications, []);
+        });
+        it('should include all fields in verification response', async () => {
+            const mockEvent = {
+                timestamp: '2026-01-31T12:00:00Z',
+                sessionId: 'sess-123',
+                verificationType: 'override',
+                traceId: 'trace-456',
+                verifierId: 'user-001',
+                originalOutput: 'Original AI text',
+                verifiedOutput: 'Corrected text',
+                metadata: { reason: 'factual error' },
+            };
+            const mockQueryFn = mock.fn(async () => [mockEvent]);
+            const options = { queryFn: mockQueryFn };
+            const result = await queryVerifications({}, options);
+            const verification = result.verifications[0];
+            assert.strictEqual(verification.timestamp, '2026-01-31T12:00:00Z');
+            assert.strictEqual(verification.sessionId, 'sess-123');
+            assert.strictEqual(verification.verificationType, 'override');
+            assert.strictEqual(verification.traceId, 'trace-456');
+            assert.strictEqual(verification.verifierId, 'user-001');
+            assert.strictEqual(verification.originalOutput, 'Original AI text');
+            assert.strictEqual(verification.verifiedOutput, 'Corrected text');
+            assert.deepStrictEqual(verification.metadata, { reason: 'factual error' });
+        });
+        it('should limit returned verifications to 50', async () => {
+            const manyEvents = Array.from({ length: 60 }, (_, i) => ({
+                timestamp: `2026-01-31T12:${String(i).padStart(2, '0')}:00Z`,
+                sessionId: `sess-${i}`,
+                verificationType: 'approval',
+            }));
+            const mockQueryFn = mock.fn(async () => manyEvents);
+            const options = { queryFn: mockQueryFn };
+            const result = await queryVerifications({}, options);
+            assert.strictEqual(result.count, 60);
+            assert.strictEqual(result.verifications.length, 50);
+        });
+        it('should pass filter options to backend', async () => {
+            let capturedOptions;
+            const mockQueryFn = mock.fn(async (opts) => {
+                capturedOptions = opts;
+                return [];
+            });
+            const options = { queryFn: mockQueryFn };
+            await queryVerifications({
+                sessionId: 'sess-test',
+                verificationType: 'rejection',
+                startDate: '2026-01-01',
+                endDate: '2026-01-31',
+                limit: 25,
+            }, options);
+            assert.strictEqual(mockQueryFn.mock.calls.length, 1);
+            assert.ok(capturedOptions);
+            assert.strictEqual(capturedOptions.sessionId, 'sess-test');
+            assert.strictEqual(capturedOptions.verificationType, 'rejection');
+            assert.strictEqual(capturedOptions.startDate, '2026-01-01');
+            assert.strictEqual(capturedOptions.endDate, '2026-01-31');
+            assert.strictEqual(capturedOptions.limit, 25);
+        });
+    });
+    describe('tool definition', () => {
+        it('should have correct name', () => {
+            assert.strictEqual(queryVerificationsTool.name, 'obs_query_verifications');
+        });
+        it('should have description mentioning EU AI Act', () => {
+            assert.ok(queryVerificationsTool.description.includes('EU AI Act'));
+        });
+        it('should have handler function', () => {
+            assert.strictEqual(typeof queryVerificationsTool.handler, 'function');
+        });
+        it('should have inputSchema', () => {
+            assert.ok(queryVerificationsTool.inputSchema);
+        });
+    });
+});
+//# sourceMappingURL=query-verifications.test.js.map

package/dist/tools/query-verifications.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-verifications.test.js","sourceRoot":"","sources":["../../src/tools/query-verifications.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EACL,kBAAkB,EAClB,wBAAwB,EACxB,sBAAsB,EACtB,YAAY,GAEb,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,MAAM,sCAAsC,CAAC;AAEpG,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,+BAA+B;IAC/B,mBAAmB,CAAC,wBAAwB,EAAE,0BAA0B,EAAE,EAAE,CAAC,CAAC;IAC9E,uBAAuB,CAAC,wBAAwB,EAAE,0BAA0B,CAAC,CAAC;IAE9E,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,MAAM,GAAG,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG;gBACZ,SAAS,EAAE,UAAU;gBACrB,gBAAgB,EAAE,UAAU;gBAC5B,SAAS,EAAE,YAAY;gBACvB,OAAO,EAAE,YAAY;gBACrB,KAAK,EAAE,GAAG;aACX,CAAC;YACF,MAAM,MAAM,GAAG,wBAAwB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACjD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;gBACjB,wBAAwB,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,KAAK,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAU,CAAC;YACvE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,wBAAwB,CAAC,KAAK,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1E,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;YACpD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE;gBACjB,wBAAwB,CAAC,KAAK,CAAC,EAAE,gBAAgB,EAAE,SAAS,EAAE,CAAC,CAAC;YAClE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAG;gBACb,EAAE,SAAS,EAAE,sBAAsB,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,UAAmB,EAAE;gBAC7F,EAAE,SAAS,EAAE,sBAAsB,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,UAAmB,EAAE;gBAC7F,EAAE,SAAS,EAAE,sBAAsB,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,WAAoB,EAAE;aAC/F,CAAC;YAEF,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YAErC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,kBAAkB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,MAAM,GAAG;gBACb,EAAE,SAAS,EAAE,sBAAsB,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,UAAmB,EAAE,UAAU,EAAE,UAAU,EAAE;gBACrH,EAAE,SAAS,EAAE,sBAAsB,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,UAAmB,EAAE,UAAU,EAAE,UAAU,EAAE;gBACrH,EAAE,SAAS,EAAE,sBAAsB,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,WAAoB,EAAE,UAAU,EAAE,UAAU,EAAE;aACvH,CAAC;YAEF,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YAErC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;YACxD,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;YAEjC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;YACvD,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,EAA8B,CAAC,CAAC;YACxE,MAAM,OAAO,GAA8B,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAEpE,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACpC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,SAAS,GAA2B;gBACxC,SAAS,EAAE,sBAAsB;gBACjC,SAAS,EAAE,UAAU;gBACrB,gBAAgB,EAAE,UAAU;gBAC5B,OAAO,EAAE,WAAW;gBACpB,UAAU,EAAE,UAAU;gBACtB,cAAc,EAAE,kBAAkB;gBAClC,cAAc,EAAE,gBAAgB;gBAChC,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE;aACtC,CAAC;YAEF,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;YACrD,MAAM,OAAO,GAA8B,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAEpE,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACrD,MAAM,YAAY,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YAE7C,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;YACnE,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACvD,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YACtD,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YACxD,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;YACpE,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;YAClE,MAAM,CAAC,eAAe,CAAC,YAAY,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,UAAU,GAA6B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACjF,SAAS,EAAE,iBAAiB,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM;gBAC5D,SAAS,EAAE,QAAQ,CAAC,EAAE;gBACtB,gBAAgB,EAAE,UAAU;aAC7B,CAAC,CAAC,CAAC;YAEJ,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC;YACpD,MAAM,OAAO,GAA8B,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAEpE,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,IAAI,eAAqD,CAAC;YAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,IAA8B,EAAE,EAAE;gBACnE,eAAe,GAAG,IAAI,CAAC;gBACvB,OAAO,EAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YACH,MAAM,OAAO,GAA8B,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAEpE,MAAM,kBAAkB,CAAC;gBACvB,SAAS,EAAE,WAAW;gBACtB,gBAAgB,EAAE,WAAW;gBAC7B,SAAS,EAAE,YAAY;gBACvB,OAAO,EAAE,YAAY;gBACrB,KAAK,EAAE,EAAE;aACV,EAAE,OAAO,CAAC,CAAC;YAEZ,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACrD,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;YAC3B,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAC3D,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;YAClE,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAC5D,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YAC1D,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,WAAW,CAAC,sBAAsB,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,CAAC,EAAE,CAAC,sBAAsB,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,WAAW,CAAC,OAAO,sBAAsB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,EAAE,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/types/evaluation-hooks.d.ts

@@ -0,0 +1,176 @@
+/**
+ * Type definitions for LLM-as-Judge Hooks (Phase 4e)
+ *
+ * Enables external evaluators to inject quality scores via webhooks.
+ * Supports multiple authentication methods and evaluation sources.
+ */
+import type { EvaluatorType, StepScore, ToolVerification } from '../backends/index.js';
+/**
+ * Supported authentication methods for webhook endpoints
+ */
+export type WebhookAuthMethod = 'hmac-sha256' | 'bearer' | 'api-key' | 'none';
+/**
+ * Configuration for a webhook endpoint that receives evaluation scores
+ */
+export interface WebhookConfig {
+    /** Unique identifier for this webhook configuration */
+    id: string;
+    /** Human-readable name for the webhook */
+    name: string;
+    /** Authentication method */
+    authMethod: WebhookAuthMethod;
+    /** Secret for HMAC signature validation (required for hmac-sha256) */
+    secret?: string;
+    /** Expected bearer token (required for bearer auth) */
+    bearerToken?: string;
+    /** Expected API key (required for api-key auth) */
+    apiKey?: string;
+    /** Optional IP allowlist for additional security */
+    allowedIps?: string[];
+    /** Whether this webhook is enabled */
+    enabled: boolean;
+    /** Creation timestamp */
+    createdAt: string;
+    /** Last update timestamp */
+    updatedAt?: string;
+}
+/**
+ * Payload structure for webhook-delivered evaluation results.
+ * Compatible with EvaluationResult but with required fields for external injection.
+ */
+export interface WebhookEvaluationPayload {
+    /** Evaluation metric name (e.g., "relevance", "faithfulness") - REQUIRED */
+    evaluationName: string;
+    /** Numeric score (0-1 normalized recommended) - one of scoreValue or scoreLabel required */
+    scoreValue?: number;
+    /** Human-readable label (e.g., "pass", "fail", "relevant") */
+    scoreLabel?: string;
+    /** Score unit/scale (e.g., "ratio_0_1", "percentage", "stars_5") */
+    scoreUnit?: string;
+    /** Explanation/rationale for the score */
+    explanation?: string;
+    /** Identity of the evaluator - REQUIRED for external webhooks */
+    evaluator: string;
+    /** Type of evaluator */
+    evaluatorType: EvaluatorType;
+    /** Response ID for correlation */
+    responseId?: string;
+    /** Trace ID for correlation */
+    traceId?: string;
+    /** Session ID for session-scoped correlation */
+    sessionId?: string;
+    /** Subject agent ID being evaluated */
+    agentId?: string;
+    /** Subject agent name being evaluated */
+    agentName?: string;
+    /** Per-step evaluation breakdown for agent trajectory */
+    stepScores?: StepScore[];
+    /** Tool call correctness verifications */
+    toolVerifications?: ToolVerification[];
+    /** Number of steps/actions in agent trajectory */
+    trajectoryLength?: number;
+    /** Optional custom metadata from external system */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * Batch payload for multiple evaluations in a single request
+ */
+export interface WebhookBatchPayload {
+    /** Array of evaluations (max 100 per batch) */
+    evaluations: WebhookEvaluationPayload[];
+}
+/**
+ * Result of webhook payload validation and processing
+ */
+export interface HookResult {
+    /** Whether the operation succeeded */
+    success: boolean;
+    /** Human-readable message */
+    message: string;
+    /** Number of evaluations successfully processed */
+    processedCount?: number;
+    /** IDs of created evaluations (for deduplication tracking) */
+    evaluationIds?: string[];
+    /** Validation errors if any */
+    errors?: HookValidationError[];
+}
+/**
+ * Detailed validation error for a specific field or evaluation
+ */
+export interface HookValidationError {
+    /** Index in batch (for batch payloads) */
+    index?: number;
+    /** Field that failed validation */
+    field: string;
+    /** Error message */
+    message: string;
+    /** Suggested fix */
+    suggestion?: string;
+}
+/**
+ * HMAC signature verification result
+ */
+export interface SignatureVerificationResult {
+    /** Whether signature is valid */
+    valid: boolean;
+    /** Error message if invalid */
+    error?: string;
+    /** Detected algorithm (for diagnostic purposes) */
+    algorithm?: string;
+}
+/**
+ * Options for the hook executor
+ */
+export interface HookExecutorOptions {
+    /** Directory to write evaluation JSONL files */
+    telemetryDir: string;
+    /** Whether to validate signature (default: true) */
+    validateSignature?: boolean;
+    /** Maximum evaluations per batch (default: 100) */
+    maxBatchSize?: number;
+    /** Whether to deduplicate by responseId + timestamp (default: false) */
+    deduplication?: boolean;
+}
+/**
+ * Statistics for hook processing
+ */
+export interface HookStats {
+    /** Total evaluations received */
+    totalReceived: number;
+    /** Successfully processed */
+    successCount: number;
+    /** Failed due to validation */
+    validationFailures: number;
+    /** Failed due to signature */
+    signatureFailures: number;
+    /** Deduplicated (skipped) */
+    deduplicatedCount: number;
+    /** Last processed timestamp */
+    lastProcessedAt?: string;
+}
+/**
+ * Score normalization configuration
+ */
+export interface ScoreNormalizationConfig {
+    /** Unit of the input score */
+    inputUnit: string;
+    /** Target unit (usually "ratio_0_1") */
+    targetUnit: string;
+    /** Minimum value for the input scale */
+    inputMin: number;
+    /** Maximum value for the input scale */
+    inputMax: number;
+}
+/**
+ * Common score units and their normalization parameters
+ */
+export declare const SCORE_UNIT_CONFIGS: Record<string, ScoreNormalizationConfig>;
+/**
+ * Normalize a score to 0-1 range based on its unit.
+ *
+ * @param value - The score value to normalize
+ * @param unit - The unit/scale of the input (e.g., "percentage", "stars_5")
+ * @returns Normalized score clamped to [0, 1]
+ */
+export declare function normalizeScore(value: number, unit?: string): number;
+//# sourceMappingURL=evaluation-hooks.d.ts.map

package/dist/types/evaluation-hooks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-hooks.d.ts","sourceRoot":"","sources":["../../src/types/evaluation-hooks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAEvF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,aAAa,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uDAAuD;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,sCAAsC;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,4EAA4E;IAC5E,cAAc,EAAE,MAAM,CAAC;IAEvB,4FAA4F;IAC5F,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAElB,wBAAwB;IACxB,aAAa,EAAE,aAAa,CAAC;IAE7B,kCAAkC;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,yDAAyD;IACzD,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IAEzB,0CAA0C;IAC1C,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAEvC,kDAAkD;IAClD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,+CAA+C;IAC/C,WAAW,EAAE,wBAAwB,EAAE,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sCAAsC;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,6BAA6B;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,mBAAmB,EAAE,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,oBAAoB;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C,iCAAiC;IACjC,KAAK,EAAE,OAAO,CAAC;IACf,+BAA+B;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,gDAAgD;IAChD,YAAY,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,mDAAmD;IACnD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,iCAAiC;IACjC,aAAa,EAAE,MAAM,CAAC;IACtB,6BAA6B;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,8BAA8B;IAC9B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,6BAA6B;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,+BAA+B;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,eAAO,MAAM,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,wBAAwB,CAOvE,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CA2BnE"}

package/dist/types/evaluation-hooks.js

@@ -0,0 +1,49 @@
+/**
+ * Type definitions for LLM-as-Judge Hooks (Phase 4e)
+ *
+ * Enables external evaluators to inject quality scores via webhooks.
+ * Supports multiple authentication methods and evaluation sources.
+ */
+/**
+ * Common score units and their normalization parameters
+ */
+export const SCORE_UNIT_CONFIGS = {
+    ratio_0_1: { inputUnit: 'ratio_0_1', targetUnit: 'ratio_0_1', inputMin: 0, inputMax: 1 },
+    percentage: { inputUnit: 'percentage', targetUnit: 'ratio_0_1', inputMin: 0, inputMax: 100 },
+    stars_5: { inputUnit: 'stars_5', targetUnit: 'ratio_0_1', inputMin: 0, inputMax: 5 },
+    stars_10: { inputUnit: 'stars_10', targetUnit: 'ratio_0_1', inputMin: 0, inputMax: 10 },
+    likert_5: { inputUnit: 'likert_5', targetUnit: 'ratio_0_1', inputMin: 1, inputMax: 5 },
+    likert_7: { inputUnit: 'likert_7', targetUnit: 'ratio_0_1', inputMin: 1, inputMax: 7 },
+};
+/**
+ * Normalize a score to 0-1 range based on its unit.
+ *
+ * @param value - The score value to normalize
+ * @param unit - The unit/scale of the input (e.g., "percentage", "stars_5")
+ * @returns Normalized score clamped to [0, 1]
+ */
+export function normalizeScore(value, unit) {
+    // Handle NaN, Infinity, -Infinity
+    if (!Number.isFinite(value)) {
+        return 0;
+    }
+    if (!unit || unit === 'ratio_0_1') {
+        return Math.max(0, Math.min(1, value));
+    }
+    const config = SCORE_UNIT_CONFIGS[unit];
+    if (!config) {
+        // Unknown unit - assume 0-1 and clamp
+        return Math.max(0, Math.min(1, value));
+    }
+    // Handle division by zero (inputMin === inputMax)
+    const range = config.inputMax - config.inputMin;
+    if (range === 0) {
+        // Edge case: if value equals the single valid value, return 0.5
+        // Otherwise clamp to bounds
+        return value === config.inputMin ? 0.5 : Math.max(0, Math.min(1, value));
+    }
+    // Normalize to 0-1
+    const normalized = (value - config.inputMin) / range;
+    return Math.max(0, Math.min(1, normalized));
+}
+//# sourceMappingURL=evaluation-hooks.js.map

package/dist/types/evaluation-hooks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-hooks.js","sourceRoot":"","sources":["../../src/types/evaluation-hooks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAyLH;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAA6C;IAC1E,SAAS,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;IACxF,UAAU,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE;IAC5F,OAAO,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;IACpF,QAAQ,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE;IACvF,QAAQ,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;IACtF,QAAQ,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE;CACvF,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,IAAa;IACzD,kCAAkC;IAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACxC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,sCAAsC;QACtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,kDAAkD;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAChD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,gEAAgE;QAChE,4BAA4B;QAC5B,OAAO,KAAK,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED,mBAAmB;IACnB,MAAM,UAAU,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC;IACrD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;AAC9C,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "observability-toolkit",
-  "version": "1.8.4",
+  "version": "2.0.0",
   "description": "MCP server for observability tooling - query traces, metrics, logs from local JSONL or SigNoz",
   "type": "module",
   "main": "dist/server.js",
@@ -11,7 +11,8 @@
     "build": "tsc",
     "watch": "tsc --watch",
     "start": "node dist/server.js",
-    "test": "node --test --test-concurrency=4 dist/*.test.js dist/**/*.test.js"
+    "test": "node --test --test-concurrency=4 dist/*.test.js dist/**/*.test.js",
+    "verify-docs": "bash scripts/verify-docs-line-counts.sh"
   },
   "keywords": [
     "mcp",
@@ -27,12 +28,20 @@
   "license": "MIT",
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.11.2",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.52.0",
+    "@opentelemetry/sdk-node": "^0.52.0",
+    "@opentelemetry/semantic-conventions": "^1.25.0",
     "@toon-format/toon": "^2.1.0",
+    "confusables": "^1.1.1",
+    "sbd": "^1.0.19",
     "zod": "^3.24.2",
     "zod-to-json-schema": "^3.25.1"
   },
   "devDependencies": {
     "@types/node": "^22.15.21",
+    "@types/sbd": "^1.0.5",
+    "memfs": "^4.56.10",
     "typescript": "^5.8.3"
   },
   "engines": {