@open-mercato/ai-assistant 0.6.7-develop.6784.1.f80b9afce5 → 0.6.7-develop.6785.1.1dd7cfac55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/AGENTS.md +12 -29
  3. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.js +80 -0
  4. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.js.map +7 -0
  5. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.js +124 -0
  6. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.js.map +7 -0
  7. package/dist/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.js +30 -0
  8. package/dist/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.js.map +2 -2
  9. package/dist/modules/ai_assistant/api/ai/chat/route.js +7 -0
  10. package/dist/modules/ai_assistant/api/ai/chat/route.js.map +2 -2
  11. package/dist/modules/ai_assistant/api/moderation-flags/route.js +121 -0
  12. package/dist/modules/ai_assistant/api/moderation-flags/route.js.map +7 -0
  13. package/dist/modules/ai_assistant/api/settings/route.js +24 -3
  14. package/dist/modules/ai_assistant/api/settings/route.js.map +2 -2
  15. package/dist/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.js +106 -0
  16. package/dist/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.js.map +2 -2
  17. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.js +139 -0
  18. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.js.map +7 -0
  19. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.js +10 -0
  20. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.js.map +7 -0
  21. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.js +26 -0
  22. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.js.map +7 -0
  23. package/dist/modules/ai_assistant/data/entities.js +42 -0
  24. package/dist/modules/ai_assistant/data/entities.js.map +2 -2
  25. package/dist/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.js +3 -1
  26. package/dist/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.js.map +2 -2
  27. package/dist/modules/ai_assistant/data/repositories/AiModerationFlagRepository.js +44 -0
  28. package/dist/modules/ai_assistant/data/repositories/AiModerationFlagRepository.js.map +7 -0
  29. package/dist/modules/ai_assistant/di.js +6 -2
  30. package/dist/modules/ai_assistant/di.js.map +2 -2
  31. package/dist/modules/ai_assistant/events.js +6 -0
  32. package/dist/modules/ai_assistant/events.js.map +2 -2
  33. package/dist/modules/ai_assistant/i18n/de.json +26 -0
  34. package/dist/modules/ai_assistant/i18n/en.json +26 -0
  35. package/dist/modules/ai_assistant/i18n/es.json +26 -0
  36. package/dist/modules/ai_assistant/i18n/pl.json +26 -0
  37. package/dist/modules/ai_assistant/lib/agent-runtime.js +158 -5
  38. package/dist/modules/ai_assistant/lib/agent-runtime.js.map +2 -2
  39. package/dist/modules/ai_assistant/lib/ai-agent-definition.js.map +2 -2
  40. package/dist/modules/ai_assistant/lib/llm-adapters/anthropic.js +3 -0
  41. package/dist/modules/ai_assistant/lib/llm-adapters/anthropic.js.map +2 -2
  42. package/dist/modules/ai_assistant/lib/llm-adapters/openai.js +8 -1
  43. package/dist/modules/ai_assistant/lib/llm-adapters/openai.js.map +2 -2
  44. package/dist/modules/ai_assistant/lib/model-factory.js +6 -1
  45. package/dist/modules/ai_assistant/lib/model-factory.js.map +2 -2
  46. package/dist/modules/ai_assistant/lib/moderation-flag-recorder.js +43 -0
  47. package/dist/modules/ai_assistant/lib/moderation-flag-recorder.js.map +7 -0
  48. package/dist/modules/ai_assistant/lib/moderation-policy.js +24 -0
  49. package/dist/modules/ai_assistant/lib/moderation-policy.js.map +7 -0
  50. package/dist/modules/ai_assistant/lib/moderation.js +109 -0
  51. package/dist/modules/ai_assistant/lib/moderation.js.map +7 -0
  52. package/dist/modules/ai_assistant/lib/openai-compatible-presets.js +2 -0
  53. package/dist/modules/ai_assistant/lib/openai-compatible-presets.js.map +2 -2
  54. package/dist/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.js +19 -0
  55. package/dist/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.js.map +7 -0
  56. package/generated/entities/ai_agent_runtime_override/index.ts +1 -0
  57. package/generated/entities/ai_moderation_flag/index.ts +9 -0
  58. package/generated/entities.ids.generated.ts +2 -1
  59. package/generated/entity-fields-registry.ts +13 -1
  60. package/package.json +6 -6
  61. package/src/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.ts +126 -0
  62. package/src/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.ts +161 -0
  63. package/src/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.ts +48 -0
  64. package/src/modules/ai_assistant/api/ai/chat/route.ts +10 -0
  65. package/src/modules/ai_assistant/api/moderation-flags/__tests__/route.test.ts +122 -0
  66. package/src/modules/ai_assistant/api/moderation-flags/route.ts +137 -0
  67. package/src/modules/ai_assistant/api/settings/__tests__/route.test.ts +50 -0
  68. package/src/modules/ai_assistant/api/settings/route.ts +23 -0
  69. package/src/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.tsx +146 -0
  70. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.tsx +186 -0
  71. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.ts +24 -0
  72. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.tsx +12 -0
  73. package/src/modules/ai_assistant/data/entities.ts +56 -0
  74. package/src/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.ts +4 -0
  75. package/src/modules/ai_assistant/data/repositories/AiModerationFlagRepository.ts +79 -0
  76. package/src/modules/ai_assistant/di.ts +5 -1
  77. package/src/modules/ai_assistant/events.ts +21 -0
  78. package/src/modules/ai_assistant/i18n/de.json +26 -0
  79. package/src/modules/ai_assistant/i18n/en.json +26 -0
  80. package/src/modules/ai_assistant/i18n/es.json +26 -0
  81. package/src/modules/ai_assistant/i18n/pl.json +26 -0
  82. package/src/modules/ai_assistant/lib/__tests__/input-moderation-gate.test.ts +121 -0
  83. package/src/modules/ai_assistant/lib/__tests__/llm-adapters-anthropic.test.ts +13 -0
  84. package/src/modules/ai_assistant/lib/__tests__/llm-adapters-openai.test.ts +41 -0
  85. package/src/modules/ai_assistant/lib/__tests__/model-factory.test.ts +60 -0
  86. package/src/modules/ai_assistant/lib/__tests__/moderation-flag-recorder.test.ts +76 -0
  87. package/src/modules/ai_assistant/lib/__tests__/moderation-policy.test.ts +86 -0
  88. package/src/modules/ai_assistant/lib/__tests__/moderation.test.ts +114 -0
  89. package/src/modules/ai_assistant/lib/agent-runtime.ts +259 -0
  90. package/src/modules/ai_assistant/lib/ai-agent-definition.ts +10 -0
  91. package/src/modules/ai_assistant/lib/llm-adapters/anthropic.ts +10 -0
  92. package/src/modules/ai_assistant/lib/llm-adapters/openai.ts +32 -1
  93. package/src/modules/ai_assistant/lib/model-factory.ts +33 -0
  94. package/src/modules/ai_assistant/lib/moderation-flag-recorder.ts +75 -0
  95. package/src/modules/ai_assistant/lib/moderation-policy.ts +77 -0
  96. package/src/modules/ai_assistant/lib/moderation.ts +179 -0
  97. package/src/modules/ai_assistant/lib/openai-compatible-presets.ts +2 -0
  98. package/src/modules/ai_assistant/migrations/.snapshot-open-mercato.json +388 -16
  99. package/src/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.ts +21 -0
@@ -0,0 +1,121 @@
1
+ import { extractLatestUserText, runInputModerationGate } from '../agent-runtime'
2
+ import {
3
+ AiModerationBlockedError,
4
+ AiModerationUnavailableError,
5
+ type ModerationResult,
6
+ type ModerationService,
7
+ } from '../moderation'
8
+
9
+ function serviceReturning(result: ModerationResult): ModerationService {
10
+ return { checkInput: jest.fn(async () => result) }
11
+ }
12
+ function serviceThrowing(error: unknown): ModerationService {
13
+ return { checkInput: jest.fn(async () => { throw error }) }
14
+ }
15
+
16
+ const CLEAN: ModerationResult = { flagged: false, categories: { hate: { flagged: false, score: 0.01 } } }
17
+ const FLAGGED: ModerationResult = { flagged: true, categories: { hate: { flagged: true, score: 0.98 } } }
18
+
19
+ const base = {
20
+ supportsInputModeration: true,
21
+ userText: 'hello there',
22
+ resolveApiKey: () => 'sk-test' as string | null,
23
+ env: {} as Record<string, string | undefined>,
24
+ }
25
+
26
+ describe('extractLatestUserText', () => {
27
+ it('returns the text parts of the most recent user message', () => {
28
+ const messages = [
29
+ { role: 'user', parts: [{ type: 'text', text: 'first' }] },
30
+ { role: 'assistant', parts: [{ type: 'text', text: 'reply' }] },
31
+ { role: 'user', parts: [{ type: 'text', text: 'second' }, { type: 'text', text: 'line' }] },
32
+ ] as never
33
+ expect(extractLatestUserText(messages)).toBe('second\nline')
34
+ })
35
+
36
+ it('falls back to string content and returns empty when no user message', () => {
37
+ expect(extractLatestUserText([{ role: 'user', content: 'plain' }] as never)).toBe('plain')
38
+ expect(extractLatestUserText([{ role: 'assistant', parts: [{ type: 'text', text: 'x' }] }] as never)).toBe('')
39
+ })
40
+ })
41
+
42
+ describe('runInputModerationGate', () => {
43
+ it('skips entirely when policy resolves to off (no untrusted, no override, no env)', async () => {
44
+ const service = serviceReturning(FLAGGED)
45
+ await expect(runInputModerationGate({ ...base, service })).resolves.toBeUndefined()
46
+ expect(service.checkInput).not.toHaveBeenCalled()
47
+ })
48
+
49
+ it('skips when the provider does not support moderation, even on an enforced surface', async () => {
50
+ const service = serviceReturning(FLAGGED)
51
+ await expect(
52
+ runInputModerationGate({ ...base, untrustedInput: true, supportsInputModeration: false, service }),
53
+ ).resolves.toBeUndefined()
54
+ expect(service.checkInput).not.toHaveBeenCalled()
55
+ })
56
+
57
+ it('skips when the user text is empty', async () => {
58
+ const service = serviceReturning(FLAGGED)
59
+ await expect(
60
+ runInputModerationGate({ ...base, untrustedInput: true, userText: ' ', service }),
61
+ ).resolves.toBeUndefined()
62
+ expect(service.checkInput).not.toHaveBeenCalled()
63
+ })
64
+
65
+ it('passes clean input through on an enabled surface', async () => {
66
+ const service = serviceReturning(CLEAN)
67
+ await expect(
68
+ runInputModerationGate({ ...base, perAgentOverride: true, service }),
69
+ ).resolves.toBeUndefined()
70
+ expect(service.checkInput).toHaveBeenCalledTimes(1)
71
+ })
72
+
73
+ it('throws AiModerationBlockedError (with categories) on flagged input', async () => {
74
+ const service = serviceReturning(FLAGGED)
75
+ const error = await runInputModerationGate({ ...base, untrustedInput: true, service }).catch((e) => e)
76
+ expect(error).toBeInstanceOf(AiModerationBlockedError)
77
+ expect((error as AiModerationBlockedError).categories.hate.flagged).toBe(true)
78
+ })
79
+
80
+ it('runs the onFlagged side effect before throwing, and the throw still applies if it fails', async () => {
81
+ const service = serviceReturning(FLAGGED)
82
+ const onFlagged = jest.fn(async () => { throw new Error('db down') })
83
+ const error = await runInputModerationGate({ ...base, untrustedInput: true, service, onFlagged }).catch((e) => e)
84
+ expect(onFlagged).toHaveBeenCalledWith(FLAGGED.categories)
85
+ expect(error).toBeInstanceOf(AiModerationBlockedError)
86
+ })
87
+
88
+ it('fails CLOSED on an enforced surface when the endpoint is unavailable', async () => {
89
+ const service = serviceThrowing(new AiModerationUnavailableError('endpoint down'))
90
+ await expect(
91
+ runInputModerationGate({ ...base, untrustedInput: true, service }),
92
+ ).rejects.toBeInstanceOf(AiModerationUnavailableError)
93
+ })
94
+
95
+ it('fails OPEN on an opt-in (on) surface when the endpoint is unavailable', async () => {
96
+ const service = serviceThrowing(new AiModerationUnavailableError('endpoint down'))
97
+ await expect(
98
+ runInputModerationGate({ ...base, perAgentOverride: true, service }),
99
+ ).resolves.toBeUndefined()
100
+ })
101
+
102
+ it('fails CLOSED on enforced when the service is missing from the container', async () => {
103
+ await expect(
104
+ runInputModerationGate({ ...base, untrustedInput: true, service: null }),
105
+ ).rejects.toBeInstanceOf(AiModerationUnavailableError)
106
+ })
107
+
108
+ it('fails OPEN on opt-in when the API key is missing', async () => {
109
+ const service = serviceReturning(CLEAN)
110
+ await expect(
111
+ runInputModerationGate({ ...base, perAgentOverride: true, resolveApiKey: () => null, service }),
112
+ ).resolves.toBeUndefined()
113
+ expect(service.checkInput).not.toHaveBeenCalled()
114
+ })
115
+
116
+ it('does not resolve the API key on the common skip path (policy off)', async () => {
117
+ const resolveApiKey = jest.fn(() => 'sk-test' as string | null)
118
+ await expect(runInputModerationGate({ ...base, resolveApiKey, service: serviceReturning(CLEAN) })).resolves.toBeUndefined()
119
+ expect(resolveApiKey).not.toHaveBeenCalled()
120
+ })
121
+ })
@@ -87,4 +87,17 @@ describe('AnthropicAdapter', () => {
87
87
  })
88
88
  expect(model).toBeDefined()
89
89
  })
90
+
91
+ it('maps an end-user identifier into the Anthropic metadata.userId providerOptions fragment', () => {
92
+ // Key MUST be the AI SDK provider-option name `userId` (camelCase); the SDK
93
+ // translates it to the `metadata.user_id` request-body field and strips
94
+ // unknown providerOptions keys, so a snake_case key would be dropped.
95
+ expect(adapter.mapEndUserIdentifier?.('hashed-id')).toEqual({
96
+ anthropic: { metadata: { userId: 'hashed-id' } },
97
+ })
98
+ })
99
+
100
+ it('does not advertise input moderation support (Anthropic has no moderation endpoint)', () => {
101
+ expect(adapter.supportsInputModeration).toBeFalsy()
102
+ })
90
103
  })
@@ -330,3 +330,44 @@ describe('LM Studio preset', () => {
330
330
  expect(model).toBeDefined()
331
331
  })
332
332
  })
333
+
334
+ describe('OpenAIAdapter — safety identifier + moderation capability', () => {
335
+ it('maps an end-user identifier into the OpenAI safetyIdentifier providerOptions fragment', () => {
336
+ // Key MUST be the AI SDK provider-option name `safetyIdentifier` (camelCase);
337
+ // the SDK translates it to the `safety_identifier` request-body field and
338
+ // strips unknown providerOptions keys, so a snake_case key would be dropped.
339
+ const openaiPreset = OPENAI_COMPATIBLE_PRESETS.find((preset) => preset.id === 'openai')!
340
+ const provider = createOpenAICompatibleProvider(openaiPreset)
341
+ expect(provider.mapEndUserIdentifier?.('hashed-id')).toEqual({
342
+ openai: { safetyIdentifier: 'hashed-id' },
343
+ })
344
+ })
345
+
346
+ it('does not map a safety identifier for OpenAI-compatible backends (OpenAI-only param)', () => {
347
+ // `safety_identifier` is an OpenAI-specific body field; compatible/self-hosted
348
+ // backends (Groq, DeepInfra, Azure, Ollama, …) may reject the unknown param,
349
+ // so they must not advertise the mapping.
350
+ const provider = createOpenAICompatibleProvider(DEEPINFRA_PRESET)
351
+ expect(provider.mapEndUserIdentifier).toBeUndefined()
352
+ for (const preset of OPENAI_COMPATIBLE_PRESETS) {
353
+ const built = createOpenAICompatibleProvider(preset)
354
+ if (preset.id === 'openai') {
355
+ expect(built.mapEndUserIdentifier).toBeDefined()
356
+ } else {
357
+ expect(built.mapEndUserIdentifier).toBeUndefined()
358
+ }
359
+ }
360
+ })
361
+
362
+ it('marks the native OpenAI preset as supporting input moderation', () => {
363
+ const openaiPreset = OPENAI_COMPATIBLE_PRESETS.find((preset) => preset.id === 'openai')
364
+ expect(openaiPreset).toBeDefined()
365
+ const provider = createOpenAICompatibleProvider(openaiPreset!)
366
+ expect(provider.supportsInputModeration).toBe(true)
367
+ })
368
+
369
+ it('leaves OpenAI-compatible backends without moderation support', () => {
370
+ const provider = createOpenAICompatibleProvider(DEEPINFRA_PRESET)
371
+ expect(provider.supportsInputModeration).toBe(false)
372
+ })
373
+ })
@@ -1216,6 +1216,66 @@ describe('Phase 4a — tenantOverride, requestOverride, allowRuntimeOverride (re
1216
1216
  })
1217
1217
  })
1218
1218
 
1219
+ describe('createModelFactory — end-user safety identifier + moderation capability', () => {
1220
+ // The shared makeProvider helper lists fields explicitly and does not carry
1221
+ // the optional mapEndUserIdentifier / supportsInputModeration members, so we
1222
+ // build providers inline here.
1223
+ type IdentifierProvider = FakeProvider & {
1224
+ mapEndUserIdentifier?: (identifier: string) => Record<string, unknown>
1225
+ supportsInputModeration?: boolean
1226
+ }
1227
+ function makeIdentifierProvider(overrides: Partial<IdentifierProvider> = {}): IdentifierProvider {
1228
+ return {
1229
+ id: 'test-provider',
1230
+ defaultModel: 'provider-default-model',
1231
+ resolveApiKey: () => 'test-api-key',
1232
+ createModel: (options) => ({ kind: 'fake-model', ...options }),
1233
+ isConfigured: () => true,
1234
+ ...overrides,
1235
+ }
1236
+ }
1237
+
1238
+ it('emits the provider-mapped providerOptions fragment when an identifier is supplied', () => {
1239
+ const mapEndUserIdentifier = jest.fn((id: string) => ({ openai: { safetyIdentifier: id } }))
1240
+ const provider = makeIdentifierProvider({ mapEndUserIdentifier, supportsInputModeration: true })
1241
+ const factory = createModelFactory(fakeContainer, makeFactoryDeps(provider))
1242
+ const resolution = factory.resolveModel({ endUserIdentifier: 'hashed-user' })
1243
+ expect(mapEndUserIdentifier).toHaveBeenCalledWith('hashed-user')
1244
+ expect(resolution.providerOptions).toEqual({ openai: { safetyIdentifier: 'hashed-user' } })
1245
+ expect(resolution.supportsInputModeration).toBe(true)
1246
+ })
1247
+
1248
+ it('forwards the identifier to the adapter createModel options', () => {
1249
+ const createModel = jest.fn((options: { modelId: string; apiKey: string }) => ({ ...options }))
1250
+ const provider = makeIdentifierProvider({
1251
+ createModel,
1252
+ mapEndUserIdentifier: (id: string) => ({ openai: { safetyIdentifier: id } }),
1253
+ })
1254
+ const factory = createModelFactory(fakeContainer, makeFactoryDeps(provider))
1255
+ factory.resolveModel({ endUserIdentifier: 'hashed-user' })
1256
+ expect(createModel).toHaveBeenCalledWith(
1257
+ expect.objectContaining({ endUserIdentifier: 'hashed-user' }),
1258
+ )
1259
+ })
1260
+
1261
+ it('omits providerOptions when no identifier is supplied', () => {
1262
+ const mapEndUserIdentifier = jest.fn((id: string) => ({ openai: { safety_identifier: id } }))
1263
+ const provider = makeIdentifierProvider({ mapEndUserIdentifier })
1264
+ const factory = createModelFactory(fakeContainer, makeFactoryDeps(provider))
1265
+ const resolution = factory.resolveModel({})
1266
+ expect(mapEndUserIdentifier).not.toHaveBeenCalled()
1267
+ expect(resolution.providerOptions).toBeUndefined()
1268
+ })
1269
+
1270
+ it('omits providerOptions when the provider has no mapping, and reports moderation unsupported', () => {
1271
+ const provider = makeIdentifierProvider()
1272
+ const factory = createModelFactory(fakeContainer, makeFactoryDeps(provider))
1273
+ const resolution = factory.resolveModel({ endUserIdentifier: 'hashed-user' })
1274
+ expect(resolution.providerOptions).toBeUndefined()
1275
+ expect(resolution.supportsInputModeration).toBe(false)
1276
+ })
1277
+ })
1278
+
1219
1279
  describe('createModelFactory — vendor-prefix gateways', () => {
1220
1280
  function gatewayProviders(
1221
1281
  overrides: { openrouterConfigured?: boolean } = {},
@@ -0,0 +1,76 @@
1
+ import type { AwilixContainer } from 'awilix'
2
+
3
+ const createMock = jest.fn()
4
+ const emitMock = jest.fn()
5
+
6
+ jest.mock('../../events', () => ({
7
+ emitAiAssistantEvent: (...args: unknown[]) => emitMock(...args),
8
+ }))
9
+ jest.mock('../../data/repositories/AiModerationFlagRepository', () => ({
10
+ AiModerationFlagRepository: jest.fn().mockImplementation(() => ({ create: createMock })),
11
+ }))
12
+
13
+ import { recordModerationFlag } from '../moderation-flag-recorder'
14
+
15
+ function fakeContainer(): AwilixContainer {
16
+ return {
17
+ resolve: () => ({ fork: () => ({}) }),
18
+ } as unknown as AwilixContainer
19
+ }
20
+
21
+ const baseInput = {
22
+ tenantId: 'tenant-1',
23
+ organizationId: 'org-1',
24
+ agentId: 'customers.account_assistant',
25
+ userId: 'user-1',
26
+ providerId: 'openai',
27
+ modelId: 'gpt-5-mini',
28
+ categories: {
29
+ hate: { flagged: true, score: 0.97 },
30
+ violence: { flagged: false, score: 0.01 },
31
+ },
32
+ }
33
+
34
+ describe('recordModerationFlag', () => {
35
+ beforeEach(() => {
36
+ createMock.mockReset()
37
+ emitMock.mockReset()
38
+ createMock.mockResolvedValue({ id: 'flag-1' })
39
+ emitMock.mockResolvedValue(undefined)
40
+ })
41
+
42
+ it('persists the audit row and emits the event with only flagged categories', async () => {
43
+ await recordModerationFlag(baseInput, fakeContainer())
44
+ expect(createMock).toHaveBeenCalledWith(
45
+ expect.objectContaining({ tenantId: 'tenant-1', agentId: 'customers.account_assistant', providerId: 'openai' }),
46
+ )
47
+ expect(emitMock).toHaveBeenCalledWith(
48
+ 'ai_assistant.moderation_flag.created',
49
+ expect.objectContaining({ id: 'flag-1', tenantId: 'tenant-1', categories: ['hate'] }),
50
+ )
51
+ })
52
+
53
+ it('skips entirely when tenantId is null (cannot tenant-scope the audit row)', async () => {
54
+ await recordModerationFlag({ ...baseInput, tenantId: null }, fakeContainer())
55
+ expect(createMock).not.toHaveBeenCalled()
56
+ expect(emitMock).not.toHaveBeenCalled()
57
+ })
58
+
59
+ it('skips when no container is available', async () => {
60
+ await recordModerationFlag(baseInput, undefined)
61
+ expect(createMock).not.toHaveBeenCalled()
62
+ expect(emitMock).not.toHaveBeenCalled()
63
+ })
64
+
65
+ it('swallows a persistence failure (never throws) and does not emit', async () => {
66
+ createMock.mockRejectedValue(new Error('db down'))
67
+ await expect(recordModerationFlag(baseInput, fakeContainer())).resolves.toBeUndefined()
68
+ expect(emitMock).not.toHaveBeenCalled()
69
+ })
70
+
71
+ it('swallows an emit failure after a successful persist (never throws)', async () => {
72
+ emitMock.mockRejectedValue(new Error('bus down'))
73
+ await expect(recordModerationFlag(baseInput, fakeContainer())).resolves.toBeUndefined()
74
+ expect(createMock).toHaveBeenCalledTimes(1)
75
+ })
76
+ })
@@ -0,0 +1,86 @@
1
+ import {
2
+ isModerationActive,
3
+ resolveModerationPolicy,
4
+ shouldFailClosed,
5
+ } from '../moderation-policy'
6
+
7
+ describe('resolveModerationPolicy — 5-step precedence', () => {
8
+ it('1. untrustedInput wins over everything → enforced', () => {
9
+ expect(
10
+ resolveModerationPolicy({
11
+ untrustedInput: true,
12
+ perAgentOverride: false,
13
+ tenantWideOverride: false,
14
+ env: { OM_AI_INPUT_MODERATION: 'false' },
15
+ }),
16
+ ).toBe('enforced')
17
+ })
18
+
19
+ it('2. per-agent override wins over tenant-wide + env', () => {
20
+ expect(
21
+ resolveModerationPolicy({
22
+ perAgentOverride: true,
23
+ tenantWideOverride: false,
24
+ env: { OM_AI_INPUT_MODERATION: 'false' },
25
+ }),
26
+ ).toBe('on')
27
+ expect(
28
+ resolveModerationPolicy({
29
+ perAgentOverride: false,
30
+ tenantWideOverride: true,
31
+ env: { OM_AI_INPUT_MODERATION: 'true' },
32
+ }),
33
+ ).toBe('off')
34
+ })
35
+
36
+ it('3. tenant-wide override wins over env when per-agent is inherit', () => {
37
+ expect(
38
+ resolveModerationPolicy({
39
+ perAgentOverride: null,
40
+ tenantWideOverride: true,
41
+ env: { OM_AI_INPUT_MODERATION: 'false' },
42
+ }),
43
+ ).toBe('on')
44
+ expect(
45
+ resolveModerationPolicy({
46
+ perAgentOverride: undefined,
47
+ tenantWideOverride: false,
48
+ env: { OM_AI_INPUT_MODERATION: 'true' },
49
+ }),
50
+ ).toBe('off')
51
+ })
52
+
53
+ it('4. env default applies when both overrides inherit', () => {
54
+ expect(
55
+ resolveModerationPolicy({ perAgentOverride: null, tenantWideOverride: null, env: { OM_AI_INPUT_MODERATION: 'true' } }),
56
+ ).toBe('on')
57
+ expect(
58
+ resolveModerationPolicy({ env: { OM_AI_INPUT_MODERATION: '1' } }),
59
+ ).toBe('on')
60
+ })
61
+
62
+ it('5. defaults to off when nothing is set', () => {
63
+ expect(resolveModerationPolicy({ env: {} })).toBe('off')
64
+ expect(resolveModerationPolicy({ perAgentOverride: null, tenantWideOverride: null, env: {} })).toBe('off')
65
+ })
66
+
67
+ it('treats untrustedInput=false the same as unset (falls through)', () => {
68
+ expect(
69
+ resolveModerationPolicy({ untrustedInput: false, tenantWideOverride: true, env: {} }),
70
+ ).toBe('on')
71
+ })
72
+ })
73
+
74
+ describe('moderation policy helpers', () => {
75
+ it('isModerationActive is true for enforced and on, false for off', () => {
76
+ expect(isModerationActive('enforced')).toBe(true)
77
+ expect(isModerationActive('on')).toBe(true)
78
+ expect(isModerationActive('off')).toBe(false)
79
+ })
80
+
81
+ it('shouldFailClosed is true only for enforced (opt-in on fails open)', () => {
82
+ expect(shouldFailClosed('enforced')).toBe(true)
83
+ expect(shouldFailClosed('on')).toBe(false)
84
+ expect(shouldFailClosed('off')).toBe(false)
85
+ })
86
+ })
@@ -0,0 +1,114 @@
1
+ import {
2
+ AiModerationUnavailableError,
3
+ createModerationService,
4
+ DEFAULT_MODERATION_MODEL,
5
+ } from '../moderation'
6
+
7
+ function jsonResponse(body: unknown, status = 200): Response {
8
+ return {
9
+ ok: status >= 200 && status < 300,
10
+ status,
11
+ json: async () => body,
12
+ } as unknown as Response
13
+ }
14
+
15
+ function moderationBody(flagged: boolean): unknown {
16
+ return {
17
+ results: [
18
+ {
19
+ flagged,
20
+ categories: { hate: flagged, violence: false },
21
+ category_scores: { hate: flagged ? 0.97 : 0.01, violence: 0.002 },
22
+ },
23
+ ],
24
+ }
25
+ }
26
+
27
+ describe('createModerationService.checkInput', () => {
28
+ const service = createModerationService()
29
+ let fetchSpy: jest.SpyInstance
30
+
31
+ beforeEach(() => {
32
+ fetchSpy = jest.spyOn(globalThis, 'fetch')
33
+ })
34
+ afterEach(() => {
35
+ fetchSpy.mockRestore()
36
+ })
37
+
38
+ it('returns flagged=false with mapped categories for clean input', async () => {
39
+ fetchSpy.mockResolvedValue(jsonResponse(moderationBody(false)))
40
+ const result = await service.checkInput({ text: 'hello', apiKey: 'sk-test' })
41
+ expect(result.flagged).toBe(false)
42
+ expect(result.categories.hate).toEqual({ flagged: false, score: 0.01 })
43
+ expect(result.categories.violence).toEqual({ flagged: false, score: 0.002 })
44
+ expect(fetchSpy).toHaveBeenCalledTimes(1)
45
+ })
46
+
47
+ it('returns flagged=true with category flags + scores for flagged input', async () => {
48
+ fetchSpy.mockResolvedValue(jsonResponse(moderationBody(true)))
49
+ const result = await service.checkInput({ text: 'bad', apiKey: 'sk-test' })
50
+ expect(result.flagged).toBe(true)
51
+ expect(result.categories.hate).toEqual({ flagged: true, score: 0.97 })
52
+ })
53
+
54
+ it('sends the configured model + base URL and bearer auth', async () => {
55
+ fetchSpy.mockResolvedValue(jsonResponse(moderationBody(false)))
56
+ await service.checkInput({
57
+ text: 'hi',
58
+ apiKey: 'sk-secret',
59
+ baseURL: 'https://proxy.example.com/v1/',
60
+ model: 'omni-moderation-2026',
61
+ })
62
+ const [url, init] = fetchSpy.mock.calls[0] as [string, RequestInit]
63
+ expect(url).toBe('https://proxy.example.com/v1/moderations')
64
+ expect((init.headers as Record<string, string>).Authorization).toBe('Bearer sk-secret')
65
+ expect(JSON.parse(init.body as string)).toEqual({ model: 'omni-moderation-2026', input: 'hi' })
66
+ })
67
+
68
+ it('defaults to omni-moderation-latest when no model is supplied', async () => {
69
+ fetchSpy.mockResolvedValue(jsonResponse(moderationBody(false)))
70
+ await service.checkInput({ text: 'hi', apiKey: 'sk-test' })
71
+ const init = fetchSpy.mock.calls[0][1] as RequestInit
72
+ expect(JSON.parse(init.body as string).model).toBe(DEFAULT_MODERATION_MODEL)
73
+ })
74
+
75
+ it('retries once on a 5xx and succeeds', async () => {
76
+ fetchSpy
77
+ .mockResolvedValueOnce(jsonResponse({ error: 'boom' }, 503))
78
+ .mockResolvedValueOnce(jsonResponse(moderationBody(false)))
79
+ const result = await service.checkInput({ text: 'hi', apiKey: 'sk-test' })
80
+ expect(result.flagged).toBe(false)
81
+ expect(fetchSpy).toHaveBeenCalledTimes(2)
82
+ })
83
+
84
+ it('throws AiModerationUnavailableError after a 5xx on both attempts', async () => {
85
+ fetchSpy.mockResolvedValue(jsonResponse({ error: 'boom' }, 500))
86
+ await expect(service.checkInput({ text: 'hi', apiKey: 'sk-test' })).rejects.toBeInstanceOf(
87
+ AiModerationUnavailableError,
88
+ )
89
+ expect(fetchSpy).toHaveBeenCalledTimes(2)
90
+ })
91
+
92
+ it('throws AiModerationUnavailableError on repeated network/timeout errors', async () => {
93
+ fetchSpy.mockRejectedValue(new Error('AbortError: timed out'))
94
+ await expect(service.checkInput({ text: 'hi', apiKey: 'sk-test' })).rejects.toBeInstanceOf(
95
+ AiModerationUnavailableError,
96
+ )
97
+ expect(fetchSpy).toHaveBeenCalledTimes(2)
98
+ })
99
+
100
+ it('does not retry a 4xx (credential/config error) and fails closed', async () => {
101
+ fetchSpy.mockResolvedValue(jsonResponse({ error: 'unauthorized' }, 401))
102
+ await expect(service.checkInput({ text: 'hi', apiKey: 'bad' })).rejects.toBeInstanceOf(
103
+ AiModerationUnavailableError,
104
+ )
105
+ expect(fetchSpy).toHaveBeenCalledTimes(1)
106
+ })
107
+
108
+ it('throws AiModerationUnavailableError when the response schema is unexpected', async () => {
109
+ fetchSpy.mockResolvedValue(jsonResponse({ unexpected: true }))
110
+ await expect(service.checkInput({ text: 'hi', apiKey: 'sk-test' })).rejects.toBeInstanceOf(
111
+ AiModerationUnavailableError,
112
+ )
113
+ })
114
+ })