@open-mercato/ai-assistant 0.6.7-develop.6784.1.f80b9afce5 → 0.6.7-develop.6785.1.1dd7cfac55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/AGENTS.md +12 -29
  3. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.js +80 -0
  4. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.js.map +7 -0
  5. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.js +124 -0
  6. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.js.map +7 -0
  7. package/dist/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.js +30 -0
  8. package/dist/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.js.map +2 -2
  9. package/dist/modules/ai_assistant/api/ai/chat/route.js +7 -0
  10. package/dist/modules/ai_assistant/api/ai/chat/route.js.map +2 -2
  11. package/dist/modules/ai_assistant/api/moderation-flags/route.js +121 -0
  12. package/dist/modules/ai_assistant/api/moderation-flags/route.js.map +7 -0
  13. package/dist/modules/ai_assistant/api/settings/route.js +24 -3
  14. package/dist/modules/ai_assistant/api/settings/route.js.map +2 -2
  15. package/dist/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.js +106 -0
  16. package/dist/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.js.map +2 -2
  17. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.js +139 -0
  18. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.js.map +7 -0
  19. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.js +10 -0
  20. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.js.map +7 -0
  21. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.js +26 -0
  22. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.js.map +7 -0
  23. package/dist/modules/ai_assistant/data/entities.js +42 -0
  24. package/dist/modules/ai_assistant/data/entities.js.map +2 -2
  25. package/dist/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.js +3 -1
  26. package/dist/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.js.map +2 -2
  27. package/dist/modules/ai_assistant/data/repositories/AiModerationFlagRepository.js +44 -0
  28. package/dist/modules/ai_assistant/data/repositories/AiModerationFlagRepository.js.map +7 -0
  29. package/dist/modules/ai_assistant/di.js +6 -2
  30. package/dist/modules/ai_assistant/di.js.map +2 -2
  31. package/dist/modules/ai_assistant/events.js +6 -0
  32. package/dist/modules/ai_assistant/events.js.map +2 -2
  33. package/dist/modules/ai_assistant/i18n/de.json +26 -0
  34. package/dist/modules/ai_assistant/i18n/en.json +26 -0
  35. package/dist/modules/ai_assistant/i18n/es.json +26 -0
  36. package/dist/modules/ai_assistant/i18n/pl.json +26 -0
  37. package/dist/modules/ai_assistant/lib/agent-runtime.js +158 -5
  38. package/dist/modules/ai_assistant/lib/agent-runtime.js.map +2 -2
  39. package/dist/modules/ai_assistant/lib/ai-agent-definition.js.map +2 -2
  40. package/dist/modules/ai_assistant/lib/llm-adapters/anthropic.js +3 -0
  41. package/dist/modules/ai_assistant/lib/llm-adapters/anthropic.js.map +2 -2
  42. package/dist/modules/ai_assistant/lib/llm-adapters/openai.js +8 -1
  43. package/dist/modules/ai_assistant/lib/llm-adapters/openai.js.map +2 -2
  44. package/dist/modules/ai_assistant/lib/model-factory.js +6 -1
  45. package/dist/modules/ai_assistant/lib/model-factory.js.map +2 -2
  46. package/dist/modules/ai_assistant/lib/moderation-flag-recorder.js +43 -0
  47. package/dist/modules/ai_assistant/lib/moderation-flag-recorder.js.map +7 -0
  48. package/dist/modules/ai_assistant/lib/moderation-policy.js +24 -0
  49. package/dist/modules/ai_assistant/lib/moderation-policy.js.map +7 -0
  50. package/dist/modules/ai_assistant/lib/moderation.js +109 -0
  51. package/dist/modules/ai_assistant/lib/moderation.js.map +7 -0
  52. package/dist/modules/ai_assistant/lib/openai-compatible-presets.js +2 -0
  53. package/dist/modules/ai_assistant/lib/openai-compatible-presets.js.map +2 -2
  54. package/dist/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.js +19 -0
  55. package/dist/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.js.map +7 -0
  56. package/generated/entities/ai_agent_runtime_override/index.ts +1 -0
  57. package/generated/entities/ai_moderation_flag/index.ts +9 -0
  58. package/generated/entities.ids.generated.ts +2 -1
  59. package/generated/entity-fields-registry.ts +13 -1
  60. package/package.json +6 -6
  61. package/src/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.ts +126 -0
  62. package/src/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.ts +161 -0
  63. package/src/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.ts +48 -0
  64. package/src/modules/ai_assistant/api/ai/chat/route.ts +10 -0
  65. package/src/modules/ai_assistant/api/moderation-flags/__tests__/route.test.ts +122 -0
  66. package/src/modules/ai_assistant/api/moderation-flags/route.ts +137 -0
  67. package/src/modules/ai_assistant/api/settings/__tests__/route.test.ts +50 -0
  68. package/src/modules/ai_assistant/api/settings/route.ts +23 -0
  69. package/src/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.tsx +146 -0
  70. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.tsx +186 -0
  71. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.ts +24 -0
  72. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.tsx +12 -0
  73. package/src/modules/ai_assistant/data/entities.ts +56 -0
  74. package/src/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.ts +4 -0
  75. package/src/modules/ai_assistant/data/repositories/AiModerationFlagRepository.ts +79 -0
  76. package/src/modules/ai_assistant/di.ts +5 -1
  77. package/src/modules/ai_assistant/events.ts +21 -0
  78. package/src/modules/ai_assistant/i18n/de.json +26 -0
  79. package/src/modules/ai_assistant/i18n/en.json +26 -0
  80. package/src/modules/ai_assistant/i18n/es.json +26 -0
  81. package/src/modules/ai_assistant/i18n/pl.json +26 -0
  82. package/src/modules/ai_assistant/lib/__tests__/input-moderation-gate.test.ts +121 -0
  83. package/src/modules/ai_assistant/lib/__tests__/llm-adapters-anthropic.test.ts +13 -0
  84. package/src/modules/ai_assistant/lib/__tests__/llm-adapters-openai.test.ts +41 -0
  85. package/src/modules/ai_assistant/lib/__tests__/model-factory.test.ts +60 -0
  86. package/src/modules/ai_assistant/lib/__tests__/moderation-flag-recorder.test.ts +76 -0
  87. package/src/modules/ai_assistant/lib/__tests__/moderation-policy.test.ts +86 -0
  88. package/src/modules/ai_assistant/lib/__tests__/moderation.test.ts +114 -0
  89. package/src/modules/ai_assistant/lib/agent-runtime.ts +259 -0
  90. package/src/modules/ai_assistant/lib/ai-agent-definition.ts +10 -0
  91. package/src/modules/ai_assistant/lib/llm-adapters/anthropic.ts +10 -0
  92. package/src/modules/ai_assistant/lib/llm-adapters/openai.ts +32 -1
  93. package/src/modules/ai_assistant/lib/model-factory.ts +33 -0
  94. package/src/modules/ai_assistant/lib/moderation-flag-recorder.ts +75 -0
  95. package/src/modules/ai_assistant/lib/moderation-policy.ts +77 -0
  96. package/src/modules/ai_assistant/lib/moderation.ts +179 -0
  97. package/src/modules/ai_assistant/lib/openai-compatible-presets.ts +2 -0
  98. package/src/modules/ai_assistant/migrations/.snapshot-open-mercato.json +388 -16
  99. package/src/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.ts +21 -0
@@ -171,6 +171,54 @@ export async function deleteTenantAllowlistInDb(tenantId: string | null): Promis
171
171
  });
172
172
  }
173
173
 
174
+ export interface SeedModerationFlagInput {
175
+ tenantId: string;
176
+ organizationId?: string | null;
177
+ agentId?: string;
178
+ userId?: string;
179
+ providerId?: string;
180
+ modelId?: string;
181
+ /** Defaults to a single flagged `hate` category. */
182
+ categories?: Record<string, { flagged: boolean; score: number }>;
183
+ }
184
+
185
+ /**
186
+ * Inserts an `ai_moderation_flags` audit row directly. A real flag is only ever
187
+ * born from the gate during a flagged LLM turn (needs a live moderation call),
188
+ * so seeding the row keeps audit-listing + tenant-isolation coverage
189
+ * deterministic and provider-free. Returns the new id for assertions/cleanup.
190
+ */
191
+ export async function seedModerationFlagInDb(input: SeedModerationFlagInput): Promise<string> {
192
+ const categories = JSON.stringify(input.categories ?? { hate: { flagged: true, score: 0.97 } });
193
+ return withClient(async (client) => {
194
+ const result = await client.query<{ id: string }>(
195
+ `insert into ai_moderation_flags
196
+ (id, tenant_id, organization_id, agent_id, user_id, provider_id, model_id, categories, created_at)
197
+ values
198
+ (gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7::jsonb, now())
199
+ returning id`,
200
+ [
201
+ input.tenantId,
202
+ input.organizationId ?? null,
203
+ input.agentId ?? 'it_agent.moderation_fixture',
204
+ input.userId ?? `it-user-${randomUUID()}`,
205
+ input.providerId ?? 'openai',
206
+ input.modelId ?? 'gpt-5-mini',
207
+ categories,
208
+ ],
209
+ );
210
+ return result.rows[0].id;
211
+ });
212
+ }
213
+
214
+ /** Hard-deletes every moderation-flag row for a tenant (best-effort cleanup). */
215
+ export async function deleteModerationFlagsInDb(tenantId: string | null): Promise<void> {
216
+ if (!tenantId) return;
217
+ await withClient(async (client) => {
218
+ await client.query('delete from ai_moderation_flags where tenant_id = $1', [tenantId]);
219
+ });
220
+ }
221
+
174
222
  /**
175
223
  * Hard-deletes a conversation and its participant + message rows by the
176
224
  * client-facing `conversation_id`. The DELETE route only soft-deletes, so this
@@ -15,6 +15,7 @@ import {
15
15
  type AiAgentLoopBudgetPreset,
16
16
  } from '../../../lib/agent-runtime'
17
17
  import { AgentPolicyError } from '../../../lib/agent-tools'
18
+ import { AiModerationBlockedError, AiModerationUnavailableError } from '../../../lib/moderation'
18
19
  import { readBaseurlAllowlist, isBaseurlAllowlisted } from '../../../lib/baseurl-allowlist'
19
20
  import {
20
21
  canonicalProviderId,
@@ -700,6 +701,15 @@ export async function POST(req: NextRequest): Promise<Response> {
700
701
  if (error instanceof AgentPolicyError) {
701
702
  return jsonError(statusForDenyCode(error.code), error.message, error.code)
702
703
  }
704
+ // Input pre-moderation outcomes (spec 2026-06-04). The blocked categories
705
+ // are NEVER sent to the client — only the typed code, which the chat UI
706
+ // maps to a generic translated message.
707
+ if (error instanceof AiModerationBlockedError) {
708
+ return jsonError(400, 'Input rejected by the content safety filter.', 'moderation_blocked')
709
+ }
710
+ if (error instanceof AiModerationUnavailableError) {
711
+ return jsonError(503, 'Content safety check temporarily unavailable.', 'moderation_unavailable')
712
+ }
703
713
  logger.error('AI Chat Agent — Dispatch failure', { err: error })
704
714
  return jsonError(
705
715
  500,
@@ -0,0 +1,122 @@
1
+ const authMock = jest.fn()
2
+ const loadAclMock = jest.fn()
3
+ const createRequestContainerMock = jest.fn()
4
+ const listMock = jest.fn()
5
+
6
+ jest.mock('@open-mercato/shared/lib/auth/server', () => ({
7
+ getAuthFromRequest: (...args: unknown[]) => authMock(...args),
8
+ }))
9
+
10
+ jest.mock('@open-mercato/shared/lib/di/container', () => ({
11
+ createRequestContainer: (...args: unknown[]) => createRequestContainerMock(...args),
12
+ }))
13
+
14
+ jest.mock('../../../data/repositories/AiModerationFlagRepository', () => ({
15
+ AiModerationFlagRepository: jest.fn().mockImplementation(() => ({ list: listMock })),
16
+ }))
17
+
18
+ import { GET } from '../route'
19
+
20
+ function buildRequest(params: Record<string, string> = {}) {
21
+ const url = new URL('http://localhost/api/ai_assistant/moderation-flags')
22
+ for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v)
23
+ return new Request(url.toString(), { method: 'GET' })
24
+ }
25
+
26
+ function makeRow(overrides: Record<string, unknown> = {}) {
27
+ return {
28
+ id: 'flag-1',
29
+ tenantId: 'tenant-1',
30
+ organizationId: null,
31
+ agentId: 'catalog.assistant',
32
+ userId: 'user-9',
33
+ providerId: 'openai',
34
+ modelId: 'gpt-5-mini',
35
+ categories: { hate: { flagged: true, score: 0.97 } },
36
+ createdAt: new Date('2026-06-10T12:00:00Z'),
37
+ ...overrides,
38
+ }
39
+ }
40
+
41
+ describe('GET /api/ai_assistant/moderation-flags', () => {
42
+ let consoleErrorSpy: jest.SpyInstance
43
+
44
+ beforeEach(() => {
45
+ jest.clearAllMocks()
46
+ consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
47
+ authMock.mockResolvedValue({ sub: 'user-1', tenantId: 'tenant-1', orgId: 'org-1' })
48
+ loadAclMock.mockResolvedValue({ features: ['ai_assistant.settings.manage'], isSuperAdmin: false })
49
+ createRequestContainerMock.mockResolvedValue({
50
+ resolve: (name: string) => {
51
+ if (name === 'rbacService') {
52
+ return {
53
+ loadAcl: loadAclMock,
54
+ hasAllFeatures: (req: string[], have: string[]) => req.every((r) => have.includes(r)),
55
+ }
56
+ }
57
+ if (name === 'em') return {}
58
+ throw new Error(`Unknown token: ${name}`)
59
+ },
60
+ })
61
+ listMock.mockResolvedValue({ items: [], total: 0 })
62
+ })
63
+
64
+ afterEach(() => {
65
+ consoleErrorSpy.mockRestore()
66
+ })
67
+
68
+ it('returns 401 when unauthenticated', async () => {
69
+ authMock.mockResolvedValue(null)
70
+ const res = await GET(buildRequest() as never)
71
+ expect(res.status).toBe(401)
72
+ })
73
+
74
+ it('returns 403 when the caller lacks ai_assistant.settings.manage', async () => {
75
+ loadAclMock.mockResolvedValue({ features: ['ai_assistant.view'], isSuperAdmin: false })
76
+ const res = await GET(buildRequest() as never)
77
+ expect(res.status).toBe(403)
78
+ expect((await res.json()).code).toBe('forbidden')
79
+ })
80
+
81
+ it('rejects pageSize above the 100 cap with a validation error', async () => {
82
+ const res = await GET(buildRequest({ pageSize: '101' }) as never)
83
+ expect(res.status).toBe(400)
84
+ expect((await res.json()).code).toBe('validation_error')
85
+ expect(listMock).not.toHaveBeenCalled()
86
+ })
87
+
88
+ it('lists tenant-scoped rows and echoes pagination', async () => {
89
+ listMock.mockResolvedValue({ items: [makeRow()], total: 1 })
90
+ const res = await GET(buildRequest({ page: '2', pageSize: '25' }) as never)
91
+ expect(res.status).toBe(200)
92
+ const json = await res.json()
93
+ expect(json.total).toBe(1)
94
+ expect(json.page).toBe(2)
95
+ expect(json.pageSize).toBe(25)
96
+ expect(json.items[0]).toMatchObject({ id: 'flag-1', userId: 'user-9', categories: { hate: { flagged: true, score: 0.97 } } })
97
+ // Tenant-scoped only — the listing does NOT narrow by organization_id.
98
+ expect(listMock).toHaveBeenCalledWith(
99
+ expect.objectContaining({ tenantId: 'tenant-1', page: 2, pageSize: 25 }),
100
+ )
101
+ expect(listMock.mock.calls[0][0].organizationId).toBeUndefined()
102
+ })
103
+
104
+ it('passes agentId/userId/date filters through to the repository', async () => {
105
+ await GET(buildRequest({ agentId: 'catalog.assistant', userId: 'user-9', from: '2026-06-01', to: '2026-06-10' }) as never)
106
+ const call = listMock.mock.calls[0][0]
107
+ expect(call.agentId).toBe('catalog.assistant')
108
+ expect(call.userId).toBe('user-9')
109
+ expect(call.from).toBeInstanceOf(Date)
110
+ expect(call.to).toBeInstanceOf(Date)
111
+ // `to` is extended to end-of-day so same-day rows are included.
112
+ expect((call.to as Date).toISOString()).toBe('2026-06-10T23:59:59.999Z')
113
+ })
114
+
115
+ it('returns an empty page (no repo call) when the caller has no tenant scope', async () => {
116
+ authMock.mockResolvedValue({ sub: 'user-1', tenantId: null, orgId: null })
117
+ const res = await GET(buildRequest() as never)
118
+ expect(res.status).toBe(200)
119
+ expect((await res.json()).items).toEqual([])
120
+ expect(listMock).not.toHaveBeenCalled()
121
+ })
122
+ })
@@ -0,0 +1,137 @@
1
+ import { NextResponse, type NextRequest } from 'next/server'
2
+ import { z } from 'zod'
3
+ import { createLogger } from '@open-mercato/shared/lib/logger'
4
+ import type { EntityManager } from '@mikro-orm/postgresql'
5
+ import type { OpenApiRouteDoc } from '@open-mercato/shared/lib/openapi'
6
+ import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
7
+ import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
8
+ import type { RbacService } from '@open-mercato/core/modules/auth/services/rbacService'
9
+ import { AiModerationFlagRepository } from '../../data/repositories/AiModerationFlagRepository'
10
+ import { hasRequiredFeatures } from '../../lib/auth'
11
+
12
+ const logger = createLogger('ai_assistant')
13
+
14
+ const REQUIRED_FEATURE = 'ai_assistant.settings.manage'
15
+ const MAX_PAGE_SIZE = 100
16
+
17
+ const querySchema = z.object({
18
+ page: z.coerce.number().int().min(1).default(1),
19
+ pageSize: z.coerce.number().int().min(1).max(MAX_PAGE_SIZE).default(50),
20
+ agentId: z.string().min(1).max(256).optional(),
21
+ userId: z.string().min(1).max(256).optional(),
22
+ from: z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'from must be a date in YYYY-MM-DD format').optional(),
23
+ to: z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'to must be a date in YYYY-MM-DD format').optional(),
24
+ })
25
+
26
+ export const openApi: OpenApiRouteDoc = {
27
+ tag: 'AI Assistant',
28
+ summary: 'Input moderation flags audit log',
29
+ methods: {
30
+ GET: {
31
+ operationId: 'aiAssistantModerationFlags',
32
+ summary: 'List input-moderation audit flags for the current tenant.',
33
+ description:
34
+ 'Returns the append-only `ai_moderation_flags` audit rows (category flags + scores only — never ' +
35
+ 'prompt content) for the caller\'s tenant. Always tenant-scoped; cross-tenant access is impossible. ' +
36
+ 'Optionally filtered by `agentId`, `userId`, and a `from`/`to` date window. Paginated (`pageSize` ≤ 100). ' +
37
+ 'Requires `ai_assistant.settings.manage`.',
38
+ query: querySchema,
39
+ responses: [
40
+ {
41
+ status: 200,
42
+ description: 'Paginated moderation-flag rows: `{ items, total, page, pageSize }`.',
43
+ mediaType: 'application/json',
44
+ },
45
+ ],
46
+ errors: [
47
+ { status: 400, description: 'Invalid query parameters.' },
48
+ { status: 401, description: 'Unauthenticated caller.' },
49
+ { status: 403, description: 'Caller lacks `ai_assistant.settings.manage`.' },
50
+ { status: 500, description: 'Internal failure.' },
51
+ ],
52
+ },
53
+ },
54
+ }
55
+
56
+ export const metadata = {
57
+ path: '/ai_assistant/moderation-flags',
58
+ GET: { requireAuth: true, requireFeatures: [REQUIRED_FEATURE] },
59
+ }
60
+
61
+ function jsonError(status: number, message: string, code: string, extra?: Record<string, unknown>): NextResponse {
62
+ return NextResponse.json({ error: message, code, ...(extra ?? {}) }, { status })
63
+ }
64
+
65
+ export async function GET(req: NextRequest): Promise<Response> {
66
+ const auth = await getAuthFromRequest(req)
67
+ if (!auth) {
68
+ return jsonError(401, 'Unauthorized', 'unauthenticated')
69
+ }
70
+
71
+ const { searchParams } = new URL(req.url)
72
+ const queryResult = querySchema.safeParse({
73
+ page: searchParams.get('page') ?? undefined,
74
+ pageSize: searchParams.get('pageSize') ?? undefined,
75
+ agentId: searchParams.get('agentId') ?? undefined,
76
+ userId: searchParams.get('userId') ?? undefined,
77
+ from: searchParams.get('from') ?? undefined,
78
+ to: searchParams.get('to') ?? undefined,
79
+ })
80
+ if (!queryResult.success) {
81
+ return jsonError(400, 'Invalid query parameters.', 'validation_error', {
82
+ issues: queryResult.error.issues,
83
+ })
84
+ }
85
+
86
+ const { page, pageSize, agentId, userId, from, to } = queryResult.data
87
+
88
+ try {
89
+ const container = await createRequestContainer()
90
+ const rbacService = container.resolve<RbacService>('rbacService')
91
+ const acl = await rbacService.loadAcl(auth.sub, {
92
+ tenantId: auth.tenantId,
93
+ organizationId: auth.orgId,
94
+ })
95
+
96
+ if (!hasRequiredFeatures([REQUIRED_FEATURE], acl.features, acl.isSuperAdmin, rbacService)) {
97
+ return jsonError(403, `Caller lacks required feature "${REQUIRED_FEATURE}".`, 'forbidden')
98
+ }
99
+
100
+ if (!auth.tenantId) {
101
+ return NextResponse.json({ items: [], total: 0, page, pageSize })
102
+ }
103
+
104
+ const em = container.resolve<EntityManager>('em')
105
+ const repo = new AiModerationFlagRepository(em)
106
+ const { items, total } = await repo.list({
107
+ tenantId: auth.tenantId,
108
+ // Tenant-scoped abuse audit: do NOT narrow by organization_id. Tenant is
109
+ // the hard isolation boundary; portal-surface flags often carry no org,
110
+ // and a tenant admin reviewing abuse needs every flag in the tenant.
111
+ agentId,
112
+ userId,
113
+ // `to` is an inclusive day; extend to end-of-day so same-day rows match.
114
+ from: from ? new Date(`${from}T00:00:00.000Z`) : undefined,
115
+ to: to ? new Date(`${to}T23:59:59.999Z`) : undefined,
116
+ page,
117
+ pageSize,
118
+ })
119
+
120
+ const serialized = items.map((row) => ({
121
+ id: row.id,
122
+ tenantId: row.tenantId,
123
+ organizationId: row.organizationId ?? null,
124
+ agentId: row.agentId,
125
+ userId: row.userId,
126
+ providerId: row.providerId,
127
+ modelId: row.modelId,
128
+ categories: row.categories,
129
+ createdAt: row.createdAt.toISOString(),
130
+ }))
131
+
132
+ return NextResponse.json({ items: serialized, total, page, pageSize })
133
+ } catch (error) {
134
+ logger.error('Moderation flags GET failed', { err: error })
135
+ return jsonError(500, 'Failed to fetch moderation flags.', 'internal_error')
136
+ }
137
+ }
@@ -161,6 +161,56 @@ describe('PUT /api/ai_assistant/settings', () => {
161
161
  )
162
162
  })
163
163
 
164
+ it('forwards inputModeration to the override upsert and echoes it back', async () => {
165
+ upsertDefaultMock.mockResolvedValueOnce({
166
+ id: 'row-1',
167
+ tenantId: 'tenant-1',
168
+ organizationId: 'org-1',
169
+ agentId: 'catalog.catalog_assistant',
170
+ providerId: null,
171
+ modelId: null,
172
+ baseUrl: null,
173
+ inputModeration: true,
174
+ updatedAt: new Date('2026-06-10T00:00:00Z'),
175
+ })
176
+
177
+ const response = await PUT(
178
+ buildRequest('PUT', { agentId: 'catalog.catalog_assistant', inputModeration: true }) as any,
179
+ )
180
+
181
+ expect(response.status).toBe(200)
182
+ const json = await response.json()
183
+ expect(json.inputModeration).toBe(true)
184
+ expect(upsertDefaultMock).toHaveBeenCalledWith(
185
+ expect.objectContaining({ agentId: 'catalog.catalog_assistant', inputModeration: true }),
186
+ expect.objectContaining({ tenantId: 'tenant-1' }),
187
+ )
188
+ })
189
+
190
+ it('passes inputModeration: null (inherit) through to the upsert', async () => {
191
+ upsertDefaultMock.mockResolvedValueOnce({
192
+ id: 'row-1',
193
+ tenantId: 'tenant-1',
194
+ organizationId: 'org-1',
195
+ agentId: 'catalog.catalog_assistant',
196
+ providerId: null,
197
+ modelId: null,
198
+ baseUrl: null,
199
+ inputModeration: null,
200
+ updatedAt: new Date('2026-06-10T00:00:00Z'),
201
+ })
202
+
203
+ const response = await PUT(
204
+ buildRequest('PUT', { agentId: 'catalog.catalog_assistant', inputModeration: null }) as any,
205
+ )
206
+
207
+ expect(response.status).toBe(200)
208
+ expect(upsertDefaultMock).toHaveBeenCalledWith(
209
+ expect.objectContaining({ inputModeration: null }),
210
+ expect.anything(),
211
+ )
212
+ })
213
+
164
214
  it('saves per-agent chat override allowlist without requiring a model override', async () => {
165
215
  upsertDefaultMock.mockResolvedValueOnce({
166
216
  id: 'row-1',
@@ -17,6 +17,7 @@ import {
17
17
  isOpenCodeProviderConfigured,
18
18
  } from '@open-mercato/shared/lib/ai/opencode-provider'
19
19
  import { AiAgentRuntimeOverrideRepository, AiAgentRuntimeOverrideValidationError } from '../../data/repositories/AiAgentRuntimeOverrideRepository'
20
+ import { resolveModerationPolicy } from '../../lib/moderation-policy'
20
21
  import { AiTenantModelAllowlistRepository } from '../../data/repositories/AiTenantModelAllowlistRepository'
21
22
  import { isBaseurlAllowlisted, readBaseurlAllowlist } from '../../lib/baseurl-allowlist'
22
23
  import { loadAgentRegistry, listAgents } from '../../lib/agent-registry'
@@ -59,6 +60,9 @@ const runtimeOverrideUpsertSchema = z.object({
59
60
  allowedOverrideModelsByProvider: z
60
61
  .record(z.string().min(1).max(64), z.array(z.string().min(1).max(256)))
61
62
  .optional(),
63
+ // Input moderation override: true = on, false = off, null = inherit.
64
+ // Spec 2026-06-04-ai-input-moderation-and-safety-identifiers.
65
+ inputModeration: z.boolean().nullable().optional(),
62
66
  })
63
67
 
64
68
  const runtimeOverrideClearSchema = z.object({
@@ -248,6 +252,7 @@ export async function GET(req: NextRequest) {
248
252
  const em = container.resolve<EntityManager>('em')
249
253
  const repo = new AiAgentRuntimeOverrideRepository(em)
250
254
  const overrideRow = await repo.getDefault({ tenantId, organizationId, agentId: null })
255
+ const tenantWideInputModeration = overrideRow?.inputModeration ?? null
251
256
  if (overrideRow) {
252
257
  tenantOverride = {
253
258
  providerId: overrideRow.providerId ?? null,
@@ -363,6 +368,20 @@ export async function GET(req: NextRequest) {
363
368
  modelId: agentResolution.modelId,
364
369
  baseURL: agentResolution.baseURL ?? null,
365
370
  source: agentResolution.source,
371
+ moderation: {
372
+ // `untrustedInput` agents enforce moderation; the UI renders a
373
+ // non-editable "Enforced" badge for them.
374
+ enforced: agent.untrustedInput === true,
375
+ // Per-agent control value: null = inherit, true = on, false = off.
376
+ override: agentOverrideRow?.inputModeration ?? null,
377
+ // Resolved runtime policy (enforced | on | off) after precedence.
378
+ effective: resolveModerationPolicy({
379
+ untrustedInput: agent.untrustedInput,
380
+ perAgentOverride: agentOverrideRow?.inputModeration ?? null,
381
+ tenantWideOverride: tenantWideInputModeration,
382
+ env,
383
+ }),
384
+ },
366
385
  }
367
386
  })
368
387
  agentResolutions = await Promise.all(agentResolutionPromises)
@@ -719,6 +738,9 @@ export async function PUT(req: NextRequest) {
719
738
  ...(allowedOverrideModelsByProvider !== undefined
720
739
  ? { allowedOverrideModelsByProvider }
721
740
  : {}),
741
+ ...(Object.prototype.hasOwnProperty.call(bodyResult.data, 'inputModeration')
742
+ ? { inputModeration: bodyResult.data.inputModeration ?? null }
743
+ : {}),
722
744
  }
723
745
  const row = await repo.upsertDefault(
724
746
  upsertInput,
@@ -734,6 +756,7 @@ export async function PUT(req: NextRequest) {
734
756
  baseURL: row.baseUrl,
735
757
  allowedOverrideProviders: row.allowedOverrideProviders ?? null,
736
758
  allowedOverrideModelsByProvider: row.allowedOverrideModelsByProvider ?? {},
759
+ inputModeration: row.inputModeration ?? null,
737
760
  updatedAt: row.updatedAt,
738
761
  })
739
762
  } catch (error) {
@@ -121,6 +121,14 @@ type AgentResolution = {
121
121
  modelId: string
122
122
  baseURL: string | null
123
123
  source: string
124
+ moderation?: {
125
+ /** `untrustedInput` agents enforce moderation (non-editable). */
126
+ enforced: boolean
127
+ /** Per-agent control value: null = inherit, true = on, false = off. */
128
+ override: boolean | null
129
+ /** Resolved runtime policy after the precedence chain. */
130
+ effective: 'enforced' | 'on' | 'off'
131
+ }
124
132
  }
125
133
 
126
134
  type TenantAllowlist = {
@@ -875,6 +883,142 @@ function MutationPolicySection({ agent }: { agent: AgentSettings }) {
875
883
  )
876
884
  }
877
885
 
886
+ type ModerationControlValue = 'inherit' | 'on' | 'off'
887
+
888
+ function ModerationSection({ agent }: { agent: AgentSettings }) {
889
+ const t = useT()
890
+ const queryClient = useQueryClient()
891
+ const settingsQuery = useQuery<RuntimeSettingsResponse>({
892
+ queryKey: ['ai_assistant', 'agent_settings', 'runtime_settings'],
893
+ queryFn: fetchRuntimeSettings,
894
+ retry: false,
895
+ })
896
+ const agentResolution = settingsQuery.data?.agents.find((entry) => entry.agentId === agent.id) ?? null
897
+ const moderation = agentResolution?.moderation ?? null
898
+ const enforced = moderation?.enforced === true
899
+ const currentValue: ModerationControlValue =
900
+ moderation?.override === true ? 'on' : moderation?.override === false ? 'off' : 'inherit'
901
+ const [isSaving, setIsSaving] = React.useState(false)
902
+ const { runMutation } = useGuardedMutation({ contextId: `ai-agent-moderation-${agent.id}` })
903
+
904
+ const save = React.useCallback(
905
+ async (value: ModerationControlValue) => {
906
+ if (isSaving || enforced) return
907
+ setIsSaving(true)
908
+ try {
909
+ const inputModeration = value === 'inherit' ? null : value === 'on'
910
+ await runMutation({
911
+ operation: async () => {
912
+ const { ok, status, result } = await apiCall<{ error?: string }>(
913
+ '/api/ai_assistant/settings',
914
+ {
915
+ method: 'PUT',
916
+ headers: { 'content-type': 'application/json' },
917
+ credentials: 'include',
918
+ body: JSON.stringify({ agentId: agent.id, inputModeration }),
919
+ },
920
+ )
921
+ if (!ok) {
922
+ throw new Error(result?.error ?? `[internal] Failed to save moderation setting (${status}).`)
923
+ }
924
+ },
925
+ context: {},
926
+ })
927
+ flash(t('ai_assistant.agents.moderation.saved', 'Input moderation setting saved.'), 'success')
928
+ await queryClient.invalidateQueries({
929
+ queryKey: ['ai_assistant', 'agent_settings', 'runtime_settings'],
930
+ })
931
+ } catch (err) {
932
+ flash(err instanceof Error ? err.message : String(err), 'error')
933
+ } finally {
934
+ setIsSaving(false)
935
+ }
936
+ },
937
+ [agent.id, enforced, isSaving, queryClient, runMutation, t],
938
+ )
939
+
940
+ return (
941
+ <section
942
+ className="rounded-lg border border-border bg-background p-4"
943
+ data-ai-agent-moderation={agent.id}
944
+ >
945
+ <header className="flex flex-wrap items-start justify-between gap-3 border-b border-border pb-3">
946
+ <div className="flex min-w-0 items-start gap-2">
947
+ <ShieldAlert className="size-4 text-muted-foreground" aria-hidden />
948
+ <div className="min-w-0">
949
+ <h3 className="text-sm font-semibold">
950
+ {t('ai_assistant.agents.moderation.title', 'Input moderation')}
951
+ </h3>
952
+ <p className="text-xs text-muted-foreground">
953
+ {t(
954
+ 'ai_assistant.agents.moderation.subtitle',
955
+ 'Screen user input for unsafe content before the model runs. Applies only when the resolved provider supports moderation.',
956
+ )}
957
+ </p>
958
+ </div>
959
+ </div>
960
+ {moderation ? (
961
+ <StatusBadge
962
+ variant={moderation.effective === 'off' ? 'neutral' : 'info'}
963
+ dot
964
+ data-ai-agent-moderation-effective
965
+ >
966
+ {moderation.effective === 'enforced'
967
+ ? t('ai_assistant.agents.moderation.effectiveEnforced', 'Enforced')
968
+ : moderation.effective === 'on'
969
+ ? t('ai_assistant.agents.moderation.effectiveOn', 'On')
970
+ : t('ai_assistant.agents.moderation.effectiveOff', 'Off')}
971
+ </StatusBadge>
972
+ ) : null}
973
+ </header>
974
+
975
+ <div className="mt-3">
976
+ {enforced ? (
977
+ <div className="flex flex-wrap items-center gap-2" data-ai-agent-moderation-enforced>
978
+ <StatusBadge variant="warning" dot>
979
+ {t('ai_assistant.agents.moderation.enforcedBadge', 'Enforced')}
980
+ </StatusBadge>
981
+ <p className="text-xs text-muted-foreground">
982
+ {t(
983
+ 'ai_assistant.agents.moderation.enforcedHint',
984
+ 'This agent accepts untrusted input, so moderation is always on and cannot be disabled here.',
985
+ )}
986
+ </p>
987
+ </div>
988
+ ) : (
989
+ <div className="max-w-xs">
990
+ <span className="text-overline font-semibold uppercase tracking-wider text-muted-foreground">
991
+ {t('ai_assistant.agents.moderation.controlLabel', 'Moderation')}
992
+ </span>
993
+ <div className="mt-1">
994
+ <Select
995
+ value={currentValue}
996
+ onValueChange={(value) => save(value as ModerationControlValue)}
997
+ disabled={isSaving || !agentResolution}
998
+ >
999
+ <SelectTrigger aria-label={t('ai_assistant.agents.moderation.controlLabel', 'Moderation')}>
1000
+ <SelectValue />
1001
+ </SelectTrigger>
1002
+ <SelectContent>
1003
+ <SelectItem value="inherit">
1004
+ {t('ai_assistant.agents.moderation.inheritLabel', 'Inherit')}
1005
+ </SelectItem>
1006
+ <SelectItem value="on">
1007
+ {t('ai_assistant.agents.moderation.onLabel', 'On')}
1008
+ </SelectItem>
1009
+ <SelectItem value="off">
1010
+ {t('ai_assistant.agents.moderation.offLabel', 'Off')}
1011
+ </SelectItem>
1012
+ </SelectContent>
1013
+ </Select>
1014
+ </div>
1015
+ </div>
1016
+ )}
1017
+ </div>
1018
+ </section>
1019
+ )
1020
+ }
1021
+
878
1022
  function AgentModelOverrideSection({ agent }: { agent: AgentSettings }) {
879
1023
  const t = useT()
880
1024
  const queryClient = useQueryClient()
@@ -2063,6 +2207,8 @@ function AgentDetailPanel({ agent }: { agent: AgentSettings }) {
2063
2207
 
2064
2208
  <AgentModelOverrideSection agent={agent} />
2065
2209
 
2210
+ <ModerationSection agent={agent} />
2211
+
2066
2212
  <MutationPolicySection agent={agent} />
2067
2213
 
2068
2214
  <LoopPolicySection agent={agent} />