@open-mercato/ai-assistant 0.6.7-develop.6784.1.f80b9afce5 → 0.6.7-develop.6785.1.1dd7cfac55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/AGENTS.md +12 -29
- package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.js +80 -0
- package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.js.map +7 -0
- package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.js +124 -0
- package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.js.map +7 -0
- package/dist/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.js +30 -0
- package/dist/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.js.map +2 -2
- package/dist/modules/ai_assistant/api/ai/chat/route.js +7 -0
- package/dist/modules/ai_assistant/api/ai/chat/route.js.map +2 -2
- package/dist/modules/ai_assistant/api/moderation-flags/route.js +121 -0
- package/dist/modules/ai_assistant/api/moderation-flags/route.js.map +7 -0
- package/dist/modules/ai_assistant/api/settings/route.js +24 -3
- package/dist/modules/ai_assistant/api/settings/route.js.map +2 -2
- package/dist/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.js +106 -0
- package/dist/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.js.map +2 -2
- package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.js +139 -0
- package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.js.map +7 -0
- package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.js +10 -0
- package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.js.map +7 -0
- package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.js +26 -0
- package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.js.map +7 -0
- package/dist/modules/ai_assistant/data/entities.js +42 -0
- package/dist/modules/ai_assistant/data/entities.js.map +2 -2
- package/dist/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.js +3 -1
- package/dist/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.js.map +2 -2
- package/dist/modules/ai_assistant/data/repositories/AiModerationFlagRepository.js +44 -0
- package/dist/modules/ai_assistant/data/repositories/AiModerationFlagRepository.js.map +7 -0
- package/dist/modules/ai_assistant/di.js +6 -2
- package/dist/modules/ai_assistant/di.js.map +2 -2
- package/dist/modules/ai_assistant/events.js +6 -0
- package/dist/modules/ai_assistant/events.js.map +2 -2
- package/dist/modules/ai_assistant/i18n/de.json +26 -0
- package/dist/modules/ai_assistant/i18n/en.json +26 -0
- package/dist/modules/ai_assistant/i18n/es.json +26 -0
- package/dist/modules/ai_assistant/i18n/pl.json +26 -0
- package/dist/modules/ai_assistant/lib/agent-runtime.js +158 -5
- package/dist/modules/ai_assistant/lib/agent-runtime.js.map +2 -2
- package/dist/modules/ai_assistant/lib/ai-agent-definition.js.map +2 -2
- package/dist/modules/ai_assistant/lib/llm-adapters/anthropic.js +3 -0
- package/dist/modules/ai_assistant/lib/llm-adapters/anthropic.js.map +2 -2
- package/dist/modules/ai_assistant/lib/llm-adapters/openai.js +8 -1
- package/dist/modules/ai_assistant/lib/llm-adapters/openai.js.map +2 -2
- package/dist/modules/ai_assistant/lib/model-factory.js +6 -1
- package/dist/modules/ai_assistant/lib/model-factory.js.map +2 -2
- package/dist/modules/ai_assistant/lib/moderation-flag-recorder.js +43 -0
- package/dist/modules/ai_assistant/lib/moderation-flag-recorder.js.map +7 -0
- package/dist/modules/ai_assistant/lib/moderation-policy.js +24 -0
- package/dist/modules/ai_assistant/lib/moderation-policy.js.map +7 -0
- package/dist/modules/ai_assistant/lib/moderation.js +109 -0
- package/dist/modules/ai_assistant/lib/moderation.js.map +7 -0
- package/dist/modules/ai_assistant/lib/openai-compatible-presets.js +2 -0
- package/dist/modules/ai_assistant/lib/openai-compatible-presets.js.map +2 -2
- package/dist/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.js +19 -0
- package/dist/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.js.map +7 -0
- package/generated/entities/ai_agent_runtime_override/index.ts +1 -0
- package/generated/entities/ai_moderation_flag/index.ts +9 -0
- package/generated/entities.ids.generated.ts +2 -1
- package/generated/entity-fields-registry.ts +13 -1
- package/package.json +6 -6
- package/src/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.ts +126 -0
- package/src/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.ts +161 -0
- package/src/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.ts +48 -0
- package/src/modules/ai_assistant/api/ai/chat/route.ts +10 -0
- package/src/modules/ai_assistant/api/moderation-flags/__tests__/route.test.ts +122 -0
- package/src/modules/ai_assistant/api/moderation-flags/route.ts +137 -0
- package/src/modules/ai_assistant/api/settings/__tests__/route.test.ts +50 -0
- package/src/modules/ai_assistant/api/settings/route.ts +23 -0
- package/src/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.tsx +146 -0
- package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.tsx +186 -0
- package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.ts +24 -0
- package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.tsx +12 -0
- package/src/modules/ai_assistant/data/entities.ts +56 -0
- package/src/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.ts +4 -0
- package/src/modules/ai_assistant/data/repositories/AiModerationFlagRepository.ts +79 -0
- package/src/modules/ai_assistant/di.ts +5 -1
- package/src/modules/ai_assistant/events.ts +21 -0
- package/src/modules/ai_assistant/i18n/de.json +26 -0
- package/src/modules/ai_assistant/i18n/en.json +26 -0
- package/src/modules/ai_assistant/i18n/es.json +26 -0
- package/src/modules/ai_assistant/i18n/pl.json +26 -0
- package/src/modules/ai_assistant/lib/__tests__/input-moderation-gate.test.ts +121 -0
- package/src/modules/ai_assistant/lib/__tests__/llm-adapters-anthropic.test.ts +13 -0
- package/src/modules/ai_assistant/lib/__tests__/llm-adapters-openai.test.ts +41 -0
- package/src/modules/ai_assistant/lib/__tests__/model-factory.test.ts +60 -0
- package/src/modules/ai_assistant/lib/__tests__/moderation-flag-recorder.test.ts +76 -0
- package/src/modules/ai_assistant/lib/__tests__/moderation-policy.test.ts +86 -0
- package/src/modules/ai_assistant/lib/__tests__/moderation.test.ts +114 -0
- package/src/modules/ai_assistant/lib/agent-runtime.ts +259 -0
- package/src/modules/ai_assistant/lib/ai-agent-definition.ts +10 -0
- package/src/modules/ai_assistant/lib/llm-adapters/anthropic.ts +10 -0
- package/src/modules/ai_assistant/lib/llm-adapters/openai.ts +32 -1
- package/src/modules/ai_assistant/lib/model-factory.ts +33 -0
- package/src/modules/ai_assistant/lib/moderation-flag-recorder.ts +75 -0
- package/src/modules/ai_assistant/lib/moderation-policy.ts +77 -0
- package/src/modules/ai_assistant/lib/moderation.ts +179 -0
- package/src/modules/ai_assistant/lib/openai-compatible-presets.ts +2 -0
- package/src/modules/ai_assistant/migrations/.snapshot-open-mercato.json +388 -16
- package/src/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.ts +21 -0
|
@@ -171,6 +171,54 @@ export async function deleteTenantAllowlistInDb(tenantId: string | null): Promis
|
|
|
171
171
|
});
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
+
export interface SeedModerationFlagInput {
|
|
175
|
+
tenantId: string;
|
|
176
|
+
organizationId?: string | null;
|
|
177
|
+
agentId?: string;
|
|
178
|
+
userId?: string;
|
|
179
|
+
providerId?: string;
|
|
180
|
+
modelId?: string;
|
|
181
|
+
/** Defaults to a single flagged `hate` category. */
|
|
182
|
+
categories?: Record<string, { flagged: boolean; score: number }>;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Inserts an `ai_moderation_flags` audit row directly. A real flag is only ever
|
|
187
|
+
* born from the gate during a flagged LLM turn (needs a live moderation call),
|
|
188
|
+
* so seeding the row keeps audit-listing + tenant-isolation coverage
|
|
189
|
+
* deterministic and provider-free. Returns the new id for assertions/cleanup.
|
|
190
|
+
*/
|
|
191
|
+
export async function seedModerationFlagInDb(input: SeedModerationFlagInput): Promise<string> {
|
|
192
|
+
const categories = JSON.stringify(input.categories ?? { hate: { flagged: true, score: 0.97 } });
|
|
193
|
+
return withClient(async (client) => {
|
|
194
|
+
const result = await client.query<{ id: string }>(
|
|
195
|
+
`insert into ai_moderation_flags
|
|
196
|
+
(id, tenant_id, organization_id, agent_id, user_id, provider_id, model_id, categories, created_at)
|
|
197
|
+
values
|
|
198
|
+
(gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7::jsonb, now())
|
|
199
|
+
returning id`,
|
|
200
|
+
[
|
|
201
|
+
input.tenantId,
|
|
202
|
+
input.organizationId ?? null,
|
|
203
|
+
input.agentId ?? 'it_agent.moderation_fixture',
|
|
204
|
+
input.userId ?? `it-user-${randomUUID()}`,
|
|
205
|
+
input.providerId ?? 'openai',
|
|
206
|
+
input.modelId ?? 'gpt-5-mini',
|
|
207
|
+
categories,
|
|
208
|
+
],
|
|
209
|
+
);
|
|
210
|
+
return result.rows[0].id;
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Hard-deletes every moderation-flag row for a tenant (best-effort cleanup). */
|
|
215
|
+
export async function deleteModerationFlagsInDb(tenantId: string | null): Promise<void> {
|
|
216
|
+
if (!tenantId) return;
|
|
217
|
+
await withClient(async (client) => {
|
|
218
|
+
await client.query('delete from ai_moderation_flags where tenant_id = $1', [tenantId]);
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
|
|
174
222
|
/**
|
|
175
223
|
* Hard-deletes a conversation and its participant + message rows by the
|
|
176
224
|
* client-facing `conversation_id`. The DELETE route only soft-deletes, so this
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
type AiAgentLoopBudgetPreset,
|
|
16
16
|
} from '../../../lib/agent-runtime'
|
|
17
17
|
import { AgentPolicyError } from '../../../lib/agent-tools'
|
|
18
|
+
import { AiModerationBlockedError, AiModerationUnavailableError } from '../../../lib/moderation'
|
|
18
19
|
import { readBaseurlAllowlist, isBaseurlAllowlisted } from '../../../lib/baseurl-allowlist'
|
|
19
20
|
import {
|
|
20
21
|
canonicalProviderId,
|
|
@@ -700,6 +701,15 @@ export async function POST(req: NextRequest): Promise<Response> {
|
|
|
700
701
|
if (error instanceof AgentPolicyError) {
|
|
701
702
|
return jsonError(statusForDenyCode(error.code), error.message, error.code)
|
|
702
703
|
}
|
|
704
|
+
// Input pre-moderation outcomes (spec 2026-06-04). The blocked categories
|
|
705
|
+
// are NEVER sent to the client — only the typed code, which the chat UI
|
|
706
|
+
// maps to a generic translated message.
|
|
707
|
+
if (error instanceof AiModerationBlockedError) {
|
|
708
|
+
return jsonError(400, 'Input rejected by the content safety filter.', 'moderation_blocked')
|
|
709
|
+
}
|
|
710
|
+
if (error instanceof AiModerationUnavailableError) {
|
|
711
|
+
return jsonError(503, 'Content safety check temporarily unavailable.', 'moderation_unavailable')
|
|
712
|
+
}
|
|
703
713
|
logger.error('AI Chat Agent — Dispatch failure', { err: error })
|
|
704
714
|
return jsonError(
|
|
705
715
|
500,
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
const authMock = jest.fn()
|
|
2
|
+
const loadAclMock = jest.fn()
|
|
3
|
+
const createRequestContainerMock = jest.fn()
|
|
4
|
+
const listMock = jest.fn()
|
|
5
|
+
|
|
6
|
+
jest.mock('@open-mercato/shared/lib/auth/server', () => ({
|
|
7
|
+
getAuthFromRequest: (...args: unknown[]) => authMock(...args),
|
|
8
|
+
}))
|
|
9
|
+
|
|
10
|
+
jest.mock('@open-mercato/shared/lib/di/container', () => ({
|
|
11
|
+
createRequestContainer: (...args: unknown[]) => createRequestContainerMock(...args),
|
|
12
|
+
}))
|
|
13
|
+
|
|
14
|
+
jest.mock('../../../data/repositories/AiModerationFlagRepository', () => ({
|
|
15
|
+
AiModerationFlagRepository: jest.fn().mockImplementation(() => ({ list: listMock })),
|
|
16
|
+
}))
|
|
17
|
+
|
|
18
|
+
import { GET } from '../route'
|
|
19
|
+
|
|
20
|
+
function buildRequest(params: Record<string, string> = {}) {
|
|
21
|
+
const url = new URL('http://localhost/api/ai_assistant/moderation-flags')
|
|
22
|
+
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v)
|
|
23
|
+
return new Request(url.toString(), { method: 'GET' })
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function makeRow(overrides: Record<string, unknown> = {}) {
|
|
27
|
+
return {
|
|
28
|
+
id: 'flag-1',
|
|
29
|
+
tenantId: 'tenant-1',
|
|
30
|
+
organizationId: null,
|
|
31
|
+
agentId: 'catalog.assistant',
|
|
32
|
+
userId: 'user-9',
|
|
33
|
+
providerId: 'openai',
|
|
34
|
+
modelId: 'gpt-5-mini',
|
|
35
|
+
categories: { hate: { flagged: true, score: 0.97 } },
|
|
36
|
+
createdAt: new Date('2026-06-10T12:00:00Z'),
|
|
37
|
+
...overrides,
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
describe('GET /api/ai_assistant/moderation-flags', () => {
|
|
42
|
+
let consoleErrorSpy: jest.SpyInstance
|
|
43
|
+
|
|
44
|
+
beforeEach(() => {
|
|
45
|
+
jest.clearAllMocks()
|
|
46
|
+
consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
47
|
+
authMock.mockResolvedValue({ sub: 'user-1', tenantId: 'tenant-1', orgId: 'org-1' })
|
|
48
|
+
loadAclMock.mockResolvedValue({ features: ['ai_assistant.settings.manage'], isSuperAdmin: false })
|
|
49
|
+
createRequestContainerMock.mockResolvedValue({
|
|
50
|
+
resolve: (name: string) => {
|
|
51
|
+
if (name === 'rbacService') {
|
|
52
|
+
return {
|
|
53
|
+
loadAcl: loadAclMock,
|
|
54
|
+
hasAllFeatures: (req: string[], have: string[]) => req.every((r) => have.includes(r)),
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (name === 'em') return {}
|
|
58
|
+
throw new Error(`Unknown token: ${name}`)
|
|
59
|
+
},
|
|
60
|
+
})
|
|
61
|
+
listMock.mockResolvedValue({ items: [], total: 0 })
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
afterEach(() => {
|
|
65
|
+
consoleErrorSpy.mockRestore()
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
it('returns 401 when unauthenticated', async () => {
|
|
69
|
+
authMock.mockResolvedValue(null)
|
|
70
|
+
const res = await GET(buildRequest() as never)
|
|
71
|
+
expect(res.status).toBe(401)
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
it('returns 403 when the caller lacks ai_assistant.settings.manage', async () => {
|
|
75
|
+
loadAclMock.mockResolvedValue({ features: ['ai_assistant.view'], isSuperAdmin: false })
|
|
76
|
+
const res = await GET(buildRequest() as never)
|
|
77
|
+
expect(res.status).toBe(403)
|
|
78
|
+
expect((await res.json()).code).toBe('forbidden')
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
it('rejects pageSize above the 100 cap with a validation error', async () => {
|
|
82
|
+
const res = await GET(buildRequest({ pageSize: '101' }) as never)
|
|
83
|
+
expect(res.status).toBe(400)
|
|
84
|
+
expect((await res.json()).code).toBe('validation_error')
|
|
85
|
+
expect(listMock).not.toHaveBeenCalled()
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
it('lists tenant-scoped rows and echoes pagination', async () => {
|
|
89
|
+
listMock.mockResolvedValue({ items: [makeRow()], total: 1 })
|
|
90
|
+
const res = await GET(buildRequest({ page: '2', pageSize: '25' }) as never)
|
|
91
|
+
expect(res.status).toBe(200)
|
|
92
|
+
const json = await res.json()
|
|
93
|
+
expect(json.total).toBe(1)
|
|
94
|
+
expect(json.page).toBe(2)
|
|
95
|
+
expect(json.pageSize).toBe(25)
|
|
96
|
+
expect(json.items[0]).toMatchObject({ id: 'flag-1', userId: 'user-9', categories: { hate: { flagged: true, score: 0.97 } } })
|
|
97
|
+
// Tenant-scoped only — the listing does NOT narrow by organization_id.
|
|
98
|
+
expect(listMock).toHaveBeenCalledWith(
|
|
99
|
+
expect.objectContaining({ tenantId: 'tenant-1', page: 2, pageSize: 25 }),
|
|
100
|
+
)
|
|
101
|
+
expect(listMock.mock.calls[0][0].organizationId).toBeUndefined()
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
it('passes agentId/userId/date filters through to the repository', async () => {
|
|
105
|
+
await GET(buildRequest({ agentId: 'catalog.assistant', userId: 'user-9', from: '2026-06-01', to: '2026-06-10' }) as never)
|
|
106
|
+
const call = listMock.mock.calls[0][0]
|
|
107
|
+
expect(call.agentId).toBe('catalog.assistant')
|
|
108
|
+
expect(call.userId).toBe('user-9')
|
|
109
|
+
expect(call.from).toBeInstanceOf(Date)
|
|
110
|
+
expect(call.to).toBeInstanceOf(Date)
|
|
111
|
+
// `to` is extended to end-of-day so same-day rows are included.
|
|
112
|
+
expect((call.to as Date).toISOString()).toBe('2026-06-10T23:59:59.999Z')
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
it('returns an empty page (no repo call) when the caller has no tenant scope', async () => {
|
|
116
|
+
authMock.mockResolvedValue({ sub: 'user-1', tenantId: null, orgId: null })
|
|
117
|
+
const res = await GET(buildRequest() as never)
|
|
118
|
+
expect(res.status).toBe(200)
|
|
119
|
+
expect((await res.json()).items).toEqual([])
|
|
120
|
+
expect(listMock).not.toHaveBeenCalled()
|
|
121
|
+
})
|
|
122
|
+
})
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { NextResponse, type NextRequest } from 'next/server'
|
|
2
|
+
import { z } from 'zod'
|
|
3
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
4
|
+
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
5
|
+
import type { OpenApiRouteDoc } from '@open-mercato/shared/lib/openapi'
|
|
6
|
+
import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
|
|
7
|
+
import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
|
|
8
|
+
import type { RbacService } from '@open-mercato/core/modules/auth/services/rbacService'
|
|
9
|
+
import { AiModerationFlagRepository } from '../../data/repositories/AiModerationFlagRepository'
|
|
10
|
+
import { hasRequiredFeatures } from '../../lib/auth'
|
|
11
|
+
|
|
12
|
+
const logger = createLogger('ai_assistant')
|
|
13
|
+
|
|
14
|
+
const REQUIRED_FEATURE = 'ai_assistant.settings.manage'
|
|
15
|
+
const MAX_PAGE_SIZE = 100
|
|
16
|
+
|
|
17
|
+
const querySchema = z.object({
|
|
18
|
+
page: z.coerce.number().int().min(1).default(1),
|
|
19
|
+
pageSize: z.coerce.number().int().min(1).max(MAX_PAGE_SIZE).default(50),
|
|
20
|
+
agentId: z.string().min(1).max(256).optional(),
|
|
21
|
+
userId: z.string().min(1).max(256).optional(),
|
|
22
|
+
from: z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'from must be a date in YYYY-MM-DD format').optional(),
|
|
23
|
+
to: z.string().regex(/^\d{4}-\d{2}-\d{2}$/, 'to must be a date in YYYY-MM-DD format').optional(),
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
export const openApi: OpenApiRouteDoc = {
|
|
27
|
+
tag: 'AI Assistant',
|
|
28
|
+
summary: 'Input moderation flags audit log',
|
|
29
|
+
methods: {
|
|
30
|
+
GET: {
|
|
31
|
+
operationId: 'aiAssistantModerationFlags',
|
|
32
|
+
summary: 'List input-moderation audit flags for the current tenant.',
|
|
33
|
+
description:
|
|
34
|
+
'Returns the append-only `ai_moderation_flags` audit rows (category flags + scores only — never ' +
|
|
35
|
+
'prompt content) for the caller\'s tenant. Always tenant-scoped; cross-tenant access is impossible. ' +
|
|
36
|
+
'Optionally filtered by `agentId`, `userId`, and a `from`/`to` date window. Paginated (`pageSize` ≤ 100). ' +
|
|
37
|
+
'Requires `ai_assistant.settings.manage`.',
|
|
38
|
+
query: querySchema,
|
|
39
|
+
responses: [
|
|
40
|
+
{
|
|
41
|
+
status: 200,
|
|
42
|
+
description: 'Paginated moderation-flag rows: `{ items, total, page, pageSize }`.',
|
|
43
|
+
mediaType: 'application/json',
|
|
44
|
+
},
|
|
45
|
+
],
|
|
46
|
+
errors: [
|
|
47
|
+
{ status: 400, description: 'Invalid query parameters.' },
|
|
48
|
+
{ status: 401, description: 'Unauthenticated caller.' },
|
|
49
|
+
{ status: 403, description: 'Caller lacks `ai_assistant.settings.manage`.' },
|
|
50
|
+
{ status: 500, description: 'Internal failure.' },
|
|
51
|
+
],
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const metadata = {
|
|
57
|
+
path: '/ai_assistant/moderation-flags',
|
|
58
|
+
GET: { requireAuth: true, requireFeatures: [REQUIRED_FEATURE] },
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function jsonError(status: number, message: string, code: string, extra?: Record<string, unknown>): NextResponse {
|
|
62
|
+
return NextResponse.json({ error: message, code, ...(extra ?? {}) }, { status })
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export async function GET(req: NextRequest): Promise<Response> {
|
|
66
|
+
const auth = await getAuthFromRequest(req)
|
|
67
|
+
if (!auth) {
|
|
68
|
+
return jsonError(401, 'Unauthorized', 'unauthenticated')
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const { searchParams } = new URL(req.url)
|
|
72
|
+
const queryResult = querySchema.safeParse({
|
|
73
|
+
page: searchParams.get('page') ?? undefined,
|
|
74
|
+
pageSize: searchParams.get('pageSize') ?? undefined,
|
|
75
|
+
agentId: searchParams.get('agentId') ?? undefined,
|
|
76
|
+
userId: searchParams.get('userId') ?? undefined,
|
|
77
|
+
from: searchParams.get('from') ?? undefined,
|
|
78
|
+
to: searchParams.get('to') ?? undefined,
|
|
79
|
+
})
|
|
80
|
+
if (!queryResult.success) {
|
|
81
|
+
return jsonError(400, 'Invalid query parameters.', 'validation_error', {
|
|
82
|
+
issues: queryResult.error.issues,
|
|
83
|
+
})
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const { page, pageSize, agentId, userId, from, to } = queryResult.data
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
const container = await createRequestContainer()
|
|
90
|
+
const rbacService = container.resolve<RbacService>('rbacService')
|
|
91
|
+
const acl = await rbacService.loadAcl(auth.sub, {
|
|
92
|
+
tenantId: auth.tenantId,
|
|
93
|
+
organizationId: auth.orgId,
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
if (!hasRequiredFeatures([REQUIRED_FEATURE], acl.features, acl.isSuperAdmin, rbacService)) {
|
|
97
|
+
return jsonError(403, `Caller lacks required feature "${REQUIRED_FEATURE}".`, 'forbidden')
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (!auth.tenantId) {
|
|
101
|
+
return NextResponse.json({ items: [], total: 0, page, pageSize })
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const em = container.resolve<EntityManager>('em')
|
|
105
|
+
const repo = new AiModerationFlagRepository(em)
|
|
106
|
+
const { items, total } = await repo.list({
|
|
107
|
+
tenantId: auth.tenantId,
|
|
108
|
+
// Tenant-scoped abuse audit: do NOT narrow by organization_id. Tenant is
|
|
109
|
+
// the hard isolation boundary; portal-surface flags often carry no org,
|
|
110
|
+
// and a tenant admin reviewing abuse needs every flag in the tenant.
|
|
111
|
+
agentId,
|
|
112
|
+
userId,
|
|
113
|
+
// `to` is an inclusive day; extend to end-of-day so same-day rows match.
|
|
114
|
+
from: from ? new Date(`${from}T00:00:00.000Z`) : undefined,
|
|
115
|
+
to: to ? new Date(`${to}T23:59:59.999Z`) : undefined,
|
|
116
|
+
page,
|
|
117
|
+
pageSize,
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
const serialized = items.map((row) => ({
|
|
121
|
+
id: row.id,
|
|
122
|
+
tenantId: row.tenantId,
|
|
123
|
+
organizationId: row.organizationId ?? null,
|
|
124
|
+
agentId: row.agentId,
|
|
125
|
+
userId: row.userId,
|
|
126
|
+
providerId: row.providerId,
|
|
127
|
+
modelId: row.modelId,
|
|
128
|
+
categories: row.categories,
|
|
129
|
+
createdAt: row.createdAt.toISOString(),
|
|
130
|
+
}))
|
|
131
|
+
|
|
132
|
+
return NextResponse.json({ items: serialized, total, page, pageSize })
|
|
133
|
+
} catch (error) {
|
|
134
|
+
logger.error('Moderation flags GET failed', { err: error })
|
|
135
|
+
return jsonError(500, 'Failed to fetch moderation flags.', 'internal_error')
|
|
136
|
+
}
|
|
137
|
+
}
|
|
@@ -161,6 +161,56 @@ describe('PUT /api/ai_assistant/settings', () => {
|
|
|
161
161
|
)
|
|
162
162
|
})
|
|
163
163
|
|
|
164
|
+
it('forwards inputModeration to the override upsert and echoes it back', async () => {
|
|
165
|
+
upsertDefaultMock.mockResolvedValueOnce({
|
|
166
|
+
id: 'row-1',
|
|
167
|
+
tenantId: 'tenant-1',
|
|
168
|
+
organizationId: 'org-1',
|
|
169
|
+
agentId: 'catalog.catalog_assistant',
|
|
170
|
+
providerId: null,
|
|
171
|
+
modelId: null,
|
|
172
|
+
baseUrl: null,
|
|
173
|
+
inputModeration: true,
|
|
174
|
+
updatedAt: new Date('2026-06-10T00:00:00Z'),
|
|
175
|
+
})
|
|
176
|
+
|
|
177
|
+
const response = await PUT(
|
|
178
|
+
buildRequest('PUT', { agentId: 'catalog.catalog_assistant', inputModeration: true }) as any,
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
expect(response.status).toBe(200)
|
|
182
|
+
const json = await response.json()
|
|
183
|
+
expect(json.inputModeration).toBe(true)
|
|
184
|
+
expect(upsertDefaultMock).toHaveBeenCalledWith(
|
|
185
|
+
expect.objectContaining({ agentId: 'catalog.catalog_assistant', inputModeration: true }),
|
|
186
|
+
expect.objectContaining({ tenantId: 'tenant-1' }),
|
|
187
|
+
)
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
it('passes inputModeration: null (inherit) through to the upsert', async () => {
|
|
191
|
+
upsertDefaultMock.mockResolvedValueOnce({
|
|
192
|
+
id: 'row-1',
|
|
193
|
+
tenantId: 'tenant-1',
|
|
194
|
+
organizationId: 'org-1',
|
|
195
|
+
agentId: 'catalog.catalog_assistant',
|
|
196
|
+
providerId: null,
|
|
197
|
+
modelId: null,
|
|
198
|
+
baseUrl: null,
|
|
199
|
+
inputModeration: null,
|
|
200
|
+
updatedAt: new Date('2026-06-10T00:00:00Z'),
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
const response = await PUT(
|
|
204
|
+
buildRequest('PUT', { agentId: 'catalog.catalog_assistant', inputModeration: null }) as any,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
expect(response.status).toBe(200)
|
|
208
|
+
expect(upsertDefaultMock).toHaveBeenCalledWith(
|
|
209
|
+
expect.objectContaining({ inputModeration: null }),
|
|
210
|
+
expect.anything(),
|
|
211
|
+
)
|
|
212
|
+
})
|
|
213
|
+
|
|
164
214
|
it('saves per-agent chat override allowlist without requiring a model override', async () => {
|
|
165
215
|
upsertDefaultMock.mockResolvedValueOnce({
|
|
166
216
|
id: 'row-1',
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
isOpenCodeProviderConfigured,
|
|
18
18
|
} from '@open-mercato/shared/lib/ai/opencode-provider'
|
|
19
19
|
import { AiAgentRuntimeOverrideRepository, AiAgentRuntimeOverrideValidationError } from '../../data/repositories/AiAgentRuntimeOverrideRepository'
|
|
20
|
+
import { resolveModerationPolicy } from '../../lib/moderation-policy'
|
|
20
21
|
import { AiTenantModelAllowlistRepository } from '../../data/repositories/AiTenantModelAllowlistRepository'
|
|
21
22
|
import { isBaseurlAllowlisted, readBaseurlAllowlist } from '../../lib/baseurl-allowlist'
|
|
22
23
|
import { loadAgentRegistry, listAgents } from '../../lib/agent-registry'
|
|
@@ -59,6 +60,9 @@ const runtimeOverrideUpsertSchema = z.object({
|
|
|
59
60
|
allowedOverrideModelsByProvider: z
|
|
60
61
|
.record(z.string().min(1).max(64), z.array(z.string().min(1).max(256)))
|
|
61
62
|
.optional(),
|
|
63
|
+
// Input moderation override: true = on, false = off, null = inherit.
|
|
64
|
+
// Spec 2026-06-04-ai-input-moderation-and-safety-identifiers.
|
|
65
|
+
inputModeration: z.boolean().nullable().optional(),
|
|
62
66
|
})
|
|
63
67
|
|
|
64
68
|
const runtimeOverrideClearSchema = z.object({
|
|
@@ -248,6 +252,7 @@ export async function GET(req: NextRequest) {
|
|
|
248
252
|
const em = container.resolve<EntityManager>('em')
|
|
249
253
|
const repo = new AiAgentRuntimeOverrideRepository(em)
|
|
250
254
|
const overrideRow = await repo.getDefault({ tenantId, organizationId, agentId: null })
|
|
255
|
+
const tenantWideInputModeration = overrideRow?.inputModeration ?? null
|
|
251
256
|
if (overrideRow) {
|
|
252
257
|
tenantOverride = {
|
|
253
258
|
providerId: overrideRow.providerId ?? null,
|
|
@@ -363,6 +368,20 @@ export async function GET(req: NextRequest) {
|
|
|
363
368
|
modelId: agentResolution.modelId,
|
|
364
369
|
baseURL: agentResolution.baseURL ?? null,
|
|
365
370
|
source: agentResolution.source,
|
|
371
|
+
moderation: {
|
|
372
|
+
// `untrustedInput` agents enforce moderation; the UI renders a
|
|
373
|
+
// non-editable "Enforced" badge for them.
|
|
374
|
+
enforced: agent.untrustedInput === true,
|
|
375
|
+
// Per-agent control value: null = inherit, true = on, false = off.
|
|
376
|
+
override: agentOverrideRow?.inputModeration ?? null,
|
|
377
|
+
// Resolved runtime policy (enforced | on | off) after precedence.
|
|
378
|
+
effective: resolveModerationPolicy({
|
|
379
|
+
untrustedInput: agent.untrustedInput,
|
|
380
|
+
perAgentOverride: agentOverrideRow?.inputModeration ?? null,
|
|
381
|
+
tenantWideOverride: tenantWideInputModeration,
|
|
382
|
+
env,
|
|
383
|
+
}),
|
|
384
|
+
},
|
|
366
385
|
}
|
|
367
386
|
})
|
|
368
387
|
agentResolutions = await Promise.all(agentResolutionPromises)
|
|
@@ -719,6 +738,9 @@ export async function PUT(req: NextRequest) {
|
|
|
719
738
|
...(allowedOverrideModelsByProvider !== undefined
|
|
720
739
|
? { allowedOverrideModelsByProvider }
|
|
721
740
|
: {}),
|
|
741
|
+
...(Object.prototype.hasOwnProperty.call(bodyResult.data, 'inputModeration')
|
|
742
|
+
? { inputModeration: bodyResult.data.inputModeration ?? null }
|
|
743
|
+
: {}),
|
|
722
744
|
}
|
|
723
745
|
const row = await repo.upsertDefault(
|
|
724
746
|
upsertInput,
|
|
@@ -734,6 +756,7 @@ export async function PUT(req: NextRequest) {
|
|
|
734
756
|
baseURL: row.baseUrl,
|
|
735
757
|
allowedOverrideProviders: row.allowedOverrideProviders ?? null,
|
|
736
758
|
allowedOverrideModelsByProvider: row.allowedOverrideModelsByProvider ?? {},
|
|
759
|
+
inputModeration: row.inputModeration ?? null,
|
|
737
760
|
updatedAt: row.updatedAt,
|
|
738
761
|
})
|
|
739
762
|
} catch (error) {
|
package/src/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.tsx
CHANGED
|
@@ -121,6 +121,14 @@ type AgentResolution = {
|
|
|
121
121
|
modelId: string
|
|
122
122
|
baseURL: string | null
|
|
123
123
|
source: string
|
|
124
|
+
moderation?: {
|
|
125
|
+
/** `untrustedInput` agents enforce moderation (non-editable). */
|
|
126
|
+
enforced: boolean
|
|
127
|
+
/** Per-agent control value: null = inherit, true = on, false = off. */
|
|
128
|
+
override: boolean | null
|
|
129
|
+
/** Resolved runtime policy after the precedence chain. */
|
|
130
|
+
effective: 'enforced' | 'on' | 'off'
|
|
131
|
+
}
|
|
124
132
|
}
|
|
125
133
|
|
|
126
134
|
type TenantAllowlist = {
|
|
@@ -875,6 +883,142 @@ function MutationPolicySection({ agent }: { agent: AgentSettings }) {
|
|
|
875
883
|
)
|
|
876
884
|
}
|
|
877
885
|
|
|
886
|
+
type ModerationControlValue = 'inherit' | 'on' | 'off'
|
|
887
|
+
|
|
888
|
+
function ModerationSection({ agent }: { agent: AgentSettings }) {
|
|
889
|
+
const t = useT()
|
|
890
|
+
const queryClient = useQueryClient()
|
|
891
|
+
const settingsQuery = useQuery<RuntimeSettingsResponse>({
|
|
892
|
+
queryKey: ['ai_assistant', 'agent_settings', 'runtime_settings'],
|
|
893
|
+
queryFn: fetchRuntimeSettings,
|
|
894
|
+
retry: false,
|
|
895
|
+
})
|
|
896
|
+
const agentResolution = settingsQuery.data?.agents.find((entry) => entry.agentId === agent.id) ?? null
|
|
897
|
+
const moderation = agentResolution?.moderation ?? null
|
|
898
|
+
const enforced = moderation?.enforced === true
|
|
899
|
+
const currentValue: ModerationControlValue =
|
|
900
|
+
moderation?.override === true ? 'on' : moderation?.override === false ? 'off' : 'inherit'
|
|
901
|
+
const [isSaving, setIsSaving] = React.useState(false)
|
|
902
|
+
const { runMutation } = useGuardedMutation({ contextId: `ai-agent-moderation-${agent.id}` })
|
|
903
|
+
|
|
904
|
+
const save = React.useCallback(
|
|
905
|
+
async (value: ModerationControlValue) => {
|
|
906
|
+
if (isSaving || enforced) return
|
|
907
|
+
setIsSaving(true)
|
|
908
|
+
try {
|
|
909
|
+
const inputModeration = value === 'inherit' ? null : value === 'on'
|
|
910
|
+
await runMutation({
|
|
911
|
+
operation: async () => {
|
|
912
|
+
const { ok, status, result } = await apiCall<{ error?: string }>(
|
|
913
|
+
'/api/ai_assistant/settings',
|
|
914
|
+
{
|
|
915
|
+
method: 'PUT',
|
|
916
|
+
headers: { 'content-type': 'application/json' },
|
|
917
|
+
credentials: 'include',
|
|
918
|
+
body: JSON.stringify({ agentId: agent.id, inputModeration }),
|
|
919
|
+
},
|
|
920
|
+
)
|
|
921
|
+
if (!ok) {
|
|
922
|
+
throw new Error(result?.error ?? `[internal] Failed to save moderation setting (${status}).`)
|
|
923
|
+
}
|
|
924
|
+
},
|
|
925
|
+
context: {},
|
|
926
|
+
})
|
|
927
|
+
flash(t('ai_assistant.agents.moderation.saved', 'Input moderation setting saved.'), 'success')
|
|
928
|
+
await queryClient.invalidateQueries({
|
|
929
|
+
queryKey: ['ai_assistant', 'agent_settings', 'runtime_settings'],
|
|
930
|
+
})
|
|
931
|
+
} catch (err) {
|
|
932
|
+
flash(err instanceof Error ? err.message : String(err), 'error')
|
|
933
|
+
} finally {
|
|
934
|
+
setIsSaving(false)
|
|
935
|
+
}
|
|
936
|
+
},
|
|
937
|
+
[agent.id, enforced, isSaving, queryClient, runMutation, t],
|
|
938
|
+
)
|
|
939
|
+
|
|
940
|
+
return (
|
|
941
|
+
<section
|
|
942
|
+
className="rounded-lg border border-border bg-background p-4"
|
|
943
|
+
data-ai-agent-moderation={agent.id}
|
|
944
|
+
>
|
|
945
|
+
<header className="flex flex-wrap items-start justify-between gap-3 border-b border-border pb-3">
|
|
946
|
+
<div className="flex min-w-0 items-start gap-2">
|
|
947
|
+
<ShieldAlert className="size-4 text-muted-foreground" aria-hidden />
|
|
948
|
+
<div className="min-w-0">
|
|
949
|
+
<h3 className="text-sm font-semibold">
|
|
950
|
+
{t('ai_assistant.agents.moderation.title', 'Input moderation')}
|
|
951
|
+
</h3>
|
|
952
|
+
<p className="text-xs text-muted-foreground">
|
|
953
|
+
{t(
|
|
954
|
+
'ai_assistant.agents.moderation.subtitle',
|
|
955
|
+
'Screen user input for unsafe content before the model runs. Applies only when the resolved provider supports moderation.',
|
|
956
|
+
)}
|
|
957
|
+
</p>
|
|
958
|
+
</div>
|
|
959
|
+
</div>
|
|
960
|
+
{moderation ? (
|
|
961
|
+
<StatusBadge
|
|
962
|
+
variant={moderation.effective === 'off' ? 'neutral' : 'info'}
|
|
963
|
+
dot
|
|
964
|
+
data-ai-agent-moderation-effective
|
|
965
|
+
>
|
|
966
|
+
{moderation.effective === 'enforced'
|
|
967
|
+
? t('ai_assistant.agents.moderation.effectiveEnforced', 'Enforced')
|
|
968
|
+
: moderation.effective === 'on'
|
|
969
|
+
? t('ai_assistant.agents.moderation.effectiveOn', 'On')
|
|
970
|
+
: t('ai_assistant.agents.moderation.effectiveOff', 'Off')}
|
|
971
|
+
</StatusBadge>
|
|
972
|
+
) : null}
|
|
973
|
+
</header>
|
|
974
|
+
|
|
975
|
+
<div className="mt-3">
|
|
976
|
+
{enforced ? (
|
|
977
|
+
<div className="flex flex-wrap items-center gap-2" data-ai-agent-moderation-enforced>
|
|
978
|
+
<StatusBadge variant="warning" dot>
|
|
979
|
+
{t('ai_assistant.agents.moderation.enforcedBadge', 'Enforced')}
|
|
980
|
+
</StatusBadge>
|
|
981
|
+
<p className="text-xs text-muted-foreground">
|
|
982
|
+
{t(
|
|
983
|
+
'ai_assistant.agents.moderation.enforcedHint',
|
|
984
|
+
'This agent accepts untrusted input, so moderation is always on and cannot be disabled here.',
|
|
985
|
+
)}
|
|
986
|
+
</p>
|
|
987
|
+
</div>
|
|
988
|
+
) : (
|
|
989
|
+
<div className="max-w-xs">
|
|
990
|
+
<span className="text-overline font-semibold uppercase tracking-wider text-muted-foreground">
|
|
991
|
+
{t('ai_assistant.agents.moderation.controlLabel', 'Moderation')}
|
|
992
|
+
</span>
|
|
993
|
+
<div className="mt-1">
|
|
994
|
+
<Select
|
|
995
|
+
value={currentValue}
|
|
996
|
+
onValueChange={(value) => save(value as ModerationControlValue)}
|
|
997
|
+
disabled={isSaving || !agentResolution}
|
|
998
|
+
>
|
|
999
|
+
<SelectTrigger aria-label={t('ai_assistant.agents.moderation.controlLabel', 'Moderation')}>
|
|
1000
|
+
<SelectValue />
|
|
1001
|
+
</SelectTrigger>
|
|
1002
|
+
<SelectContent>
|
|
1003
|
+
<SelectItem value="inherit">
|
|
1004
|
+
{t('ai_assistant.agents.moderation.inheritLabel', 'Inherit')}
|
|
1005
|
+
</SelectItem>
|
|
1006
|
+
<SelectItem value="on">
|
|
1007
|
+
{t('ai_assistant.agents.moderation.onLabel', 'On')}
|
|
1008
|
+
</SelectItem>
|
|
1009
|
+
<SelectItem value="off">
|
|
1010
|
+
{t('ai_assistant.agents.moderation.offLabel', 'Off')}
|
|
1011
|
+
</SelectItem>
|
|
1012
|
+
</SelectContent>
|
|
1013
|
+
</Select>
|
|
1014
|
+
</div>
|
|
1015
|
+
</div>
|
|
1016
|
+
)}
|
|
1017
|
+
</div>
|
|
1018
|
+
</section>
|
|
1019
|
+
)
|
|
1020
|
+
}
|
|
1021
|
+
|
|
878
1022
|
function AgentModelOverrideSection({ agent }: { agent: AgentSettings }) {
|
|
879
1023
|
const t = useT()
|
|
880
1024
|
const queryClient = useQueryClient()
|
|
@@ -2063,6 +2207,8 @@ function AgentDetailPanel({ agent }: { agent: AgentSettings }) {
|
|
|
2063
2207
|
|
|
2064
2208
|
<AgentModelOverrideSection agent={agent} />
|
|
2065
2209
|
|
|
2210
|
+
<ModerationSection agent={agent} />
|
|
2211
|
+
|
|
2066
2212
|
<MutationPolicySection agent={agent} />
|
|
2067
2213
|
|
|
2068
2214
|
<LoopPolicySection agent={agent} />
|