@open-mercato/ai-assistant 0.6.7-develop.6784.1.f80b9afce5 → 0.6.7-develop.6785.1.1dd7cfac55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/AGENTS.md +12 -29
  3. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.js +80 -0
  4. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.js.map +7 -0
  5. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.js +124 -0
  6. package/dist/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.js.map +7 -0
  7. package/dist/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.js +30 -0
  8. package/dist/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.js.map +2 -2
  9. package/dist/modules/ai_assistant/api/ai/chat/route.js +7 -0
  10. package/dist/modules/ai_assistant/api/ai/chat/route.js.map +2 -2
  11. package/dist/modules/ai_assistant/api/moderation-flags/route.js +121 -0
  12. package/dist/modules/ai_assistant/api/moderation-flags/route.js.map +7 -0
  13. package/dist/modules/ai_assistant/api/settings/route.js +24 -3
  14. package/dist/modules/ai_assistant/api/settings/route.js.map +2 -2
  15. package/dist/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.js +106 -0
  16. package/dist/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.js.map +2 -2
  17. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.js +139 -0
  18. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.js.map +7 -0
  19. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.js +10 -0
  20. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.js.map +7 -0
  21. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.js +26 -0
  22. package/dist/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.js.map +7 -0
  23. package/dist/modules/ai_assistant/data/entities.js +42 -0
  24. package/dist/modules/ai_assistant/data/entities.js.map +2 -2
  25. package/dist/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.js +3 -1
  26. package/dist/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.js.map +2 -2
  27. package/dist/modules/ai_assistant/data/repositories/AiModerationFlagRepository.js +44 -0
  28. package/dist/modules/ai_assistant/data/repositories/AiModerationFlagRepository.js.map +7 -0
  29. package/dist/modules/ai_assistant/di.js +6 -2
  30. package/dist/modules/ai_assistant/di.js.map +2 -2
  31. package/dist/modules/ai_assistant/events.js +6 -0
  32. package/dist/modules/ai_assistant/events.js.map +2 -2
  33. package/dist/modules/ai_assistant/i18n/de.json +26 -0
  34. package/dist/modules/ai_assistant/i18n/en.json +26 -0
  35. package/dist/modules/ai_assistant/i18n/es.json +26 -0
  36. package/dist/modules/ai_assistant/i18n/pl.json +26 -0
  37. package/dist/modules/ai_assistant/lib/agent-runtime.js +158 -5
  38. package/dist/modules/ai_assistant/lib/agent-runtime.js.map +2 -2
  39. package/dist/modules/ai_assistant/lib/ai-agent-definition.js.map +2 -2
  40. package/dist/modules/ai_assistant/lib/llm-adapters/anthropic.js +3 -0
  41. package/dist/modules/ai_assistant/lib/llm-adapters/anthropic.js.map +2 -2
  42. package/dist/modules/ai_assistant/lib/llm-adapters/openai.js +8 -1
  43. package/dist/modules/ai_assistant/lib/llm-adapters/openai.js.map +2 -2
  44. package/dist/modules/ai_assistant/lib/model-factory.js +6 -1
  45. package/dist/modules/ai_assistant/lib/model-factory.js.map +2 -2
  46. package/dist/modules/ai_assistant/lib/moderation-flag-recorder.js +43 -0
  47. package/dist/modules/ai_assistant/lib/moderation-flag-recorder.js.map +7 -0
  48. package/dist/modules/ai_assistant/lib/moderation-policy.js +24 -0
  49. package/dist/modules/ai_assistant/lib/moderation-policy.js.map +7 -0
  50. package/dist/modules/ai_assistant/lib/moderation.js +109 -0
  51. package/dist/modules/ai_assistant/lib/moderation.js.map +7 -0
  52. package/dist/modules/ai_assistant/lib/openai-compatible-presets.js +2 -0
  53. package/dist/modules/ai_assistant/lib/openai-compatible-presets.js.map +2 -2
  54. package/dist/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.js +19 -0
  55. package/dist/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.js.map +7 -0
  56. package/generated/entities/ai_agent_runtime_override/index.ts +1 -0
  57. package/generated/entities/ai_moderation_flag/index.ts +9 -0
  58. package/generated/entities.ids.generated.ts +2 -1
  59. package/generated/entity-fields-registry.ts +13 -1
  60. package/package.json +6 -6
  61. package/src/modules/ai_assistant/__integration__/TC-AI-MODERATION-008-settings-and-audit.spec.ts +126 -0
  62. package/src/modules/ai_assistant/__integration__/TC-AI-MODERATION-009-ui.spec.ts +161 -0
  63. package/src/modules/ai_assistant/__integration__/helpers/aiAssistantFixtures.ts +48 -0
  64. package/src/modules/ai_assistant/api/ai/chat/route.ts +10 -0
  65. package/src/modules/ai_assistant/api/moderation-flags/__tests__/route.test.ts +122 -0
  66. package/src/modules/ai_assistant/api/moderation-flags/route.ts +137 -0
  67. package/src/modules/ai_assistant/api/settings/__tests__/route.test.ts +50 -0
  68. package/src/modules/ai_assistant/api/settings/route.ts +23 -0
  69. package/src/modules/ai_assistant/backend/config/ai-assistant/agents/AiAgentSettingsPageClient.tsx +146 -0
  70. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/AiModerationFlagsPageClient.tsx +186 -0
  71. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.meta.ts +24 -0
  72. package/src/modules/ai_assistant/backend/config/ai-assistant/moderation-flags/page.tsx +12 -0
  73. package/src/modules/ai_assistant/data/entities.ts +56 -0
  74. package/src/modules/ai_assistant/data/repositories/AiAgentRuntimeOverrideRepository.ts +4 -0
  75. package/src/modules/ai_assistant/data/repositories/AiModerationFlagRepository.ts +79 -0
  76. package/src/modules/ai_assistant/di.ts +5 -1
  77. package/src/modules/ai_assistant/events.ts +21 -0
  78. package/src/modules/ai_assistant/i18n/de.json +26 -0
  79. package/src/modules/ai_assistant/i18n/en.json +26 -0
  80. package/src/modules/ai_assistant/i18n/es.json +26 -0
  81. package/src/modules/ai_assistant/i18n/pl.json +26 -0
  82. package/src/modules/ai_assistant/lib/__tests__/input-moderation-gate.test.ts +121 -0
  83. package/src/modules/ai_assistant/lib/__tests__/llm-adapters-anthropic.test.ts +13 -0
  84. package/src/modules/ai_assistant/lib/__tests__/llm-adapters-openai.test.ts +41 -0
  85. package/src/modules/ai_assistant/lib/__tests__/model-factory.test.ts +60 -0
  86. package/src/modules/ai_assistant/lib/__tests__/moderation-flag-recorder.test.ts +76 -0
  87. package/src/modules/ai_assistant/lib/__tests__/moderation-policy.test.ts +86 -0
  88. package/src/modules/ai_assistant/lib/__tests__/moderation.test.ts +114 -0
  89. package/src/modules/ai_assistant/lib/agent-runtime.ts +259 -0
  90. package/src/modules/ai_assistant/lib/ai-agent-definition.ts +10 -0
  91. package/src/modules/ai_assistant/lib/llm-adapters/anthropic.ts +10 -0
  92. package/src/modules/ai_assistant/lib/llm-adapters/openai.ts +32 -1
  93. package/src/modules/ai_assistant/lib/model-factory.ts +33 -0
  94. package/src/modules/ai_assistant/lib/moderation-flag-recorder.ts +75 -0
  95. package/src/modules/ai_assistant/lib/moderation-policy.ts +77 -0
  96. package/src/modules/ai_assistant/lib/moderation.ts +179 -0
  97. package/src/modules/ai_assistant/lib/openai-compatible-presets.ts +2 -0
  98. package/src/modules/ai_assistant/migrations/.snapshot-open-mercato.json +388 -16
  99. package/src/modules/ai_assistant/migrations/Migration20260610134045_ai_assistant.ts +21 -0
@@ -0,0 +1,186 @@
1
+ 'use client'
2
+
3
+ import * as React from 'react'
4
+ import { useQuery } from '@tanstack/react-query'
5
+ import type { ColumnDef } from '@tanstack/react-table'
6
+ import { useT } from '@open-mercato/shared/lib/i18n/context'
7
+ import { DataTable } from '@open-mercato/ui/backend/DataTable'
8
+ import { Button } from '@open-mercato/ui/primitives/button'
9
+ import { Input } from '@open-mercato/ui/primitives/input'
10
+ import { Label } from '@open-mercato/ui/primitives/label'
11
+ import { StatusBadge } from '@open-mercato/ui/primitives/status-badge'
12
+ import { apiCallOrThrow } from '@open-mercato/ui/backend/utils/apiCall'
13
+
14
+ const PAGE_SIZE = 50
15
+
16
+ type ModerationCategory = { flagged: boolean; score: number }
17
+
18
+ type ModerationFlagRow = {
19
+ id: string
20
+ tenantId: string
21
+ organizationId: string | null
22
+ agentId: string
23
+ userId: string
24
+ providerId: string
25
+ modelId: string
26
+ categories: Record<string, ModerationCategory>
27
+ createdAt: string
28
+ }
29
+
30
+ type ModerationFlagsResponse = {
31
+ items: ModerationFlagRow[]
32
+ total: number
33
+ page: number
34
+ pageSize: number
35
+ }
36
+
37
+ async function fetchModerationFlags(params: {
38
+ page: number
39
+ from: string
40
+ to: string
41
+ agentId: string
42
+ userId: string
43
+ }): Promise<ModerationFlagsResponse> {
44
+ const search = new URLSearchParams()
45
+ search.set('page', String(params.page))
46
+ search.set('pageSize', String(PAGE_SIZE))
47
+ if (params.from) search.set('from', params.from)
48
+ if (params.to) search.set('to', params.to)
49
+ if (params.agentId) search.set('agentId', params.agentId)
50
+ if (params.userId) search.set('userId', params.userId)
51
+ const { result, status } = await apiCallOrThrow<ModerationFlagsResponse>(
52
+ `/api/ai_assistant/moderation-flags?${search.toString()}`,
53
+ undefined,
54
+ { errorMessage: 'Failed to load moderation flags' },
55
+ )
56
+ if (!result) throw new Error(`Failed to load moderation flags (${status})`)
57
+ return result
58
+ }
59
+
60
+ function flaggedCategoryNames(categories: Record<string, ModerationCategory>): string[] {
61
+ return Object.entries(categories)
62
+ .filter(([, value]) => value.flagged)
63
+ .map(([name]) => name)
64
+ }
65
+
66
+ export function AiModerationFlagsPageClient() {
67
+ const t = useT()
68
+ const [page, setPage] = React.useState(1)
69
+ const [from, setFrom] = React.useState('')
70
+ const [to, setTo] = React.useState('')
71
+ const [agentId, setAgentId] = React.useState('')
72
+ const [userId, setUserId] = React.useState('')
73
+ const [applied, setApplied] = React.useState({ from: '', to: '', agentId: '', userId: '' })
74
+
75
+ const query = useQuery<ModerationFlagsResponse>({
76
+ queryKey: ['ai_assistant', 'moderation_flags', page, applied],
77
+ queryFn: () => fetchModerationFlags({ page, ...applied }),
78
+ retry: false,
79
+ })
80
+
81
+ const applyFilters = React.useCallback(() => {
82
+ setPage(1)
83
+ setApplied({ from, to, agentId, userId })
84
+ }, [from, to, agentId, userId])
85
+
86
+ const columns = React.useMemo<ColumnDef<ModerationFlagRow, unknown>[]>(
87
+ () => [
88
+ {
89
+ accessorKey: 'agentId',
90
+ header: t('ai_assistant.moderationFlags.columns.agent', 'Agent'),
91
+ },
92
+ {
93
+ accessorKey: 'userId',
94
+ header: t('ai_assistant.moderationFlags.columns.user', 'User'),
95
+ meta: { truncate: true, maxWidth: 220 },
96
+ },
97
+ {
98
+ accessorKey: 'categories',
99
+ header: t('ai_assistant.moderationFlags.columns.categories', 'Categories'),
100
+ cell: ({ row }) => {
101
+ const names = flaggedCategoryNames(row.original.categories)
102
+ if (names.length === 0) {
103
+ return <span className="text-muted-foreground">—</span>
104
+ }
105
+ return (
106
+ <div className="flex flex-wrap gap-1">
107
+ {names.map((name) => (
108
+ <StatusBadge key={name} variant="error" dot>
109
+ {name}
110
+ </StatusBadge>
111
+ ))}
112
+ </div>
113
+ )
114
+ },
115
+ },
116
+ {
117
+ accessorKey: 'createdAt',
118
+ header: t('ai_assistant.moderationFlags.columns.createdAt', 'Flagged at'),
119
+ cell: ({ row }) => new Date(row.original.createdAt).toLocaleString(),
120
+ },
121
+ ],
122
+ [t],
123
+ )
124
+
125
+ const total = query.data?.total ?? 0
126
+
127
+ return (
128
+ <div className="flex flex-col gap-4">
129
+ <div>
130
+ <h2 className="text-base font-semibold">
131
+ {t('ai_assistant.moderationFlags.title', 'Moderation flags')}
132
+ </h2>
133
+ <p className="text-xs text-muted-foreground">
134
+ {t(
135
+ 'ai_assistant.moderationFlags.subtitle',
136
+ 'Inputs blocked by the content safety filter. Category flags and scores only — no prompt content is stored.',
137
+ )}
138
+ </p>
139
+ </div>
140
+
141
+ <div className="flex flex-wrap items-end gap-3">
142
+ <div className="flex flex-col gap-1">
143
+ <Label htmlFor="moderation-from">{t('ai_assistant.moderationFlags.filters.from', 'From')}</Label>
144
+ <Input id="moderation-from" type="date" value={from} onChange={(e) => setFrom(e.target.value)} />
145
+ </div>
146
+ <div className="flex flex-col gap-1">
147
+ <Label htmlFor="moderation-to">{t('ai_assistant.moderationFlags.filters.to', 'To')}</Label>
148
+ <Input id="moderation-to" type="date" value={to} onChange={(e) => setTo(e.target.value)} />
149
+ </div>
150
+ <div className="flex flex-col gap-1">
151
+ <Label htmlFor="moderation-agent">{t('ai_assistant.moderationFlags.columns.agent', 'Agent')}</Label>
152
+ <Input
153
+ id="moderation-agent"
154
+ value={agentId}
155
+ onChange={(e) => setAgentId(e.target.value)}
156
+ placeholder={t('ai_assistant.moderationFlags.filters.agentPlaceholder', 'module.agent')}
157
+ />
158
+ </div>
159
+ <div className="flex flex-col gap-1">
160
+ <Label htmlFor="moderation-user">{t('ai_assistant.moderationFlags.columns.user', 'User')}</Label>
161
+ <Input id="moderation-user" value={userId} onChange={(e) => setUserId(e.target.value)} />
162
+ </div>
163
+ <Button type="button" variant="outline" onClick={applyFilters}>
164
+ {t('ai_assistant.moderationFlags.filters.apply', 'Apply')}
165
+ </Button>
166
+ </div>
167
+
168
+ <DataTable<ModerationFlagRow>
169
+ columns={columns}
170
+ data={query.data?.items ?? []}
171
+ isLoading={query.isLoading}
172
+ error={query.error ? (query.error as Error).message : undefined}
173
+ emptyState={t('ai_assistant.moderationFlags.empty', 'No flagged messages.')}
174
+ pagination={{
175
+ page,
176
+ pageSize: PAGE_SIZE,
177
+ total,
178
+ totalPages: Math.max(1, Math.ceil(total / PAGE_SIZE)),
179
+ onPageChange: setPage,
180
+ }}
181
+ />
182
+ </div>
183
+ )
184
+ }
185
+
186
+ export default AiModerationFlagsPageClient
@@ -0,0 +1,24 @@
1
+ import React from 'react'
2
+
3
+ const moderationIcon = React.createElement(
4
+ 'svg',
5
+ { width: 16, height: 16, viewBox: '0 0 24 24', fill: 'none', stroke: 'currentColor', strokeWidth: 2, strokeLinecap: 'round', strokeLinejoin: 'round' },
6
+ React.createElement('path', { d: 'M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10z' }),
7
+ React.createElement('path', { d: 'M12 8v4' }),
8
+ React.createElement('path', { d: 'M12 16h.01' }),
9
+ )
10
+
11
+ export const metadata = {
12
+ requireAuth: true,
13
+ requireFeatures: ['ai_assistant.settings.manage'],
14
+ pageTitle: 'Moderation flags',
15
+ pageTitleKey: 'ai_assistant.moderationFlags.navTitle',
16
+ pageGroup: 'Module Configs',
17
+ pageGroupKey: 'settings.sections.moduleConfigs',
18
+ pageOrder: 432,
19
+ icon: moderationIcon,
20
+ pageContext: 'settings' as const,
21
+ breadcrumb: [
22
+ { label: 'Moderation flags', labelKey: 'ai_assistant.moderationFlags.navTitle' },
23
+ ],
24
+ } as const
@@ -0,0 +1,12 @@
1
+ import { Page, PageBody } from '@open-mercato/ui/backend/Page'
2
+ import { AiModerationFlagsPageClient } from './AiModerationFlagsPageClient'
3
+
4
+ export default async function AiModerationFlagsPage() {
5
+ return (
6
+ <Page>
7
+ <PageBody>
8
+ <AiModerationFlagsPageClient />
9
+ </PageBody>
10
+ </Page>
11
+ )
12
+ }
@@ -361,6 +361,17 @@ export class AiAgentRuntimeOverride {
361
361
  */
362
362
  @Property({ name: 'loop_active_tools_json', type: 'jsonb', nullable: true })
363
363
  loopActiveToolsJson?: unknown | null
364
+
365
+ /**
366
+ * Per-agent (or tenant-wide when `agent_id` is null) input-moderation
367
+ * override. `true` = on, `false` = off, `null` = inherit (env default →
368
+ * off). Additive nullable column; existing rows inherit. The `untrustedInput`
369
+ * agent flag still forces moderation regardless of this value.
370
+ *
371
+ * Spec `2026-06-04-ai-input-moderation-and-safety-identifiers`.
372
+ */
373
+ @Property({ name: 'input_moderation', type: 'boolean', nullable: true })
374
+ inputModeration?: boolean | null
364
375
  }
365
376
 
366
377
  /**
@@ -933,3 +944,48 @@ export class AiChatMessage {
933
944
  @Property({ name: 'deleted_at', type: Date, nullable: true })
934
945
  deletedAt?: Date | null
935
946
  }
947
+
948
+ /**
949
+ * Append-only audit record of an input that the moderation gate blocked.
950
+ *
951
+ * Stores ONLY category flags + numeric scores — never the prompt content or
952
+ * any PII (Q4 data-minimization decision). Enough to identify repeat abusers
953
+ * and tune policy. No `updated_at` / `deleted_at`: rows are immutable. Two
954
+ * indexes back the audit listing (`tenant_id, created_at`) and the per-user
955
+ * abuse review (`tenant_id, user_id`); every read filters `tenant_id`.
956
+ *
957
+ * Spec `2026-06-04-ai-input-moderation-and-safety-identifiers`.
958
+ */
959
+ @Entity({ tableName: 'ai_moderation_flags' })
960
+ @Index({ name: 'ai_moderation_flags_tenant_created_idx', properties: ['tenantId', 'createdAt'] })
961
+ @Index({ name: 'ai_moderation_flags_tenant_user_idx', properties: ['tenantId', 'userId'] })
962
+ export class AiModerationFlag {
963
+ [OptionalProps]?: 'id' | 'createdAt' | 'organizationId'
964
+
965
+ @PrimaryKey({ type: 'uuid', defaultRaw: 'gen_random_uuid()' })
966
+ id!: string
967
+
968
+ @Property({ name: 'tenant_id', type: 'uuid' })
969
+ tenantId!: string
970
+
971
+ @Property({ name: 'organization_id', type: 'uuid', nullable: true })
972
+ organizationId?: string | null
973
+
974
+ @Property({ name: 'agent_id', type: 'text' })
975
+ agentId!: string
976
+
977
+ @Property({ name: 'user_id', type: 'text' })
978
+ userId!: string
979
+
980
+ @Property({ name: 'provider_id', type: 'text' })
981
+ providerId!: string
982
+
983
+ @Property({ name: 'model_id', type: 'text' })
984
+ modelId!: string
985
+
986
+ @Property({ name: 'categories', type: 'jsonb' })
987
+ categories!: Record<string, { flagged: boolean; score: number }>
988
+
989
+ @Property({ name: 'created_at', type: Date, onCreate: () => new Date() })
990
+ createdAt: Date = new Date()
991
+ }
@@ -35,6 +35,8 @@ export interface AiAgentRuntimeOverrideInput extends AiAgentRuntimeOverrideLoopI
35
35
  baseURL?: string | null
36
36
  allowedOverrideProviders?: string[] | null
37
37
  allowedOverrideModelsByProvider?: Record<string, string[]>
38
+ /** Input-moderation override: true = on, false = off, null = inherit. */
39
+ inputModeration?: boolean | null
38
40
  /**
39
41
  * Optional: the agent's declared allowedTools. When provided, loopActiveToolsJson
40
42
  * is validated to be a subset. When omitted, allowlist validation is skipped
@@ -270,6 +272,7 @@ export class AiAgentRuntimeOverrideRepository {
270
272
  if ('loopMaxTokens' in input) existing.loopMaxTokens = input.loopMaxTokens ?? null
271
273
  if ('loopStopWhenJson' in input) existing.loopStopWhenJson = input.loopStopWhenJson ?? null
272
274
  if ('loopActiveToolsJson' in input) existing.loopActiveToolsJson = input.loopActiveToolsJson ?? null
275
+ if ('inputModeration' in input) existing.inputModeration = input.inputModeration ?? null
273
276
  await tx.persist(existing).flush()
274
277
  return existing
275
278
  }
@@ -295,6 +298,7 @@ export class AiAgentRuntimeOverrideRepository {
295
298
  loopMaxTokens: input.loopMaxTokens ?? null,
296
299
  loopStopWhenJson: input.loopStopWhenJson ?? null,
297
300
  loopActiveToolsJson: input.loopActiveToolsJson ?? null,
301
+ inputModeration: input.inputModeration ?? null,
298
302
  } as unknown as AiAgentRuntimeOverride)
299
303
  await tx.persist(row).flush()
300
304
  return row
@@ -0,0 +1,79 @@
1
+ import type { EntityManager } from '@mikro-orm/postgresql'
2
+ import { AiModerationFlag } from '../entities'
3
+ import type { ModerationCategoryResult } from '../../lib/moderation'
4
+
5
+ export interface CreateModerationFlagInput {
6
+ tenantId: string
7
+ organizationId?: string | null
8
+ agentId: string
9
+ userId: string
10
+ providerId: string
11
+ modelId: string
12
+ categories: Record<string, ModerationCategoryResult>
13
+ }
14
+
15
+ export interface ListModerationFlagsFilter {
16
+ tenantId: string
17
+ organizationId?: string | null
18
+ agentId?: string
19
+ userId?: string
20
+ /** Inclusive lower bound on `created_at`. */
21
+ from?: Date
22
+ /** Inclusive upper bound on `created_at`. */
23
+ to?: Date
24
+ page: number
25
+ pageSize: number
26
+ }
27
+
28
+ export interface ModerationFlagListResult {
29
+ items: AiModerationFlag[]
30
+ total: number
31
+ }
32
+
33
+ /**
34
+ * Repository for the append-only `ai_moderation_flags` audit table. Every read
35
+ * is tenant-scoped — cross-tenant access is structurally impossible because
36
+ * `tenantId` is always part of the filter.
37
+ *
38
+ * Spec `2026-06-04-ai-input-moderation-and-safety-identifiers`.
39
+ */
40
+ export class AiModerationFlagRepository {
41
+ constructor(private readonly em: EntityManager) {}
42
+
43
+ async create(input: CreateModerationFlagInput): Promise<AiModerationFlag> {
44
+ const flag = this.em.create(AiModerationFlag, {
45
+ tenantId: input.tenantId,
46
+ organizationId: input.organizationId ?? null,
47
+ agentId: input.agentId,
48
+ userId: input.userId,
49
+ providerId: input.providerId,
50
+ modelId: input.modelId,
51
+ categories: input.categories,
52
+ })
53
+ this.em.persist(flag)
54
+ await this.em.flush()
55
+ return flag
56
+ }
57
+
58
+ async list(filter: ListModerationFlagsFilter): Promise<ModerationFlagListResult> {
59
+ const where: Record<string, unknown> = { tenantId: filter.tenantId }
60
+ if (filter.organizationId !== undefined && filter.organizationId !== null) {
61
+ where.organizationId = filter.organizationId
62
+ }
63
+ if (filter.agentId) where.agentId = filter.agentId
64
+ if (filter.userId) where.userId = filter.userId
65
+ if (filter.from || filter.to) {
66
+ const createdAt: Record<string, Date> = {}
67
+ if (filter.from) createdAt.$gte = filter.from
68
+ if (filter.to) createdAt.$lte = filter.to
69
+ where.createdAt = createdAt
70
+ }
71
+
72
+ const [items, total] = await this.em.findAndCount(AiModerationFlag, where, {
73
+ orderBy: { createdAt: 'desc' },
74
+ limit: filter.pageSize,
75
+ offset: (filter.page - 1) * filter.pageSize,
76
+ })
77
+ return { items, total }
78
+ }
79
+ }
@@ -1,9 +1,13 @@
1
- import { asValue } from 'awilix'
1
+ import { asFunction, asValue } from 'awilix'
2
2
  import type { AwilixContainer } from 'awilix'
3
3
  import { toolRegistry } from './lib/tool-registry'
4
+ import { createModerationService } from './lib/moderation'
4
5
 
5
6
  export function register(container: AwilixContainer): void {
6
7
  container.register({
7
8
  mcpToolRegistry: asValue(toolRegistry),
9
+ // Input pre-moderation service (OpenAI /v1/moderations). Singleton +
10
+ // overridable by downstream apps via the module overrides DI seam.
11
+ moderationService: asFunction(() => createModerationService()).singleton(),
8
12
  })
9
13
  }
@@ -61,6 +61,12 @@ const events = [
61
61
  category: 'lifecycle' as const,
62
62
  clientBroadcast: true,
63
63
  },
64
+ {
65
+ id: 'ai_assistant.moderation_flag.created',
66
+ label: 'AI Moderation Flag Created',
67
+ entity: 'ai_moderation_flag',
68
+ category: 'system' as const,
69
+ },
64
70
  ] as const
65
71
 
66
72
  export const eventsConfig = createModuleEvents({
@@ -148,4 +154,19 @@ export interface AiConversationUnsharedPayload {
148
154
  participantUserId: string
149
155
  }
150
156
 
157
+ /**
158
+ * Emitted (best-effort) when the input moderation gate blocks a turn and the
159
+ * audit row is persisted. Carries category flags only — never prompt content.
160
+ *
161
+ * Spec `2026-06-04-ai-input-moderation-and-safety-identifiers`.
162
+ */
163
+ export interface AiModerationFlagCreatedPayload {
164
+ id: string
165
+ tenantId: string
166
+ organizationId: string | null
167
+ agentId: string
168
+ userId: string
169
+ categories: string[]
170
+ }
171
+
151
172
  export default eventsConfig
@@ -62,6 +62,18 @@
62
62
  "ai_assistant.agents.model_override.subtitle": "Überschreibe Standardanbieter und -modell für diesen Mandanten und Agenten.",
63
63
  "ai_assistant.agents.model_override.tenantOverride": "Mandanten-Überschreibung",
64
64
  "ai_assistant.agents.model_override.title": "Anbieter und Modell",
65
+ "ai_assistant.agents.moderation.controlLabel": "Moderation",
66
+ "ai_assistant.agents.moderation.effectiveEnforced": "Erzwungen",
67
+ "ai_assistant.agents.moderation.effectiveOff": "Aus",
68
+ "ai_assistant.agents.moderation.effectiveOn": "Ein",
69
+ "ai_assistant.agents.moderation.enforcedBadge": "Erzwungen",
70
+ "ai_assistant.agents.moderation.enforcedHint": "Dieser Agent akzeptiert nicht vertrauenswürdige Eingaben, daher ist die Moderation immer aktiv und kann hier nicht deaktiviert werden.",
71
+ "ai_assistant.agents.moderation.inheritLabel": "Erben",
72
+ "ai_assistant.agents.moderation.offLabel": "Aus",
73
+ "ai_assistant.agents.moderation.onLabel": "Ein",
74
+ "ai_assistant.agents.moderation.saved": "Eingabe-Moderationseinstellung gespeichert.",
75
+ "ai_assistant.agents.moderation.subtitle": "Benutzereingaben vor dem Modelllauf auf unsichere Inhalte prüfen. Gilt nur, wenn der aufgelöste Anbieter Moderation unterstützt.",
76
+ "ai_assistant.agents.moderation.title": "Eingabe-Moderation",
65
77
  "ai_assistant.agents.mutation_policy.clear": "Überschreibung löschen",
66
78
  "ai_assistant.agents.mutation_policy.clearedMessage": "Überschreibung der Mutationsrichtlinie gelöscht; Agent verwendet die im Code deklarierte Richtlinie.",
67
79
  "ai_assistant.agents.mutation_policy.codeDeclared": "Im Code deklariert",
@@ -297,6 +309,8 @@
297
309
  "ai_assistant.dock.left": "Links andocken",
298
310
  "ai_assistant.dock.minimize": "Minimieren",
299
311
  "ai_assistant.dock.right": "Rechts andocken",
312
+ "ai_assistant.errors.moderationBlocked": "Ihre Nachricht wurde vom Inhaltssicherheitsfilter blockiert. Bitte formulieren Sie sie um und versuchen Sie es erneut.",
313
+ "ai_assistant.errors.moderationUnavailable": "Die Inhaltssicherheitsprüfung ist vorübergehend nicht verfügbar. Bitte versuchen Sie es in einem Moment erneut.",
300
314
  "ai_assistant.launcher.composerPlaceholder": "Fragen Sie mich etwas…",
301
315
  "ai_assistant.launcher.dialogDescription": "Wähle einen Assistenten. Mit ↑/↓ navigieren, Enter startet, Esc schließt.",
302
316
  "ai_assistant.launcher.dialogTitle": "AI assistants",
@@ -342,6 +356,18 @@
342
356
  "ai_assistant.modelPicker.triggerAriaLabel": "KI-Modell auswählen",
343
357
  "ai_assistant.modelPicker.useDefault": "Standard des Agenten verwenden",
344
358
  "ai_assistant.modelPicker.useDefaultWithModel": "Standard des Agenten verwenden: {{model}}",
359
+ "ai_assistant.moderationFlags.columns.agent": "Agent",
360
+ "ai_assistant.moderationFlags.columns.categories": "Kategorien",
361
+ "ai_assistant.moderationFlags.columns.createdAt": "Markiert am",
362
+ "ai_assistant.moderationFlags.columns.user": "Benutzer",
363
+ "ai_assistant.moderationFlags.empty": "Keine markierten Nachrichten.",
364
+ "ai_assistant.moderationFlags.filters.agentPlaceholder": "modul.agent",
365
+ "ai_assistant.moderationFlags.filters.apply": "Anwenden",
366
+ "ai_assistant.moderationFlags.filters.from": "Von",
367
+ "ai_assistant.moderationFlags.filters.to": "Bis",
368
+ "ai_assistant.moderationFlags.navTitle": "Moderations-Flags",
369
+ "ai_assistant.moderationFlags.subtitle": "Vom Inhaltssicherheitsfilter blockierte Eingaben. Nur Kategorie-Flags und Scores — es wird kein Prompt-Inhalt gespeichert.",
370
+ "ai_assistant.moderationFlags.title": "Moderations-Flags",
345
371
  "ai_assistant.notifications.conversation_shared.body": "Eine KI-Konversation wurde mit Ihnen geteilt.",
346
372
  "ai_assistant.notifications.conversation_shared.title": "Konversation mit Ihnen geteilt",
347
373
  "ai_assistant.notifications.conversation_shared.view_button": "Konversation öffnen",
@@ -62,6 +62,18 @@
62
62
  "ai_assistant.agents.model_override.subtitle": "Override the default provider and model for this tenant and agent.",
63
63
  "ai_assistant.agents.model_override.tenantOverride": "Tenant override",
64
64
  "ai_assistant.agents.model_override.title": "Provider and model",
65
+ "ai_assistant.agents.moderation.controlLabel": "Moderation",
66
+ "ai_assistant.agents.moderation.effectiveEnforced": "Enforced",
67
+ "ai_assistant.agents.moderation.effectiveOff": "Off",
68
+ "ai_assistant.agents.moderation.effectiveOn": "On",
69
+ "ai_assistant.agents.moderation.enforcedBadge": "Enforced",
70
+ "ai_assistant.agents.moderation.enforcedHint": "This agent accepts untrusted input, so moderation is always on and cannot be disabled here.",
71
+ "ai_assistant.agents.moderation.inheritLabel": "Inherit",
72
+ "ai_assistant.agents.moderation.offLabel": "Off",
73
+ "ai_assistant.agents.moderation.onLabel": "On",
74
+ "ai_assistant.agents.moderation.saved": "Input moderation setting saved.",
75
+ "ai_assistant.agents.moderation.subtitle": "Screen user input for unsafe content before the model runs. Applies only when the resolved provider supports moderation.",
76
+ "ai_assistant.agents.moderation.title": "Input moderation",
65
77
  "ai_assistant.agents.mutation_policy.clear": "Clear override",
66
78
  "ai_assistant.agents.mutation_policy.clearedMessage": "Mutation policy override cleared; agent is using its code-declared policy.",
67
79
  "ai_assistant.agents.mutation_policy.codeDeclared": "Code-declared",
@@ -297,6 +309,8 @@
297
309
  "ai_assistant.dock.left": "Dock Left",
298
310
  "ai_assistant.dock.minimize": "Minimize",
299
311
  "ai_assistant.dock.right": "Dock Right",
312
+ "ai_assistant.errors.moderationBlocked": "Your message was blocked by the content safety filter. Please rephrase and try again.",
313
+ "ai_assistant.errors.moderationUnavailable": "The content safety check is temporarily unavailable. Please try again in a moment.",
300
314
  "ai_assistant.launcher.composerPlaceholder": "Ask anything…",
301
315
  "ai_assistant.launcher.dialogDescription": "Pick an assistant. Use ↑/↓ to navigate, Enter to launch, Esc to close.",
302
316
  "ai_assistant.launcher.dialogTitle": "AI assistants",
@@ -342,6 +356,18 @@
342
356
  "ai_assistant.modelPicker.triggerAriaLabel": "Select AI model",
343
357
  "ai_assistant.modelPicker.useDefault": "Use agent default",
344
358
  "ai_assistant.modelPicker.useDefaultWithModel": "Use agent default: {{model}}",
359
+ "ai_assistant.moderationFlags.columns.agent": "Agent",
360
+ "ai_assistant.moderationFlags.columns.categories": "Categories",
361
+ "ai_assistant.moderationFlags.columns.createdAt": "Flagged at",
362
+ "ai_assistant.moderationFlags.columns.user": "User",
363
+ "ai_assistant.moderationFlags.empty": "No flagged messages.",
364
+ "ai_assistant.moderationFlags.filters.agentPlaceholder": "module.agent",
365
+ "ai_assistant.moderationFlags.filters.apply": "Apply",
366
+ "ai_assistant.moderationFlags.filters.from": "From",
367
+ "ai_assistant.moderationFlags.filters.to": "To",
368
+ "ai_assistant.moderationFlags.navTitle": "Moderation flags",
369
+ "ai_assistant.moderationFlags.subtitle": "Inputs blocked by the content safety filter. Category flags and scores only — no prompt content is stored.",
370
+ "ai_assistant.moderationFlags.title": "Moderation flags",
345
371
  "ai_assistant.notifications.conversation_shared.body": "An AI conversation has been shared with you.",
346
372
  "ai_assistant.notifications.conversation_shared.title": "Conversation shared with you",
347
373
  "ai_assistant.notifications.conversation_shared.view_button": "View Conversation",
@@ -62,6 +62,18 @@
62
62
  "ai_assistant.agents.model_override.subtitle": "Anula el proveedor y el modelo predeterminados para este inquilino y agente.",
63
63
  "ai_assistant.agents.model_override.tenantOverride": "Sobrescritura del tenant",
64
64
  "ai_assistant.agents.model_override.title": "Proveedor y modelo",
65
+ "ai_assistant.agents.moderation.controlLabel": "Moderación",
66
+ "ai_assistant.agents.moderation.effectiveEnforced": "Forzada",
67
+ "ai_assistant.agents.moderation.effectiveOff": "Desactivada",
68
+ "ai_assistant.agents.moderation.effectiveOn": "Activada",
69
+ "ai_assistant.agents.moderation.enforcedBadge": "Forzada",
70
+ "ai_assistant.agents.moderation.enforcedHint": "Este agente acepta entradas no confiables, por lo que la moderación siempre está activada y no se puede desactivar aquí.",
71
+ "ai_assistant.agents.moderation.inheritLabel": "Heredar",
72
+ "ai_assistant.agents.moderation.offLabel": "Desactivada",
73
+ "ai_assistant.agents.moderation.onLabel": "Activada",
74
+ "ai_assistant.agents.moderation.saved": "Ajuste de moderación de entrada guardado.",
75
+ "ai_assistant.agents.moderation.subtitle": "Filtra la entrada del usuario en busca de contenido no seguro antes de ejecutar el modelo. Solo aplica cuando el proveedor resuelto admite moderación.",
76
+ "ai_assistant.agents.moderation.title": "Moderación de entrada",
65
77
  "ai_assistant.agents.mutation_policy.clear": "Eliminar sobrescritura",
66
78
  "ai_assistant.agents.mutation_policy.clearedMessage": "Sobrescritura de política de mutación eliminada; el agente usa la política declarada en el código.",
67
79
  "ai_assistant.agents.mutation_policy.codeDeclared": "Declarada en el código",
@@ -297,6 +309,8 @@
297
309
  "ai_assistant.dock.left": "Anclar a la izquierda",
298
310
  "ai_assistant.dock.minimize": "Minimizar",
299
311
  "ai_assistant.dock.right": "Anclar a la derecha",
312
+ "ai_assistant.errors.moderationBlocked": "Tu mensaje fue bloqueado por el filtro de seguridad de contenido. Reformúlalo e inténtalo de nuevo.",
313
+ "ai_assistant.errors.moderationUnavailable": "La verificación de seguridad de contenido no está disponible temporalmente. Inténtalo de nuevo en un momento.",
300
314
  "ai_assistant.launcher.composerPlaceholder": "Pregunte lo que necesite…",
301
315
  "ai_assistant.launcher.dialogDescription": "Elige un asistente. Usa ↑/↓ para navegar, Enter para iniciar, Esc para cerrar.",
302
316
  "ai_assistant.launcher.dialogTitle": "AI assistants",
@@ -342,6 +356,18 @@
342
356
  "ai_assistant.modelPicker.triggerAriaLabel": "Selector de modelo de IA",
343
357
  "ai_assistant.modelPicker.useDefault": "Usar predeterminado",
344
358
  "ai_assistant.modelPicker.useDefaultWithModel": "Usar predeterminado del agente: {{model}}",
359
+ "ai_assistant.moderationFlags.columns.agent": "Agente",
360
+ "ai_assistant.moderationFlags.columns.categories": "Categorías",
361
+ "ai_assistant.moderationFlags.columns.createdAt": "Marcado el",
362
+ "ai_assistant.moderationFlags.columns.user": "Usuario",
363
+ "ai_assistant.moderationFlags.empty": "No hay mensajes marcados.",
364
+ "ai_assistant.moderationFlags.filters.agentPlaceholder": "modulo.agente",
365
+ "ai_assistant.moderationFlags.filters.apply": "Aplicar",
366
+ "ai_assistant.moderationFlags.filters.from": "Desde",
367
+ "ai_assistant.moderationFlags.filters.to": "Hasta",
368
+ "ai_assistant.moderationFlags.navTitle": "Marcas de moderación",
369
+ "ai_assistant.moderationFlags.subtitle": "Entradas bloqueadas por el filtro de seguridad de contenido. Solo indicadores de categoría y puntuaciones — no se almacena el contenido del prompt.",
370
+ "ai_assistant.moderationFlags.title": "Marcas de moderación",
345
371
  "ai_assistant.notifications.conversation_shared.body": "Se ha compartido una conversación de IA con usted.",
346
372
  "ai_assistant.notifications.conversation_shared.title": "Conversación compartida con usted",
347
373
  "ai_assistant.notifications.conversation_shared.view_button": "Ver conversación",
@@ -62,6 +62,18 @@
62
62
  "ai_assistant.agents.model_override.subtitle": "Nadpisz domyślnego dostawcę i model dla tego najemcy i agenta.",
63
63
  "ai_assistant.agents.model_override.tenantOverride": "Nadpisanie tenant",
64
64
  "ai_assistant.agents.model_override.title": "Dostawca i model",
65
+ "ai_assistant.agents.moderation.controlLabel": "Moderacja",
66
+ "ai_assistant.agents.moderation.effectiveEnforced": "Wymuszona",
67
+ "ai_assistant.agents.moderation.effectiveOff": "Wyłączona",
68
+ "ai_assistant.agents.moderation.effectiveOn": "Włączona",
69
+ "ai_assistant.agents.moderation.enforcedBadge": "Wymuszona",
70
+ "ai_assistant.agents.moderation.enforcedHint": "Ten agent przyjmuje niezaufane dane wejściowe, więc moderacja jest zawsze włączona i nie można jej tutaj wyłączyć.",
71
+ "ai_assistant.agents.moderation.inheritLabel": "Dziedzicz",
72
+ "ai_assistant.agents.moderation.offLabel": "Wyłączona",
73
+ "ai_assistant.agents.moderation.onLabel": "Włączona",
74
+ "ai_assistant.agents.moderation.saved": "Zapisano ustawienie moderacji danych wejściowych.",
75
+ "ai_assistant.agents.moderation.subtitle": "Filtruj dane wejściowe użytkownika pod kątem niebezpiecznych treści przed uruchomieniem modelu. Działa tylko, gdy wybrany dostawca obsługuje moderację.",
76
+ "ai_assistant.agents.moderation.title": "Moderacja danych wejściowych",
65
77
  "ai_assistant.agents.mutation_policy.clear": "Wyczyść nadpisanie",
66
78
  "ai_assistant.agents.mutation_policy.clearedMessage": "Nadpisanie polityki mutacji usunięte; agent korzysta z polityki zadeklarowanej w kodzie.",
67
79
  "ai_assistant.agents.mutation_policy.codeDeclared": "Zadeklarowane w kodzie",
@@ -297,6 +309,8 @@
297
309
  "ai_assistant.dock.left": "Dokuj po lewej",
298
310
  "ai_assistant.dock.minimize": "Minimalizuj",
299
311
  "ai_assistant.dock.right": "Dokuj po prawej",
312
+ "ai_assistant.errors.moderationBlocked": "Twoja wiadomość została zablokowana przez filtr bezpieczeństwa treści. Przeformułuj ją i spróbuj ponownie.",
313
+ "ai_assistant.errors.moderationUnavailable": "Sprawdzanie bezpieczeństwa treści jest chwilowo niedostępne. Spróbuj ponownie za chwilę.",
300
314
  "ai_assistant.launcher.composerPlaceholder": "Zapytaj o cokolwiek…",
301
315
  "ai_assistant.launcher.dialogDescription": "Wybierz asystenta. Nawiguj klawiszami ↑/↓, Enter uruchamia, Esc zamyka.",
302
316
  "ai_assistant.launcher.dialogTitle": "Asystenci AI",
@@ -342,6 +356,18 @@
342
356
  "ai_assistant.modelPicker.triggerAriaLabel": "Wybierz model AI",
343
357
  "ai_assistant.modelPicker.useDefault": "Użyj domyślnego agenta",
344
358
  "ai_assistant.modelPicker.useDefaultWithModel": "Użyj domyślnego agenta: {{model}}",
359
+ "ai_assistant.moderationFlags.columns.agent": "Agent",
360
+ "ai_assistant.moderationFlags.columns.categories": "Kategorie",
361
+ "ai_assistant.moderationFlags.columns.createdAt": "Oznaczono",
362
+ "ai_assistant.moderationFlags.columns.user": "Użytkownik",
363
+ "ai_assistant.moderationFlags.empty": "Brak oznaczonych wiadomości.",
364
+ "ai_assistant.moderationFlags.filters.agentPlaceholder": "moduł.agent",
365
+ "ai_assistant.moderationFlags.filters.apply": "Zastosuj",
366
+ "ai_assistant.moderationFlags.filters.from": "Od",
367
+ "ai_assistant.moderationFlags.filters.to": "Do",
368
+ "ai_assistant.moderationFlags.navTitle": "Flagi moderacji",
369
+ "ai_assistant.moderationFlags.subtitle": "Dane wejściowe zablokowane przez filtr bezpieczeństwa treści. Tylko flagi kategorii i wyniki — treść promptu nie jest przechowywana.",
370
+ "ai_assistant.moderationFlags.title": "Flagi moderacji",
345
371
  "ai_assistant.notifications.conversation_shared.body": "Udostępniono Ci konwersację z asystentem AI.",
346
372
  "ai_assistant.notifications.conversation_shared.title": "Udostępniono Ci konwersację",
347
373
  "ai_assistant.notifications.conversation_shared.view_button": "Otwórz konwersację",