@umituz/react-native-ai-generation-content 1.12.3 → 1.12.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/src/domains/content-moderation/domain/entities/moderation.types.ts +84 -0
- package/src/domains/content-moderation/domain/interfaces/content-filter.interface.ts +24 -0
- package/src/domains/content-moderation/index.ts +67 -0
- package/src/domains/content-moderation/infrastructure/rules/default-rules.data.ts +144 -0
- package/src/domains/content-moderation/infrastructure/rules/rules-registry.ts +75 -0
- package/src/domains/content-moderation/infrastructure/services/content-moderation.service.ts +150 -0
- package/src/domains/content-moderation/infrastructure/services/index.ts +8 -0
- package/src/domains/content-moderation/infrastructure/services/moderators/base.moderator.ts +62 -0
- package/src/domains/content-moderation/infrastructure/services/moderators/image.moderator.ts +64 -0
- package/src/domains/content-moderation/infrastructure/services/moderators/index.ts +10 -0
- package/src/domains/content-moderation/infrastructure/services/moderators/text.moderator.ts +144 -0
- package/src/domains/content-moderation/infrastructure/services/moderators/video.moderator.ts +64 -0
- package/src/domains/content-moderation/infrastructure/services/moderators/voice.moderator.ts +74 -0
- package/src/domains/content-moderation/infrastructure/services/pattern-matcher.service.ts +51 -0
- package/src/domains/content-moderation/presentation/exceptions/content-policy-violation.exception.ts +48 -0
- package/src/index.ts +6 -0
package/package.json
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@umituz/react-native-ai-generation-content",
|
|
3
|
-
"version": "1.12.
|
|
3
|
+
"version": "1.12.4",
|
|
4
4
|
"description": "Provider-agnostic AI generation orchestration for React Native",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
7
7
|
"exports": {
|
|
8
8
|
".": "./src/index.ts",
|
|
9
|
-
"./prompts": "./src/domains/prompts/index.ts"
|
|
9
|
+
"./prompts": "./src/domains/prompts/index.ts",
|
|
10
|
+
"./content-moderation": "./src/domains/content-moderation/index.ts"
|
|
10
11
|
},
|
|
11
12
|
"files": [
|
|
12
13
|
"src"
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content Moderation Types
|
|
3
|
+
* Core type definitions for content moderation system
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export type ContentType = "text" | "image" | "video" | "voice";
|
|
7
|
+
|
|
8
|
+
export type ModerationSeverity = "low" | "medium" | "high" | "critical";
|
|
9
|
+
|
|
10
|
+
export type AgeRating = "G" | "PG" | "PG-13" | "R" | "NC-17" | "18+";
|
|
11
|
+
|
|
12
|
+
export type ViolationType =
|
|
13
|
+
| "explicit_content"
|
|
14
|
+
| "violence"
|
|
15
|
+
| "hate_speech"
|
|
16
|
+
| "harassment"
|
|
17
|
+
| "illegal_activity"
|
|
18
|
+
| "spam"
|
|
19
|
+
| "copyright"
|
|
20
|
+
| "personal_info"
|
|
21
|
+
| "dangerous_content";
|
|
22
|
+
|
|
23
|
+
export interface ModerationRule {
|
|
24
|
+
id: string;
|
|
25
|
+
name: string;
|
|
26
|
+
description: string;
|
|
27
|
+
contentTypes: ContentType[];
|
|
28
|
+
severity: ModerationSeverity;
|
|
29
|
+
violationType: ViolationType;
|
|
30
|
+
patterns: string[];
|
|
31
|
+
enabled: boolean;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface ModerationResult {
|
|
35
|
+
isAllowed: boolean;
|
|
36
|
+
violations: Violation[];
|
|
37
|
+
confidence: number;
|
|
38
|
+
suggestedAction: "allow" | "warn" | "block";
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface Violation {
|
|
42
|
+
ruleId: string;
|
|
43
|
+
ruleName: string;
|
|
44
|
+
violationType: ViolationType;
|
|
45
|
+
severity: ModerationSeverity;
|
|
46
|
+
matchedPattern: string;
|
|
47
|
+
context: string;
|
|
48
|
+
suggestion: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface ModerationContext {
|
|
52
|
+
userId?: string;
|
|
53
|
+
contentType: ContentType;
|
|
54
|
+
content: string;
|
|
55
|
+
metadata?: Record<string, unknown>;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface ModerationConfig {
|
|
59
|
+
strictMode: boolean;
|
|
60
|
+
autoBlock: boolean;
|
|
61
|
+
logViolations: boolean;
|
|
62
|
+
notifyUser: boolean;
|
|
63
|
+
customRules?: ModerationRule[];
|
|
64
|
+
suggestionMessages?: SuggestionMessages;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface SuggestionMessages {
|
|
68
|
+
explicit_content?: string;
|
|
69
|
+
violence?: string;
|
|
70
|
+
hate_speech?: string;
|
|
71
|
+
harassment?: string;
|
|
72
|
+
illegal_activity?: string;
|
|
73
|
+
spam?: string;
|
|
74
|
+
copyright?: string;
|
|
75
|
+
personal_info?: string;
|
|
76
|
+
dangerous_content?: string;
|
|
77
|
+
default?: string;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface ValidationLimits {
|
|
81
|
+
maxTextLength?: number;
|
|
82
|
+
maxVoiceTextLength?: number;
|
|
83
|
+
maxUriLength?: number;
|
|
84
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content Filter Interface
|
|
3
|
+
* Contract for content filtering implementations
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { Violation } from "../entities/moderation.types";
|
|
7
|
+
|
|
8
|
+
export interface ContentFilterResult {
|
|
9
|
+
isAllowed: boolean;
|
|
10
|
+
violations: Violation[];
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface IContentFilter {
|
|
14
|
+
filter(content: string): ContentFilterResult;
|
|
15
|
+
addBlockedWord(word: string): void;
|
|
16
|
+
addBlockedPattern(pattern: string): void;
|
|
17
|
+
removeBlockedWord(word: string): void;
|
|
18
|
+
removeBlockedPattern(pattern: string): void;
|
|
19
|
+
clearFilters(): void;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface IModerator {
|
|
23
|
+
moderate(content: string): ContentFilterResult;
|
|
24
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @umituz/react-native-ai-content-moderation
|
|
3
|
+
* Content moderation service for AI applications
|
|
4
|
+
*
|
|
5
|
+
* Usage:
|
|
6
|
+
* import {
|
|
7
|
+
* contentModerationService,
|
|
8
|
+
* ContentPolicyViolationError,
|
|
9
|
+
* rulesRegistry
|
|
10
|
+
* } from '@umituz/react-native-ai-content-moderation';
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
// =============================================================================
|
|
14
|
+
// DOMAIN LAYER - Types
|
|
15
|
+
// =============================================================================
|
|
16
|
+
|
|
17
|
+
export type {
|
|
18
|
+
ContentType,
|
|
19
|
+
ModerationSeverity,
|
|
20
|
+
AgeRating,
|
|
21
|
+
ViolationType,
|
|
22
|
+
ModerationRule,
|
|
23
|
+
ModerationResult,
|
|
24
|
+
Violation,
|
|
25
|
+
ModerationContext,
|
|
26
|
+
ModerationConfig,
|
|
27
|
+
SuggestionMessages,
|
|
28
|
+
ValidationLimits,
|
|
29
|
+
} from "./domain/entities/moderation.types";
|
|
30
|
+
|
|
31
|
+
export type {
|
|
32
|
+
ContentFilterResult,
|
|
33
|
+
IContentFilter,
|
|
34
|
+
IModerator,
|
|
35
|
+
} from "./domain/interfaces/content-filter.interface";
|
|
36
|
+
|
|
37
|
+
// =============================================================================
|
|
38
|
+
// INFRASTRUCTURE LAYER - Services
|
|
39
|
+
// =============================================================================
|
|
40
|
+
|
|
41
|
+
export {
|
|
42
|
+
contentModerationService,
|
|
43
|
+
patternMatcherService,
|
|
44
|
+
textModerator,
|
|
45
|
+
imageModerator,
|
|
46
|
+
videoModerator,
|
|
47
|
+
voiceModerator,
|
|
48
|
+
BaseModerator,
|
|
49
|
+
} from "./infrastructure/services";
|
|
50
|
+
|
|
51
|
+
export type {
|
|
52
|
+
PatternMatch,
|
|
53
|
+
ModerationResult as ModeratorResult,
|
|
54
|
+
} from "./infrastructure/services";
|
|
55
|
+
|
|
56
|
+
// =============================================================================
|
|
57
|
+
// INFRASTRUCTURE LAYER - Rules
|
|
58
|
+
// =============================================================================
|
|
59
|
+
|
|
60
|
+
export { rulesRegistry } from "./infrastructure/rules/rules-registry";
|
|
61
|
+
export { defaultModerationRules } from "./infrastructure/rules/default-rules.data";
|
|
62
|
+
|
|
63
|
+
// =============================================================================
|
|
64
|
+
// PRESENTATION LAYER - Exceptions
|
|
65
|
+
// =============================================================================
|
|
66
|
+
|
|
67
|
+
export { ContentPolicyViolationError } from "./presentation/exceptions/content-policy-violation.exception";
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default Moderation Rules
|
|
3
|
+
* Built-in rules for content filtering - can be extended via config
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { ModerationRule } from "../../domain/entities/moderation.types";
|
|
7
|
+
|
|
8
|
+
const explicitContentRules: ModerationRule[] = [
|
|
9
|
+
{
|
|
10
|
+
id: "explicit-001",
|
|
11
|
+
name: "Sexual Content",
|
|
12
|
+
description: "Detects explicit sexual content and nudity",
|
|
13
|
+
contentTypes: ["text", "image"],
|
|
14
|
+
severity: "critical",
|
|
15
|
+
violationType: "explicit_content",
|
|
16
|
+
patterns: [
|
|
17
|
+
"\\bnude\\b",
|
|
18
|
+
"\\bnaked\\b",
|
|
19
|
+
"\\bnsfw\\b",
|
|
20
|
+
"\\bsexual\\b",
|
|
21
|
+
"\\berotic\\b",
|
|
22
|
+
"\\bporn\\b",
|
|
23
|
+
"\\bxxx\\b",
|
|
24
|
+
"\\badult content\\b",
|
|
25
|
+
"\\bexplicit\\b",
|
|
26
|
+
],
|
|
27
|
+
enabled: true,
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
id: "explicit-002",
|
|
31
|
+
name: "Body Parts Focus",
|
|
32
|
+
description: "Detects inappropriate body part references",
|
|
33
|
+
contentTypes: ["text", "image"],
|
|
34
|
+
severity: "high",
|
|
35
|
+
violationType: "explicit_content",
|
|
36
|
+
patterns: [
|
|
37
|
+
"\\bbreasts?\\b",
|
|
38
|
+
"\\bbutt\\b",
|
|
39
|
+
"\\bass\\b",
|
|
40
|
+
"\\bchest\\b.*\\b(exposed|revealing|bare)\\b",
|
|
41
|
+
"\\bcleavage\\b",
|
|
42
|
+
],
|
|
43
|
+
enabled: true,
|
|
44
|
+
},
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
const violenceRules: ModerationRule[] = [
|
|
48
|
+
{
|
|
49
|
+
id: "violence-001",
|
|
50
|
+
name: "Graphic Violence",
|
|
51
|
+
description: "Detects graphic violence and gore",
|
|
52
|
+
contentTypes: ["text", "image", "video"],
|
|
53
|
+
severity: "critical",
|
|
54
|
+
violationType: "violence",
|
|
55
|
+
patterns: [
|
|
56
|
+
"\\bgore\\b",
|
|
57
|
+
"\\bblood\\b.*\\b(splatter|dripping|pool)\\b",
|
|
58
|
+
"\\bmurder\\b",
|
|
59
|
+
"\\bkilling\\b",
|
|
60
|
+
"\\btorture\\b",
|
|
61
|
+
"\\bdismember\\b",
|
|
62
|
+
"\\bbeheading\\b",
|
|
63
|
+
],
|
|
64
|
+
enabled: true,
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
id: "violence-002",
|
|
68
|
+
name: "Weapons Focus",
|
|
69
|
+
description: "Detects inappropriate weapon usage context",
|
|
70
|
+
contentTypes: ["text", "image", "video"],
|
|
71
|
+
severity: "medium",
|
|
72
|
+
violationType: "violence",
|
|
73
|
+
patterns: [
|
|
74
|
+
"\\bgun\\b.*\\b(pointing|aimed|shooting)\\b",
|
|
75
|
+
"\\bweapon\\b.*\\b(attack|assault|harm)\\b",
|
|
76
|
+
"\\bknife\\b.*\\b(stabbing|cutting|slashing)\\b",
|
|
77
|
+
],
|
|
78
|
+
enabled: true,
|
|
79
|
+
},
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
const hateSpeechRules: ModerationRule[] = [
|
|
83
|
+
{
|
|
84
|
+
id: "hate-001",
|
|
85
|
+
name: "Discriminatory Language",
|
|
86
|
+
description: "Detects hate speech and discriminatory content",
|
|
87
|
+
contentTypes: ["text", "voice"],
|
|
88
|
+
severity: "critical",
|
|
89
|
+
violationType: "hate_speech",
|
|
90
|
+
patterns: [
|
|
91
|
+
"\\bhate\\b.*\\b(speech|crime|group)\\b",
|
|
92
|
+
"\\bracist\\b",
|
|
93
|
+
"\\bsexist\\b",
|
|
94
|
+
"\\bbigot\\b",
|
|
95
|
+
"\\bxenophobia\\b",
|
|
96
|
+
],
|
|
97
|
+
enabled: true,
|
|
98
|
+
},
|
|
99
|
+
];
|
|
100
|
+
|
|
101
|
+
const illegalActivityRules: ModerationRule[] = [
|
|
102
|
+
{
|
|
103
|
+
id: "illegal-001",
|
|
104
|
+
name: "Drug Content",
|
|
105
|
+
description: "Detects drug-related content",
|
|
106
|
+
contentTypes: ["text", "image", "video"],
|
|
107
|
+
severity: "high",
|
|
108
|
+
violationType: "illegal_activity",
|
|
109
|
+
patterns: [
|
|
110
|
+
"\\bdrug\\b.*\\b(dealing|trafficking|manufacturing)\\b",
|
|
111
|
+
"\\bcocaine\\b",
|
|
112
|
+
"\\bheroin\\b",
|
|
113
|
+
"\\bmethamphetamine\\b",
|
|
114
|
+
"\\billegal\\b.*\\b(substance|drug)\\b",
|
|
115
|
+
],
|
|
116
|
+
enabled: true,
|
|
117
|
+
},
|
|
118
|
+
];
|
|
119
|
+
|
|
120
|
+
const personalInfoRules: ModerationRule[] = [
|
|
121
|
+
{
|
|
122
|
+
id: "pii-001",
|
|
123
|
+
name: "Personal Information",
|
|
124
|
+
description: "Detects personal identifiable information",
|
|
125
|
+
contentTypes: ["text", "image"],
|
|
126
|
+
severity: "medium",
|
|
127
|
+
violationType: "personal_info",
|
|
128
|
+
patterns: [
|
|
129
|
+
"\\b\\d{3}-\\d{2}-\\d{4}\\b",
|
|
130
|
+
"\\b\\d{16}\\b",
|
|
131
|
+
"\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}\\b",
|
|
132
|
+
"\\b\\d{3}[-.]?\\d{3}[-.]?\\d{4}\\b",
|
|
133
|
+
],
|
|
134
|
+
enabled: true,
|
|
135
|
+
},
|
|
136
|
+
];
|
|
137
|
+
|
|
138
|
+
export const defaultModerationRules: ModerationRule[] = [
|
|
139
|
+
...explicitContentRules,
|
|
140
|
+
...violenceRules,
|
|
141
|
+
...hateSpeechRules,
|
|
142
|
+
...illegalActivityRules,
|
|
143
|
+
...personalInfoRules,
|
|
144
|
+
];
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rules Registry
|
|
3
|
+
* Manages moderation rules - allows adding/removing custom rules
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
ContentType,
|
|
8
|
+
ModerationRule,
|
|
9
|
+
} from "../../domain/entities/moderation.types";
|
|
10
|
+
import { defaultModerationRules } from "./default-rules.data";
|
|
11
|
+
|
|
12
|
+
class RulesRegistry {
|
|
13
|
+
private rules: ModerationRule[] = [...defaultModerationRules];
|
|
14
|
+
|
|
15
|
+
getRules(): ModerationRule[] {
|
|
16
|
+
return this.rules;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
getRulesByContentType(contentType: ContentType): ModerationRule[] {
|
|
20
|
+
return this.rules.filter(
|
|
21
|
+
(rule) => rule.enabled && rule.contentTypes.includes(contentType)
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
getRuleById(id: string): ModerationRule | undefined {
|
|
26
|
+
return this.rules.find((rule) => rule.id === id);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
addRule(rule: ModerationRule): void {
|
|
30
|
+
const existing = this.getRuleById(rule.id);
|
|
31
|
+
if (existing) {
|
|
32
|
+
this.updateRule(rule.id, rule);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
this.rules.push(rule);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
addRules(rules: ModerationRule[]): void {
|
|
39
|
+
rules.forEach((rule) => this.addRule(rule));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
updateRule(id: string, updates: Partial<ModerationRule>): boolean {
|
|
43
|
+
const index = this.rules.findIndex((rule) => rule.id === id);
|
|
44
|
+
if (index === -1) return false;
|
|
45
|
+
|
|
46
|
+
this.rules[index] = { ...this.rules[index], ...updates };
|
|
47
|
+
return true;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
removeRule(id: string): boolean {
|
|
51
|
+
const index = this.rules.findIndex((rule) => rule.id === id);
|
|
52
|
+
if (index === -1) return false;
|
|
53
|
+
|
|
54
|
+
this.rules.splice(index, 1);
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
enableRule(id: string): boolean {
|
|
59
|
+
return this.updateRule(id, { enabled: true });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
disableRule(id: string): boolean {
|
|
63
|
+
return this.updateRule(id, { enabled: false });
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
resetToDefaults(): void {
|
|
67
|
+
this.rules = [...defaultModerationRules];
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
clearAllRules(): void {
|
|
71
|
+
this.rules = [];
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export const rulesRegistry = new RulesRegistry();
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content Moderation Service
|
|
3
|
+
* Orchestrates content moderation by type
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
ContentType,
|
|
8
|
+
ModerationResult,
|
|
9
|
+
ModerationContext,
|
|
10
|
+
Violation,
|
|
11
|
+
SuggestionMessages,
|
|
12
|
+
ValidationLimits,
|
|
13
|
+
ModerationRule,
|
|
14
|
+
} from "../../domain/entities/moderation.types";
|
|
15
|
+
import { textModerator } from "./moderators/text.moderator";
|
|
16
|
+
import { imageModerator } from "./moderators/image.moderator";
|
|
17
|
+
import { videoModerator } from "./moderators/video.moderator";
|
|
18
|
+
import { voiceModerator } from "./moderators/voice.moderator";
|
|
19
|
+
import { rulesRegistry } from "../rules/rules-registry";
|
|
20
|
+
|
|
21
|
+
declare const __DEV__: boolean;
|
|
22
|
+
|
|
23
|
+
interface ServiceConfig {
|
|
24
|
+
suggestionMessages?: SuggestionMessages;
|
|
25
|
+
validationLimits?: ValidationLimits;
|
|
26
|
+
customRules?: ModerationRule[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
class ContentModerationService {
|
|
30
|
+
configure(config: ServiceConfig): void {
|
|
31
|
+
if (typeof __DEV__ !== "undefined" && __DEV__) {
|
|
32
|
+
// eslint-disable-next-line no-console
|
|
33
|
+
console.log("[ContentModeration] Configure:", {
|
|
34
|
+
hasSuggestionMessages: !!config.suggestionMessages,
|
|
35
|
+
hasValidationLimits: !!config.validationLimits,
|
|
36
|
+
customRulesCount: config.customRules?.length ?? 0,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (config.suggestionMessages) {
|
|
41
|
+
textModerator.setSuggestionMessages(config.suggestionMessages);
|
|
42
|
+
voiceModerator.setSuggestionMessages(config.suggestionMessages);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
if (config.validationLimits) {
|
|
46
|
+
const { maxTextLength, maxVoiceTextLength, maxUriLength } =
|
|
47
|
+
config.validationLimits;
|
|
48
|
+
|
|
49
|
+
if (maxTextLength) textModerator.setMaxLength(maxTextLength);
|
|
50
|
+
if (maxVoiceTextLength) voiceModerator.setMaxLength(maxVoiceTextLength);
|
|
51
|
+
if (maxUriLength) {
|
|
52
|
+
imageModerator.setMaxUriLength(maxUriLength);
|
|
53
|
+
videoModerator.setMaxUriLength(maxUriLength);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (config.customRules) {
|
|
58
|
+
rulesRegistry.addRules(config.customRules);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async moderate(context: ModerationContext): Promise<ModerationResult> {
|
|
63
|
+
if (typeof __DEV__ !== "undefined" && __DEV__) {
|
|
64
|
+
// eslint-disable-next-line no-console
|
|
65
|
+
console.log("[ContentModeration] Moderate started:", {
|
|
66
|
+
contentType: context.contentType,
|
|
67
|
+
contentLength: context.content?.length ?? 0,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const result = this.moderateByType(context.contentType, context.content);
|
|
72
|
+
const moderationResult: ModerationResult = {
|
|
73
|
+
isAllowed: result.isAllowed,
|
|
74
|
+
violations: result.violations,
|
|
75
|
+
confidence: this.calculateConfidence(result.violations),
|
|
76
|
+
suggestedAction: this.determineAction(result.violations),
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
if (typeof __DEV__ !== "undefined" && __DEV__) {
|
|
80
|
+
// eslint-disable-next-line no-console
|
|
81
|
+
console.log("[ContentModeration] Moderate completed:", {
|
|
82
|
+
contentType: context.contentType,
|
|
83
|
+
isAllowed: moderationResult.isAllowed,
|
|
84
|
+
violationsCount: moderationResult.violations?.length ?? 0,
|
|
85
|
+
suggestedAction: moderationResult.suggestedAction,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return moderationResult;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async isContentAllowed(
|
|
93
|
+
content: string,
|
|
94
|
+
contentType: ContentType
|
|
95
|
+
): Promise<boolean> {
|
|
96
|
+
const result = this.moderateByType(contentType, content);
|
|
97
|
+
return result.isAllowed;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
moderateSync(
|
|
101
|
+
content: string,
|
|
102
|
+
contentType: ContentType
|
|
103
|
+
): { isAllowed: boolean; violations: Violation[] } {
|
|
104
|
+
return this.moderateByType(contentType, content);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private moderateByType(
|
|
108
|
+
contentType: ContentType,
|
|
109
|
+
content: string
|
|
110
|
+
): { isAllowed: boolean; violations: Violation[] } {
|
|
111
|
+
switch (contentType) {
|
|
112
|
+
case "text":
|
|
113
|
+
return textModerator.moderate(content);
|
|
114
|
+
case "image":
|
|
115
|
+
return imageModerator.moderate(content);
|
|
116
|
+
case "video":
|
|
117
|
+
return videoModerator.moderate(content);
|
|
118
|
+
case "voice":
|
|
119
|
+
return voiceModerator.moderate(content);
|
|
120
|
+
default:
|
|
121
|
+
return { isAllowed: true, violations: [] };
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
private calculateConfidence(violations: Violation[]): number {
|
|
126
|
+
if (violations.length === 0) return 1.0;
|
|
127
|
+
|
|
128
|
+
const weights = { critical: 1.0, high: 0.75, medium: 0.5, low: 0.25 };
|
|
129
|
+
const score = violations.reduce(
|
|
130
|
+
(sum, v) => sum + (weights[v.severity] || 0.25),
|
|
131
|
+
0
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
return Math.min(1.0, score / 2);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
private determineAction(
|
|
138
|
+
violations: Violation[]
|
|
139
|
+
): "allow" | "warn" | "block" {
|
|
140
|
+
if (violations.length === 0) return "allow";
|
|
141
|
+
|
|
142
|
+
const hasCritical = violations.some((v) => v.severity === "critical");
|
|
143
|
+
const hasHigh = violations.some((v) => v.severity === "high");
|
|
144
|
+
|
|
145
|
+
if (hasCritical || hasHigh) return "block";
|
|
146
|
+
return "warn";
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export const contentModerationService = new ContentModerationService();
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Base Moderator
|
|
3
|
+
* Shared functionality for all content moderators
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
Violation,
|
|
8
|
+
ViolationType,
|
|
9
|
+
SuggestionMessages,
|
|
10
|
+
} from "../../../domain/entities/moderation.types";
|
|
11
|
+
|
|
12
|
+
export interface ModerationResult {
|
|
13
|
+
isAllowed: boolean;
|
|
14
|
+
violations: Violation[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const DEFAULT_SUGGESTIONS: Record<string, string> = {
|
|
18
|
+
explicit_content: "Remove explicit content",
|
|
19
|
+
violence: "Remove violent content",
|
|
20
|
+
hate_speech: "Remove discriminatory language",
|
|
21
|
+
harassment: "Remove harassing content",
|
|
22
|
+
illegal_activity: "Remove illegal content references",
|
|
23
|
+
spam: "Remove spam content",
|
|
24
|
+
copyright: "Remove copyrighted content",
|
|
25
|
+
personal_info: "Remove personal information",
|
|
26
|
+
dangerous_content: "Remove dangerous content",
|
|
27
|
+
default: "Modify content to comply with guidelines",
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export abstract class BaseModerator {
|
|
31
|
+
protected customSuggestions?: SuggestionMessages;
|
|
32
|
+
|
|
33
|
+
setSuggestionMessages(messages: SuggestionMessages): void {
|
|
34
|
+
this.customSuggestions = messages;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
protected getSuggestion(type: ViolationType): string {
|
|
38
|
+
if (this.customSuggestions?.[type]) {
|
|
39
|
+
return this.customSuggestions[type] as string;
|
|
40
|
+
}
|
|
41
|
+
return DEFAULT_SUGGESTIONS[type] || DEFAULT_SUGGESTIONS.default;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
protected createViolation(
|
|
45
|
+
id: string,
|
|
46
|
+
ruleName: string,
|
|
47
|
+
message: string,
|
|
48
|
+
violationType: ViolationType = "illegal_activity"
|
|
49
|
+
): Violation {
|
|
50
|
+
return {
|
|
51
|
+
ruleId: id,
|
|
52
|
+
ruleName,
|
|
53
|
+
violationType,
|
|
54
|
+
severity: "high",
|
|
55
|
+
matchedPattern: "validation",
|
|
56
|
+
context: "",
|
|
57
|
+
suggestion: message,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
abstract moderate(content: string): ModerationResult;
|
|
62
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Image Content Moderator
|
|
3
|
+
* Validates and moderates image URIs
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { Violation } from "../../../domain/entities/moderation.types";
|
|
7
|
+
import { BaseModerator, type ModerationResult } from "./base.moderator";
|
|
8
|
+
|
|
9
|
+
const DEFAULT_PROTOCOLS = ["http:", "https:", "file:", "data:"];
|
|
10
|
+
const DEFAULT_MAX_URI_LENGTH = 2048;
|
|
11
|
+
|
|
12
|
+
class ImageModerator extends BaseModerator {
|
|
13
|
+
private allowedProtocols = DEFAULT_PROTOCOLS;
|
|
14
|
+
private maxUriLength = DEFAULT_MAX_URI_LENGTH;
|
|
15
|
+
|
|
16
|
+
setAllowedProtocols(protocols: string[]): void {
|
|
17
|
+
this.allowedProtocols = protocols;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
setMaxUriLength(length: number): void {
|
|
21
|
+
this.maxUriLength = length;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
moderate(imageUri: string): ModerationResult {
|
|
25
|
+
const validationError = this.validate(imageUri);
|
|
26
|
+
if (validationError) {
|
|
27
|
+
return { isAllowed: false, violations: [validationError] };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return { isAllowed: true, violations: [] };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
private validate(uri: string): Violation | null {
|
|
34
|
+
if (!uri || typeof uri !== "string") {
|
|
35
|
+
return this.createViolation("empty-uri", "Image Validation", "empty URI");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (uri.length > this.maxUriLength) {
|
|
39
|
+
return this.createViolation(
|
|
40
|
+
"uri-too-long",
|
|
41
|
+
"Image Validation",
|
|
42
|
+
"URI too long"
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (!this.hasValidProtocol(uri)) {
|
|
47
|
+
return this.createViolation(
|
|
48
|
+
"invalid-protocol",
|
|
49
|
+
"Image Validation",
|
|
50
|
+
"invalid protocol"
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
private hasValidProtocol(uri: string): boolean {
|
|
58
|
+
return this.allowedProtocols.some((protocol) =>
|
|
59
|
+
uri.toLowerCase().startsWith(protocol)
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export const imageModerator = new ImageModerator();
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Moderators Index
|
|
3
|
+
* Exports all content moderators
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export { textModerator } from "./text.moderator";
|
|
7
|
+
export { imageModerator } from "./image.moderator";
|
|
8
|
+
export { videoModerator } from "./video.moderator";
|
|
9
|
+
export { voiceModerator } from "./voice.moderator";
|
|
10
|
+
export { BaseModerator, type ModerationResult } from "./base.moderator";
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text Content Moderator
|
|
3
|
+
* Validates and moderates text content
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { Violation } from "../../../domain/entities/moderation.types";
|
|
7
|
+
import { patternMatcherService } from "../pattern-matcher.service";
|
|
8
|
+
import { rulesRegistry } from "../../rules/rules-registry";
|
|
9
|
+
import { BaseModerator, type ModerationResult } from "./base.moderator";
|
|
10
|
+
|
|
11
|
+
declare const __DEV__: boolean;
|
|
12
|
+
|
|
13
|
+
const DEFAULT_MAX_LENGTH = 10000;
|
|
14
|
+
|
|
15
|
+
const MALICIOUS_CODE_PATTERNS = [
|
|
16
|
+
/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,
|
|
17
|
+
/javascript:/gi,
|
|
18
|
+
/on\w+\s*=/gi,
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
const PROMPT_INJECTION_PATTERNS = [
|
|
22
|
+
/ignore\s+(all\s+)?(previous|prior|above)\s+(instructions?|prompts?|rules?)/gi,
|
|
23
|
+
/disregard\s+(all\s+)?(previous|prior|above)\s+(instructions?|prompts?)/gi,
|
|
24
|
+
/forget\s+(all\s+)?(previous|prior|your)\s+(instructions?|prompts?|rules?)/gi,
|
|
25
|
+
/you\s+are\s+now\s+(a|an)\s+/gi,
|
|
26
|
+
/act\s+as\s+(if|though)\s+you/gi,
|
|
27
|
+
/pretend\s+(you\s+are|to\s+be)/gi,
|
|
28
|
+
/bypass\s+(your\s+)?(safety|content|moderation)/gi,
|
|
29
|
+
/override\s+(your\s+)?(restrictions?|limitations?|rules?)/gi,
|
|
30
|
+
/jailbreak/gi,
|
|
31
|
+
/DAN\s*mode/gi,
|
|
32
|
+
/developer\s+mode\s+(enabled|on|activated)/gi,
|
|
33
|
+
/system\s*:\s*/gi,
|
|
34
|
+
/\[system\]/gi,
|
|
35
|
+
/<<\s*sys\s*>>/gi,
|
|
36
|
+
];
|
|
37
|
+
|
|
38
|
+
class TextModerator extends BaseModerator {
|
|
39
|
+
private maxLength = DEFAULT_MAX_LENGTH;
|
|
40
|
+
|
|
41
|
+
setMaxLength(length: number): void {
|
|
42
|
+
this.maxLength = length;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
moderate(content: string): ModerationResult {
|
|
46
|
+
if (typeof __DEV__ !== "undefined" && __DEV__) {
|
|
47
|
+
// eslint-disable-next-line no-console
|
|
48
|
+
console.log("[TextModerator] moderate() called", {
|
|
49
|
+
contentLength: content?.length ?? 0,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const validationError = this.validate(content);
|
|
54
|
+
if (validationError) {
|
|
55
|
+
if (typeof __DEV__ !== "undefined" && __DEV__) {
|
|
56
|
+
// eslint-disable-next-line no-console
|
|
57
|
+
console.log("[TextModerator] validation failed", {
|
|
58
|
+
ruleId: validationError.ruleId,
|
|
59
|
+
violationType: validationError.violationType,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
return { isAllowed: false, violations: [validationError] };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const violations = this.evaluateRules(content);
|
|
66
|
+
|
|
67
|
+
if (typeof __DEV__ !== "undefined" && __DEV__) {
|
|
68
|
+
// eslint-disable-next-line no-console
|
|
69
|
+
console.log("[TextModerator] moderate() completed", {
|
|
70
|
+
isAllowed: violations.length === 0,
|
|
71
|
+
violationsCount: violations.length,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return { isAllowed: violations.length === 0, violations };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private validate(content: string): Violation | null {
|
|
79
|
+
if (!content || typeof content !== "string") {
|
|
80
|
+
return this.createViolation("empty-content", "Validation", "empty");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (content.length > this.maxLength) {
|
|
84
|
+
return this.createViolation("too-long", "Validation", "length exceeded");
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (this.containsMaliciousCode(content)) {
|
|
88
|
+
return this.createViolation(
|
|
89
|
+
"malicious",
|
|
90
|
+
"Security",
|
|
91
|
+
"malicious code",
|
|
92
|
+
"dangerous_content"
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (this.containsPromptInjection(content)) {
|
|
97
|
+
return this.createViolation(
|
|
98
|
+
"prompt-injection",
|
|
99
|
+
"Security",
|
|
100
|
+
"prompt injection",
|
|
101
|
+
"dangerous_content"
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private containsMaliciousCode(content: string): boolean {
|
|
109
|
+
return MALICIOUS_CODE_PATTERNS.some((pattern) => pattern.test(content));
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
private containsPromptInjection(content: string): boolean {
|
|
113
|
+
return PROMPT_INJECTION_PATTERNS.some((pattern) => pattern.test(content));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
private evaluateRules(content: string): Violation[] {
|
|
117
|
+
const rules = rulesRegistry.getRulesByContentType("text");
|
|
118
|
+
const violations: Violation[] = [];
|
|
119
|
+
|
|
120
|
+
for (const rule of rules) {
|
|
121
|
+
const matches = patternMatcherService.matchAnyPattern(
|
|
122
|
+
content,
|
|
123
|
+
rule.patterns
|
|
124
|
+
);
|
|
125
|
+
const matched = matches.find((m) => m.matched);
|
|
126
|
+
|
|
127
|
+
if (matched) {
|
|
128
|
+
violations.push({
|
|
129
|
+
ruleId: rule.id,
|
|
130
|
+
ruleName: rule.name,
|
|
131
|
+
violationType: rule.violationType,
|
|
132
|
+
severity: rule.severity,
|
|
133
|
+
matchedPattern: matched.matchedText || "",
|
|
134
|
+
context: content.slice(0, 100),
|
|
135
|
+
suggestion: this.getSuggestion(rule.violationType),
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return violations;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export const textModerator = new TextModerator();
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Video Content Moderator
|
|
3
|
+
* Validates and moderates video URIs
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { Violation } from "../../../domain/entities/moderation.types";
|
|
7
|
+
import { BaseModerator, type ModerationResult } from "./base.moderator";
|
|
8
|
+
|
|
9
|
+
const DEFAULT_PROTOCOLS = ["http:", "https:", "file:"];
|
|
10
|
+
const DEFAULT_MAX_URI_LENGTH = 2048;
|
|
11
|
+
|
|
12
|
+
class VideoModerator extends BaseModerator {
|
|
13
|
+
private allowedProtocols = DEFAULT_PROTOCOLS;
|
|
14
|
+
private maxUriLength = DEFAULT_MAX_URI_LENGTH;
|
|
15
|
+
|
|
16
|
+
setAllowedProtocols(protocols: string[]): void {
|
|
17
|
+
this.allowedProtocols = protocols;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
setMaxUriLength(length: number): void {
|
|
21
|
+
this.maxUriLength = length;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
moderate(videoUri: string): ModerationResult {
|
|
25
|
+
const validationError = this.validate(videoUri);
|
|
26
|
+
if (validationError) {
|
|
27
|
+
return { isAllowed: false, violations: [validationError] };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return { isAllowed: true, violations: [] };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
private validate(uri: string): Violation | null {
|
|
34
|
+
if (!uri || typeof uri !== "string") {
|
|
35
|
+
return this.createViolation("empty-uri", "Video Validation", "empty URI");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (uri.length > this.maxUriLength) {
|
|
39
|
+
return this.createViolation(
|
|
40
|
+
"uri-too-long",
|
|
41
|
+
"Video Validation",
|
|
42
|
+
"URI too long"
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (!this.hasValidProtocol(uri)) {
|
|
47
|
+
return this.createViolation(
|
|
48
|
+
"invalid-protocol",
|
|
49
|
+
"Video Validation",
|
|
50
|
+
"invalid protocol"
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
private hasValidProtocol(uri: string): boolean {
|
|
58
|
+
return this.allowedProtocols.some((protocol) =>
|
|
59
|
+
uri.toLowerCase().startsWith(protocol)
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export const videoModerator = new VideoModerator();
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Voice Content Moderator
|
|
3
|
+
* Validates and moderates voice/TTS text content
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { Violation } from "../../../domain/entities/moderation.types";
|
|
7
|
+
import { patternMatcherService } from "../pattern-matcher.service";
|
|
8
|
+
import { rulesRegistry } from "../../rules/rules-registry";
|
|
9
|
+
import { BaseModerator, type ModerationResult } from "./base.moderator";
|
|
10
|
+
|
|
11
|
+
const DEFAULT_MAX_LENGTH = 5000;
|
|
12
|
+
|
|
13
|
+
class VoiceModerator extends BaseModerator {
|
|
14
|
+
private maxLength = DEFAULT_MAX_LENGTH;
|
|
15
|
+
|
|
16
|
+
setMaxLength(length: number): void {
|
|
17
|
+
this.maxLength = length;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
moderate(text: string): ModerationResult {
|
|
21
|
+
const validationError = this.validate(text);
|
|
22
|
+
if (validationError) {
|
|
23
|
+
return { isAllowed: false, violations: [validationError] };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const violations = this.evaluateRules(text);
|
|
27
|
+
return { isAllowed: violations.length === 0, violations };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
private validate(text: string): Violation | null {
|
|
31
|
+
if (!text || typeof text !== "string") {
|
|
32
|
+
return this.createViolation("empty-text", "Voice Validation", "empty");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (text.length > this.maxLength) {
|
|
36
|
+
return this.createViolation(
|
|
37
|
+
"too-long",
|
|
38
|
+
"Voice Validation",
|
|
39
|
+
"length exceeded"
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
private evaluateRules(text: string): Violation[] {
|
|
47
|
+
const rules = rulesRegistry.getRulesByContentType("voice");
|
|
48
|
+
const violations: Violation[] = [];
|
|
49
|
+
|
|
50
|
+
for (const rule of rules) {
|
|
51
|
+
const matches = patternMatcherService.matchAnyPattern(
|
|
52
|
+
text,
|
|
53
|
+
rule.patterns
|
|
54
|
+
);
|
|
55
|
+
const matched = matches.find((m) => m.matched);
|
|
56
|
+
|
|
57
|
+
if (matched) {
|
|
58
|
+
violations.push({
|
|
59
|
+
ruleId: rule.id,
|
|
60
|
+
ruleName: rule.name,
|
|
61
|
+
violationType: rule.violationType,
|
|
62
|
+
severity: rule.severity,
|
|
63
|
+
matchedPattern: matched.matchedText || "",
|
|
64
|
+
context: text.slice(0, 100),
|
|
65
|
+
suggestion: this.getSuggestion(rule.violationType),
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return violations;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export const voiceModerator = new VoiceModerator();
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pattern Matcher Service
|
|
3
|
+
* Utility service for regex pattern matching
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export interface PatternMatch {
|
|
7
|
+
pattern: string;
|
|
8
|
+
matched: boolean;
|
|
9
|
+
matchedText?: string;
|
|
10
|
+
position?: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
class PatternMatcherService {
|
|
14
|
+
matchPattern(content: string, pattern: string): PatternMatch {
|
|
15
|
+
try {
|
|
16
|
+
const regex = new RegExp(pattern, "gi");
|
|
17
|
+
const match = regex.exec(content);
|
|
18
|
+
|
|
19
|
+
if (match) {
|
|
20
|
+
return {
|
|
21
|
+
pattern,
|
|
22
|
+
matched: true,
|
|
23
|
+
matchedText: match[0],
|
|
24
|
+
position: match.index,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return { pattern, matched: false };
|
|
29
|
+
} catch {
|
|
30
|
+
return { pattern, matched: false };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
matchAnyPattern(content: string, patterns: string[]): PatternMatch[] {
|
|
35
|
+
return patterns.map((pattern) => this.matchPattern(content, pattern));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
matchAllPatterns(content: string, patterns: string[]): boolean {
|
|
39
|
+
return patterns.every(
|
|
40
|
+
(pattern) => this.matchPattern(content, pattern).matched
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
hasAnyMatch(content: string, patterns: string[]): boolean {
|
|
45
|
+
return patterns.some(
|
|
46
|
+
(pattern) => this.matchPattern(content, pattern).matched
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export const patternMatcherService = new PatternMatcherService();
|
package/src/domains/content-moderation/presentation/exceptions/content-policy-violation.exception.ts
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Content Policy Violation Exception
|
|
3
|
+
* Custom error class for content policy violations
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
Violation,
|
|
8
|
+
ViolationType,
|
|
9
|
+
} from "../../domain/entities/moderation.types";
|
|
10
|
+
|
|
11
|
+
export class ContentPolicyViolationError extends Error {
|
|
12
|
+
public readonly violations: Violation[];
|
|
13
|
+
|
|
14
|
+
constructor(violations: Violation[], message?: string) {
|
|
15
|
+
const defaultMessage = `Content policy violation: ${violations
|
|
16
|
+
.map((v) => v.ruleName)
|
|
17
|
+
.join(", ")}`;
|
|
18
|
+
super(message || defaultMessage);
|
|
19
|
+
this.name = "ContentPolicyViolationError";
|
|
20
|
+
this.violations = violations;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
getUserMessage(): string {
|
|
24
|
+
if (this.violations.length === 0) {
|
|
25
|
+
return "Content policy violation detected.";
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const firstViolation = this.violations[0];
|
|
29
|
+
return firstViolation?.suggestion || "Please modify your content.";
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
getViolationTypes(): ViolationType[] {
|
|
33
|
+
return this.violations.map((v) => v.violationType);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
hasViolationType(type: ViolationType): boolean {
|
|
37
|
+
return this.violations.some((v) => v.violationType === type);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
getSeverityLevel(): "low" | "medium" | "high" | "critical" {
|
|
41
|
+
const severities = this.violations.map((v) => v.severity);
|
|
42
|
+
|
|
43
|
+
if (severities.includes("critical")) return "critical";
|
|
44
|
+
if (severities.includes("high")) return "high";
|
|
45
|
+
if (severities.includes("medium")) return "medium";
|
|
46
|
+
return "low";
|
|
47
|
+
}
|
|
48
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -227,3 +227,9 @@ export type {
|
|
|
227
227
|
// =============================================================================
|
|
228
228
|
|
|
229
229
|
export * from "./domains/prompts";
|
|
230
|
+
|
|
231
|
+
// =============================================================================
|
|
232
|
+
// DOMAINS - Content Moderation
|
|
233
|
+
// =============================================================================
|
|
234
|
+
|
|
235
|
+
export * from "./domains/content-moderation";
|