@voltro/plugin-moderation 0.85.0 → 0.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5987 -0
- package/THIRD-PARTY-NOTICES.md +4723 -519
- package/dist/index.js +41 -86
- package/dist/providers.js +51 -0
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,99 +1,54 @@
|
|
|
1
1
|
import { ContentRejected as e } from "./errors.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
//#region src/
|
|
6
|
-
var
|
|
7
|
-
let t = (e) => e !== void 0 && /[\p{L}\p{N}_]/u.test(e), n = t(e[0]) ? "\\b" : "", r = t(e[e.length - 1]) ? "\\b" : "";
|
|
8
|
-
return RegExp(`${n}${e.replace(l, "\\$&")}${r}`, "u");
|
|
9
|
-
}, d = (e) => {
|
|
10
|
-
let t = e.filter((e) => e.length > 0).map((e) => ({
|
|
11
|
-
term: e.toLowerCase(),
|
|
12
|
-
re: u(e.toLowerCase())
|
|
13
|
-
}));
|
|
14
|
-
return (e) => n.sync(() => {
|
|
15
|
-
let n = e.toLowerCase(), r = t.filter((e) => e.re.test(n)).map((e) => e.term);
|
|
16
|
-
return r.length > 0 ? {
|
|
17
|
-
flagged: !0,
|
|
18
|
-
categories: r,
|
|
19
|
-
reason: `matched denied term(s): ${r.join(", ")}`
|
|
20
|
-
} : c;
|
|
21
|
-
});
|
|
22
|
-
}, f = (e = {}) => {
|
|
23
|
-
let t = e.threshold ?? .7, r = e.categories ?? [
|
|
24
|
-
"hate",
|
|
25
|
-
"harassment",
|
|
26
|
-
"sexual",
|
|
27
|
-
"violence",
|
|
28
|
-
"self-harm",
|
|
29
|
-
"illicit"
|
|
30
|
-
], i = e.load ?? (() => import("@voltro/ai")), a = !1;
|
|
31
|
-
return (e) => n.gen(function* () {
|
|
32
|
-
let o = yield* n.tryPromise(i).pipe(n.catch(() => n.succeed(null)));
|
|
33
|
-
return o === null ? (a || (a = !0, s.warn("aiProvider configured but @voltro/ai could not be loaded — content is NOT being moderated; install @voltro/ai")), c) : yield* n.tryPromise(async () => {
|
|
34
|
-
let { Schema: i } = await import("effect"), a = i.Struct({
|
|
35
|
-
flagged: i.Boolean,
|
|
36
|
-
topCategory: i.NullOr(i.String),
|
|
37
|
-
score: i.Number
|
|
38
|
-
}), { object: s } = await n.runPromise(o.generateObject({
|
|
39
|
-
schema: a,
|
|
40
|
-
system: `You are a content moderator. Categories: ${r.join(", ")}. Return whether the content violates policy, the worst category, and a 0..1 severity score.`,
|
|
41
|
-
prompt: e
|
|
42
|
-
})), c = s.flagged || s.score >= t;
|
|
43
|
-
return {
|
|
44
|
-
flagged: c,
|
|
45
|
-
score: s.score,
|
|
46
|
-
...s.topCategory ? { categories: [s.topCategory] } : {},
|
|
47
|
-
...c ? { reason: `ai moderation: ${s.topCategory ?? "policy"} (${s.score.toFixed(2)})` } : {}
|
|
48
|
-
};
|
|
49
|
-
}).pipe(n.catch(() => n.succeed(c)));
|
|
50
|
-
});
|
|
51
|
-
}, p = class extends t.Service()("@voltro/plugin-moderation/ModerationService") {}, m = (e) => n.flatMap(p, (t) => t.moderate(e)), h = (e, t) => {
|
|
2
|
+
import { aiProvider as t, keywordProvider as n } from "./providers.js";
|
|
3
|
+
import { Context as r, Effect as i, Layer as a } from "effect";
|
|
4
|
+
import { definePlugin as o, pluginInstanceName as s } from "@voltro/protocol";
|
|
5
|
+
//#region src/index.ts
|
|
6
|
+
var c = class extends r.Service()("@voltro/plugin-moderation/ModerationService") {}, l = (e) => i.flatMap(c, (t) => t.moderate(e)), u = (e, t) => {
|
|
52
7
|
if (typeof e != "object" || !e) return "";
|
|
53
8
|
let n = e;
|
|
54
9
|
return t.map((e) => n[e]).filter((e) => typeof e == "string").join("\n");
|
|
55
|
-
},
|
|
56
|
-
let
|
|
57
|
-
base:
|
|
10
|
+
}, d = "@voltro/plugin-moderation", f = (t) => {
|
|
11
|
+
let n = s({
|
|
12
|
+
base: d,
|
|
58
13
|
alias: t.alias,
|
|
59
14
|
instance: t.name
|
|
60
|
-
}),
|
|
15
|
+
}), r = t.rules.map((e) => {
|
|
61
16
|
let t = e.match;
|
|
62
17
|
if (typeof t == "string") return (e) => t === e;
|
|
63
18
|
let n = t.global || t.sticky ? new RegExp(t.source, t.flags.replace(/[gy]/g, "")) : t;
|
|
64
19
|
return (e) => n.test(e);
|
|
65
|
-
}),
|
|
66
|
-
|
|
67
|
-
id: `mod_${Date.now().toString(36)}_${
|
|
20
|
+
}), l = (e) => t.rules.filter((t, n) => r[n](e)), f = t.onFlag, p = () => void 0, m = [], h = 0, g = 0, _ = 0, v = (e, t, n) => {
|
|
21
|
+
m.unshift({
|
|
22
|
+
id: `mod_${Date.now().toString(36)}_${h++}`,
|
|
68
23
|
tag: e,
|
|
69
24
|
categories: n.categories ?? [],
|
|
70
25
|
reason: n.reason ?? "content policy violation",
|
|
71
26
|
subjectId: t,
|
|
72
27
|
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
73
28
|
status: "pending"
|
|
74
|
-
}),
|
|
75
|
-
}, y = (
|
|
76
|
-
let a =
|
|
77
|
-
if (a.length === 0) return
|
|
78
|
-
let o =
|
|
79
|
-
return
|
|
80
|
-
for (let
|
|
81
|
-
let a =
|
|
29
|
+
}), m.length > 200 && (m.length = 200);
|
|
30
|
+
}, y = (n, r) => {
|
|
31
|
+
let a = l(r.tag);
|
|
32
|
+
if (a.length === 0) return n;
|
|
33
|
+
let o = r.subject?.id ?? null;
|
|
34
|
+
return i.gen(function* () {
|
|
35
|
+
for (let n of a) {
|
|
36
|
+
let a = u(r.input, n.fields);
|
|
82
37
|
if (a.length === 0) continue;
|
|
83
38
|
let s = yield* t.provider(a);
|
|
84
39
|
if (!s.flagged) continue;
|
|
85
|
-
let c = s.categories ?? [],
|
|
86
|
-
if ((
|
|
87
|
-
tag:
|
|
40
|
+
let c = s.categories ?? [], l = s.reason ?? "content policy violation";
|
|
41
|
+
if ((n.action ?? "block") === "block") return g++, v(r.tag, o, s), yield* i.fail(new e({
|
|
42
|
+
tag: r.tag,
|
|
88
43
|
categories: c,
|
|
89
|
-
reason:
|
|
44
|
+
reason: l
|
|
90
45
|
}));
|
|
91
|
-
_++, v(
|
|
92
|
-
tag:
|
|
46
|
+
_++, v(r.tag, o, s), p(r.tag, s), f?.({
|
|
47
|
+
tag: r.tag,
|
|
93
48
|
verdict: s
|
|
94
49
|
});
|
|
95
50
|
}
|
|
96
|
-
return yield*
|
|
51
|
+
return yield* n;
|
|
97
52
|
});
|
|
98
53
|
}, b = (e) => ({
|
|
99
54
|
kind: "json",
|
|
@@ -107,19 +62,19 @@ var s = o({ scope: "@voltro/plugin-moderation" }), c = { flagged: !1 }, l = /[.*
|
|
|
107
62
|
path: "/flagged",
|
|
108
63
|
scope: "process",
|
|
109
64
|
description: "The content-moderation review queue (newest first).",
|
|
110
|
-
handler: () =>
|
|
111
|
-
items:
|
|
65
|
+
handler: () => i.succeed(b({
|
|
66
|
+
items: m.slice(),
|
|
112
67
|
stats: {
|
|
113
|
-
blocked:
|
|
68
|
+
blocked: g,
|
|
114
69
|
flagged: _,
|
|
115
|
-
pending:
|
|
70
|
+
pending: m.filter((e) => e.status === "pending").length
|
|
116
71
|
}
|
|
117
72
|
}))
|
|
118
73
|
}, {
|
|
119
74
|
method: "POST",
|
|
120
75
|
path: "/resolve",
|
|
121
76
|
description: "Resolve a flagged item — confirm (real violation) or dismiss (false positive).",
|
|
122
|
-
handler: (e) =>
|
|
77
|
+
handler: (e) => i.sync(() => {
|
|
123
78
|
let t;
|
|
124
79
|
try {
|
|
125
80
|
t = JSON.parse(e.body || "{}");
|
|
@@ -128,7 +83,7 @@ var s = o({ scope: "@voltro/plugin-moderation" }), c = { flagged: !1 }, l = /[.*
|
|
|
128
83
|
}
|
|
129
84
|
if (typeof t != "object" || !t || Array.isArray(t)) return x(400, "expected a JSON object");
|
|
130
85
|
if (!("id" in t) || typeof t.id != "string" || !t.id.trim() || !("action" in t) || t.action !== "confirm" && t.action !== "dismiss") return x(400, "id + action (confirm|dismiss) required");
|
|
131
|
-
let n =
|
|
86
|
+
let n = m.find((e) => e.id === t.id);
|
|
132
87
|
return n ? n.status === "pending" ? (n.status = t.action === "confirm" ? "confirmed" : "dismissed", b({
|
|
133
88
|
ok: !0,
|
|
134
89
|
item: n
|
|
@@ -140,10 +95,10 @@ var s = o({ scope: "@voltro/plugin-moderation" }), c = { flagged: !1 }, l = /[.*
|
|
|
140
95
|
"rpc:intercept:workflow",
|
|
141
96
|
"inspect:read",
|
|
142
97
|
"inspect:write"
|
|
143
|
-
], w =
|
|
144
|
-
return
|
|
145
|
-
name:
|
|
146
|
-
baseName:
|
|
98
|
+
], w = a.succeed(c, { moderate: (e) => t.provider(e) });
|
|
99
|
+
return o({
|
|
100
|
+
name: n,
|
|
101
|
+
baseName: d,
|
|
147
102
|
description: "Content moderation — block or flag user-content writes via an AI / keyword / custom check.",
|
|
148
103
|
permissions: C,
|
|
149
104
|
errorSchemas: [{
|
|
@@ -158,14 +113,14 @@ var s = o({ scope: "@voltro/plugin-moderation" }), c = { flagged: !1 }, l = /[.*
|
|
|
158
113
|
interceptWorkflow: y,
|
|
159
114
|
inspectEndpoints: S,
|
|
160
115
|
services: w,
|
|
161
|
-
onActivate: (e) =>
|
|
162
|
-
|
|
116
|
+
onActivate: (e) => i.sync(() => {
|
|
117
|
+
p = (t, n) => e.logger.warn("content flagged", {
|
|
163
118
|
tag: t,
|
|
164
119
|
categories: n.categories,
|
|
165
120
|
score: n.score
|
|
166
|
-
}),
|
|
121
|
+
}), f = t.onFlag, e.logger.info("moderation active", { rules: t.rules.length });
|
|
167
122
|
})
|
|
168
123
|
});
|
|
169
124
|
};
|
|
170
125
|
//#endregion
|
|
171
|
-
export { e as ContentRejected,
|
|
126
|
+
export { e as ContentRejected, c as ModerationService, t as aiProvider, n as keywordProvider, l as moderate, f as moderationPlugin };
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { Effect as e } from "effect";
|
|
2
|
+
import { createLogger as t } from "@voltro/logger";
|
|
3
|
+
//#region src/providers.ts
|
|
4
|
+
var n = t({ scope: "@voltro/plugin-moderation" }), r = { flagged: !1 }, i = /[.*+?^${}()|[\]\\]/g, a = (e) => {
|
|
5
|
+
let t = (e) => e !== void 0 && /[\p{L}\p{N}_]/u.test(e), n = t(e[0]) ? "\\b" : "", r = t(e[e.length - 1]) ? "\\b" : "";
|
|
6
|
+
return RegExp(`${n}${e.replace(i, "\\$&")}${r}`, "u");
|
|
7
|
+
}, o = (t) => {
|
|
8
|
+
let n = t.filter((e) => e.length > 0).map((e) => ({
|
|
9
|
+
term: e.toLowerCase(),
|
|
10
|
+
re: a(e.toLowerCase())
|
|
11
|
+
}));
|
|
12
|
+
return (t) => e.sync(() => {
|
|
13
|
+
let e = t.toLowerCase(), i = n.filter((t) => t.re.test(e)).map((e) => e.term);
|
|
14
|
+
return i.length > 0 ? {
|
|
15
|
+
flagged: !0,
|
|
16
|
+
categories: i,
|
|
17
|
+
reason: `matched denied term(s): ${i.join(", ")}`
|
|
18
|
+
} : r;
|
|
19
|
+
});
|
|
20
|
+
}, s = (t = {}) => {
|
|
21
|
+
let i = t.threshold ?? .7, a = t.categories ?? [
|
|
22
|
+
"hate",
|
|
23
|
+
"harassment",
|
|
24
|
+
"sexual",
|
|
25
|
+
"violence",
|
|
26
|
+
"self-harm",
|
|
27
|
+
"illicit"
|
|
28
|
+
], o = t.load ?? (() => import("@voltro/ai")), s = !1;
|
|
29
|
+
return (t) => e.gen(function* () {
|
|
30
|
+
let c = yield* e.tryPromise(o).pipe(e.catch(() => e.succeed(null)));
|
|
31
|
+
return c === null ? (s || (s = !0, n.warn("aiProvider configured but @voltro/ai could not be loaded — content is NOT being moderated; install @voltro/ai")), r) : yield* e.tryPromise(async () => {
|
|
32
|
+
let { Schema: n } = await import("effect"), r = n.Struct({
|
|
33
|
+
flagged: n.Boolean,
|
|
34
|
+
topCategory: n.NullOr(n.String),
|
|
35
|
+
score: n.Number
|
|
36
|
+
}), { object: o } = await e.runPromise(c.generateObject({
|
|
37
|
+
schema: r,
|
|
38
|
+
system: `You are a content moderator. Categories: ${a.join(", ")}. Return whether the content violates policy, the worst category, and a 0..1 severity score.`,
|
|
39
|
+
prompt: t
|
|
40
|
+
})), s = o.flagged || o.score >= i;
|
|
41
|
+
return {
|
|
42
|
+
flagged: s,
|
|
43
|
+
score: o.score,
|
|
44
|
+
...o.topCategory ? { categories: [o.topCategory] } : {},
|
|
45
|
+
...s ? { reason: `ai moderation: ${o.topCategory ?? "policy"} (${o.score.toFixed(2)})` } : {}
|
|
46
|
+
};
|
|
47
|
+
}).pipe(e.catch(() => e.succeed(r)));
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
//#endregion
|
|
51
|
+
export { s as aiProvider, o as keywordProvider };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voltro/plugin-moderation",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.87.0",
|
|
4
4
|
"description": "Content moderation — run an AI / keyword / custom check on user-content fields and block or flag a write before it commits. Declarative per-rpc rules + an in-handler moderate() helper for redact/inline control.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"voltro",
|
|
@@ -38,11 +38,11 @@
|
|
|
38
38
|
"node": ">=24.0.0"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@voltro/logger": "0.
|
|
42
|
-
"@voltro/protocol": "0.
|
|
41
|
+
"@voltro/logger": "0.87.0",
|
|
42
|
+
"@voltro/protocol": "0.87.0"
|
|
43
43
|
},
|
|
44
44
|
"optionalDependencies": {
|
|
45
|
-
"@voltro/ai": "0.
|
|
45
|
+
"@voltro/ai": "0.87.0"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
48
|
"effect": "4.0.1"
|