@voltro/plugin-moderation 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1968 -0
- package/dist/index.d.ts +20 -1
- package/dist/index.js +73 -57
- package/package.json +4 -4
package/dist/index.d.ts
CHANGED
|
@@ -54,7 +54,8 @@ export declare interface FlaggedItem {
|
|
|
54
54
|
status: 'pending' | 'confirmed' | 'dismissed';
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
/** Zero-dep denylist matcher. Case-insensitive whole-word
|
|
57
|
+
/** Zero-dep denylist matcher. Case-insensitive; whole-word where the term has
|
|
58
|
+
* word edges (see {@link termMatcher}). */
|
|
58
59
|
export declare const keywordProvider: (deny: ReadonlyArray<string>) => ModerationProvider;
|
|
59
60
|
|
|
60
61
|
/** Moderate arbitrary text in a handler (e.g. to redact before writing) — the
|
|
@@ -72,6 +73,24 @@ export declare interface ModerationPluginOptions {
|
|
|
72
73
|
tag: string;
|
|
73
74
|
verdict: ModerationVerdict;
|
|
74
75
|
}) => void;
|
|
76
|
+
/**
|
|
77
|
+
* Namespace for this plugin's inspect endpoints. Default `moderation`.
|
|
78
|
+
*
|
|
79
|
+
* Set it when your app already publishes under that name — an exact tag
|
|
80
|
+
* collision is fatal at codegen, and this is the way out. Orthogonal to
|
|
81
|
+
* `name` below: `alias` REPLACES the namespace, `name` distinguishes two
|
|
82
|
+
* installations within it.
|
|
83
|
+
*
|
|
84
|
+
* The cost, stated because nothing else states it: the local and cloud
|
|
85
|
+
* dashboards fetch this plugin's panel at the DEFAULT slug, so an aliased
|
|
86
|
+
* install keeps working while its dashboard panel 404s. Alias to escape a
|
|
87
|
+
* collision, not for taste.
|
|
88
|
+
*/
|
|
89
|
+
readonly alias?: string;
|
|
90
|
+
/**
|
|
91
|
+
* Discriminator for a SECOND installation of this plugin, when one app runs
|
|
92
|
+
* two (`@voltro/plugin-moderation#analytics`). Not a rename — for that use `alias`.
|
|
93
|
+
*/
|
|
75
94
|
readonly name?: string;
|
|
76
95
|
}
|
|
77
96
|
|
package/dist/index.js
CHANGED
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
import { ContentRejected as e } from "./errors.js";
|
|
2
2
|
import { Context as t, Effect as n, Layer as r } from "effect";
|
|
3
|
-
import { definePlugin as i } from "@voltro/protocol";
|
|
4
|
-
import { createLogger as
|
|
3
|
+
import { definePlugin as i, pluginInstanceName as a } from "@voltro/protocol";
|
|
4
|
+
import { createLogger as o } from "@voltro/logger";
|
|
5
5
|
//#region src/providers.ts
|
|
6
|
-
var
|
|
7
|
-
let t = e.
|
|
6
|
+
var s = o({ scope: "@voltro/plugin-moderation" }), c = { flagged: !1 }, l = /[.*+?^${}()|[\]\\]/g, u = (e) => {
|
|
7
|
+
let t = (e) => e !== void 0 && /[\p{L}\p{N}_]/u.test(e), n = t(e[0]) ? "\\b" : "", r = t(e[e.length - 1]) ? "\\b" : "";
|
|
8
|
+
return RegExp(`${n}${e.replace(l, "\\$&")}${r}`, "u");
|
|
9
|
+
}, d = (e) => {
|
|
10
|
+
let t = e.filter((e) => e.length > 0).map((e) => ({
|
|
11
|
+
term: e.toLowerCase(),
|
|
12
|
+
re: u(e.toLowerCase())
|
|
13
|
+
}));
|
|
8
14
|
return (e) => n.sync(() => {
|
|
9
|
-
let n = e.toLowerCase(), r = t.filter((e) =>
|
|
15
|
+
let n = e.toLowerCase(), r = t.filter((e) => e.re.test(n)).map((e) => e.term);
|
|
10
16
|
return r.length > 0 ? {
|
|
11
17
|
flagged: !0,
|
|
12
18
|
categories: r,
|
|
13
19
|
reason: `matched denied term(s): ${r.join(", ")}`
|
|
14
|
-
} :
|
|
20
|
+
} : c;
|
|
15
21
|
});
|
|
16
|
-
},
|
|
22
|
+
}, f = (e = {}) => {
|
|
17
23
|
let t = e.threshold ?? .7, r = e.categories ?? [
|
|
18
24
|
"hate",
|
|
19
25
|
"harassment",
|
|
@@ -23,80 +29,89 @@ var o = a({ scope: "@voltro/plugin-moderation" }), s = { flagged: !1 }, c = (e)
|
|
|
23
29
|
"illicit"
|
|
24
30
|
], i = e.load ?? (() => import("@voltro/ai")), a = !1;
|
|
25
31
|
return (e) => n.gen(function* () {
|
|
26
|
-
let
|
|
27
|
-
return
|
|
32
|
+
let o = yield* n.tryPromise(i).pipe(n.catchAll(() => n.succeed(null)));
|
|
33
|
+
return o === null ? (a || (a = !0, s.warn("aiProvider configured but @voltro/ai could not be loaded — content is NOT being moderated; install @voltro/ai")), c) : yield* n.tryPromise(async () => {
|
|
28
34
|
let { Schema: i } = await import("effect"), a = i.Struct({
|
|
29
35
|
flagged: i.Boolean,
|
|
30
36
|
topCategory: i.NullOr(i.String),
|
|
31
37
|
score: i.Number
|
|
32
|
-
}), { object:
|
|
38
|
+
}), { object: s } = await n.runPromise(o.generateObject({
|
|
33
39
|
schema: a,
|
|
34
40
|
system: `You are a content moderator. Categories: ${r.join(", ")}. Return whether the content violates policy, the worst category, and a 0..1 severity score.`,
|
|
35
41
|
prompt: e
|
|
36
|
-
})),
|
|
42
|
+
})), c = s.flagged || s.score >= t;
|
|
37
43
|
return {
|
|
38
|
-
flagged:
|
|
39
|
-
score:
|
|
40
|
-
...
|
|
41
|
-
...
|
|
44
|
+
flagged: c,
|
|
45
|
+
score: s.score,
|
|
46
|
+
...s.topCategory ? { categories: [s.topCategory] } : {},
|
|
47
|
+
...c ? { reason: `ai moderation: ${s.topCategory ?? "policy"} (${s.score.toFixed(2)})` } : {}
|
|
42
48
|
};
|
|
43
|
-
}).pipe(n.catchAll(() => n.succeed(
|
|
49
|
+
}).pipe(n.catchAll(() => n.succeed(c)));
|
|
44
50
|
});
|
|
45
|
-
},
|
|
51
|
+
}, p = class extends t.Tag("@voltro/plugin-moderation/ModerationService")() {}, m = (e) => n.flatMap(p, (t) => t.moderate(e)), h = (e, t) => {
|
|
46
52
|
if (typeof e != "object" || !e) return "";
|
|
47
53
|
let n = e;
|
|
48
54
|
return t.map((e) => n[e]).filter((e) => typeof e == "string").join("\n");
|
|
49
|
-
},
|
|
50
|
-
let
|
|
51
|
-
|
|
52
|
-
|
|
55
|
+
}, g = "@voltro/plugin-moderation", _ = (t) => {
|
|
56
|
+
let o = a({
|
|
57
|
+
base: g,
|
|
58
|
+
alias: t.alias,
|
|
59
|
+
instance: t.name
|
|
60
|
+
}), s = t.rules.map((e) => {
|
|
61
|
+
let t = e.match;
|
|
62
|
+
if (typeof t == "string") return (e) => t === e;
|
|
63
|
+
let n = t.global || t.sticky ? new RegExp(t.source, t.flags.replace(/[gy]/g, "")) : t;
|
|
64
|
+
return (e) => n.test(e);
|
|
65
|
+
}), c = (e) => t.rules.filter((t, n) => s[n](e)), l = t.onFlag, u = () => void 0, d = [], f = 0, m = 0, _ = 0, v = (e, t, n) => {
|
|
66
|
+
d.unshift({
|
|
67
|
+
id: `mod_${Date.now().toString(36)}_${f++}`,
|
|
53
68
|
tag: e,
|
|
54
69
|
categories: n.categories ?? [],
|
|
55
70
|
reason: n.reason ?? "content policy violation",
|
|
56
71
|
subjectId: t,
|
|
57
72
|
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
58
73
|
status: "pending"
|
|
59
|
-
}),
|
|
60
|
-
},
|
|
61
|
-
let a =
|
|
74
|
+
}), d.length > 200 && (d.length = 200);
|
|
75
|
+
}, y = (r, i) => {
|
|
76
|
+
let a = c(i.tag);
|
|
62
77
|
if (a.length === 0) return r;
|
|
63
|
-
let
|
|
78
|
+
let o = i.subject?.id ?? null;
|
|
64
79
|
return n.gen(function* () {
|
|
65
80
|
for (let r of a) {
|
|
66
|
-
let a =
|
|
81
|
+
let a = h(i.input, r.fields);
|
|
67
82
|
if (a.length === 0) continue;
|
|
68
|
-
let
|
|
69
|
-
if (!
|
|
70
|
-
let
|
|
71
|
-
if ((r.action ?? "block") === "block") return
|
|
83
|
+
let s = yield* t.provider(a);
|
|
84
|
+
if (!s.flagged) continue;
|
|
85
|
+
let c = s.categories ?? [], d = s.reason ?? "content policy violation";
|
|
86
|
+
if ((r.action ?? "block") === "block") return m++, v(i.tag, o, s), yield* n.fail(new e({
|
|
72
87
|
tag: i.tag,
|
|
73
|
-
categories:
|
|
88
|
+
categories: c,
|
|
74
89
|
reason: d
|
|
75
90
|
}));
|
|
76
|
-
|
|
91
|
+
_++, v(i.tag, o, s), u(i.tag, s), l?.({
|
|
77
92
|
tag: i.tag,
|
|
78
|
-
verdict:
|
|
93
|
+
verdict: s
|
|
79
94
|
});
|
|
80
95
|
}
|
|
81
96
|
return yield* r;
|
|
82
97
|
});
|
|
83
|
-
},
|
|
98
|
+
}, b = (e) => ({
|
|
84
99
|
kind: "json",
|
|
85
100
|
data: e
|
|
86
|
-
}),
|
|
101
|
+
}), x = (e, t) => ({
|
|
87
102
|
kind: "json",
|
|
88
103
|
status: e,
|
|
89
104
|
data: { error: t }
|
|
90
|
-
}),
|
|
105
|
+
}), S = [{
|
|
91
106
|
method: "GET",
|
|
92
107
|
path: "/flagged",
|
|
93
108
|
description: "The content-moderation review queue (newest first).",
|
|
94
|
-
handler: () => n.succeed(
|
|
95
|
-
items:
|
|
109
|
+
handler: () => n.succeed(b({
|
|
110
|
+
items: d.slice(0, 100),
|
|
96
111
|
stats: {
|
|
97
|
-
blocked:
|
|
98
|
-
flagged:
|
|
99
|
-
pending:
|
|
112
|
+
blocked: m,
|
|
113
|
+
flagged: _,
|
|
114
|
+
pending: d.filter((e) => e.status === "pending").length
|
|
100
115
|
}
|
|
101
116
|
}))
|
|
102
117
|
}, {
|
|
@@ -108,25 +123,26 @@ var o = a({ scope: "@voltro/plugin-moderation" }), s = { flagged: !1 }, c = (e)
|
|
|
108
123
|
try {
|
|
109
124
|
t = JSON.parse(e.body || "{}");
|
|
110
125
|
} catch {
|
|
111
|
-
return
|
|
126
|
+
return x(400, "invalid JSON body");
|
|
112
127
|
}
|
|
113
|
-
if (!t.id || t.action !== "confirm" && t.action !== "dismiss") return
|
|
114
|
-
let n =
|
|
115
|
-
return n ? (n.status = t.action === "confirm" ? "confirmed" : "dismissed",
|
|
128
|
+
if (!t.id || t.action !== "confirm" && t.action !== "dismiss") return x(400, "id + action (confirm|dismiss) required");
|
|
129
|
+
let n = d.find((e) => e.id === t.id);
|
|
130
|
+
return n ? (n.status = t.action === "confirm" ? "confirmed" : "dismissed", b({
|
|
116
131
|
ok: !0,
|
|
117
132
|
item: n
|
|
118
|
-
})) :
|
|
133
|
+
})) : x(404, `unknown item "${t.id}"`);
|
|
119
134
|
})
|
|
120
|
-
}],
|
|
135
|
+
}], C = [
|
|
121
136
|
"rpc:intercept:mutation",
|
|
122
137
|
"rpc:intercept:action",
|
|
123
138
|
"inspect:read",
|
|
124
139
|
"inspect:write"
|
|
125
|
-
],
|
|
140
|
+
], w = r.succeed(p, { moderate: (e) => t.provider(e) });
|
|
126
141
|
return i({
|
|
127
|
-
name:
|
|
142
|
+
name: o,
|
|
143
|
+
baseName: g,
|
|
128
144
|
description: "Content moderation — block or flag user-content writes via an AI / keyword / custom check.",
|
|
129
|
-
permissions:
|
|
145
|
+
permissions: C,
|
|
130
146
|
errorSchemas: [{
|
|
131
147
|
schema: e,
|
|
132
148
|
import: {
|
|
@@ -134,18 +150,18 @@ var o = a({ scope: "@voltro/plugin-moderation" }), s = { flagged: !1 }, c = (e)
|
|
|
134
150
|
name: "ContentRejected"
|
|
135
151
|
}
|
|
136
152
|
}],
|
|
137
|
-
interceptMutation:
|
|
138
|
-
interceptAction:
|
|
139
|
-
inspectEndpoints:
|
|
140
|
-
services:
|
|
153
|
+
interceptMutation: y,
|
|
154
|
+
interceptAction: y,
|
|
155
|
+
inspectEndpoints: S,
|
|
156
|
+
services: w,
|
|
141
157
|
onActivate: (e) => n.sync(() => {
|
|
142
|
-
|
|
158
|
+
u = (t, n) => e.logger.warn("content flagged", {
|
|
143
159
|
tag: t,
|
|
144
160
|
categories: n.categories,
|
|
145
161
|
score: n.score
|
|
146
|
-
}),
|
|
162
|
+
}), l = t.onFlag, e.logger.info("moderation active", { rules: t.rules.length });
|
|
147
163
|
})
|
|
148
164
|
});
|
|
149
165
|
};
|
|
150
166
|
//#endregion
|
|
151
|
-
export { e as ContentRejected,
|
|
167
|
+
export { e as ContentRejected, p as ModerationService, f as aiProvider, d as keywordProvider, m as moderate, _ as moderationPlugin };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voltro/plugin-moderation",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.35.0",
|
|
4
4
|
"description": "Content moderation — run an AI / keyword / custom check on user-content fields and block or flag a write before it commits. Declarative per-rpc rules + an in-handler moderate() helper for redact/inline control.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"voltro",
|
|
@@ -38,11 +38,11 @@
|
|
|
38
38
|
"node": ">=24.0.0"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@voltro/logger": "0.
|
|
42
|
-
"@voltro/protocol": "0.
|
|
41
|
+
"@voltro/logger": "0.35.0",
|
|
42
|
+
"@voltro/protocol": "0.35.0"
|
|
43
43
|
},
|
|
44
44
|
"optionalDependencies": {
|
|
45
|
-
"@voltro/ai": "0.
|
|
45
|
+
"@voltro/ai": "0.35.0"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
48
|
"effect": "^3.22.0"
|