@vibesdotdev/moderation 0.0.0-stage → 1.0.0-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -0
- package/README.md +83 -2
- package/dist/decide.d.ts +16 -0
- package/dist/decide.d.ts.map +1 -0
- package/dist/decide.js +81 -0
- package/dist/decide.js.map +1 -0
- package/dist/docs/moderation.decisions.docs.descriptor.d.ts +24 -0
- package/dist/docs/moderation.decisions.docs.descriptor.d.ts.map +1 -0
- package/dist/docs/moderation.decisions.docs.descriptor.js +94 -0
- package/dist/docs/moderation.decisions.docs.descriptor.js.map +1 -0
- package/dist/docs/moderation.docs.descriptor.d.ts +22 -0
- package/dist/docs/moderation.docs.descriptor.d.ts.map +1 -0
- package/dist/docs/moderation.docs.descriptor.js +68 -0
- package/dist/docs/moderation.docs.descriptor.js.map +1 -0
- package/dist/docs/moderation.subjects.docs.descriptor.d.ts +23 -0
- package/dist/docs/moderation.subjects.docs.descriptor.d.ts.map +1 -0
- package/dist/docs/moderation.subjects.docs.descriptor.js +99 -0
- package/dist/docs/moderation.subjects.docs.descriptor.js.map +1 -0
- package/dist/errors.d.ts +17 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +36 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/kinds/subject.kind.d.ts +10 -0
- package/dist/kinds/subject.kind.d.ts.map +1 -0
- package/dist/kinds/subject.kind.js +16 -0
- package/dist/kinds/subject.kind.js.map +1 -0
- package/dist/moderate.d.ts +17 -0
- package/dist/moderate.d.ts.map +1 -0
- package/dist/moderate.js +38 -0
- package/dist/moderate.js.map +1 -0
- package/dist/moderation.plugin.d.ts +12 -0
- package/dist/moderation.plugin.d.ts.map +1 -0
- package/dist/moderation.plugin.js +23 -0
- package/dist/moderation.plugin.js.map +1 -0
- package/dist/orchestration/labels.d.ts +5 -0
- package/dist/orchestration/labels.d.ts.map +1 -0
- package/dist/orchestration/labels.js +34 -0
- package/dist/orchestration/labels.js.map +1 -0
- package/dist/orchestration/resolve.d.ts +11 -0
- package/dist/orchestration/resolve.d.ts.map +1 -0
- package/dist/orchestration/resolve.js +90 -0
- package/dist/orchestration/resolve.js.map +1 -0
- package/dist/orchestration/state.d.ts +10 -0
- package/dist/orchestration/state.d.ts.map +1 -0
- package/dist/orchestration/state.js +58 -0
- package/dist/orchestration/state.js.map +1 -0
- package/dist/schemas/decision.d.ts +131 -0
- package/dist/schemas/decision.d.ts.map +1 -0
- package/dist/schemas/decision.js +97 -0
- package/dist/schemas/decision.js.map +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.d.ts.map +1 -0
- package/dist/schemas/errors.js +11 -0
- package/dist/schemas/errors.js.map +1 -0
- package/dist/schemas/evaluation.d.ts +47 -0
- package/dist/schemas/evaluation.d.ts.map +1 -0
- package/dist/schemas/evaluation.js +14 -0
- package/dist/schemas/evaluation.js.map +1 -0
- package/dist/schemas/index.d.ts +6 -0
- package/dist/schemas/index.d.ts.map +1 -0
- package/dist/schemas/index.js +9 -0
- package/dist/schemas/index.js.map +1 -0
- package/dist/schemas/input.d.ts +30 -0
- package/dist/schemas/input.d.ts.map +1 -0
- package/dist/schemas/input.js +20 -0
- package/dist/schemas/input.js.map +1 -0
- package/dist/schemas/labels.d.ts +19 -0
- package/dist/schemas/labels.d.ts.map +1 -0
- package/dist/schemas/labels.js +18 -0
- package/dist/schemas/labels.js.map +1 -0
- package/dist/schemas/subject.d.ts +79 -0
- package/dist/schemas/subject.d.ts.map +1 -0
- package/dist/schemas/subject.js +66 -0
- package/dist/schemas/subject.js.map +1 -0
- package/package.json +100 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 VIBES DEV LLC
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
|
6
|
+
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
|
7
|
+
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
8
|
+
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
|
9
|
+
following conditions:
|
|
10
|
+
|
|
11
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial
|
|
12
|
+
portions of the Software.
|
|
13
|
+
|
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
|
15
|
+
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
|
16
|
+
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
17
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
18
|
+
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,84 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @vibesdotdev/moderation
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Judge user content with an AI evaluation battery and turn the scores into `allow`, `review` or `hide`.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
`bun add @vibesdotdev/moderation` using the Vibes package registry.
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
|
|
11
|
+
1. **Register a battery** in the feature that owns the content: one `noul` head per label, named exactly as the label, and a non-empty `requiredState`.
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
import { createAIEvaluationSchemaDescriptor } from '@vibesdotdev/ai/schemas';
|
|
15
|
+
|
|
16
|
+
export default createAIEvaluationSchemaDescriptor({
|
|
17
|
+
id: 'posts.moderation',
|
|
18
|
+
name: 'Post moderation',
|
|
19
|
+
description: 'Scores a post against the moderation labels',
|
|
20
|
+
requiredState: ['post'],
|
|
21
|
+
questions: {
|
|
22
|
+
spam: { type: 'noul', instructions: 'The post is unsolicited bulk, repetitive or off-topic promotion.' },
|
|
23
|
+
scam: { type: 'noul', instructions: 'The post tries to obtain money, credentials or personal data by fraud.' }
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
2. **Register a subject** for the content, naming the battery and the thresholds.
|
|
29
|
+
|
|
30
|
+
```ts
|
|
31
|
+
import { createModerationSubjectDescriptor } from '@vibesdotdev/moderation/schemas';
|
|
32
|
+
|
|
33
|
+
export default createModerationSubjectDescriptor({
|
|
34
|
+
id: 'posts.post',
|
|
35
|
+
name: 'Post',
|
|
36
|
+
description: 'A published post',
|
|
37
|
+
batteryId: 'posts.moderation',
|
|
38
|
+
labels: ['spam', 'scam'],
|
|
39
|
+
thresholds: { default: { reviewAt: 0.5, hideAt: 0.85 } }
|
|
40
|
+
});
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
3. **Moderate** with the runtime that composes `moderationPlugin` (from `@vibesdotdev/moderation/plugin`), the feature's plugin and the canonical public `AIClient`. AI owns battery and answer contracts; moderation owns label coverage and thresholds. Pass a plain JSON state object; moderation snapshots it before asynchronous work.
|
|
44
|
+
|
|
45
|
+
```ts
|
|
46
|
+
import { moderateSubject } from '@vibesdotdev/moderation';
|
|
47
|
+
|
|
48
|
+
const decision = await moderateSubject(runtime, {
|
|
49
|
+
subject: { type: 'posts.post', id: post.id },
|
|
50
|
+
state: { post: post.body }
|
|
51
|
+
});
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
4. **Override thresholds** in a host by registering the host's own subject id against the same battery, then passing that id as `subject.type`. A feature's subject cannot be re-registered.
|
|
55
|
+
|
|
56
|
+
The `.` entrypoint exports the isomorphic contract, policy and orchestration.
|
|
57
|
+
`./plugin` exports moderationPlugin; `./schemas` exports moderation schemas,
|
|
58
|
+
inferred types and descriptor factories without runtime orchestration.
|
|
59
|
+
|
|
60
|
+
`decideModeration` validates its policy before applying thresholds. Invalid
|
|
61
|
+
ranges, inverted thresholds and undeclared overrides throw `ZodError`.
|
|
62
|
+
Malformed orchestration envelopes, references and options (including null) also throw `ZodError`. Omitted or undefined options use the defaults.
|
|
63
|
+
Domain refusals throw `ModerationError` with a standard CapabilityError code,
|
|
64
|
+
moderationCode and kebab-case reason. Accessors and cycles are refused before
|
|
65
|
+
inference; caller-object exceptions and unrelated AI failures propagate unchanged.
|
|
66
|
+
Adapted answers can review but cannot hide.
|
|
67
|
+
|
|
68
|
+
Run `bun run check`, `bun run test` and `bun run build` in this package.
|
|
69
|
+
Tests exercise public source entrypoints on owned runtimes and check schemas,
|
|
70
|
+
declared export mappings, publication exclusions and universal source closures.
|
|
71
|
+
The check project includes tests; the build project emits production JavaScript
|
|
72
|
+
and declarations. Tests require no prebuilt dist and run in process.
|
|
73
|
+
|
|
74
|
+
Distributed-artifact verification belongs to the canonical release preflight in
|
|
75
|
+
`@vibesdotdev/infra-deploy` (`stagePackageForPublish` and
|
|
76
|
+
`verifyPackedPackage`). Actual packed contents, published dependency range
|
|
77
|
+
conversion, Node compatibility and emitted-declaration consumption require
|
|
78
|
+
release evidence; source/manifest checks and compilation do not prove them.
|
|
79
|
+
|
|
80
|
+
See [SPEC.md](./SPEC.md) for the contract and the error codes.
|
|
81
|
+
|
|
82
|
+
## License
|
|
83
|
+
|
|
84
|
+
MIT. See [LICENSE](./LICENSE).
|
package/dist/decide.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type ModerationDecision, type ModerationLabel } from './schemas/decision.ts';
|
|
2
|
+
import { type ModerationSubjectDescriptor } from './schemas/subject.ts';
|
|
3
|
+
/**
|
|
4
|
+
* The threshold policy. Pure: no runtime, no clock, no I/O.
|
|
5
|
+
*
|
|
6
|
+
* Each label uses its subject's per-label thresholds, else the default. A
|
|
7
|
+
* native score at or above `hideAt` is a hide trigger; otherwise a score at
|
|
8
|
+
* or above `reviewAt` is a review trigger. Adapted (JSON-mode) scores are
|
|
9
|
+
* uncalibrated, so they can reach review but never hide.
|
|
10
|
+
*
|
|
11
|
+
* The labels must cover exactly the subject's labels, for one subject, from
|
|
12
|
+
* one battery and one answer source; anything else throws
|
|
13
|
+
* MODERATION_EVALUATION_INVALID.
|
|
14
|
+
*/
|
|
15
|
+
export declare function decideModeration(subject: ModerationSubjectDescriptor, labels: readonly ModerationLabel[], decidedAt: Date): ModerationDecision;
|
|
16
|
+
//# sourceMappingURL=decide.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"decide.d.ts","sourceRoot":"","sources":["../src/decide.ts"],"names":[],"mappings":"AACA,OAAO,EAEN,KAAK,kBAAkB,EACvB,KAAK,eAAe,EAEpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAqC,KAAK,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AAE3G;;;;;;;;;;;GAWG;AACH,wBAAgB,gBAAgB,CAC/B,OAAO,EAAE,2BAA2B,EACpC,MAAM,EAAE,SAAS,eAAe,EAAE,EAClC,SAAS,EAAE,IAAI,GACb,kBAAkB,CA+EpB"}
|
package/dist/decide.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { ModerationError } from "./errors.js";
|
|
2
|
+
import { ModerationDecisionSchema } from "./schemas/decision.js";
|
|
3
|
+
import { ModerationSubjectDescriptorSchema } from "./schemas/subject.js";
|
|
4
|
+
/**
|
|
5
|
+
* The threshold policy. Pure: no runtime, no clock, no I/O.
|
|
6
|
+
*
|
|
7
|
+
* Each label uses its subject's per-label thresholds, else the default. A
|
|
8
|
+
* native score at or above `hideAt` is a hide trigger; otherwise a score at
|
|
9
|
+
* or above `reviewAt` is a review trigger. Adapted (JSON-mode) scores are
|
|
10
|
+
* uncalibrated, so they can reach review but never hide.
|
|
11
|
+
*
|
|
12
|
+
* The labels must cover exactly the subject's labels, for one subject, from
|
|
13
|
+
* one battery and one answer source; anything else throws
|
|
14
|
+
* MODERATION_EVALUATION_INVALID.
|
|
15
|
+
*/
|
|
16
|
+
export function decideModeration(subject, labels, decidedAt) {
|
|
17
|
+
subject = ModerationSubjectDescriptorSchema.parse(subject);
|
|
18
|
+
const invalid = (reason) => new ModerationError('MODERATION_EVALUATION_INVALID', `labels do not cover subject '${subject.id}' labels exactly: ${reason}`);
|
|
19
|
+
const [first] = labels;
|
|
20
|
+
if (!first)
|
|
21
|
+
throw invalid('no labels');
|
|
22
|
+
const values = labels.map((label) => label.value);
|
|
23
|
+
const covers = values.length === subject.labels.length &&
|
|
24
|
+
new Set(values).size === values.length &&
|
|
25
|
+
subject.labels.every((value) => values.includes(value));
|
|
26
|
+
if (!covers) {
|
|
27
|
+
throw invalid(`got [${values.join(', ')}], expected [${subject.labels.join(', ')}]`);
|
|
28
|
+
}
|
|
29
|
+
const ref = first.subject;
|
|
30
|
+
if (ref.type !== subject.id) {
|
|
31
|
+
throw invalid(`labels name '${ref.type}', not policy subject '${subject.id}'`);
|
|
32
|
+
}
|
|
33
|
+
if (!labels.every((label) => label.subject.type === ref.type &&
|
|
34
|
+
label.subject.id === ref.id &&
|
|
35
|
+
label.subject.containerId === ref.containerId)) {
|
|
36
|
+
throw invalid('labels disagree on subject');
|
|
37
|
+
}
|
|
38
|
+
const source = first.source;
|
|
39
|
+
if (!labels.every((label) => label.source.batteryId === source.batteryId &&
|
|
40
|
+
label.source.answerSource === source.answerSource)) {
|
|
41
|
+
throw invalid('labels disagree on source');
|
|
42
|
+
}
|
|
43
|
+
if (source.batteryId !== subject.batteryId) {
|
|
44
|
+
throw invalid(`labels come from battery '${source.batteryId}', not '${subject.batteryId}'`);
|
|
45
|
+
}
|
|
46
|
+
const triggers = [];
|
|
47
|
+
for (const label of labels) {
|
|
48
|
+
const thresholds = subject.thresholds.labels?.[label.value] ?? subject.thresholds.default;
|
|
49
|
+
if (source.answerSource === 'native' && label.score >= thresholds.hideAt) {
|
|
50
|
+
triggers.push({
|
|
51
|
+
label: label.value,
|
|
52
|
+
score: label.score,
|
|
53
|
+
threshold: thresholds.hideAt,
|
|
54
|
+
level: 'hide'
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
else if (label.score >= thresholds.reviewAt) {
|
|
58
|
+
triggers.push({
|
|
59
|
+
label: label.value,
|
|
60
|
+
score: label.score,
|
|
61
|
+
threshold: thresholds.reviewAt,
|
|
62
|
+
level: 'review'
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
triggers.sort((a, b) => b.score - a.score);
|
|
67
|
+
const action = triggers.some((trigger) => trigger.level === 'hide')
|
|
68
|
+
? 'hide'
|
|
69
|
+
: triggers.length > 0
|
|
70
|
+
? 'review'
|
|
71
|
+
: 'allow';
|
|
72
|
+
return ModerationDecisionSchema.parse({
|
|
73
|
+
subject: ref,
|
|
74
|
+
action,
|
|
75
|
+
answerSource: source.answerSource,
|
|
76
|
+
labels: [...labels],
|
|
77
|
+
triggers,
|
|
78
|
+
decidedAt: decidedAt.toISOString()
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=decide.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"decide.js","sourceRoot":"","sources":["../src/decide.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EACN,wBAAwB,EAIxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iCAAiC,EAAoC,MAAM,sBAAsB,CAAC;AAE3G;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,gBAAgB,CAC/B,OAAoC,EACpC,MAAkC,EAClC,SAAe;IAEf,OAAO,GAAG,iCAAiC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,CAAC,MAAc,EAAmB,EAAE,CACnD,IAAI,eAAe,CAClB,+BAA+B,EAC/B,gCAAgC,OAAO,CAAC,EAAE,qBAAqB,MAAM,EAAE,CACvE,CAAC;IACH,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,KAAK;QAAE,MAAM,OAAO,CAAC,WAAW,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClD,MAAM,MAAM,GACX,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM;QACvC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;QACtC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,OAAO,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC;IAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,CAAC,gBAAgB,GAAG,CAAC,IAAI,0BAA0B,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;IAChF,CAAC;IACD,IACC,CAAC,MAAM,CAAC,KAAK,CACZ,CAAC,KAAK,EAAE,EAAE,CACT,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI;QAC/B,KAAK,CAAC,OAAO,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE;QAC3B,KAAK,CAAC,OAAO,CAAC,WAAW,KAAK,GAAG,CAAC,WAAW,CAC9C,EACA,CAAC;QACF,MAAM,OAAO,CAAC,4BAA4B,CAAC,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,IACC,CAAC,MAAM,CAAC,KAAK,CACZ,CAAC,KAAK,EAAE,EAAE,CACT,KAAK,CAAC,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS;QAC3C,KAAK,CAAC,MAAM,CAAC,YAAY,KAAK,MAAM,CAAC,YAAY,CAClD,EACA,CAAC;QACF,MAAM,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC;QAC5C,MAAM,OAAO,CAAC,6BAA6B,MAAM,CAAC,SAAS,WAAW,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;IAC7F,CAAC;IAED,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;QAC1F,IAAI,MAAM,CAAC,YAAY,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YAC1E,QAAQ,CAAC,IAAI,CAAC;gBACb,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,UAAU,CAAC,MAAM;gBAC5B,KAAK,EAAE,MAAM;aACb,CAAC,CAAC;QACJ,CAAC;aAAM,IAAI,KAAK,CAAC,KAAK,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC/C,QAAQ,CAAC,IAAI,CAAC;gBACb,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,UAAU,CAAC,QAAQ;gBAC9B,KAAK,EAAE,QAAQ;aACf,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IACD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,MAAM,CAAC;QAClE,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;YACpB,CAAC,CAAC,QAAQ;YACV,CAAC,CAAC,OAAO,CAAC;IAEZ,OAAO,wBAAwB,CAAC,KAAK,CAAC;QACrC,OAAO,EAAE,GAAG;QACZ,MAAM;QACN,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC;QACnB,QAAQ;QACR,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;KAClC,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Descriptor for moderation.decisions — moderateSubject, the threshold
|
|
3
|
+
* policy, the label and decision records, and the error codes. Pure data:
|
|
4
|
+
* no imports.
|
|
5
|
+
*/
|
|
6
|
+
declare const descriptor: {
|
|
7
|
+
kind: "docs/topic";
|
|
8
|
+
id: string;
|
|
9
|
+
title: string;
|
|
10
|
+
summary: string;
|
|
11
|
+
parent: string;
|
|
12
|
+
body: {
|
|
13
|
+
type: "markdown";
|
|
14
|
+
sourceType: "raw";
|
|
15
|
+
source: string;
|
|
16
|
+
};
|
|
17
|
+
order: number;
|
|
18
|
+
tags: string[];
|
|
19
|
+
surfaces: string[];
|
|
20
|
+
hardware: string[];
|
|
21
|
+
enabled: boolean;
|
|
22
|
+
};
|
|
23
|
+
export default descriptor;
|
|
24
|
+
//# sourceMappingURL=moderation.decisions.docs.descriptor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moderation.decisions.docs.descriptor.d.ts","sourceRoot":"","sources":["../../src/docs/moderation.decisions.docs.descriptor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAyEH,QAAA,MAAM,UAAU;;;;;;;;;;;;;;;;CAiBf,CAAC;AAEF,eAAe,UAAU,CAAC"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Descriptor for moderation.decisions — moderateSubject, the threshold
|
|
3
|
+
* policy, the label and decision records, and the error codes. Pure data:
|
|
4
|
+
* no imports.
|
|
5
|
+
*/
|
|
6
|
+
const moderationDecisionsMarkdown = `---
|
|
7
|
+
title: Moderation Decisions
|
|
8
|
+
summary: moderateSubject, the threshold policy, the label and decision records, and every moderation error code
|
|
9
|
+
tags: [moderation, decisions, labels, thresholds, errors]
|
|
10
|
+
parent: moderation
|
|
11
|
+
order: 2
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
# Moderation Decisions
|
|
15
|
+
|
|
16
|
+
## moderateSubject
|
|
17
|
+
|
|
18
|
+
\`moderateSubject(runtime, { subject, state }, { now? })\` judges one subject on the runtime you pass and returns a validated decision. It runs in this order, and every precondition fails before any inference is spent:
|
|
19
|
+
|
|
20
|
+
1. Validate the envelope derived from \`ModerateSubjectInputSchema\`, options with \`ModerateSubjectOptionsSchema\`, and \`subject\` with \`ModerationSubjectRefSchema\` and snapshot plain JSON \`state\` before the first await. A malformed ref throws the parse error; non-JSON state throws \`MODERATION_STATE_INVALID\`. An accessor property is refused without being read and a cycle is refused by its path; any other non-JSON value fails the state schema, whose error is the \`cause\`. A failure the caller's own object raises while it is read (a proxy trap, or nesting deeper than the call stack) propagates unchanged.
|
|
21
|
+
2. Resolve the enabled \`moderation/subject\` whose id is \`subject.type\`.
|
|
22
|
+
3. Resolve the subject's \`ai/evaluation-schema\` battery once.
|
|
23
|
+
4. Check the battery contract: heads equal the labels, every head is \`noul\`, \`requiredState\` is non-empty.
|
|
24
|
+
5. Check the owned state snapshot: every \`requiredState\` key is present and not null, blank or an empty list. Caller replacement or nested mutation cannot alter the observation passed to inference.
|
|
25
|
+
6. Resolve runtime/client \`ai\`. If the runtime has no \`runtime/client\` kind, moderation refuses rather than registering one. If \`ai\` is not registered, it refuses with that absence as the \`cause\`; any other failure while loading the client propagates unchanged with its own code.
|
|
26
|
+
7. Call \`evaluate({ schemaId, questions, state })\` with the already-resolved questions, so the battery is not resolved a second time. \`schemaId\` keeps scorecard attribution.
|
|
27
|
+
8. Validate the answers: one \`noul\` in \`[0, 1]\` per label, no missing and no undeclared heads.
|
|
28
|
+
9. Apply \`decideModeration\` and return the decision.
|
|
29
|
+
|
|
30
|
+
Evaluator errors, including "no evaluation model", propagate unchanged. Nothing is stored or enqueued.
|
|
31
|
+
|
|
32
|
+
## The threshold policy
|
|
33
|
+
|
|
34
|
+
\`decideModeration(subject, labels, decidedAt)\` first validates the policy with \`ModerationSubjectDescriptorSchema\`. Out-of-range or inverted thresholds and invalid label overrides throw \`ZodError\` before a decision. It is pure. For each label it uses the per-label thresholds, else the default:
|
|
35
|
+
|
|
36
|
+
- a **native** score at or above \`hideAt\` is a \`hide\` trigger at that threshold;
|
|
37
|
+
- otherwise a score at or above \`reviewAt\` is a \`review\` trigger.
|
|
38
|
+
|
|
39
|
+
The action is \`hide\` if any trigger hides, \`review\` if any trigger exists, else \`allow\`. Triggers are sorted by score, highest first.
|
|
40
|
+
|
|
41
|
+
**Adapted answers never hide.** A JSON-mode model running the battery returns \`source: 'adapted'\`; its numbers are not calibrated probabilities. Adapted scores can reach \`review\` but never \`hide\`, and the decision records \`answerSource\` so a reader can tell. An absent source reads as \`native\`.
|
|
42
|
+
|
|
43
|
+
The labels must cover exactly the subject's labels, with their subject type equal to the policy descriptor's id, from the subject's battery and one answer source. Sharing a battery does not authorize another subject's thresholds. Anything else throws \`MODERATION_EVALUATION_INVALID\` ("labels do not cover subject … labels exactly").
|
|
44
|
+
|
|
45
|
+
Decision records reject duplicate trigger labels, scores that differ from the carried label, and scores below the recorded threshold. Validation does not reconstruct an absent policy or infer missing triggers.
|
|
46
|
+
|
|
47
|
+
## Records
|
|
48
|
+
|
|
49
|
+
| Schema | Shape |
|
|
50
|
+
|---|---|
|
|
51
|
+
| \`ModerationAnswerSourceSchema\` | \`'native'\` or \`'adapted'\` |
|
|
52
|
+
| \`ModerationLabelSchema\` | \`{ subject, value, score, source: { batteryId, answerSource } }\` |
|
|
53
|
+
| \`ModerationActionSchema\` | \`'allow'\`, \`'review'\` or \`'hide'\` |
|
|
54
|
+
| \`ModerationTriggerSchema\` | \`{ label, score, threshold, level: 'review' \\| 'hide' }\` |
|
|
55
|
+
| \`ModerationDecisionSchema\` | \`{ subject, action, answerSource, labels, triggers, decidedAt }\` |
|
|
56
|
+
|
|
57
|
+
\`ModerationDecisionSchema\` refuses inconsistent decisions: \`allow\` exactly when there are no triggers, \`hide\` exactly when a trigger hides, no hide trigger from adapted answers, unique label values, every label about the decision's subject and from its answer source, and every trigger naming a carried label.
|
|
58
|
+
|
|
59
|
+
Input contracts also expose \`ModerationStateSchema\` and \`ModerationStateInputSchema\`. Orchestration leaves caller state opaque until safe inspection and snapshot validation; the public input schema describes JSON state. Public types are inferred; \`ModerationErrorCodeSchema\` validates refusal codes.
|
|
60
|
+
|
|
61
|
+
Only omitted or undefined options use defaults; null is malformed. Malformed envelopes, references and options throw \`ZodError\`. Caller-object exceptions propagate unchanged.
|
|
62
|
+
|
|
63
|
+
## Errors
|
|
64
|
+
|
|
65
|
+
\`ModerationError\` is a \`CapabilityError\`. Its \`moderationCode\` comes from \`MODERATION_ERROR_CODES\`, its \`reason\` is that code in kebab case, and \`details.moderationCode\` repeats it. Its \`code\` stays the standard one, \`NOT_FOUND\` for the two unknown codes, \`UNAVAILABLE\` for \`MODERATION_EVALUATOR_UNAVAILABLE\` and \`INVALID_INPUT\` for the rest, so \`exitCode\`, \`httpStatus\` and \`toEnvelope()\` map every failure on the CLI and the web. Its message starts with \`<moderationCode>: \`, and a wrapped failure is its \`cause\`.
|
|
66
|
+
|
|
67
|
+
| Code | Raised when |
|
|
68
|
+
|---|---|
|
|
69
|
+
| \`MODERATION_SUBJECT_UNKNOWN\` | No enabled subject has the requested id, the kind is not mounted, or the registered subject fails the subject schema |
|
|
70
|
+
| \`MODERATION_BATTERY_UNKNOWN\` | The subject's battery is not registered or is disabled |
|
|
71
|
+
| \`MODERATION_BATTERY_MISMATCH\` | The battery is not a readable evaluation battery, or breaks the contract above |
|
|
72
|
+
| \`MODERATION_STATE_INVALID\` | \`state\` is not a plain JSON object (an accessor or a cycle is named by path), or a required key is missing or empty; the message names the keys |
|
|
73
|
+
| \`MODERATION_EVALUATOR_UNAVAILABLE\` | runtime/client is not composed, \`ai\` is not registered on it, or the client has no \`evaluate\` |
|
|
74
|
+
| \`MODERATION_EVALUATION_INVALID\` | The answers do not match the battery, or the labels do not cover the subject |
|
|
75
|
+
`;
|
|
76
|
+
const descriptor = {
|
|
77
|
+
kind: 'docs/topic',
|
|
78
|
+
id: 'moderation.decisions',
|
|
79
|
+
title: 'Moderation Decisions',
|
|
80
|
+
summary: 'moderateSubject, the threshold policy, the label and decision records, and every moderation error code',
|
|
81
|
+
parent: 'moderation',
|
|
82
|
+
body: {
|
|
83
|
+
type: 'markdown',
|
|
84
|
+
sourceType: 'raw',
|
|
85
|
+
source: moderationDecisionsMarkdown
|
|
86
|
+
},
|
|
87
|
+
order: 2,
|
|
88
|
+
tags: ['moderation', 'decisions', 'labels', 'thresholds', 'errors'],
|
|
89
|
+
surfaces: ['cli', 'web', 'in-app'],
|
|
90
|
+
hardware: ['consumer', 'cloud'],
|
|
91
|
+
enabled: true
|
|
92
|
+
};
|
|
93
|
+
export default descriptor;
|
|
94
|
+
//# sourceMappingURL=moderation.decisions.docs.descriptor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moderation.decisions.docs.descriptor.js","sourceRoot":"","sources":["../../src/docs/moderation.decisions.docs.descriptor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,2BAA2B,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqEnC,CAAC;AAEF,MAAM,UAAU,GAAG;IAClB,IAAI,EAAE,YAAqB;IAC3B,EAAE,EAAE,sBAAsB;IAC1B,KAAK,EAAE,sBAAsB;IAC7B,OAAO,EACN,wGAAwG;IACzG,MAAM,EAAE,YAAY;IACpB,IAAI,EAAE;QACL,IAAI,EAAE,UAAmB;QACzB,UAAU,EAAE,KAAc;QAC1B,MAAM,EAAE,2BAA2B;KACnC;IACD,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,CAAC;IACnE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC;IAClC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC;IAC/B,OAAO,EAAE,IAAI;CACb,CAAC;AAEF,eAAe,UAAU,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Descriptor for moderation — the root topic of the moderation docs tree.
|
|
3
|
+
* Pure data: no imports.
|
|
4
|
+
*/
|
|
5
|
+
declare const descriptor: {
|
|
6
|
+
kind: "docs/topic";
|
|
7
|
+
id: string;
|
|
8
|
+
title: string;
|
|
9
|
+
summary: string;
|
|
10
|
+
body: {
|
|
11
|
+
type: "markdown";
|
|
12
|
+
sourceType: "raw";
|
|
13
|
+
source: string;
|
|
14
|
+
};
|
|
15
|
+
order: number;
|
|
16
|
+
tags: string[];
|
|
17
|
+
surfaces: string[];
|
|
18
|
+
hardware: string[];
|
|
19
|
+
enabled: boolean;
|
|
20
|
+
};
|
|
21
|
+
export default descriptor;
|
|
22
|
+
//# sourceMappingURL=moderation.docs.descriptor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moderation.docs.descriptor.d.ts","sourceRoot":"","sources":["../../src/docs/moderation.docs.descriptor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAiDH,QAAA,MAAM,UAAU;;;;;;;;;;;;;;;CAgBf,CAAC;AAEF,eAAe,UAAU,CAAC"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Descriptor for moderation — the root topic of the moderation docs tree.
|
|
3
|
+
* Pure data: no imports.
|
|
4
|
+
*/
|
|
5
|
+
const moderationMarkdown = `---
|
|
6
|
+
title: Moderation
|
|
7
|
+
summary: Judge user content with an AI evaluation battery and turn the scores into allow, review or hide
|
|
8
|
+
tags: [moderation, safety, labels, overview]
|
|
9
|
+
order: 20
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
# Moderation
|
|
13
|
+
|
|
14
|
+
\`@vibesdotdev/moderation\` is the one owner of content-moderation policy: which labels exist, how a feature declares a moderatable content type, and how label scores become an action. Features own their content, their battery and their subject. Model choice stays in \`ai\`: any structured-evaluation provider can answer a moderation battery.
|
|
15
|
+
|
|
16
|
+
## When to use
|
|
17
|
+
|
|
18
|
+
- A feature accepts user content (a chat message, a post, a comment) and needs a label-by-label judgment of it.
|
|
19
|
+
- A host needs different thresholds from a feature's default without forking the feature.
|
|
20
|
+
- Code needs a validated label or decision record to store, show or act on.
|
|
21
|
+
|
|
22
|
+
## The pieces
|
|
23
|
+
|
|
24
|
+
| Piece | What it is | Who provides it |
|
|
25
|
+
|---|---|---|
|
|
26
|
+
| \`moderation/subject\` | A moderatable content type: its battery, labels and thresholds | The feature that owns the content, or a host overriding it |
|
|
27
|
+
| \`ai/evaluation-schema\` battery | One \`noul\` question per label, named exactly as the label | The same feature |
|
|
28
|
+
| runtime/client \`ai\` | The evaluator that answers the battery | The host's AI composition |
|
|
29
|
+
|
|
30
|
+
\`\`\`ts
|
|
31
|
+
import { moderateSubject } from '@vibesdotdev/moderation';
|
|
32
|
+
|
|
33
|
+
const decision = await moderateSubject(runtime, {
|
|
34
|
+
subject: { type: 'chat.message', id: message.id, containerId: room.id },
|
|
35
|
+
state: { message: message.content }
|
|
36
|
+
});
|
|
37
|
+
// decision.action is 'allow', 'review' or 'hide'
|
|
38
|
+
\`\`\`
|
|
39
|
+
|
|
40
|
+
\`moderateSubject\` resolves everything on the runtime you pass. It persists nothing and enqueues nothing: storing decisions, queues and the moderator console are separate.
|
|
41
|
+
|
|
42
|
+
## Registering
|
|
43
|
+
|
|
44
|
+
Compose \`moderationPlugin\` (the default export of \`@vibesdotdev/moderation/plugin\`, id \`moderation\`) once. It registers the \`moderation/subject\` kind and these docs. Features register their subjects and batteries from their own plugins; the descriptors queue until the kind registers, so order does not matter.
|
|
45
|
+
|
|
46
|
+
:::card{title="In this section"}
|
|
47
|
+
- [\`moderation.subjects\`](moderation.subjects) — the subject kind, the label vocabulary, the battery contract, thresholds and the host override
|
|
48
|
+
- [\`moderation.decisions\`](moderation.decisions) — \`moderateSubject\`, the threshold policy, the label and decision records, and the error codes
|
|
49
|
+
:::
|
|
50
|
+
`;
|
|
51
|
+
const descriptor = {
|
|
52
|
+
kind: 'docs/topic',
|
|
53
|
+
id: 'moderation',
|
|
54
|
+
title: 'Moderation',
|
|
55
|
+
summary: 'Judge user content with an AI evaluation battery and turn the scores into allow, review or hide',
|
|
56
|
+
body: {
|
|
57
|
+
type: 'markdown',
|
|
58
|
+
sourceType: 'raw',
|
|
59
|
+
source: moderationMarkdown
|
|
60
|
+
},
|
|
61
|
+
order: 20,
|
|
62
|
+
tags: ['moderation', 'safety', 'labels', 'overview'],
|
|
63
|
+
surfaces: ['cli', 'web', 'in-app'],
|
|
64
|
+
hardware: ['consumer', 'cloud'],
|
|
65
|
+
enabled: true
|
|
66
|
+
};
|
|
67
|
+
export default descriptor;
|
|
68
|
+
//# sourceMappingURL=moderation.docs.descriptor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moderation.docs.descriptor.js","sourceRoot":"","sources":["../../src/docs/moderation.docs.descriptor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,kBAAkB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6C1B,CAAC;AAEF,MAAM,UAAU,GAAG;IAClB,IAAI,EAAE,YAAqB;IAC3B,EAAE,EAAE,YAAY;IAChB,KAAK,EAAE,YAAY;IACnB,OAAO,EACN,iGAAiG;IAClG,IAAI,EAAE;QACL,IAAI,EAAE,UAAmB;QACzB,UAAU,EAAE,KAAc;QAC1B,MAAM,EAAE,kBAAkB;KAC1B;IACD,KAAK,EAAE,EAAE;IACT,IAAI,EAAE,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC;IACpD,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC;IAClC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC;IAC/B,OAAO,EAAE,IAAI;CACb,CAAC;AAEF,eAAe,UAAU,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Descriptor for moderation.subjects — the `moderation/subject` kind, the
|
|
3
|
+
* label vocabulary and the battery contract. Pure data: no imports.
|
|
4
|
+
*/
|
|
5
|
+
declare const descriptor: {
|
|
6
|
+
kind: "docs/topic";
|
|
7
|
+
id: string;
|
|
8
|
+
title: string;
|
|
9
|
+
summary: string;
|
|
10
|
+
parent: string;
|
|
11
|
+
body: {
|
|
12
|
+
type: "markdown";
|
|
13
|
+
sourceType: "raw";
|
|
14
|
+
source: string;
|
|
15
|
+
};
|
|
16
|
+
order: number;
|
|
17
|
+
tags: string[];
|
|
18
|
+
surfaces: string[];
|
|
19
|
+
hardware: string[];
|
|
20
|
+
enabled: boolean;
|
|
21
|
+
};
|
|
22
|
+
export default descriptor;
|
|
23
|
+
//# sourceMappingURL=moderation.subjects.docs.descriptor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moderation.subjects.docs.descriptor.d.ts","sourceRoot":"","sources":["../../src/docs/moderation.subjects.docs.descriptor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA+EH,QAAA,MAAM,UAAU;;;;;;;;;;;;;;;;CAiBf,CAAC;AAEF,eAAe,UAAU,CAAC"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Descriptor for moderation.subjects — the `moderation/subject` kind, the
|
|
3
|
+
* label vocabulary and the battery contract. Pure data: no imports.
|
|
4
|
+
*/
|
|
5
|
+
const moderationSubjectsMarkdown = `---
|
|
6
|
+
title: Moderation Subjects
|
|
7
|
+
summary: The moderation/subject kind — a feature's content type, its battery, its labels and its thresholds
|
|
8
|
+
tags: [moderation, subjects, labels, thresholds, battery]
|
|
9
|
+
parent: moderation
|
|
10
|
+
order: 1
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# Moderation Subjects
|
|
14
|
+
|
|
15
|
+
A subject is one moderatable content type, registered as a \`moderation/subject\` descriptor (\`MODERATION_SUBJECT_KIND_ID\`) by the feature that owns the content. The kind object is \`moderationSubjectKind\`: descriptor-only, single-owner, with no default implementation and no loaders.
|
|
16
|
+
|
|
17
|
+
## The descriptor
|
|
18
|
+
|
|
19
|
+
\`ModerationSubjectDescriptorSchema\` extends the runtime descriptor with:
|
|
20
|
+
|
|
21
|
+
| Field | Meaning |
|
|
22
|
+
|---|---|
|
|
23
|
+
| \`id\` | \`<feature>.<subject>\`, for example \`chat.message\` |
|
|
24
|
+
| \`name\`, \`description\` | Non-empty |
|
|
25
|
+
| \`batteryId\` | The \`ai/evaluation-schema\` battery that judges this content |
|
|
26
|
+
| \`labels\` | The labels this subject scores; unique, at least one |
|
|
27
|
+
| \`thresholds.default\` | \`{ reviewAt, hideAt }\` for every label |
|
|
28
|
+
| \`thresholds.labels\` | Optional per-label overrides; each key must be one of \`labels\` |
|
|
29
|
+
|
|
30
|
+
\`ModerationThresholdsSchema\` holds two scores in \`[0, 1]\` with \`reviewAt <= hideAt\`. Build a descriptor with \`createModerationSubjectDescriptor\`, which parses the input and stamps the kind:
|
|
31
|
+
|
|
32
|
+
\`\`\`ts
|
|
33
|
+
import { createModerationSubjectDescriptor } from '@vibesdotdev/moderation/schemas';
|
|
34
|
+
|
|
35
|
+
export default createModerationSubjectDescriptor({
|
|
36
|
+
id: 'chat.message',
|
|
37
|
+
name: 'Chat message',
|
|
38
|
+
description: 'A message posted to a chat room',
|
|
39
|
+
batteryId: 'chat.moderation',
|
|
40
|
+
labels: ['harassment', 'hate', 'sexual', 'sexual_minors', 'violence', 'self_harm', 'spam', 'scam'],
|
|
41
|
+
thresholds: {
|
|
42
|
+
default: { reviewAt: 0.5, hideAt: 0.85 },
|
|
43
|
+
labels: { sexual_minors: { reviewAt: 0.2, hideAt: 0.5 } }
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
\`\`\`
|
|
47
|
+
|
|
48
|
+
\`ModerationSubjectRefSchema\` names one piece of content: \`{ type, id, containerId? }\`, where \`type\` is a subject id.
|
|
49
|
+
|
|
50
|
+
## Labels
|
|
51
|
+
|
|
52
|
+
\`MODERATION_LABELS\` is a closed vocabulary of eight: \`harassment\`, \`hate\`, \`sexual\`, \`sexual_minors\`, \`violence\`, \`self_harm\`, \`spam\` and \`scam\`. \`ModerationLabelValueSchema\` is its enum. A feature that needs another label amends this package.
|
|
53
|
+
|
|
54
|
+
## The battery contract
|
|
55
|
+
|
|
56
|
+
Battery schemas belong to public \`@vibesdotdev/ai/evaluation-schemas/schemas/index\`; answer schemas belong to \`@vibesdotdev/ai-schemas/generation/responses\`. Moderation adds its local probability bounds.
|
|
57
|
+
|
|
58
|
+
The subject's battery is an ordinary \`ai/evaluation-schema\` descriptor owned by the same feature. \`moderateSubject\` checks it before any inference and throws \`MODERATION_BATTERY_MISMATCH\` unless:
|
|
59
|
+
|
|
60
|
+
- its question names equal the subject's labels exactly — the label IS the head name, with no mapping table;
|
|
61
|
+
- every question is a \`noul\` (a calibrated yes-probability);
|
|
62
|
+
- \`requiredState\` is a non-empty list of the state keys the questions read.
|
|
63
|
+
|
|
64
|
+
Write each question as one atomic positive statement, for example \`harassment\`: "The content insults, threatens, bullies or demeans a specific person."
|
|
65
|
+
|
|
66
|
+
## Overriding a feature's thresholds
|
|
67
|
+
|
|
68
|
+
A feature's thresholds are its default policy. Subjects are single-owner, so a host cannot re-register \`chat.message\`. To apply its own policy, a host registers its own subject id against the same battery and passes that id as \`subject.type\`:
|
|
69
|
+
|
|
70
|
+
\`\`\`ts
|
|
71
|
+
createModerationSubjectDescriptor({
|
|
72
|
+
id: 'cinderlink.chat-message',
|
|
73
|
+
name: 'Cinderlink chat message',
|
|
74
|
+
description: 'Chat messages under Cinderlink community policy',
|
|
75
|
+
batteryId: 'chat.moderation',
|
|
76
|
+
labels: ['harassment', 'hate', 'sexual', 'sexual_minors', 'violence', 'self_harm', 'spam', 'scam'],
|
|
77
|
+
thresholds: { default: { reviewAt: 0.3, hideAt: 0.6 } }
|
|
78
|
+
});
|
|
79
|
+
\`\`\`
|
|
80
|
+
`;
|
|
81
|
+
const descriptor = {
|
|
82
|
+
kind: 'docs/topic',
|
|
83
|
+
id: 'moderation.subjects',
|
|
84
|
+
title: 'Moderation Subjects',
|
|
85
|
+
summary: "The moderation/subject kind — a feature's content type, its battery, its labels and its thresholds",
|
|
86
|
+
parent: 'moderation',
|
|
87
|
+
body: {
|
|
88
|
+
type: 'markdown',
|
|
89
|
+
sourceType: 'raw',
|
|
90
|
+
source: moderationSubjectsMarkdown
|
|
91
|
+
},
|
|
92
|
+
order: 1,
|
|
93
|
+
tags: ['moderation', 'subjects', 'labels', 'thresholds', 'battery'],
|
|
94
|
+
surfaces: ['cli', 'web', 'in-app'],
|
|
95
|
+
hardware: ['consumer', 'cloud'],
|
|
96
|
+
enabled: true
|
|
97
|
+
};
|
|
98
|
+
export default descriptor;
|
|
99
|
+
//# sourceMappingURL=moderation.subjects.docs.descriptor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moderation.subjects.docs.descriptor.js","sourceRoot":"","sources":["../../src/docs/moderation.subjects.docs.descriptor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,0BAA0B,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2ElC,CAAC;AAEF,MAAM,UAAU,GAAG;IAClB,IAAI,EAAE,YAAqB;IAC3B,EAAE,EAAE,qBAAqB;IACzB,KAAK,EAAE,qBAAqB;IAC5B,OAAO,EACN,oGAAoG;IACrG,MAAM,EAAE,YAAY;IACpB,IAAI,EAAE;QACL,IAAI,EAAE,UAAmB;QACzB,UAAU,EAAE,KAAc;QAC1B,MAAM,EAAE,0BAA0B;KAClC;IACD,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,CAAC;IACnE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC;IAClC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC;IAC/B,OAAO,EAAE,IAAI;CACb,CAAC;AAEF,eAAe,UAAU,CAAC"}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { CapabilityError } from '@vibesdotdev/kernel';
|
|
2
|
+
import { type ModerationErrorCode } from './schemas/errors.ts';
|
|
3
|
+
export { MODERATION_ERROR_CODES, type ModerationErrorCode } from './schemas/errors.ts';
|
|
4
|
+
/**
|
|
5
|
+
* A named moderation failure. `moderationCode` names it, `reason` is that code
|
|
6
|
+
* in kebab case, and `code` stays the standard CapabilityError code, so
|
|
7
|
+
* `exitCode`, `httpStatus` and `toEnvelope()` map it on every surface. The
|
|
8
|
+
* message starts with `<moderationCode>: `. When the failure wraps another
|
|
9
|
+
* error, that error is the standard `cause`.
|
|
10
|
+
*/
|
|
11
|
+
export declare class ModerationError extends CapabilityError {
|
|
12
|
+
readonly moderationCode: ModerationErrorCode;
|
|
13
|
+
constructor(code: ModerationErrorCode, detail: string, options?: {
|
|
14
|
+
cause?: unknown;
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAA4B,MAAM,qBAAqB,CAAC;AAEhF,OAAO,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAgBvF;;;;;;GAMG;AACH,qBAAa,eAAgB,SAAQ,eAAe;IACnD,QAAQ,CAAC,cAAc,EAAE,mBAAmB,CAAC;gBAEjC,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CASpF"}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { CapabilityError } from '@vibesdotdev/kernel';
|
|
2
|
+
import {} from "./schemas/errors.js";
|
|
3
|
+
export { MODERATION_ERROR_CODES } from "./schemas/errors.js";
|
|
4
|
+
function toCapabilityCode(code) {
|
|
5
|
+
switch (code) {
|
|
6
|
+
case 'MODERATION_SUBJECT_UNKNOWN':
|
|
7
|
+
case 'MODERATION_BATTERY_UNKNOWN':
|
|
8
|
+
return 'NOT_FOUND';
|
|
9
|
+
case 'MODERATION_BATTERY_MISMATCH':
|
|
10
|
+
case 'MODERATION_STATE_INVALID':
|
|
11
|
+
case 'MODERATION_EVALUATION_INVALID':
|
|
12
|
+
return 'INVALID_INPUT';
|
|
13
|
+
case 'MODERATION_EVALUATOR_UNAVAILABLE':
|
|
14
|
+
return 'UNAVAILABLE';
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A named moderation failure. `moderationCode` names it, `reason` is that code
|
|
19
|
+
* in kebab case, and `code` stays the standard CapabilityError code, so
|
|
20
|
+
* `exitCode`, `httpStatus` and `toEnvelope()` map it on every surface. The
|
|
21
|
+
* message starts with `<moderationCode>: `. When the failure wraps another
|
|
22
|
+
* error, that error is the standard `cause`.
|
|
23
|
+
*/
|
|
24
|
+
export class ModerationError extends CapabilityError {
|
|
25
|
+
moderationCode;
|
|
26
|
+
constructor(code, detail, options) {
|
|
27
|
+
super(`${code}: ${detail}`, toCapabilityCode(code), {
|
|
28
|
+
reason: code.toLowerCase().replaceAll('_', '-'),
|
|
29
|
+
details: { moderationCode: code },
|
|
30
|
+
cause: options?.cause
|
|
31
|
+
});
|
|
32
|
+
this.name = 'ModerationError';
|
|
33
|
+
this.moderationCode = code;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAA4B,MAAM,qBAAqB,CAAC;AAEhF,OAAO,EAA4B,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAA4B,MAAM,qBAAqB,CAAC;AAEvF,SAAS,gBAAgB,CAAC,IAAyB;IAClD,QAAQ,IAAI,EAAE,CAAC;QACd,KAAK,4BAA4B,CAAC;QAClC,KAAK,4BAA4B;YAChC,OAAO,WAAW,CAAC;QACpB,KAAK,6BAA6B,CAAC;QACnC,KAAK,0BAA0B,CAAC;QAChC,KAAK,+BAA+B;YACnC,OAAO,eAAe,CAAC;QACxB,KAAK,kCAAkC;YACtC,OAAO,aAAa,CAAC;IACvB,CAAC;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,eAAgB,SAAQ,eAAe;IAC1C,cAAc,CAAsB;IAE7C,YAAY,IAAyB,EAAE,MAAc,EAAE,OAA6B;QACnF,KAAK,CAAC,GAAG,IAAI,KAAK,MAAM,EAAE,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE;YACnD,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;YAC/C,OAAO,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE;YACjC,KAAK,EAAE,OAAO,EAAE,KAAK;SACrB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;IAC5B,CAAC;CACD"}
|