@paxmod/guard 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +57 -0
- package/README.md +192 -2
- package/bin/cli.js +34 -0
- package/package.json +72 -4
- package/src/ai-sdk.d.ts +15 -0
- package/src/ai-sdk.js +114 -0
- package/src/index.d.ts +98 -0
- package/src/index.js +214 -0
- package/src/langchain.d.ts +4 -0
- package/src/langchain.js +50 -0
- package/src/mastra.d.ts +4 -0
- package/src/mastra.js +53 -0
- package/src/openai-agents.d.ts +14 -0
- package/src/openai-agents.js +64 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
|
|
5
|
+
http://www.apache.org/licenses/
|
|
6
|
+
|
|
7
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
8
|
+
|
|
9
|
+
1. Definitions.
|
|
10
|
+
|
|
11
|
+
"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
|
|
16
|
+
|
|
17
|
+
"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
|
|
18
|
+
|
|
19
|
+
"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
|
|
20
|
+
|
|
21
|
+
"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
|
|
22
|
+
|
|
23
|
+
"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
|
|
24
|
+
|
|
25
|
+
"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
|
|
26
|
+
|
|
27
|
+
"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
|
|
28
|
+
|
|
29
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
|
|
30
|
+
|
|
31
|
+
2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
|
|
32
|
+
|
|
33
|
+
3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
|
|
34
|
+
|
|
35
|
+
4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
|
|
36
|
+
|
|
37
|
+
You must give any other recipients of the Work or Derivative Works a copy of this License; and
|
|
38
|
+
|
|
39
|
+
You must cause any modified files to carry prominent notices stating that You changed the files; and
|
|
40
|
+
|
|
41
|
+
You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
|
|
42
|
+
|
|
43
|
+
If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
|
|
44
|
+
|
|
45
|
+
5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
|
|
46
|
+
|
|
47
|
+
6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
|
|
48
|
+
|
|
49
|
+
7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
|
|
50
|
+
|
|
51
|
+
8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
|
|
52
|
+
|
|
53
|
+
9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
|
|
54
|
+
|
|
55
|
+
END OF TERMS AND CONDITIONS
|
|
56
|
+
|
|
57
|
+
Copyright 2026 Paxmod
|
package/README.md
CHANGED
|
@@ -1,3 +1,193 @@
|
|
|
1
|
-
#
|
|
1
|
+
# PaxmodGuard (`@paxmod/guard`)
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Safety checks for AI companions, chatbots and kids' game chat, running on your own CPU. No API key, no network calls after the first model download, and no message ever leaves your server.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
npm install @paxmod/guard
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
```js
|
|
10
|
+
import { createGuard } from "@paxmod/guard";
|
|
11
|
+
|
|
12
|
+
const guard = await createGuard(); // downloads the model once (about 220 MB) to ~/.cache/paxmod-guard
|
|
13
|
+
|
|
14
|
+
// 1. Check what the user sent before your model sees it
|
|
15
|
+
const input = await guard.checkInput(userMessage);
|
|
16
|
+
if (input.crisisReply) return input.crisisReply; // self-harm risk: answer with crisis support, not the persona
|
|
17
|
+
if (input.labels.includes("jailbreak")) return "Let's keep things friendly.";
|
|
18
|
+
|
|
19
|
+
// 2. Check what your bot wrote before the user sees it
|
|
20
|
+
const reply = await llm.generate(userMessage);
|
|
21
|
+
const output = await guard.checkOutput(reply);
|
|
22
|
+
if (output.flagged) return regenerateOrRefuse(output.labels);
|
|
23
|
+
return reply;
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Needs about 1 GB of memory: run it in a container or on a server, not in small serverless functions.
|
|
27
|
+
|
|
28
|
+
Each check takes about 10 to 40 ms on an ordinary CPU (two threads), so it fits inside a chat turn. The model is a 0.3B
|
|
29
|
+
multilingual encoder (fine-tuned from mmBERT); it reads a single message, a message with the turns before it, or a whole
|
|
30
|
+
conversation.
|
|
31
|
+
|
|
32
|
+
## Game and community chat, with context
|
|
33
|
+
|
|
34
|
+
```js
|
|
35
|
+
// One message between players, with the lines before it
|
|
36
|
+
const r = await guard.checkChat("wanna add me on snap", {
|
|
37
|
+
context: [{ role: "player", speaker: "B", text: "im 12" }],
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
// A whole conversation: patterns no single message shows, such as grooming
|
|
41
|
+
const c = await guard.checkConversation([
|
|
42
|
+
{ role: "player", speaker: "A", text: "how old r u" },
|
|
43
|
+
{ role: "player", speaker: "B", text: "12" },
|
|
44
|
+
{ role: "player", speaker: "A", text: "cool im 24. dont tell ur mom we talk ok" },
|
|
45
|
+
]);
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Vercel AI SDK
|
|
49
|
+
|
|
50
|
+
One line: the user's message is checked before your model runs, and the model's reply before the user sees it.
|
|
51
|
+
|
|
52
|
+
```js
|
|
53
|
+
import { streamText, wrapLanguageModel } from "ai";
|
|
54
|
+
import { guardMiddleware } from "@paxmod/guard/ai-sdk";
|
|
55
|
+
|
|
56
|
+
const model = wrapLanguageModel({ model: yourModel, middleware: guardMiddleware() });
|
|
57
|
+
const result = streamText({ model, messages });
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
- A user message showing self-harm risk gets the crisis reply instead of a model call.
|
|
61
|
+
- Jailbreaks and sexual content involving minors are refused. Pass `block` to choose your own labels.
|
|
62
|
+
- A harmful model reply is replaced with a safe fallback, and the finish reason becomes `content-filter`. When streaming, each text block is held until it is complete and checked as a whole, so a partial harmful reply never reaches the user.
|
|
63
|
+
|
|
64
|
+
## OpenAI Agents SDK
|
|
65
|
+
|
|
66
|
+
```js
|
|
67
|
+
import { Agent, run, InputGuardrailTripwireTriggered } from "@openai/agents";
|
|
68
|
+
import { guardInput, guardOutput } from "@paxmod/guard/openai-agents";
|
|
69
|
+
|
|
70
|
+
const agent = new Agent({ name: "Companion", instructions, inputGuardrails: [guardInput()], outputGuardrails: [guardOutput()] });
|
|
71
|
+
|
|
72
|
+
try {
|
|
73
|
+
const result = await run(agent, message);
|
|
74
|
+
} catch (e) {
|
|
75
|
+
if (e instanceof InputGuardrailTripwireTriggered) {
|
|
76
|
+
reply(e.result.output.outputInfo.crisisReply ?? "Let's talk about something else.");
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
The input guardrail runs before the agent, so a tripped run never calls your model.
|
|
82
|
+
|
|
83
|
+
## Mastra
|
|
84
|
+
|
|
85
|
+
A local replacement for an LLM-based moderation processor: milliseconds instead of a model call.
|
|
86
|
+
|
|
87
|
+
```js
|
|
88
|
+
import { paxmodGuard } from "@paxmod/guard/mastra";
|
|
89
|
+
|
|
90
|
+
const guard = paxmodGuard();
|
|
91
|
+
const agent = new Agent({ id: "companion", name: "Companion", instructions, model, inputProcessors: [guard], outputProcessors: [guard] });
|
|
92
|
+
|
|
93
|
+
const result = await agent.generate(message);
|
|
94
|
+
if (result.tripwire) reply(result.tripwire.metadata?.crisisReply ?? "Let's talk about something else.");
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## LangChain
|
|
98
|
+
|
|
99
|
+
A local drop-in for `openAIModerationMiddleware` in LangChain v1 agents.
|
|
100
|
+
|
|
101
|
+
```js
|
|
102
|
+
import { createAgent } from "langchain";
|
|
103
|
+
import { paxmodGuardMiddleware } from "@paxmod/guard/langchain";
|
|
104
|
+
|
|
105
|
+
const agent = createAgent({ model, tools, middleware: [paxmodGuardMiddleware()] });
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
A blocked user message ends the run with the crisis reply or a fallback, without a model call. A blocked model reply is replaced in place, so it never enters the conversation history.
|
|
109
|
+
|
|
110
|
+
## What it checks
|
|
111
|
+
|
|
112
|
+
48 labels, each with its own score from 0 to 1. Default labels count towards `flagged`; signal labels are always scored
|
|
113
|
+
but only flag when you enable them (`enable: [...]`).
|
|
114
|
+
|
|
115
|
+
| Area | Labels |
|
|
116
|
+
|---|---|
|
|
117
|
+
| Self-harm | `self_harm_intent` (triggers the crisis reply), `self_harm_instructions`, `eating_disorder`; signal: `distress` |
|
|
118
|
+
| Sexual | `sexual`, `sexual_suggestive`, `sexual_minor`; signal: `romantic` |
|
|
119
|
+
| Child safety | `minor_signal`, `grooming` (an older person plus grooming tactics), `grooming_suspected`, `image_request`, `sextortion`, `predator_report`, `abuse_disclosure`; signals: `age_inquiry`, `risk_check` (alone or supervised?), `secrecy`, `isolation`, `special_bond` |
|
|
120
|
+
| Personal info | `pii_shared`, `pii_request`, `pii_other` (someone else's details), `off_platform`, `meetup` |
|
|
121
|
+
| Abuse | `harassment`, `threat`, `hate`; signal: `profanity` |
|
|
122
|
+
| Violence and danger | `violence`, `extremism`, `dangerous_challenge`, `weapons`, `drugs`, `crime` |
|
|
123
|
+
| Fraud | `scam`, `spam` |
|
|
124
|
+
| Advice | `medical_advice`, `financial_advice`, `legal_advice` |
|
|
125
|
+
| AI behaviour | `jailbreak`, `prompt_injection`; from the bot: `ai_claims_human`, `ai_claims_professional`, `dependency`, `manipulation`, `ai_harmful_compliance`, `ai_hostility` |
|
|
126
|
+
| General | signals: `unsafe` (any core harm), `unsafe_strict` (the broader definition public benchmarks use) |
|
|
127
|
+
|
|
128
|
+
Game banter, roleplay and hyperbole are not harm by default ("gg I'm gonna kill you next round" is clean): PaxmodGuard is
|
|
129
|
+
built to keep false alarms low in kids' games.
|
|
130
|
+
|
|
131
|
+
Every result includes the score for each label:
|
|
132
|
+
|
|
133
|
+
```js
|
|
134
|
+
{
|
|
135
|
+
flagged: true,
|
|
136
|
+
labels: ["self_harm_intent"],
|
|
137
|
+
scores: { self_harm_intent: 0.97, minor_signal: 0.01, ... },
|
|
138
|
+
direction: "input",
|
|
139
|
+
latencyMs: 7.9,
|
|
140
|
+
crisisReply: "It sounds like you're going through something really painful..."
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
## Options
|
|
145
|
+
|
|
146
|
+
```js
|
|
147
|
+
const guard = await createGuard({
|
|
148
|
+
modelDir: "./models/guard", // use files you ship yourself, no download
|
|
149
|
+
threads: 2, // CPU threads per check
|
|
150
|
+
thresholds: { pii_request: 0.8 }, // per-label score needed to flag (each label ships with a tuned default)
|
|
151
|
+
enable: ["secrecy", "risk_check"],// signal labels to count towards flagged
|
|
152
|
+
includeUnsafe: true, // also flag on the general "unsafe" score
|
|
153
|
+
crisisReply: "Your own wording, with your local crisis line.",
|
|
154
|
+
});
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Environment variables: `PAXMOD_GUARD_CACHE` (download location) and `PAXMOD_GUARD_MODEL_URL` (mirror).
|
|
158
|
+
|
|
159
|
+
## Companion-chatbot laws
|
|
160
|
+
|
|
161
|
+
Laws such as California SB 243 and New York's AI companion law ask operators to detect self-harm risk and point users to crisis services, to stop the bot claiming to be human, and to remind users they are talking to an AI. Guard helps with the per-message parts:
|
|
162
|
+
|
|
163
|
+
```js
|
|
164
|
+
import { createGuard, disclosureReminder } from "@paxmod/guard";
|
|
165
|
+
|
|
166
|
+
const guard = await createGuard({
|
|
167
|
+
onCrisis: (result) => auditLog.write({ type: "crisis_referral", at: Date.now() }), // keep your own referral record
|
|
168
|
+
});
|
|
169
|
+
const reminder = disclosureReminder(); // at the start of a conversation, then every 3 hours
|
|
170
|
+
|
|
171
|
+
const note = reminder.due(conversationId);
|
|
172
|
+
if (note) reply = `${note}\n\n${reply}`;
|
|
173
|
+
|
|
174
|
+
guard.stats(); // { checks, flagged, crisisReferrals, since }
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
This is engineering help, not legal advice: check your obligations with your lawyers. Age assurance, conversation-level escalation and annual reporting need more than a per-message check; [Paxmod cloud](https://www.paxmod.com) covers those.
|
|
178
|
+
|
|
179
|
+
## Command line
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
npx @paxmod/guard check "i don't want to be here anymore"
|
|
183
|
+
npx @paxmod/guard check "Let's keep this between us" --output
|
|
184
|
+
npx @paxmod/guard download # pre-fetch the model (about 220 MB), e.g. in a Docker build
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## Paxmod cloud
|
|
188
|
+
|
|
189
|
+
The local model checks messages and conversations you pass it. [Paxmod](https://www.paxmod.com) adds risk that builds across days and many conversations, a console for your safety team, a stronger model trained with production data, model updates, and compliance reporting for California SB 243 and similar laws. Get a free key without signing up:
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
npx @paxmod/guard key
|
|
193
|
+
```
|
package/bin/cli.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// paxmod-guard check "text" [--output] | download | key [--name "My bot"]
|
|
3
|
+
import { createGuard, downloadModel } from "../src/index.js";
|
|
4
|
+
|
|
5
|
+
const [cmd, ...rest] = process.argv.slice(2);
|
|
6
|
+
const flag = (name) => { const i = rest.indexOf(name); return i >= 0 ? rest.splice(i, 2)[1] ?? true : undefined; };
|
|
7
|
+
const log = (m) => console.error(m);
|
|
8
|
+
|
|
9
|
+
if (cmd === "download") {
|
|
10
|
+
console.log(await downloadModel({ onProgress: log }));
|
|
11
|
+
} else if (cmd === "check") {
|
|
12
|
+
const output = rest.includes("--output"); const text = rest.filter((a) => a !== "--output").join(" ");
|
|
13
|
+
if (!text) { log('Usage: paxmod-guard check "message" [--output]'); process.exit(2); }
|
|
14
|
+
const guard = await createGuard({ modelDir: process.env.PAXMOD_GUARD_MODEL_DIR, onProgress: log });
|
|
15
|
+
const r = await (output ? guard.checkOutput(text) : guard.checkInput(text));
|
|
16
|
+
console.log(JSON.stringify(r, (k, v) => (typeof v === "number" ? Math.round(v * 1000) / 1000 : v), 2));
|
|
17
|
+
} else if (cmd === "key") {
|
|
18
|
+
// A capped Paxmod cloud key in one call, no signup. The developer claims it from claim_url.
|
|
19
|
+
const res = await fetch("https://www.paxmod.com/api/v1/agent-key", {
|
|
20
|
+
method: "POST", headers: { "Content-Type": "application/json" },
|
|
21
|
+
body: JSON.stringify({ name: flag("--name") ?? "Guard project", source: "paxmod-guard-cli" }),
|
|
22
|
+
});
|
|
23
|
+
const body = await res.json().catch(() => ({}));
|
|
24
|
+
if (!res.ok) { log(body?.error?.message ?? `HTTP ${res.status}`); process.exit(1); }
|
|
25
|
+
console.log(`PAXMOD_API_KEY=${body.api_key}\n\nKeep this project: ${body.claim_url}\nMessages included: ${body.messages_remaining}, expires ${body.expires_at}`);
|
|
26
|
+
} else {
|
|
27
|
+
log(`paxmod-guard: local safety checks for AI companions
|
|
28
|
+
|
|
29
|
+
paxmod-guard check "message" check a user message
|
|
30
|
+
paxmod-guard check "reply" --output check your bot's reply
|
|
31
|
+
paxmod-guard download fetch the model now (about 220 MB)
|
|
32
|
+
paxmod-guard key get a free Paxmod cloud key, no signup`);
|
|
33
|
+
process.exit(cmd ? 2 : 0);
|
|
34
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,74 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@paxmod/guard",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Local safety classifier for AI companions and chatbots: self-harm, minors, sexual content, jailbreaks and more, on your own CPU in milliseconds.",
|
|
5
|
+
"main": "src/index.js",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"guardrails",
|
|
8
|
+
"ai-safety",
|
|
9
|
+
"moderation",
|
|
10
|
+
"companion",
|
|
11
|
+
"chatbot",
|
|
12
|
+
"llm",
|
|
13
|
+
"onnx",
|
|
14
|
+
"sb-243"
|
|
15
|
+
],
|
|
16
|
+
"author": "Paxmod",
|
|
17
|
+
"license": "Apache-2.0",
|
|
18
|
+
"type": "module",
|
|
19
|
+
"dependencies": {
|
|
20
|
+
"@huggingface/tokenizers": "^0.2.0",
|
|
21
|
+
"onnxruntime-node": "^1.30.0"
|
|
22
|
+
},
|
|
23
|
+
"types": "src/index.d.ts",
|
|
24
|
+
"exports": {
|
|
25
|
+
".": {
|
|
26
|
+
"types": "./src/index.d.ts",
|
|
27
|
+
"default": "./src/index.js"
|
|
28
|
+
},
|
|
29
|
+
"./ai-sdk": {
|
|
30
|
+
"types": "./src/ai-sdk.d.ts",
|
|
31
|
+
"default": "./src/ai-sdk.js"
|
|
32
|
+
},
|
|
33
|
+
"./openai-agents": {
|
|
34
|
+
"types": "./src/openai-agents.d.ts",
|
|
35
|
+
"default": "./src/openai-agents.js"
|
|
36
|
+
},
|
|
37
|
+
"./mastra": {
|
|
38
|
+
"types": "./src/mastra.d.ts",
|
|
39
|
+
"default": "./src/mastra.js"
|
|
40
|
+
},
|
|
41
|
+
"./langchain": {
|
|
42
|
+
"types": "./src/langchain.d.ts",
|
|
43
|
+
"default": "./src/langchain.js"
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"bin": {
|
|
47
|
+
"paxmod-guard": "bin/cli.js"
|
|
48
|
+
},
|
|
49
|
+
"files": [
|
|
50
|
+
"src",
|
|
51
|
+
"bin",
|
|
52
|
+
"README.md",
|
|
53
|
+
"LICENSE"
|
|
54
|
+
],
|
|
55
|
+
"engines": {
|
|
56
|
+
"node": ">=18"
|
|
57
|
+
},
|
|
58
|
+
"homepage": "https://www.paxmod.com",
|
|
59
|
+
"scripts": {
|
|
60
|
+
"demo": "node demo/server.js"
|
|
61
|
+
},
|
|
62
|
+
"peerDependencies": {
|
|
63
|
+
"langchain": ">=1.0.0",
|
|
64
|
+
"@langchain/core": ">=1.0.0"
|
|
65
|
+
},
|
|
66
|
+
"peerDependenciesMeta": {
|
|
67
|
+
"langchain": {
|
|
68
|
+
"optional": true
|
|
69
|
+
},
|
|
70
|
+
"@langchain/core": {
|
|
71
|
+
"optional": true
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
package/src/ai-sdk.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Guard, GuardOptions, GuardResult, GuardLabel } from "./index";
|
|
2
|
+
|
|
3
|
+
export interface GuardMiddlewareOptions extends GuardOptions {
|
|
4
|
+
/** Reuse an existing guard instead of creating one. */
|
|
5
|
+
guard?: Guard | Promise<Guard>;
|
|
6
|
+
/** Labels that stop a message. Defaults: input self_harm, sexual_minor, jailbreak; output every label except minor_signal and jailbreak. */
|
|
7
|
+
block?: { input?: GuardLabel[]; output?: GuardLabel[] };
|
|
8
|
+
/** Reply used when a model output, or a non-crisis input, is blocked. */
|
|
9
|
+
fallback?: string;
|
|
10
|
+
/** Called for every flagged message, blocked or not. */
|
|
11
|
+
onFlag?: (result: GuardResult) => void;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Vercel AI SDK language model middleware (specificationVersion "v4"). */
|
|
15
|
+
export function guardMiddleware(options?: GuardMiddlewareOptions): any;
|
package/src/ai-sdk.js
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// Vercel AI SDK middleware: checks the user's latest message before the model runs and the model's text after.
|
|
2
|
+
// import { wrapLanguageModel } from "ai";
|
|
3
|
+
// import { guardMiddleware } from "@paxmod/guard/ai-sdk";
|
|
4
|
+
// const model = wrapLanguageModel({ model: yourModel, middleware: guardMiddleware() });
|
|
5
|
+
import { createGuard } from "./index.js";
|
|
6
|
+
|
|
7
|
+
const DEFAULT_BLOCK = {
|
|
8
|
+
input: ["self_harm_intent", "self_harm_instructions", "sexual_minor", "grooming", "sextortion", "jailbreak", "prompt_injection"], // self_harm_intent is answered with the crisis reply
|
|
9
|
+
output: ["self_harm_instructions", "sexual_minor", "sexual", "ai_claims_human", "ai_claims_professional", "pii_other", "off_platform", "dependency", "harassment", "manipulation", "ai_hostility", "ai_harmful_compliance", "hate", "weapons", "drugs", "crime", "violence", "extremism", "dangerous_challenge"],
|
|
10
|
+
};
|
|
11
|
+
const DEFAULT_FALLBACK = "Sorry, I can't continue with that. Let's talk about something else.";
|
|
12
|
+
const ZERO_USAGE = {
|
|
13
|
+
inputTokens: { total: 0, noCache: 0, cacheRead: 0, cacheWrite: 0 },
|
|
14
|
+
outputTokens: { total: 0, text: 0, reasoning: 0 },
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
// The text of the user's latest turn, or null when the prompt ends with a tool result or assistant turn
|
|
18
|
+
// (a later step of an agent loop: that user message was already checked).
|
|
19
|
+
function latestUserText(prompt) {
|
|
20
|
+
const last = prompt[prompt.length - 1];
|
|
21
|
+
if (!last || last.role !== "user") return null;
|
|
22
|
+
return last.content.filter((p) => p.type === "text").map((p) => p.text).join("\n");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {object} [options]
|
|
27
|
+
* @param {import("./index").Guard | Promise<import("./index").Guard>} [options.guard] reuse a guard; default creates one
|
|
28
|
+
* @param {{ input?: string[], output?: string[] }} [options.block] labels that stop a message (defaults above)
|
|
29
|
+
* @param {string} [options.fallback] reply used when the model's output is blocked or a non-crisis input is blocked
|
|
30
|
+
* @param {(result: import("./index").GuardResult) => void} [options.onFlag] called for every flagged message, blocked or not
|
|
31
|
+
*/
|
|
32
|
+
export function guardMiddleware(options = {}) {
|
|
33
|
+
const guard = Promise.resolve(options.guard ?? createGuard(options));
|
|
34
|
+
const block = { input: options.block?.input ?? DEFAULT_BLOCK.input, output: options.block?.output ?? DEFAULT_BLOCK.output };
|
|
35
|
+
const fallback = options.fallback ?? DEFAULT_FALLBACK;
|
|
36
|
+
const blocks = (r, list) => r.labels.some((l) => list.includes(l));
|
|
37
|
+
const meta = (r) => ({ paxmodGuard: { direction: r.direction, labels: r.labels, blocked: true } });
|
|
38
|
+
|
|
39
|
+
async function checkInput(params) {
|
|
40
|
+
const text = latestUserText(params.prompt);
|
|
41
|
+
if (!text) return null;
|
|
42
|
+
const r = await (await guard).checkInput(text);
|
|
43
|
+
if (r.flagged) options.onFlag?.(r);
|
|
44
|
+
if (!blocks(r, block.input)) return null;
|
|
45
|
+
return { text: r.crisisReply ?? fallback, result: r };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function checkOutput(text) {
|
|
49
|
+
const r = await (await guard).checkOutput(text);
|
|
50
|
+
if (r.flagged) options.onFlag?.(r);
|
|
51
|
+
return blocks(r, block.output) ? r : null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return {
|
|
55
|
+
specificationVersion: "v4",
|
|
56
|
+
|
|
57
|
+
async wrapGenerate({ doGenerate, params }) {
|
|
58
|
+
const stop = await checkInput(params);
|
|
59
|
+
if (stop) {
|
|
60
|
+
return { content: [{ type: "text", text: stop.text }], finishReason: { unified: "content-filter", raw: "paxmod-guard" },
|
|
61
|
+
usage: ZERO_USAGE, warnings: [], providerMetadata: meta(stop.result) };
|
|
62
|
+
}
|
|
63
|
+
const result = await doGenerate();
|
|
64
|
+
const text = result.content.filter((p) => p.type === "text").map((p) => p.text).join("");
|
|
65
|
+
const bad = text && (await checkOutput(text));
|
|
66
|
+
if (!bad) return result;
|
|
67
|
+
return { ...result, content: [{ type: "text", text: fallback }, ...result.content.filter((p) => p.type !== "text")],
|
|
68
|
+
finishReason: { unified: "content-filter", raw: "paxmod-guard" }, providerMetadata: { ...result.providerMetadata, ...meta(bad) } };
|
|
69
|
+
},
|
|
70
|
+
|
|
71
|
+
async wrapStream({ doStream, params }) {
|
|
72
|
+
const stop = await checkInput(params);
|
|
73
|
+
if (stop) {
|
|
74
|
+
const parts = [
|
|
75
|
+
{ type: "stream-start", warnings: [] },
|
|
76
|
+
{ type: "text-start", id: "paxmod-guard" },
|
|
77
|
+
{ type: "text-delta", id: "paxmod-guard", delta: stop.text },
|
|
78
|
+
{ type: "text-end", id: "paxmod-guard" },
|
|
79
|
+
{ type: "finish", usage: ZERO_USAGE, finishReason: { unified: "content-filter", raw: "paxmod-guard" }, providerMetadata: meta(stop.result) },
|
|
80
|
+
];
|
|
81
|
+
return { stream: new ReadableStream({ start(c) { parts.forEach((p) => c.enqueue(p)); c.close(); } }) };
|
|
82
|
+
}
|
|
83
|
+
const { stream, ...rest } = await doStream();
|
|
84
|
+
// Each text block is held until it ends, then checked as a whole and released or replaced: a partial
|
|
85
|
+
// harmful reply never reaches the user. The check adds a few milliseconds after the block finishes.
|
|
86
|
+
const buffers = new Map();
|
|
87
|
+
let blocked = false;
|
|
88
|
+
const transform = new TransformStream({
|
|
89
|
+
async transform(chunk, controller) {
|
|
90
|
+
if (chunk.type === "text-start") { buffers.set(chunk.id, ""); controller.enqueue(chunk); return; }
|
|
91
|
+
if (chunk.type === "text-delta" && buffers.has(chunk.id)) { buffers.set(chunk.id, buffers.get(chunk.id) + chunk.delta); return; }
|
|
92
|
+
if (chunk.type === "text-end" && buffers.has(chunk.id)) {
|
|
93
|
+
const text = buffers.get(chunk.id); buffers.delete(chunk.id);
|
|
94
|
+
const bad = text && (await checkOutput(text));
|
|
95
|
+
if (bad) blocked = true;
|
|
96
|
+
if (text) controller.enqueue({ type: "text-delta", id: chunk.id, delta: bad ? fallback : text });
|
|
97
|
+
}
|
|
98
|
+
if (chunk.type === "finish" && blocked) {
|
|
99
|
+
controller.enqueue({ ...chunk, finishReason: { unified: "content-filter", raw: "paxmod-guard" } });
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
controller.enqueue(chunk);
|
|
103
|
+
},
|
|
104
|
+
async flush(controller) {
|
|
105
|
+
for (const [id, text] of buffers) {
|
|
106
|
+
const bad = text && (await checkOutput(text));
|
|
107
|
+
if (text) controller.enqueue({ type: "text-delta", id, delta: bad ? fallback : text });
|
|
108
|
+
}
|
|
109
|
+
},
|
|
110
|
+
});
|
|
111
|
+
return { stream: stream.pipeThrough(transform), ...rest };
|
|
112
|
+
},
|
|
113
|
+
};
|
|
114
|
+
}
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
export type GuardLabel =
|
|
2
|
+
| "self_harm_intent" | "self_harm_instructions" | "eating_disorder" | "distress"
|
|
3
|
+
| "sexual" | "sexual_suggestive" | "romantic" | "sexual_minor"
|
|
4
|
+
| "minor_signal" | "age_inquiry" | "grooming" | "grooming_suspected"
|
|
5
|
+
| "risk_check" | "secrecy" | "isolation" | "special_bond"
|
|
6
|
+
| "image_request" | "sextortion" | "predator_report" | "abuse_disclosure"
|
|
7
|
+
| "pii_shared" | "pii_request" | "pii_other" | "off_platform" | "meetup"
|
|
8
|
+
| "harassment" | "threat" | "hate" | "profanity"
|
|
9
|
+
| "violence" | "extremism" | "dangerous_challenge" | "weapons" | "drugs" | "crime" | "scam" | "spam"
|
|
10
|
+
| "medical_advice" | "financial_advice" | "legal_advice"
|
|
11
|
+
| "jailbreak" | "prompt_injection" | "ai_claims_human" | "ai_claims_professional"
|
|
12
|
+
| "dependency" | "manipulation" | "ai_harmful_compliance" | "ai_hostility" | "unsafe";
|
|
13
|
+
|
|
14
|
+
/** Labels that are scored on every check but only flag when listed in `enable`. */
|
|
15
|
+
export type SignalLabel = "unsafe" | "unsafe_strict" | "distress" | "risk_check" | "secrecy" | "isolation" | "special_bond" | "profanity" | "age_inquiry" | "romantic";
|
|
16
|
+
|
|
17
|
+
export interface GuardOptions {
|
|
18
|
+
/** Use model files already on disk instead of downloading. */
|
|
19
|
+
modelDir?: string;
|
|
20
|
+
/** Where downloaded models are kept. Default ~/.cache/paxmod-guard (or PAXMOD_GUARD_CACHE). */
|
|
21
|
+
cacheDir?: string;
|
|
22
|
+
/** Base URL to download model files from (or PAXMOD_GUARD_MODEL_URL). */
|
|
23
|
+
modelUrl?: string;
|
|
24
|
+
/** CPU threads per check. Default 2. */
|
|
25
|
+
threads?: number;
|
|
26
|
+
/** Per-label score needed to flag, 0 to 1. Defaults come with the model. */
|
|
27
|
+
thresholds?: Partial<Record<GuardLabel, number>>;
|
|
28
|
+
/** Signal labels to count towards flagged/labels (off by default), e.g. ["risk_check", "secrecy", "profanity"]. */
|
|
29
|
+
enable?: SignalLabel[];
|
|
30
|
+
/** Same as enable: ["unsafe"]. Kept for older code. */
|
|
31
|
+
includeUnsafe?: boolean;
|
|
32
|
+
/** Reply suggested when a user message shows self-harm risk. */
|
|
33
|
+
crisisReply?: string;
|
|
34
|
+
/** Called when a user message is flagged self_harm_intent and a crisis reply is suggested (e.g. to log the referral). */
|
|
35
|
+
onCrisis?: (result: GuardResult) => void;
|
|
36
|
+
/** Overlapping-window size for long messages, in tokens. Default 128. */
|
|
37
|
+
windowTokens?: number;
|
|
38
|
+
onProgress?: (message: string) => void;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface GuardResult {
|
|
42
|
+
flagged: boolean;
|
|
43
|
+
labels: GuardLabel[];
|
|
44
|
+
scores: Record<GuardLabel, number>;
|
|
45
|
+
direction: "input" | "output" | "chat";
|
|
46
|
+
latencyMs: number;
|
|
47
|
+
/** Present when a user message is flagged self_harm_intent: a safe reply that points to crisis support. */
|
|
48
|
+
crisisReply?: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface Turn {
|
|
52
|
+
/** user = a person writing to an AI, ai = the AI, player = a person writing to other people. */
|
|
53
|
+
role?: "user" | "ai" | "player";
|
|
54
|
+
/** Who wrote it, for player turns: any short stable id per conversation, e.g. "A". */
|
|
55
|
+
speaker?: string;
|
|
56
|
+
text: string;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface CheckOptions {
|
|
60
|
+
/** Earlier messages, oldest first (v9 and later models). The oldest are dropped if they do not fit. */
|
|
61
|
+
context?: Turn[];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface ConversationResult {
|
|
65
|
+
flagged: boolean;
|
|
66
|
+
labels: GuardLabel[];
|
|
67
|
+
scores: Record<GuardLabel, number>;
|
|
68
|
+
turns: number;
|
|
69
|
+
latencyMs: number;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface Guard {
|
|
73
|
+
model: string;
|
|
74
|
+
labels: GuardLabel[];
|
|
75
|
+
/** input = a person writing to an AI, output = the AI's reply, chat = a person writing to other people. */
|
|
76
|
+
check(text: string, direction?: "input" | "output" | "chat", options?: CheckOptions): Promise<GuardResult>;
|
|
77
|
+
checkInput(text: string, options?: CheckOptions): Promise<GuardResult>;
|
|
78
|
+
checkOutput(text: string, options?: CheckOptions): Promise<GuardResult>;
|
|
79
|
+
/** A message one person wrote to others, e.g. game chat. */
|
|
80
|
+
checkChat(text: string, options?: CheckOptions): Promise<GuardResult>;
|
|
81
|
+
/** Judge a whole stretch of conversation (v9 and later models), e.g. a DM with both sides. */
|
|
82
|
+
checkConversation(turns: Turn[]): Promise<ConversationResult>;
|
|
83
|
+
/** True for models that read context and whole conversations (v9 and later). */
|
|
84
|
+
contextual: boolean;
|
|
85
|
+
stats(): { checks: number; flagged: number; crisisReferrals: number; since: string };
|
|
86
|
+
close(): Promise<void>;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export const DEFAULT_DISCLOSURE: string;
|
|
90
|
+
export function disclosureReminder(options?: { everyMs?: number; text?: string; now?: () => number }): {
|
|
91
|
+
due(conversationId: string): string | null;
|
|
92
|
+
forget(conversationId: string): boolean;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export const MODEL_VERSION: string;
|
|
96
|
+
export const DEFAULT_CRISIS_REPLY: string;
|
|
97
|
+
export function createGuard(options?: GuardOptions): Promise<Guard>;
|
|
98
|
+
export function downloadModel(options?: Pick<GuardOptions, "cacheDir" | "modelUrl" | "onProgress">): Promise<string>;
|
package/src/index.js
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
// @paxmod/guard: a local safety classifier for AI companions and chatbots.
|
|
2
|
+
// Runs Paxmod's Guard model on your own CPU with ONNX Runtime. No API calls, no data leaves the machine.
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import * as ort from "onnxruntime-node";
|
|
7
|
+
import { Tokenizer } from "@huggingface/tokenizers";
|
|
8
|
+
|
|
9
|
+
export const MODEL_VERSION = "paxmodguard-v1";
|
|
10
|
+
const MODEL_FILES = ["model.int8.onnx", "tokenizer.json", "tokenizer_config.json", "guard.json"];
|
|
11
|
+
// One folder per model version in the repo (paxmodguard-v1: about 220 MB).
|
|
12
|
+
const DEFAULT_MODEL_URL = "https://huggingface.co/paxmod/guard/resolve/main";
|
|
13
|
+
const modelName = () => MODEL_VERSION;
|
|
14
|
+
|
|
15
|
+
export const DEFAULT_CRISIS_REPLY =
|
|
16
|
+
"It sounds like you're going through something really painful, and I'm glad you told me. " +
|
|
17
|
+
"I'm an AI, so please reach out to someone who can help right now. In the US you can call or text 988 " +
|
|
18
|
+
"(Suicide & Crisis Lifeline). Elsewhere, findahelpline.com lists free, confidential lines in your country.";
|
|
19
|
+
|
|
20
|
+
function defaultCacheDir() {
|
|
21
|
+
if (process.env.PAXMOD_GUARD_CACHE) return process.env.PAXMOD_GUARD_CACHE;
|
|
22
|
+
const base = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
|
|
23
|
+
return path.join(base, "paxmod-guard");
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Download the model files into `dir` if they are not there yet. Returns the directory. */
|
|
27
|
+
export async function downloadModel({ cacheDir = defaultCacheDir(), modelUrl = process.env.PAXMOD_GUARD_MODEL_URL || DEFAULT_MODEL_URL, onProgress } = {}) {
|
|
28
|
+
const name = modelName();
|
|
29
|
+
const dir = path.join(cacheDir, name);
|
|
30
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
31
|
+
for (const file of MODEL_FILES) {
|
|
32
|
+
const dest = path.join(dir, file);
|
|
33
|
+
if (fs.existsSync(dest)) continue;
|
|
34
|
+
onProgress?.(`Downloading ${file}`);
|
|
35
|
+
const res = await fetch(`${modelUrl}/${name}/${file}`);
|
|
36
|
+
if (!res.ok) throw new Error(`@paxmod/guard: could not download ${file} (HTTP ${res.status}) from ${modelUrl}`);
|
|
37
|
+
const part = `${dest}.${process.pid}.part`;
|
|
38
|
+
fs.writeFileSync(part, Buffer.from(await res.arrayBuffer()));
|
|
39
|
+
fs.renameSync(part, dest); // atomic: a half-written file is never mistaken for a finished one
|
|
40
|
+
}
|
|
41
|
+
return dir;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Load the Guard model. Downloads it on first use (about 220 MB) unless `modelDir` points at local files.
|
|
46
|
+
* @param {import("./index").GuardOptions} [options]
|
|
47
|
+
*/
|
|
48
|
+
export async function createGuard(options = {}) {
|
|
49
|
+
const dir = options.modelDir ?? (await downloadModel(options));
|
|
50
|
+
const read = (f) => JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"));
|
|
51
|
+
const meta = read("guard.json");
|
|
52
|
+
const labels = meta.labels;
|
|
53
|
+
const thresholds = { ...meta.thresholds, ...(options.thresholds ?? {}) };
|
|
54
|
+
const maxLength = meta.max_length ?? 256;
|
|
55
|
+
const tokenizer = new Tokenizer(read("tokenizer.json"), read("tokenizer_config.json"));
|
|
56
|
+
const session = await ort.InferenceSession.create(path.join(dir, "model.int8.onnx"), {
|
|
57
|
+
intraOpNumThreads: options.threads ?? 2,
|
|
58
|
+
executionProviders: ["cpu"],
|
|
59
|
+
});
|
|
60
|
+
const crisisReply = options.crisisReply ?? DEFAULT_CRISIS_REPLY;
|
|
61
|
+
// Signal labels (grooming tactics, distress, swearing, age questions, romance, the general "unsafe" score) are
|
|
62
|
+
// scored on every check but only count towards flagged/labels when enabled. Older models list none.
|
|
63
|
+
const enabled = new Set(options.enable ?? []);
|
|
64
|
+
if (options.includeUnsafe) enabled.add("unsafe");
|
|
65
|
+
const off = new Set((meta.default_off ?? ["unsafe"]).filter((l) => !enabled.has(l)));
|
|
66
|
+
const counts = (l, scores) => !off.has(l) && scores[l] >= (thresholds[l] ?? 0.5);
|
|
67
|
+
// v10 renamed self_harm to self_harm_intent; either triggers the crisis reply.
|
|
68
|
+
const CRISIS = ["self_harm_intent", "self_harm"];
|
|
69
|
+
// Crisis referrals made, for the counts companion-chatbot laws ask operators to report (e.g. California SB 243).
|
|
70
|
+
const stats = { checks: 0, flagged: 0, crisisReferrals: 0, since: new Date().toISOString() };
|
|
71
|
+
|
|
72
|
+
// Long messages are split into overlapping windows at two sizes, full-length for context and short so one
|
|
73
|
+
// bad sentence in a long message is not drowned out. A label's score is its highest across all windows.
|
|
74
|
+
// Context and conversation inputs only use full-length windows: they are already structured and cut as in training.
|
|
75
|
+
// Runs of spaces are collapsed to one: the JS tokenizer splits them differently from the Python one the model
|
|
76
|
+
// was trained with, and a single space means the same thing.
|
|
77
|
+
function windows(text, short = true) {
|
|
78
|
+
const ids = tokenizer.encode(text.replace(/ {2,}/g, " ")).ids;
|
|
79
|
+
const cls = ids[0], sep = ids[ids.length - 1], body = ids.slice(1, -1);
|
|
80
|
+
const sizes = [maxLength - 2, Math.min(maxLength, options.windowTokens ?? 128) - 2].filter((n, i, a) => i === 0 || (short && n < a[0] && body.length > n));
|
|
81
|
+
const out = [];
|
|
82
|
+
for (const size of sizes) {
|
|
83
|
+
const stride = Math.floor(size * 0.75);
|
|
84
|
+
for (let i = 0; ; i += stride) {
|
|
85
|
+
out.push([cls, ...body.slice(i, i + size), sep]);
|
|
86
|
+
if (i + size >= body.length) break;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// texts: one or more inputs; a label's score is its highest over every window of every input.
|
|
93
|
+
async function score(texts, short = true) {
|
|
94
|
+
const all = [].concat(texts).flatMap((t) => windows(t, short)).sort((a, b) => a.length - b.length);
|
|
95
|
+
const best = new Float64Array(labels.length).fill(-Infinity);
|
|
96
|
+
for (let k = 0; k < all.length; k += 8) { // small batches of similar length: little padding
|
|
97
|
+
const batch = all.slice(k, k + 8), len = batch[batch.length - 1].length;
|
|
98
|
+
const ids = new BigInt64Array(batch.length * len), mask = new BigInt64Array(batch.length * len);
|
|
99
|
+
batch.forEach((w, b) => w.forEach((id, i) => { ids[b * len + i] = BigInt(id); mask[b * len + i] = 1n; }));
|
|
100
|
+
const out = await session.run({
|
|
101
|
+
input_ids: new ort.Tensor("int64", ids, [batch.length, len]),
|
|
102
|
+
attention_mask: new ort.Tensor("int64", mask, [batch.length, len]),
|
|
103
|
+
});
|
|
104
|
+
const logits = out.logits.data;
|
|
105
|
+
for (let b = 0; b < batch.length; b++) for (let j = 0; j < labels.length; j++) best[j] = Math.max(best[j], logits[b * labels.length + j]);
|
|
106
|
+
}
|
|
107
|
+
return Object.fromEntries(labels.map((l, j) => [l, 1 / (1 + Math.exp(-best[j]))]));
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// v9 models (guard-v9 and later) read the messages before the one being checked, and whole conversations.
|
|
111
|
+
// Their input tags each turn by who wrote it: [USER] to an AI, [AI], or [CHAT:<speaker>] between people.
|
|
112
|
+
const contextual = labels.includes("grooming");
|
|
113
|
+
const cut = (t, n = 300) => (t.length <= n ? t : t.slice(0, n) + "…"); // as in training: 300 for earlier turns
|
|
114
|
+
const tag = (turn) => {
|
|
115
|
+
const role = turn.role ?? (turn.direction === "output" ? "ai" : turn.direction === "chat" ? "player" : "user");
|
|
116
|
+
if (role === "ai") return "[AI]";
|
|
117
|
+
if (role === "player") return turn.speaker ? `[CHAT:${turn.speaker}]` : "[CHAT]";
|
|
118
|
+
return "[USER]";
|
|
119
|
+
};
|
|
120
|
+
const tokens = (t) => tokenizer.encode(t).ids.length;
|
|
121
|
+
|
|
122
|
+
/** Render the target with as many earlier turns as fit (the oldest are dropped first). */
|
|
123
|
+
function withContext(text, direction, context) {
|
|
124
|
+
const target = `[TARGET] ${tag({ direction })} ${cut(text, 1000)}`; // the judged turn: 1000, as in training
|
|
125
|
+
const earlier = context.map((t) => `${tag(t)} ${cut(t.text ?? "")}`);
|
|
126
|
+
for (let k = 0; k <= earlier.length; k++) {
|
|
127
|
+
const s = [...earlier.slice(k), target].join(" ");
|
|
128
|
+
if (k === earlier.length || tokens(s) <= maxLength) return s;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* @param {string} text
|
|
134
|
+
* @param {"input"|"output"|"chat"} direction input = a person writing to an AI, output = the AI, chat = a person writing to other people
|
|
135
|
+
* @param {{context?: {role?: "user"|"ai"|"player", direction?: string, speaker?: string, text: string}[]}} [opts]
|
|
136
|
+
*/
|
|
137
|
+
async function check(text, direction = "input", opts = {}) {
|
|
138
|
+
if (typeof text !== "string") throw new TypeError("@paxmod/guard: text must be a string");
|
|
139
|
+
const t0 = performance.now();
|
|
140
|
+
const input = !contextual ? `${direction === "output" ? "[AI]" : "[USER]"} ${text}`
|
|
141
|
+
: opts.context?.length ? withContext(text, direction, opts.context)
|
|
142
|
+
: `${direction === "chat" ? "[CHAT]" : tag({ direction })} ${text}`; // a single message is read whole, in windows
|
|
143
|
+
const scores = text.trim() ? await score(input, !(contextual && opts.context?.length)) : Object.fromEntries(labels.map((l) => [l, 0]));
|
|
144
|
+
const flagged = labels.filter((l) => counts(l, scores));
|
|
145
|
+
const result = { flagged: flagged.length > 0, labels: flagged, scores, direction, latencyMs: Math.round((performance.now() - t0) * 10) / 10 };
|
|
146
|
+
stats.checks++;
|
|
147
|
+
if (result.flagged) stats.flagged++;
|
|
148
|
+
if (direction !== "output" && CRISIS.some((l) => flagged.includes(l))) {
|
|
149
|
+
result.crisisReply = crisisReply;
|
|
150
|
+
stats.crisisReferrals++;
|
|
151
|
+
options.onCrisis?.(result);
|
|
152
|
+
}
|
|
153
|
+
return result;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Judge a whole stretch of conversation (v9 and later), e.g. a DM with both sides. Labels describe anything in it,
|
|
158
|
+
* including patterns no single message shows (grooming). Long conversations are read in overlapping windows.
|
|
159
|
+
* @param {{role?: "user"|"ai"|"player", direction?: string, speaker?: string, text: string}[]} turns
|
|
160
|
+
*/
|
|
161
|
+
async function checkConversation(turns) {
|
|
162
|
+
if (!contextual) throw new Error("@paxmod/guard: checkConversation needs a v9 or later model");
|
|
163
|
+
const t0 = performance.now();
|
|
164
|
+
// As in training: windows of up to 40 turns, overlapping by half, so long chats are read piece by piece.
|
|
165
|
+
const parts = [];
|
|
166
|
+
for (let i = 0; ; i += 20) {
|
|
167
|
+
parts.push("[CONVERSATION] " + turns.slice(i, i + 40).map((t) => `${tag(t)} ${cut(t.text ?? "")}`).join(" "));
|
|
168
|
+
if (i + 40 >= turns.length) break;
|
|
169
|
+
}
|
|
170
|
+
const scores = await score(parts, false);
|
|
171
|
+
const flagged = labels.filter((l) => counts(l, scores));
|
|
172
|
+
stats.checks++;
|
|
173
|
+
if (flagged.length) stats.flagged++;
|
|
174
|
+
return { flagged: flagged.length > 0, labels: flagged, scores, turns: turns.length, latencyMs: Math.round((performance.now() - t0) * 10) / 10 };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
return {
|
|
178
|
+
model: meta.version ?? modelName(),
|
|
179
|
+
labels,
|
|
180
|
+
check,
|
|
181
|
+
/** Check a message from the user before your bot sees it. */
|
|
182
|
+
checkInput: (text, opts) => check(text, "input", opts),
|
|
183
|
+
/** Check your bot's reply before the user sees it. */
|
|
184
|
+
checkOutput: (text, opts) => check(text, "output", opts),
|
|
185
|
+
/** Check a message one person wrote to others (game or community chat). */
|
|
186
|
+
checkChat: (text, opts) => check(text, "chat", opts),
|
|
187
|
+
checkConversation,
|
|
188
|
+
/** True for models that read context and whole conversations (v9 and later). */
|
|
189
|
+
contextual,
|
|
190
|
+
/** Counts since this guard was created: checks, flagged messages, crisis referrals. */
|
|
191
|
+
stats: () => ({ ...stats }),
|
|
192
|
+
close: () => session.release(),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export const DEFAULT_DISCLOSURE = "Just a reminder: I'm an AI, not a person. It might be a good moment to take a break.";
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Periodic "I'm an AI" reminders, per conversation. Some companion-chatbot laws require one at least every
|
|
200
|
+
* 3 hours (for example New York, and California for users known to be minors).
|
|
201
|
+
* Call due(conversationId) on each bot turn; it returns the reminder text when one is owed, otherwise null.
|
|
202
|
+
*/
|
|
203
|
+
export function disclosureReminder({ everyMs = 3 * 60 * 60 * 1000, text = DEFAULT_DISCLOSURE, now = () => Date.now() } = {}) {
|
|
204
|
+
const last = new Map();
|
|
205
|
+
return {
|
|
206
|
+
due(conversationId) {
|
|
207
|
+
const t = now(), prev = last.get(conversationId);
|
|
208
|
+
if (prev === undefined) { last.set(conversationId, t); return text; } // disclose at the start
|
|
209
|
+
if (t - prev >= everyMs) { last.set(conversationId, t); return text; }
|
|
210
|
+
return null;
|
|
211
|
+
},
|
|
212
|
+
forget: (conversationId) => last.delete(conversationId),
|
|
213
|
+
};
|
|
214
|
+
}
|
package/src/langchain.js
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// LangChain (v1 createAgent) middleware, a local drop-in for openAIModerationMiddleware.
|
|
2
|
+
// import { createAgent } from "langchain";
|
|
3
|
+
// import { paxmodGuardMiddleware } from "@paxmod/guard/langchain";
|
|
4
|
+
// const agent = createAgent({ model, tools, middleware: [paxmodGuardMiddleware()] });
|
|
5
|
+
// A blocked user message ends the run with the crisis reply (self-harm) or the fallback, without calling the model.
|
|
6
|
+
// A blocked model reply is replaced in place (same message id), so it never enters the history, and the run ends.
|
|
7
|
+
import { createMiddleware } from "langchain";
|
|
8
|
+
import { AIMessage, HumanMessage } from "@langchain/core/messages";
|
|
9
|
+
import { createGuard } from "./index.js";
|
|
10
|
+
|
|
11
|
+
const DEFAULT_BLOCK = {
|
|
12
|
+
input: ["self_harm_intent", "self_harm_instructions", "sexual_minor", "grooming", "sextortion", "jailbreak", "prompt_injection"], // self_harm_intent is answered with the crisis reply
|
|
13
|
+
output: ["self_harm_instructions", "sexual_minor", "sexual", "ai_claims_human", "ai_claims_professional", "pii_other", "off_platform", "dependency", "harassment", "manipulation", "ai_hostility", "ai_harmful_compliance", "hate", "weapons", "drugs", "crime", "violence", "extremism", "dangerous_challenge"],
|
|
14
|
+
};
|
|
15
|
+
const DEFAULT_FALLBACK = "Sorry, I can't continue with that. Let's talk about something else.";
|
|
16
|
+
|
|
17
|
+
/** @param {import("./ai-sdk").GuardMiddlewareOptions} [options] */
|
|
18
|
+
export function paxmodGuardMiddleware(options = {}) {
|
|
19
|
+
const guard = Promise.resolve(options.guard ?? createGuard(options));
|
|
20
|
+
const block = { input: options.block?.input ?? DEFAULT_BLOCK.input, output: options.block?.output ?? DEFAULT_BLOCK.output };
|
|
21
|
+
const fallback = options.fallback ?? DEFAULT_FALLBACK;
|
|
22
|
+
const blocks = (r, list) => r.labels.some((l) => list.includes(l));
|
|
23
|
+
|
|
24
|
+
return createMiddleware({
|
|
25
|
+
name: "PaxmodGuardMiddleware",
|
|
26
|
+
beforeModel: {
|
|
27
|
+
canJumpTo: ["end"],
|
|
28
|
+
hook: async (state) => {
|
|
29
|
+
const last = state.messages?.[state.messages.length - 1];
|
|
30
|
+
// Only on a fresh user turn: later steps of a tool loop would re-check the same message.
|
|
31
|
+
if (!last || !HumanMessage.isInstance(last) || !last.text) return;
|
|
32
|
+
const r = await (await guard).checkInput(last.text);
|
|
33
|
+
if (r.flagged) options.onFlag?.(r);
|
|
34
|
+
if (!blocks(r, block.input)) return;
|
|
35
|
+
return { jumpTo: "end", messages: [new AIMessage({ content: r.crisisReply ?? fallback, response_metadata: { paxmodGuard: r } })] };
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
afterModel: {
|
|
39
|
+
canJumpTo: ["end"],
|
|
40
|
+
hook: async (state) => {
|
|
41
|
+
const last = state.messages?.[state.messages.length - 1];
|
|
42
|
+
if (!last || !AIMessage.isInstance(last) || !last.text) return;
|
|
43
|
+
const r = await (await guard).checkOutput(last.text);
|
|
44
|
+
if (r.flagged) options.onFlag?.(r);
|
|
45
|
+
if (!blocks(r, block.output)) return;
|
|
46
|
+
return { jumpTo: "end", messages: [new AIMessage({ id: last.id, content: fallback, response_metadata: { paxmodGuard: r } })] };
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
}
|
package/src/mastra.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { GuardMiddlewareOptions } from "./ai-sdk";
|
|
2
|
+
|
|
3
|
+
/** Mastra input and output processor. On a block, the run's tripwire metadata is the GuardResult (with crisisReply on self-harm). */
|
|
4
|
+
export function paxmodGuard(options?: Omit<GuardMiddlewareOptions, "fallback">): any;
|
package/src/mastra.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// Mastra processor: a local, millisecond replacement for an LLM-based moderation processor.
|
|
2
|
+
// import { paxmodGuard } from "@paxmod/guard/mastra";
|
|
3
|
+
// const guard = paxmodGuard();
|
|
4
|
+
// const agent = new Agent({ ..., inputProcessors: [guard], outputProcessors: [guard] });
|
|
5
|
+
// const result = await agent.generate(message);
|
|
6
|
+
// if (result.tripwire) reply(result.tripwire.metadata?.crisisReply ?? "Let's talk about something else.");
|
|
7
|
+
import { createGuard } from "./index.js";
|
|
8
|
+
|
|
9
|
+
const DEFAULT_BLOCK = {
|
|
10
|
+
input: ["self_harm_intent", "self_harm_instructions", "sexual_minor", "grooming", "sextortion", "jailbreak", "prompt_injection"], // self_harm_intent is answered with the crisis reply
|
|
11
|
+
output: ["self_harm_instructions", "sexual_minor", "sexual", "ai_claims_human", "ai_claims_professional", "pii_other", "off_platform", "dependency", "harassment", "manipulation", "ai_hostility", "ai_harmful_compliance", "hate", "weapons", "drugs", "crime", "violence", "extremism", "dangerous_challenge"],
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
function text(message) {
|
|
15
|
+
const parts = message?.content?.parts;
|
|
16
|
+
const t = Array.isArray(parts) ? parts.filter((p) => p.type === "text").map((p) => p.text).join("\n") : "";
|
|
17
|
+
return t || (typeof message?.content?.content === "string" ? message.content.content : "");
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const latest = (messages, role) => [...messages].reverse().find((m) => m.role === role);
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* @param {import("./ai-sdk").GuardMiddlewareOptions} [options] same options as the other adapters (guard, block, onFlag, model options)
|
|
24
|
+
*/
|
|
25
|
+
export function paxmodGuard(options = {}) {
|
|
26
|
+
const guard = Promise.resolve(options.guard ?? createGuard(options));
|
|
27
|
+
const block = { input: options.block?.input ?? DEFAULT_BLOCK.input, output: options.block?.output ?? DEFAULT_BLOCK.output };
|
|
28
|
+
|
|
29
|
+
async function check(message, direction, abort) {
|
|
30
|
+
const t = text(message);
|
|
31
|
+
if (!t.trim()) return;
|
|
32
|
+
const g = await guard;
|
|
33
|
+
const r = await (direction === "input" ? g.checkInput(t) : g.checkOutput(t));
|
|
34
|
+
if (r.flagged) options.onFlag?.(r);
|
|
35
|
+
if (r.labels.some((l) => block[direction].includes(l))) {
|
|
36
|
+
abort(`PaxmodGuard: ${r.labels.join(", ")}`, { metadata: r });
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return {
|
|
41
|
+
id: "paxmod-guard",
|
|
42
|
+
name: "PaxmodGuard",
|
|
43
|
+
description: "Local safety classifier for companion chatbots (self-harm, minors, sexual content, jailbreaks and more).",
|
|
44
|
+
async processInput({ messages, abort }) {
|
|
45
|
+
await check(latest(messages, "user"), "input", abort);
|
|
46
|
+
return messages;
|
|
47
|
+
},
|
|
48
|
+
async processOutputResult({ messages, abort }) {
|
|
49
|
+
await check(latest(messages, "assistant"), "output", abort);
|
|
50
|
+
return messages;
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Guard, GuardOptions, GuardResult, GuardLabel } from "./index";
|
|
2
|
+
|
|
3
|
+
export interface GuardrailOptions extends GuardOptions {
|
|
4
|
+
/** Reuse an existing guard instead of creating one. Pass the same options object to guardInput and guardOutput to share one model. */
|
|
5
|
+
guard?: Guard;
|
|
6
|
+
/** Labels that trip the guardrail. */
|
|
7
|
+
block?: { input?: GuardLabel[]; output?: GuardLabel[] };
|
|
8
|
+
onFlag?: (result: GuardResult) => void;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** InputGuardrail for @openai/agents. outputInfo is the GuardResult (with crisisReply on self-harm). */
|
|
12
|
+
export function guardInput(options?: GuardrailOptions): any;
|
|
13
|
+
/** OutputGuardrail for @openai/agents. */
|
|
14
|
+
export function guardOutput(options?: GuardrailOptions): any;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// OpenAI Agents SDK guardrails. A flagged message trips the run; catch the error and use the crisis reply.
|
|
2
|
+
// import { Agent, run, InputGuardrailTripwireTriggered } from "@openai/agents";
|
|
3
|
+
// import { guardInput, guardOutput } from "@paxmod/guard/openai-agents";
|
|
4
|
+
// const agent = new Agent({ name: "Companion", instructions, inputGuardrails: [guardInput()], outputGuardrails: [guardOutput()] });
|
|
5
|
+
// try { await run(agent, message) } catch (e) {
|
|
6
|
+
// if (e instanceof InputGuardrailTripwireTriggered) reply(e.result.output.outputInfo.crisisReply ?? "Let's talk about something else.");
|
|
7
|
+
// }
|
|
8
|
+
import { createGuard } from "./index.js";
|
|
9
|
+
|
|
10
|
+
const DEFAULT_BLOCK = {
|
|
11
|
+
input: ["self_harm_intent", "self_harm_instructions", "sexual_minor", "grooming", "sextortion", "jailbreak", "prompt_injection"], // self_harm_intent is answered with the crisis reply
|
|
12
|
+
output: ["self_harm_instructions", "sexual_minor", "sexual", "ai_claims_human", "ai_claims_professional", "pii_other", "off_platform", "dependency", "harassment", "manipulation", "ai_hostility", "ai_harmful_compliance", "hate", "weapons", "drugs", "crime", "violence", "extremism", "dangerous_challenge"],
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
// The latest user message: input is a string, or a list of items where user messages carry a string or text parts.
|
|
16
|
+
function latestUserText(input) {
|
|
17
|
+
if (typeof input === "string") return input;
|
|
18
|
+
for (let i = input.length - 1; i >= 0; i--) {
|
|
19
|
+
const item = input[i];
|
|
20
|
+
if (item?.role !== "user") continue;
|
|
21
|
+
if (typeof item.content === "string") return item.content;
|
|
22
|
+
return (item.content ?? []).filter((p) => typeof p?.text === "string").map((p) => p.text).join("\n");
|
|
23
|
+
}
|
|
24
|
+
return "";
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const shared = new WeakMap();
|
|
28
|
+
function guardFor(options) {
|
|
29
|
+
if (options.guard) return Promise.resolve(options.guard);
|
|
30
|
+
// One model per options object, so guardInput(opts) and guardOutput(opts) share it.
|
|
31
|
+
if (!shared.has(options)) shared.set(options, createGuard(options));
|
|
32
|
+
return shared.get(options);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Input guardrail. Runs before the agent (not in parallel), so a tripped run never calls the model. */
|
|
36
|
+
export function guardInput(options = {}) {
|
|
37
|
+
const block = options.block?.input ?? DEFAULT_BLOCK.input;
|
|
38
|
+
return {
|
|
39
|
+
name: "paxmod-guard-input",
|
|
40
|
+
runInParallel: false,
|
|
41
|
+
async execute({ input }) {
|
|
42
|
+
const text = latestUserText(input);
|
|
43
|
+
if (!text.trim()) return { tripwireTriggered: false, outputInfo: null };
|
|
44
|
+
const r = await (await guardFor(options)).checkInput(text);
|
|
45
|
+
if (r.flagged) options.onFlag?.(r);
|
|
46
|
+
return { tripwireTriggered: r.labels.some((l) => block.includes(l)), outputInfo: r };
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Output guardrail: checks the agent's final text output. */
|
|
52
|
+
export function guardOutput(options = {}) {
|
|
53
|
+
const block = options.block?.output ?? DEFAULT_BLOCK.output;
|
|
54
|
+
return {
|
|
55
|
+
name: "paxmod-guard-output",
|
|
56
|
+
async execute({ agentOutput }) {
|
|
57
|
+
const text = typeof agentOutput === "string" ? agentOutput : JSON.stringify(agentOutput ?? "");
|
|
58
|
+
if (!text.trim()) return { tripwireTriggered: false, outputInfo: null };
|
|
59
|
+
const r = await (await guardFor(options)).checkOutput(text);
|
|
60
|
+
if (r.flagged) options.onFlag?.(r);
|
|
61
|
+
return { tripwireTriggered: r.labels.some((l) => block.includes(l)), outputInfo: r };
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|