open-safety-guard 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,25 @@
1
+ name: Node.js CI
2
+
3
+ on:
4
+ push:
5
+ branches: [ "main" ]
6
+ pull_request:
7
+ branches: [ "main" ]
8
+
9
+ jobs:
10
+ build:
11
+ runs-on: ubuntu-latest
12
+
13
+ strategy:
14
+ matrix:
15
+ node-version: [18.x, 20.x, 22.x]
16
+
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - name: Use Node.js ${{ matrix.node-version }}
20
+ uses: actions/setup-node@v4
21
+ with:
22
+ node-version: ${{ matrix.node-version }}
23
+ cache: 'npm'
24
+ - run: npm ci
25
+ - run: npm test
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,115 @@
1
+ # open-safety-guard
2
+
3
+ **A high-speed, structural LLM classification engine for Trust & Safety AI.**
4
+
5
+ [![Node.js CI](https://github.com/anatolyben/open-safety-guard/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/anatolyben/open-safety-guard/actions/workflows/ci.yml)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
7
+
8
+ `open-safety-guard` is a Node.js framework optimized for executing real-time content moderation using ultra-fast LLMs like Groq. It ships with built-in policies for **Spam**, **Conduct**, and **Topics**, and provides a simple Express middleware for protecting your application routes out of the box.
9
+
10
+ ## The Problem
11
+
12
+ Traditional content moderation relies heavily on regex pattern matching, legacy blocklists, or slow, expensive Machine Learning models. These approaches are often bypassed by bad actors using slight text variations or lack context when moderating nuanced conversation (e.g. distinguishing between a heated debate and a personal attack).
13
+
14
+ By leveraging ultra-fast, structured LLM inference (using providers like Groq), we can now execute high-quality semantic moderation in milliseconds. `open-safety-guard` forces the LLM to output deterministic JSON schemas, meaning you get a fast, structured, programmatic safety report for every message, making it easy to block unwanted content reliably.
15
+
16
+ ## Installation
17
+
18
+ ```bash
19
+ npm install open-safety-guard
20
+ ```
21
+
22
+ Ensure your Groq API key is available in your environment:
23
+ ```bash
24
+ export GROQ_API_KEY="gsk_..."
25
+ ```
26
+
27
+ ## Programmatic Usage
28
+
29
+ You can use the core logic directly in any Node.js application to evaluate a string of text across multiple guards.
30
+
31
+ ```javascript
32
+ import { ModerateText, SpamGuard, ConductGuard } from 'open-safety-guard';
33
+
34
+ async function checkMessage() {
35
+ const input = "Click here to win a free iPhone! http://scam.link";
36
+
37
+ // Evaluate the input across the configured guards
38
+ const result = await ModerateText(input, [
39
+ new SpamGuard({ sensitivity: 'high' }),
40
+ new ConductGuard()
41
+ ]);
42
+
43
+ console.log(result);
44
+ /*
45
+ {
46
+ flagged: true,
47
+ reasons: [ "[SpamGuard] The message promotes a scam." ],
48
+ results: {
49
+ SpamGuard: { flagged: true, is_spam: true, category: "scam", confidence: 0.95, ... },
50
+ ConductGuard: { flagged: false, violation: false, ... }
51
+ }
52
+ }
53
+ */
54
+ }
55
+
56
+ checkMessage();
57
+ ```
58
+
59
+ ## Express Middleware Usage
60
+
61
+ Want to protect your APIs instantly? Use the included Express middleware to build "Moderation as a Service" directly into your app.
62
+
63
+ ```javascript
64
+ import express from 'express';
65
+ import { contentFilter, SpamGuard, ConductGuard } from 'open-safety-guard';
66
+
67
+ const app = express();
68
+ app.use(express.json());
69
+
70
+ // Protect the route using open-safety-guard
71
+ app.post(
72
+ '/api/comments',
73
+ contentFilter({
74
+ guards: [new SpamGuard(), new ConductGuard()],
75
+ // Optional: tell the middleware how to extract text from the request
76
+ extractText: (req) => req.body.comment
77
+ }),
78
+ (req, res) => {
79
+ // If we reach here, the comment passed moderation!
80
+ res.json({ success: true, message: "Comment saved!" });
81
+ }
82
+ );
83
+
84
+ app.listen(3000, () => console.log('Server is running on port 3000'));
85
+ ```
86
+
87
+ ## Custom Guards
88
+
89
+ You can easily build custom LLM guards by providing your own system prompt and JSON schema. This is incredibly powerful for evaluating highly specific business rules or custom content policies.
90
+
91
+ ```javascript
92
+ import { CustomGuard, ModerateText } from 'open-safety-guard';
93
+
94
+ const MedicalAdviceGuard = new CustomGuard({
95
+ name: "MedicalAdviceGuard",
96
+ flagField: "is_medical_advice",
97
+ systemPrompt: `You are a medical safety assistant. Determine if the user is providing unauthorized medical advice.
98
+ Respond with JSON only.
99
+ Schema: {"is_medical_advice": boolean, "confidence": number, "reason": string}`
100
+ });
101
+
102
+ const result = await ModerateText("You should definitely take 400mg of ibuprofen for that.", [MedicalAdviceGuard]);
103
+
104
+ if (result.flagged) {
105
+ console.log("Blocked for unauthorized medical advice!");
106
+ }
107
+ ```
108
+
109
+ ## Fallback & Circuit Breaker (Fail-Open)
110
+
111
+ Content moderation APIs should never break your main application loop. `open-safety-guard` is built with a strict **fail-open** philosophy. If your LLM API key is missing, rate limits are hit, or the provider has an outage, the package gracefully bypasses classification (evaluating to `flagged: false`) so your application stays online.
112
+
113
+ ## License
114
+
115
+ MIT
package/package.json ADDED
@@ -0,0 +1,30 @@
1
+ {
2
+ "name": "open-safety-guard",
3
+ "version": "1.0.0",
4
+ "description": "High-speed, structural LLM classification engine for trust & safety AI moderation.",
5
+ "type": "module",
6
+ "main": "src/index.js",
7
+ "scripts": {
8
+ "start": "node src/server.js",
9
+ "test": "node --test"
10
+ },
11
+ "keywords": [
12
+ "moderation",
13
+ "trust and safety",
14
+ "llm",
15
+ "groq",
16
+ "spam",
17
+ "conduct",
18
+ "classification",
19
+ "express"
20
+ ],
21
+ "author": "",
22
+ "license": "MIT",
23
+ "dependencies": {
24
+ "express": "^4.21.2",
25
+ "openai": "^4.79.0"
26
+ },
27
+ "devDependencies": {
28
+ "supertest": "^7.0.0"
29
+ }
30
+ }
@@ -0,0 +1,93 @@
1
+ import OpenAI from "openai";
2
+
3
+ export const DEFAULT_LLM_MODEL = "llama-3.1-8b-instant";
4
+ export const DEFAULT_GROQ_BASE_URL = "https://api.groq.com/openai/v1";
5
+
6
+ /** Confidence a positive verdict must clear, per operator sensitivity. */
7
+ export const SENSITIVITY_THRESHOLDS = Object.freeze({
8
+ low: 0.9,
9
+ medium: 0.75,
10
+ high: 0.6,
11
+ });
12
+
13
+ export function resolveThreshold(sensitivity) {
14
+ return SENSITIVITY_THRESHOLDS[sensitivity] ?? SENSITIVITY_THRESHOLDS.medium;
15
+ }
16
+
17
+ export function clampConfidence(value) {
18
+ return Math.min(1, Math.max(0, value));
19
+ }
20
+
21
+ export function meetsThreshold(confidence, sensitivity) {
22
+ return confidence >= resolveThreshold(sensitivity);
23
+ }
24
+
25
+ /**
26
+ * Ensures we have an API key configured (fail-open)
27
+ */
28
+ export function ensureGroqKey(tag, apiKey) {
29
+ if (!apiKey) {
30
+ console.warn(`[${tag}] API key not provided or set in environment — skipping classification`);
31
+ return false;
32
+ }
33
+ return true;
34
+ }
35
+
36
+ /**
37
+ * Run one deterministic JSON classification.
38
+ */
39
+ export async function runJsonClassification({
40
+ tag,
41
+ label,
42
+ systemPrompt,
43
+ userPrompt,
44
+ config
45
+ }) {
46
+ const apiKey = config?.apiKey || process.env.GROQ_API_KEY;
47
+ if (!ensureGroqKey(tag, apiKey)) return null;
48
+
49
+ const client = new OpenAI({
50
+ apiKey,
51
+ baseURL: config?.baseURL || process.env.GROQ_BASE_URL || DEFAULT_GROQ_BASE_URL,
52
+ });
53
+
54
+ try {
55
+ const response = await client.chat.completions.create({
56
+ model: config?.model || process.env.LLM_MODEL || DEFAULT_LLM_MODEL,
57
+ messages: [
58
+ { role: "system", content: systemPrompt },
59
+ { role: "user", content: userPrompt },
60
+ ],
61
+ response_format: { type: "json_object" },
62
+ temperature: 0,
63
+ max_tokens: config?.maxTokens || 128,
64
+ });
65
+ return response.choices?.[0]?.message?.content ?? "";
66
+ } catch (err) {
67
+ // Fail-open strategy
68
+ console.warn(`[${tag}] ${label} error — fail-open:`, err?.message ?? err);
69
+ return null;
70
+ }
71
+ }
72
+
73
+ /**
74
+ * Parse the verdict envelope every guard shares.
75
+ */
76
+ export function parseVerdictEnvelope(raw, flagField) {
77
+ let parsed;
78
+ try {
79
+ parsed = JSON.parse(raw);
80
+ } catch {
81
+ return null;
82
+ }
83
+
84
+ if (typeof parsed?.[flagField] !== "boolean") return null;
85
+ if (typeof parsed.confidence !== "number") return null;
86
+
87
+ return {
88
+ parsed,
89
+ flagged: parsed[flagField],
90
+ confidence: clampConfidence(parsed.confidence),
91
+ reason: typeof parsed.reason === "string" ? parsed.reason.trim() : "",
92
+ };
93
+ }
@@ -0,0 +1,39 @@
1
+ export async function ModerateText(text, guards = [], config = {}) {
2
+ const results = {};
3
+
4
+ const validGuards = guards.map((guard) => {
5
+ // Determine if guard is a class constructor or an instance
6
+ const guardInstance = typeof guard === 'function' ? new guard() : guard;
7
+
8
+ if (typeof guardInstance.evaluate !== 'function') {
9
+ throw new Error(`Invalid guard: ${guardInstance?.name || 'unknown'} must implement an evaluate() method.`);
10
+ }
11
+ return guardInstance;
12
+ });
13
+
14
+ const evalPromises = validGuards.map(async (guardInstance) => {
15
+ const result = await guardInstance.evaluate(text, config);
16
+ results[guardInstance.name || guardInstance.constructor.name] = result;
17
+ });
18
+
19
+ await Promise.allSettled(evalPromises);
20
+
21
+ // Compile a summarized safety report
22
+ const report = {
23
+ flagged: false,
24
+ results,
25
+ reasons: []
26
+ };
27
+
28
+ for (const [name, result] of Object.entries(results)) {
29
+ // 'null' means skipped or fail-open allowed
30
+ if (result && result.flagged) {
31
+ report.flagged = true;
32
+ if (result.reason) {
33
+ report.reasons.push(`[${name}] ${result.reason}`);
34
+ }
35
+ }
36
+ }
37
+
38
+ return report;
39
+ }
@@ -0,0 +1,47 @@
1
+ import { ModerateText } from '../core/moderate.js';
2
+
3
+ /**
4
+ * Express middleware for 'Moderation as a Service'.
5
+ * Intercepts incoming requests, extracts text, runs LLM classifiers,
6
+ * and blocks the request (403) if policy is violated.
7
+ *
8
+ * @param {Object} options
9
+ * @param {Array} options.guards - Array of Guard instances to evaluate.
10
+ * @param {Function} [options.extractText] - Function to extract text from request (default: req => req.body.text).
11
+ * @param {Object} [options.config] - Global configuration to pass to ModerateText (e.g. apiKey).
12
+ */
13
+ export function contentFilter(options = {}) {
14
+ const {
15
+ guards = [],
16
+ extractText = (req) => req.body?.text,
17
+ config = {}
18
+ } = options;
19
+
20
+ return async (req, res, next) => {
21
+ try {
22
+ const text = extractText(req);
23
+
24
+ if (!text) {
25
+ return next();
26
+ }
27
+
28
+ const report = await ModerateText(text, guards, config);
29
+
30
+ if (report.flagged) {
31
+ return res.status(403).json({
32
+ error: "Content violated safety policy",
33
+ reasons: report.reasons,
34
+ report
35
+ });
36
+ }
37
+
38
+ // Attach report to request for downstream handlers if they need it
39
+ req.moderationReport = report;
40
+ next();
41
+ } catch (err) {
42
+ console.error("[contentFilter] Middleware error:", err);
43
+ // Fail-open strategy: if moderation fails, allow request to proceed
44
+ next();
45
+ }
46
+ };
47
+ }
@@ -0,0 +1,74 @@
1
+ import {
2
+ runJsonClassification,
3
+ parseVerdictEnvelope,
4
+ meetsThreshold
5
+ } from "../core/classifier.js";
6
+
7
+ const VIOLATION_TYPES = [
8
+ "harassment",
9
+ "personal_attack",
10
+ "threat",
11
+ "hate_speech",
12
+ "other",
13
+ ];
14
+
15
+ const MAX_TEXT_CHARS = 800;
16
+ const MAX_RULE_CHARS = 2000;
17
+
18
+ const SYSTEM_PROMPT = `You are a community conduct moderation assistant. Your only job is to decide whether a message violates the operator-defined community conduct rules.
19
+ Respond with JSON only — no markdown, no explanation outside the JSON.
20
+ Schema: {"violation":boolean,"violation_type":string|null,"confidence":number,"reason":string}
21
+ Rules:
22
+ - Judge strictly against the operator-provided conduct rules. Do not apply your own assumptions.
23
+ - violation_type must be one of: "harassment", "personal_attack", "threat", "hate_speech", "other", or null when violation is false.
24
+ - confidence is a number from 0 to 1 (1 = certain).
25
+ - reason is one short sentence (max 20 words). Keep it factual.
26
+ - Disagreements, criticism of ideas, sarcasm, and venting frustration are NOT violations unless they target a specific person with hostility.
27
+ - Borderline or ambiguous messages should be marked as violation=false with low confidence.`;
28
+
29
+ function buildUserPrompt(text, rules) {
30
+ const parts = [];
31
+ if (rules?.trim()) {
32
+ parts.push(`COMMUNITY CONDUCT RULES:\n${String(rules).slice(0, MAX_RULE_CHARS)}`);
33
+ } else {
34
+ parts.push(`COMMUNITY CONDUCT RULES:\nBe respectful. No harassment, personal attacks, threats, or hate speech.`);
35
+ }
36
+ parts.push(`MESSAGE TO EVALUATE:\n${String(text).slice(0, MAX_TEXT_CHARS)}`);
37
+ return parts.join("\n\n");
38
+ }
39
+
40
+ export class ConductGuard {
41
+ constructor(opts = {}) {
42
+ this.name = 'ConductGuard';
43
+ this.opts = opts;
44
+ }
45
+
46
+ async evaluate(text, config = {}) {
47
+ if (!text?.trim()) return null;
48
+
49
+ const sensitivity = this.opts.sensitivity ?? config.sensitivity ?? "medium";
50
+
51
+ const raw = await runJsonClassification({
52
+ tag: "conduct-guard",
53
+ label: "classifyConduct",
54
+ systemPrompt: SYSTEM_PROMPT,
55
+ userPrompt: buildUserPrompt(text, this.opts.rules),
56
+ config
57
+ });
58
+
59
+ if (raw === null) return null; // fail-open
60
+
61
+ const envelope = parseVerdictEnvelope(raw, "violation");
62
+ if (!envelope) return null;
63
+
64
+ const { parsed, flagged, confidence, reason } = envelope;
65
+ const rawType = parsed.violation_type;
66
+ const violation_type = VIOLATION_TYPES.includes(rawType) ? rawType : null;
67
+
68
+ if (!flagged || !meetsThreshold(confidence, sensitivity)) {
69
+ return { flagged: false, violation: false, violation_type: null, confidence, reason };
70
+ }
71
+
72
+ return { flagged: true, violation: true, violation_type, confidence, reason };
73
+ }
74
+ }
@@ -0,0 +1,55 @@
1
+ import {
2
+ runJsonClassification,
3
+ parseVerdictEnvelope,
4
+ meetsThreshold
5
+ } from "../core/classifier.js";
6
+
7
+ export class CustomGuard {
8
+ /**
9
+ * Create a custom guard with your own schema and prompt.
10
+ *
11
+ * @param {Object} opts
12
+ * @param {string} opts.name - Name of the guard.
13
+ * @param {string} opts.systemPrompt - The prompt containing instructions and JSON schema.
14
+ * @param {string} opts.flagField - The boolean field in the JSON response that indicates a violation (e.g. 'is_violation').
15
+ * @param {Function} [opts.buildUserPrompt] - Optional function to build user prompt `(text) => string`.
16
+ */
17
+ constructor(opts) {
18
+ if (!opts.name || !opts.systemPrompt || !opts.flagField) {
19
+ throw new Error("CustomGuard requires 'name', 'systemPrompt', and 'flagField'");
20
+ }
21
+ this.name = opts.name;
22
+ this.opts = opts;
23
+ }
24
+
25
+ async evaluate(text, config = {}) {
26
+ if (!text?.trim()) return null;
27
+
28
+ const sensitivity = this.opts.sensitivity ?? config.sensitivity ?? "medium";
29
+
30
+ const userPrompt = this.opts.buildUserPrompt
31
+ ? this.opts.buildUserPrompt(text)
32
+ : `MESSAGE TO EVALUATE:\n${text}`;
33
+
34
+ const raw = await runJsonClassification({
35
+ tag: `custom-${this.name}`,
36
+ label: "evaluate",
37
+ systemPrompt: this.opts.systemPrompt,
38
+ userPrompt,
39
+ config
40
+ });
41
+
42
+ if (raw === null) return null; // fail-open
43
+
44
+ const envelope = parseVerdictEnvelope(raw, this.opts.flagField);
45
+ if (!envelope) return null;
46
+
47
+ const { parsed, flagged, confidence, reason } = envelope;
48
+
49
+ if (!flagged || !meetsThreshold(confidence, sensitivity)) {
50
+ return { flagged: false, confidence, reason, ...parsed };
51
+ }
52
+
53
+ return { flagged: true, confidence, reason, ...parsed };
54
+ }
55
+ }
@@ -0,0 +1,98 @@
1
+ import {
2
+ runJsonClassification,
3
+ parseVerdictEnvelope,
4
+ meetsThreshold
5
+ } from "../core/classifier.js";
6
+
7
+ const SPAM_CATEGORIES = [
8
+ "scam",
9
+ "phishing",
10
+ "crypto",
11
+ "adult",
12
+ "gambling",
13
+ "impersonation",
14
+ "promotion",
15
+ "other",
16
+ ];
17
+
18
+ const MAX_TEXT_CHARS = 800;
19
+ const MAX_DEFINITION_CHARS = 2000;
20
+ const CATEGORY_LIST = SPAM_CATEGORIES.map((c) => `"${c}"`).join(", ");
21
+
22
+ const SYSTEM_PROMPT = `You are a spam moderation assistant for a group chat. Your only job is to decide whether a message is spam, judged against the operator's definition of spam for THIS community.
23
+ Respond with JSON only — no markdown, no explanation outside the JSON.
24
+ Schema: {"is_spam":boolean,"category":string|null,"confidence":number,"reason":string}
25
+ Rules:
26
+ - Judge primarily against the operator-provided spam definition. Do not apply your own assumptions about what the community wants.
27
+ - category must be one of: ${CATEGORY_LIST}, or null when is_spam is false.
28
+ - confidence is a number from 0 to 1 (1 = certain).
29
+ - reason is one short sentence (max 20 words). Keep it factual.
30
+ - On-topic discussion, questions, opinions, and a member sharing their own relevant work are NOT spam.
31
+ - Borderline or ambiguous messages should be marked is_spam=false with low confidence.`;
32
+
33
+ function buildUserPrompt(text, spamDefinition) {
34
+ const parts = [];
35
+ if (spamDefinition?.trim()) {
36
+ parts.push(
37
+ `WHAT COUNTS AS SPAM IN THIS COMMUNITY:\n${String(spamDefinition).slice(0, MAX_DEFINITION_CHARS)}`,
38
+ );
39
+ } else {
40
+ parts.push(
41
+ "WHAT COUNTS AS SPAM IN THIS COMMUNITY:\nUse general good judgement — scams, phishing, crypto shilling, adult bots, gambling, impersonation, and unsolicited promotion or advertising.",
42
+ );
43
+ }
44
+ parts.push(`MESSAGE TO EVALUATE:\n${String(text).slice(0, MAX_TEXT_CHARS)}`);
45
+ return parts.join("\n\n");
46
+ }
47
+
48
+ export function normalizeEnabledCategories(enabledCategories) {
49
+ if (!enabledCategories) return null;
50
+ if (Array.isArray(enabledCategories)) {
51
+ return enabledCategories.length ? new Set(enabledCategories) : null;
52
+ }
53
+ if (typeof enabledCategories === "object") {
54
+ const on = Object.entries(enabledCategories)
55
+ .filter(([, v]) => v)
56
+ .map(([k]) => k);
57
+ if (Object.keys(enabledCategories).length === 0) return null;
58
+ return new Set(on);
59
+ }
60
+ return null;
61
+ }
62
+
63
+ export class SpamGuard {
64
+ constructor(opts = {}) {
65
+ this.name = 'SpamGuard';
66
+ this.opts = opts;
67
+ }
68
+
69
+ async evaluate(text, config = {}) {
70
+ if (!text?.trim()) return null;
71
+
72
+ const sensitivity = this.opts.sensitivity ?? config.sensitivity ?? "medium";
73
+ const enabledSet = normalizeEnabledCategories(this.opts.enabledCategories);
74
+
75
+ const raw = await runJsonClassification({
76
+ tag: "spam-guard",
77
+ label: "classifySpam",
78
+ systemPrompt: SYSTEM_PROMPT,
79
+ userPrompt: buildUserPrompt(text, this.opts.spamDefinition),
80
+ config
81
+ });
82
+
83
+ if (raw === null) return null; // fail-open
84
+
85
+ const envelope = parseVerdictEnvelope(raw, "is_spam");
86
+ if (!envelope) return null;
87
+
88
+ const { parsed, flagged, confidence, reason } = envelope;
89
+ const rawCat = parsed.category;
90
+ const category = SPAM_CATEGORIES.includes(rawCat) ? rawCat : "other";
91
+
92
+ if (!flagged || !meetsThreshold(confidence, sensitivity) || (enabledSet && !enabledSet.has(category))) {
93
+ return { flagged: false, is_spam: false, category: null, confidence, reason };
94
+ }
95
+
96
+ return { flagged: true, is_spam: true, category, confidence, reason };
97
+ }
98
+ }
@@ -0,0 +1,82 @@
1
+ import {
2
+ runJsonClassification,
3
+ parseVerdictEnvelope,
4
+ meetsThreshold
5
+ } from "../core/classifier.js";
6
+
7
+ const MAX_TEXT_CHARS = 800;
8
+ const MAX_EXAMPLE_CHARS = 200;
9
+ const MAX_EXAMPLES = 5;
10
+
11
+ const SYSTEM_PROMPT = `You are a community moderation assistant. Your only job is to decide whether a message belongs in the operator-defined group topic.
12
+ Respond with JSON only — no markdown, no explanation outside the JSON.
13
+ Schema: {"on_topic":boolean,"confidence":number,"reason":string}
14
+ Rules:
15
+ - Judge strictly against the operator-provided topic description and examples. Do not use the group name or your own assumptions.
16
+ - Brief greetings, thanks, emoji-only messages, and casual banter are on_topic unless they clearly derail discussion.
17
+ - confidence is a number from 0 to 1 (1 = certain).
18
+ - reason is one short sentence (max 20 words). Keep it factual.`;
19
+
20
+ function buildUserPrompt(text, topic = {}) {
21
+ const parts = [];
22
+ if (topic.topic_title) {
23
+ parts.push(`GROUP TOPIC: ${topic.topic_title}`);
24
+ }
25
+ if (topic.topic_description) {
26
+ parts.push(`DESCRIPTION: ${topic.topic_description}`);
27
+ }
28
+
29
+ const allowed = (topic.allowed_examples ?? []).slice(0, MAX_EXAMPLES);
30
+ if (allowed.length > 0) {
31
+ parts.push(
32
+ `ON-TOPIC EXAMPLES:\n${allowed.map((e) => `- ${String(e).slice(0, MAX_EXAMPLE_CHARS)}`).join("\n")}`,
33
+ );
34
+ }
35
+
36
+ const offTopic = (topic.off_topic_examples ?? []).slice(0, MAX_EXAMPLES);
37
+ if (offTopic.length > 0) {
38
+ parts.push(
39
+ `OFF-TOPIC EXAMPLES:\n${offTopic.map((e) => `- ${String(e).slice(0, MAX_EXAMPLE_CHARS)}`).join("\n")}`,
40
+ );
41
+ }
42
+
43
+ parts.push(`MESSAGE TO EVALUATE:\n${String(text).slice(0, MAX_TEXT_CHARS)}`);
44
+ return parts.join("\n\n");
45
+ }
46
+
47
+ export class TopicGuard {
48
+ constructor(opts = {}) {
49
+ this.name = 'TopicGuard';
50
+ this.opts = opts;
51
+ }
52
+
53
+ async evaluate(text, config = {}) {
54
+ if (!text?.trim()) return null;
55
+
56
+ const sensitivity = this.opts.sensitivity ?? config.sensitivity ?? "medium";
57
+
58
+ const raw = await runJsonClassification({
59
+ tag: "topic-guard",
60
+ label: "classifyMessage",
61
+ systemPrompt: SYSTEM_PROMPT,
62
+ userPrompt: buildUserPrompt(text, this.opts.topic),
63
+ config
64
+ });
65
+
66
+ if (raw === null) return null; // fail-open
67
+
68
+ const envelope = parseVerdictEnvelope(raw, "on_topic");
69
+ if (!envelope) return null;
70
+
71
+ const { flagged: onTopic, confidence, reason } = envelope;
72
+
73
+ if (onTopic) return { flagged: false, on_topic: true, confidence, reason };
74
+
75
+ // Off-topic call only stands if it clears sensitivity bar
76
+ if (!meetsThreshold(confidence, sensitivity)) {
77
+ return { flagged: false, on_topic: true, confidence, reason };
78
+ }
79
+
80
+ return { flagged: true, on_topic: false, confidence, reason };
81
+ }
82
+ }
package/src/index.js ADDED
@@ -0,0 +1,6 @@
1
+ export { ModerateText } from './core/moderate.js';
2
+ export { SpamGuard } from './guards/SpamGuard.js';
3
+ export { ConductGuard } from './guards/ConductGuard.js';
4
+ export { TopicGuard } from './guards/TopicGuard.js';
5
+ export { CustomGuard } from './guards/CustomGuard.js';
6
+ export { contentFilter } from './express/contentFilter.js';
package/src/server.js ADDED
@@ -0,0 +1,41 @@
1
+ import express from 'express';
2
+ import { SpamGuard } from './guards/SpamGuard.js';
3
+ import { ConductGuard } from './guards/ConductGuard.js';
4
+ import { ModerateText } from './core/moderate.js';
5
+
6
+ const app = express();
7
+ app.use(express.json());
8
+
9
+ // Setup default guards for the microservice
10
+ const defaultGuards = [
11
+ new SpamGuard(),
12
+ new ConductGuard()
13
+ ];
14
+
15
+ // Standalone REST API endpoint
16
+ app.post('/api/moderate', async (req, res) => {
17
+ const { text } = req.body;
18
+ if (!text) {
19
+ return res.status(400).json({ error: "Missing 'text' in JSON payload" });
20
+ }
21
+
22
+ try {
23
+ const report = await ModerateText(text, defaultGuards);
24
+ res.json(report);
25
+ } catch (err) {
26
+ console.error("[Server Error]", err);
27
+ res.status(500).json({ error: "Internal Server Error" });
28
+ }
29
+ });
30
+
31
+ const PORT = process.env.PORT || 3000;
32
+
33
+ // Start server if not imported as a module (for testing)
34
+ const isMain = process.argv[1] && process.argv[1].endsWith('server.js');
35
+ if (isMain || process.env.NODE_ENV === 'development') {
36
+ app.listen(PORT, () => {
37
+ console.log(`open-safety-guard Microservice running on port ${PORT}`);
38
+ });
39
+ }
40
+
41
+ export default app;
@@ -0,0 +1,39 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert';
3
+ import { ModerateText } from '../src/core/moderate.js';
4
+ import { CustomGuard } from '../src/guards/CustomGuard.js';
5
+
6
+ test('ModerateText - Fail-open behavior when API keys are missing', async () => {
7
+ const dummyGuard = new CustomGuard({
8
+ name: "TestGuard",
9
+ systemPrompt: "Dummy prompt",
10
+ flagField: "is_bad"
11
+ });
12
+
13
+ // Call without any environment API keys
14
+ // It should quickly fail-open and not crash
15
+ const report = await ModerateText("Hello world", [dummyGuard], { apiKey: '' });
16
+
17
+ assert.strictEqual(report.flagged, false);
18
+ });
19
+
20
+ test('ModerateText - Rejects invalid guards', async () => {
21
+ const invalidGuard = { name: "BadGuard" }; // missing evaluate()
22
+
23
+ await assert.rejects(
24
+ ModerateText("Hello", [invalidGuard]),
25
+ /Invalid guard: BadGuard must implement an evaluate\(\) method./
26
+ );
27
+ });
28
+
29
+ test('CustomGuard - Returns correct fail-open on invalid response', async () => {
30
+ const guard = new CustomGuard({
31
+ name: "TestGuard",
32
+ systemPrompt: "Dummy",
33
+ flagField: "is_bad"
34
+ });
35
+
36
+ // Since we don't have a valid API key, evaluate should fail-open
37
+ const result = await guard.evaluate("Test", { apiKey: '' });
38
+ assert.strictEqual(result, null);
39
+ });
@@ -0,0 +1,49 @@
1
+ import test from 'node:test';
2
+ import assert from 'node:assert';
3
+ import request from 'supertest';
4
+ import express from 'express';
5
+ import { contentFilter } from '../src/express/contentFilter.js';
6
+
7
+ test('Express Middleware - Passes text when no policy violations occur', async () => {
8
+ const app = express();
9
+ app.use(express.json());
10
+
11
+ // A dummy guard that fails-open or always allows
12
+ const mockGuard = {
13
+ name: "MockGuard",
14
+ evaluate: async () => ({ flagged: false, reason: "" })
15
+ };
16
+
17
+ app.post('/test', contentFilter({ guards: [mockGuard] }), (req, res) => {
18
+ res.status(200).json({ success: true });
19
+ });
20
+
21
+ const response = await request(app)
22
+ .post('/test')
23
+ .send({ text: "Hello world" })
24
+ .expect(200);
25
+
26
+ assert.strictEqual(response.body.success, true);
27
+ });
28
+
29
+ test('Express Middleware - Blocks text when policy violated', async () => {
30
+ const app = express();
31
+ app.use(express.json());
32
+
33
+ const mockGuard = {
34
+ name: "MockGuard",
35
+ evaluate: async () => ({ flagged: true, reason: "Mock failure" })
36
+ };
37
+
38
+ app.post('/test', contentFilter({ guards: [mockGuard] }), (req, res) => {
39
+ res.status(200).json({ success: true });
40
+ });
41
+
42
+ const response = await request(app)
43
+ .post('/test')
44
+ .send({ text: "Bad content" })
45
+ .expect(403);
46
+
47
+ assert.strictEqual(response.body.error, "Content violated safety policy");
48
+ assert.deepStrictEqual(response.body.reasons, ["[MockGuard] Mock failure"]);
49
+ });