@zhin.js/plugin-content-moderation 1.0.0

package/middlewares/inbound.ts

@@ -0,0 +1,112 @@
+import { defineMiddleware } from '@zhin.js/middleware';
+import type { Message } from '@zhin.js/core/runtime';
+import { outboundHostToken } from '@zhin.js/plugin-runtime';
+import {
+  buildScanContext,
+  extractFromTextAndSegments,
+  getModerationEngine,
+  resolveModerationConfig,
+  shouldBypassInbound,
+  type ModerationConfig,
+} from '../src/index.js';
+
+/**
+ * Inbound content moderation.
+ * - order -100: run early before commands / other middleware
+ * - inbound redact：无法改写只读 Message，仍 next() 放行
+ */
+export default defineMiddleware<Message, ModerationConfig>({
+  target: 'inbound',
+  phase: 'before-dispatch',
+  order: -100,
+  async handle(context, next) {
+    const config = resolveModerationConfig(context.config);
+    if (!config.enabled || !config.inbound.enabled) {
+      await next();
+      return;
+    }
+
+    const message = context.input;
+    const sender = message.sender;
+    const conversationId = message.conversation.id;
+    const endpointMasters = resolveEndpointMasters(message, config);
+
+    if (shouldBypassInbound(config, {
+      sender,
+      conversationId,
+      endpointMasters,
+    })) {
+      await next();
+      return;
+    }
+
+    const engine = getModerationEngine();
+    engine.configure(config);
+
+    const extracted = extractFromTextAndSegments(message.content, message.segments);
+    if (!extracted.text && extracted.images.length === 0) {
+      await next();
+      return;
+    }
+
+    const scanInput = {
+      text: extracted.text,
+      images: extracted.images,
+      direction: 'inbound' as const,
+      context: buildScanContext({
+        adapter: message.conversation.endpoint.adapter,
+        endpoint: message.conversation.endpoint.id,
+        conversationKind: message.conversation.kind,
+        conversationId,
+        sender,
+      }),
+    };
+
+    const result = await engine.apply({
+      direction: 'inbound',
+      extracted,
+      scanInput,
+      hooks: {
+        reply: async (content) => {
+          await message.$reply(content);
+        },
+        recall: async () => {
+          const messageId = message.id;
+          if (!messageId) return false;
+          try {
+            const host = context.use(outboundHostToken);
+            if (!host.recall) return false;
+            await host.recall({
+              adapter: message.conversation.endpoint.adapter,
+              endpointId: message.conversation.endpoint.id,
+              messageId,
+            });
+            return true;
+          } catch {
+            return false;
+          }
+        },
+      },
+    });
+
+    if (result.continue) {
+      await next();
+    }
+  },
+});
+
+/**
+ * Masters from plugin config + optional metadata.endpointMaster / metadata.master.
+ */
+function resolveEndpointMasters(
+  message: Message,
+  config: ModerationConfig,
+): readonly string[] {
+  const out: string[] = [...config.masters];
+  const meta = message.metadata ?? {};
+  for (const key of ['master', 'endpointMaster', 'owner'] as const) {
+    const value = meta[key];
+    if (typeof value === 'string' && value.trim()) out.push(value.trim());
+  }
+  return Object.freeze(out);
+}

package/middlewares/outbound.js

@@ -0,0 +1,53 @@
+// Generated by build-plugin-runtime-entries.mjs. Do not edit.
+import { defineMiddleware } from '@zhin.js/middleware';
+import { buildScanContext, extractFromOutboundPayload, getModerationEngine, resolveModerationConfig, shouldBypassOutbound, } from "../lib/index.js";
+/**
+ * Outbound content moderation.
+ * - redact uses envelope.replace()
+ * - drop skips platform send (do not call next)
+ */
+export default defineMiddleware({
+    target: 'outbound',
+    phase: 'before-dispatch',
+    order: -100,
+    async handle(context, next) {
+        const config = resolveModerationConfig(context.config);
+        if (shouldBypassOutbound(config)) {
+            await next();
+            return;
+        }
+        const envelope = context.input;
+        const engine = getModerationEngine();
+        engine.configure(config);
+        const extracted = extractFromOutboundPayload(envelope.payload);
+        if (!extracted.text && extracted.images.length === 0) {
+            await next();
+            return;
+        }
+        const scanInput = {
+            text: extracted.text,
+            images: extracted.images,
+            direction: 'outbound',
+            context: buildScanContext({
+                adapter: envelope.conversation.endpoint.adapter,
+                endpoint: envelope.conversation.endpoint.id,
+                conversationKind: envelope.conversation.kind,
+                conversationId: envelope.conversation.id,
+            }),
+        };
+        const result = await engine.apply({
+            direction: 'outbound',
+            extracted,
+            scanInput,
+            hooks: {
+                getPayload: () => envelope.payload,
+                replacePayload: (payload) => {
+                    envelope.replace(payload);
+                },
+            },
+        });
+        if (result.continue) {
+            await next();
+        }
+    },
+});

package/middlewares/outbound.ts

@@ -0,0 +1,66 @@
+import { defineMiddleware } from '@zhin.js/middleware';
+import type { OutboundEnvelope } from '@zhin.js/core/runtime';
+import {
+  buildScanContext,
+  extractFromOutboundPayload,
+  getModerationEngine,
+  resolveModerationConfig,
+  shouldBypassOutbound,
+  type ModerationConfig,
+} from '../src/index.js';
+
+/**
+ * Outbound content moderation.
+ * - redact uses envelope.replace()
+ * - drop skips platform send (do not call next)
+ */
+export default defineMiddleware<OutboundEnvelope, ModerationConfig>({
+  target: 'outbound',
+  phase: 'before-dispatch',
+  order: -100,
+  async handle(context, next) {
+    const config = resolveModerationConfig(context.config);
+    if (shouldBypassOutbound(config)) {
+      await next();
+      return;
+    }
+
+    const envelope = context.input;
+    const engine = getModerationEngine();
+    engine.configure(config);
+
+    const extracted = extractFromOutboundPayload(envelope.payload);
+    if (!extracted.text && extracted.images.length === 0) {
+      await next();
+      return;
+    }
+
+    const scanInput = {
+      text: extracted.text,
+      images: extracted.images,
+      direction: 'outbound' as const,
+      context: buildScanContext({
+        adapter: envelope.conversation.endpoint.adapter,
+        endpoint: envelope.conversation.endpoint.id,
+        conversationKind: envelope.conversation.kind,
+        conversationId: envelope.conversation.id,
+      }),
+    };
+
+    const result = await engine.apply({
+      direction: 'outbound',
+      extracted,
+      scanInput,
+      hooks: {
+        getPayload: () => envelope.payload,
+        replacePayload: (payload) => {
+          envelope.replace(payload);
+        },
+      },
+    });
+
+    if (result.continue) {
+      await next();
+    }
+  },
+});

package/package.json

@@ -0,0 +1,81 @@
+{
+  "name": "@zhin.js/plugin-content-moderation",
+  "version": "1.0.0",
+  "description": "Content moderation plugin for Zhin.js — inbound/outbound multi-source scanning (Plugin Runtime)",
+  "type": "module",
+  "main": "./lib/index.js",
+  "types": "./lib/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "development": "./src/index.ts",
+      "import": "./lib/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "plugin.js",
+    "schema.json",
+    "middlewares",
+    "src",
+    "lib",
+    "README.md"
+  ],
+  "keywords": [
+    "zhin",
+    "zhin.js",
+    "bot",
+    "plugin",
+    "moderation",
+    "content-safety",
+    "chatbot",
+    "im"
+  ],
+  "author": {
+    "name": "lc-cn",
+    "email": "admin@liucl.cn",
+    "url": "https://github.com/lc-cn"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@zhin.js/core": "1.5.2",
+    "@zhin.js/logger": "1.0.75",
+    "@zhin.js/middleware": "1.0.6",
+    "@zhin.js/plugin-runtime": "1.1.3"
+  },
+  "devDependencies": {
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.10"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zhinjs/zhin.git",
+    "directory": "plugins/utils/content-moderation"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "engines": {
+    "node": "^20.19.0 || >=22.12.0"
+  },
+  "zhin": {
+    "protocol": 1,
+    "type": "plugin",
+    "entry": "./plugin.js",
+    "engine": "^1.0.0",
+    "runtime": "trusted",
+    "features": [
+      {
+        "package": "@zhin.js/middleware",
+        "api": "^1.0.0"
+      }
+    ],
+    "plugins": []
+  },
+  "scripts": {
+    "build": "tsc",
+    "clean": "rimraf lib",
+    "test": "NODE_OPTIONS=--experimental-strip-types vitest run --root ../../.. plugins/utils/content-moderation/tests"
+  }
+}

package/plugin.js

@@ -0,0 +1,8 @@
+// Generated by build-plugin-runtime-entries.mjs. Do not edit.
+import { definePlugin } from '@zhin.js/plugin-runtime';
+export default definePlugin({
+    name: 'content-moderation',
+    metadata: {
+        displayName: 'Content Moderation',
+    },
+});

package/schema.json

@@ -0,0 +1,240 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "enabled": {
+      "type": "boolean",
+      "default": true,
+      "description": "总开关"
+    },
+    "onError": {
+      "type": "string",
+      "enum": ["open", "closed"],
+      "default": "open",
+      "description": "审查源失败时的全局默认策略：open=视为通过，closed=视为 critical"
+    },
+    "maskChar": {
+      "type": "string",
+      "default": "*",
+      "minLength": 1,
+      "maxLength": 4,
+      "description": "文本打码字符"
+    },
+    "replyTemplate": {
+      "type": "string",
+      "default": "消息含不当内容，已拦截。",
+      "description": "reply 动作时的提示文案"
+    },
+    "masters": {
+      "type": "array",
+      "items": { "type": "string" },
+      "default": [],
+      "description": "额外视为 master 的 userId（兜底，与 endpoint master 合并）"
+    },
+    "inbound": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "enabled": { "type": "boolean", "default": true },
+        "bypassMasters": {
+          "type": "boolean",
+          "default": true,
+          "description": "master 跳过入站审查"
+        },
+        "whitelist": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "userIds": {
+              "type": "array",
+              "items": { "type": "string" },
+              "default": []
+            },
+            "conversationIds": {
+              "type": "array",
+              "items": { "type": "string" },
+              "default": []
+            }
+          }
+        }
+      }
+    },
+    "outbound": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "enabled": { "type": "boolean", "default": true },
+        "bypass": {
+          "type": "boolean",
+          "default": false,
+          "description": "为 true 时跳过出站审查"
+        }
+      }
+    },
+    "actions": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "severity → 动作（字符串或数组）；缺省用偏严默认表",
+      "properties": {
+        "pass": {
+          "anyOf": [
+            {
+              "type": "string",
+              "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+            },
+            {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+              }
+            }
+          ]
+        },
+        "low": {
+          "anyOf": [
+            {
+              "type": "string",
+              "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+            },
+            {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+              }
+            }
+          ]
+        },
+        "medium": {
+          "anyOf": [
+            {
+              "type": "string",
+              "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+            },
+            {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+              }
+            }
+          ]
+        },
+        "high": {
+          "anyOf": [
+            {
+              "type": "string",
+              "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+            },
+            {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+              }
+            }
+          ]
+        },
+        "critical": {
+          "anyOf": [
+            {
+              "type": "string",
+              "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+            },
+            {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "enum": ["allow", "log", "reply", "redact", "drop", "recall"]
+              }
+            }
+          ]
+        }
+      }
+    },
+    "sources": {
+      "type": "array",
+      "default": [],
+      "items": {
+        "type": "object",
+        "required": ["id", "type"],
+        "additionalProperties": false,
+        "properties": {
+          "id": { "type": "string", "minLength": 1 },
+          "type": { "type": "string", "enum": ["local", "http"] },
+          "enabled": { "type": "boolean", "default": true },
+          "onError": { "type": "string", "enum": ["open", "closed"] },
+          "includeBuiltin": {
+            "type": "boolean",
+            "default": true,
+            "description": "是否合并内置分级违禁词词库"
+          },
+          "words": {
+            "type": "array",
+            "default": [],
+            "description": "自定义词：字符串（用 defaultSeverity）或 { word, severity }",
+            "items": {
+              "anyOf": [
+                { "type": "string", "minLength": 1 },
+                {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["word"],
+                  "properties": {
+                    "word": { "type": "string", "minLength": 1 },
+                    "text": { "type": "string", "minLength": 1 },
+                    "severity": {
+                      "type": "string",
+                      "enum": ["low", "medium", "high", "critical"]
+                    }
+                  }
+                }
+              ]
+            }
+          },
+          "wordFiles": {
+            "type": "array",
+            "items": { "type": "string" },
+            "default": [],
+            "description": "词库文件；行格式 word / severity:word / word|severity"
+          },
+          "defaultSeverity": {
+            "type": "string",
+            "enum": ["low", "medium", "high", "critical"],
+            "default": "high",
+            "description": "未标注分级的自定义词默认 severity"
+          },
+          "severity": {
+            "type": "string",
+            "enum": ["low", "medium", "high", "critical"],
+            "default": "high",
+            "description": "兼容旧字段，等同 defaultSeverity"
+          },
+          "url": { "type": "string" },
+          "headers": {
+            "type": "object",
+            "additionalProperties": { "type": "string" }
+          },
+          "timeoutMs": {
+            "type": "number",
+            "default": 5000,
+            "minimum": 500,
+            "maximum": 60000
+          },
+          "forceUpload": {
+            "type": "boolean",
+            "default": false,
+            "description": "强制下载图片后上传，不传 URL"
+          }
+        }
+      }
+    }
+  }
+}

package/src/bypass.ts

@@ -0,0 +1,38 @@
+import type { ModerationConfig } from './types.js';
+
+export interface BypassInput {
+  readonly sender?: string;
+  readonly conversationId: string;
+  /** Endpoint / plugin-declared masters already resolved by caller. */
+  readonly endpointMasters?: readonly string[];
+}
+
+export function shouldBypassInbound(
+  config: ModerationConfig,
+  input: BypassInput,
+): boolean {
+  if (!config.enabled || !config.inbound.enabled) return true;
+
+  const sender = input.sender?.trim();
+  if (sender) {
+    if (config.inbound.whitelist.userIds.includes(sender)) return true;
+    if (config.inbound.bypassMasters) {
+      const masters = new Set([
+        ...config.masters,
+        ...(input.endpointMasters ?? []),
+      ].map(String));
+      if (masters.has(sender)) return true;
+    }
+  }
+
+  if (config.inbound.whitelist.conversationIds.includes(input.conversationId)) {
+    return true;
+  }
+
+  return false;
+}
+
+export function shouldBypassOutbound(config: ModerationConfig): boolean {
+  if (!config.enabled || !config.outbound.enabled) return true;
+  return config.outbound.bypass === true;
+}