@zhin.js/plugin-content-moderation 1.0.0

package/src/redact.ts

@@ -0,0 +1,141 @@
+import type { ExtractedContent } from './extract.js';
+import type { ExtractedImage, TextMatch } from './types.js';
+
+export interface RedactOptions {
+  readonly maskChar: string;
+  readonly matches: readonly TextMatch[];
+  readonly flaggedImageIndexes: readonly number[];
+}
+
+/**
+ * Mask matched spans in text. When there are no spans but redact is requested
+ * for non-empty text, replace the whole string with mask chars.
+ */
+export function redactText(
+  text: string,
+  matches: readonly TextMatch[],
+  maskChar: string,
+): string {
+  const ch = maskChar.slice(0, 1) || '*';
+  if (!text) return text;
+  if (!matches.length) {
+    const len = Math.max(3, Math.min(text.length, 16));
+    return ch.repeat(len);
+  }
+
+  const chars = [...text];
+  const sorted = [...matches].sort((a, b) => a.start - b.start);
+  for (const match of sorted) {
+    const start = Math.max(0, Math.min(chars.length, match.start));
+    const end = Math.max(start, Math.min(chars.length, match.end));
+    for (let i = start; i < end; i++) chars[i] = ch;
+  }
+  return chars.join('');
+}
+
+/**
+ * Apply text masking + remove flagged image segments from outbound payload.
+ */
+export function redactOutboundPayload(
+  payload: unknown,
+  extracted: ExtractedContent,
+  options: RedactOptions,
+): unknown {
+  const flagged = new Set(options.flaggedImageIndexes);
+  const flaggedSegmentIndexes = new Set(
+    extracted.images
+      .filter((img) => flagged.has(img.index) && img.segmentIndex != null)
+      .map((img) => img.segmentIndex as number),
+  );
+
+  if (typeof payload === 'string') {
+    return redactText(payload, options.matches, options.maskChar);
+  }
+
+  if (Array.isArray(payload)) {
+    return redactSegmentList(payload, options, flaggedSegmentIndexes);
+  }
+
+  if (payload && typeof payload === 'object' && 'type' in payload) {
+    const list = redactSegmentList([payload], options, flaggedSegmentIndexes);
+    return list.length === 1 ? list[0] : list;
+  }
+
+  if (extracted.text) {
+    return redactText(extracted.text, options.matches, options.maskChar);
+  }
+  return payload;
+}
+
+function redactSegmentList(
+  segments: readonly unknown[],
+  options: RedactOptions,
+  flaggedSegmentIndexes: ReadonlySet<number>,
+): unknown[] {
+  const out: unknown[] = [];
+  let textCursor = 0;
+
+  for (let i = 0; i < segments.length; i++) {
+    const seg = segments[i];
+    if (!seg || typeof seg !== 'object') {
+      out.push(seg);
+      continue;
+    }
+    if (flaggedSegmentIndexes.has(i)) continue;
+
+    const record = seg as Record<string, unknown>;
+    const type = String(record.type ?? '');
+    if (type === 'text') {
+      const data = asRecord(record.data);
+      const text = typeof data.text === 'string' ? data.text : '';
+      const localMatches = shiftMatches(options.matches, textCursor, text.length);
+      // No spans overall → whole-segment mask; otherwise only mask local hits.
+      const nextText = options.matches.length === 0
+        ? redactText(text, [], options.maskChar)
+        : localMatches.length > 0
+          ? redactText(text, localMatches, options.maskChar)
+          : text;
+      out.push({
+        ...record,
+        data: { ...data, text: nextText },
+      });
+      textCursor += text.length;
+      continue;
+    }
+
+    out.push(seg);
+  }
+
+  return out;
+}
+
+function shiftMatches(
+  matches: readonly TextMatch[],
+  offset: number,
+  length: number,
+): TextMatch[] {
+  const end = offset + length;
+  const out: TextMatch[] = [];
+  for (const m of matches) {
+    if (m.end <= offset || m.start >= end) continue;
+    out.push({
+      start: Math.max(0, m.start - offset),
+      end: Math.min(length, m.end - offset),
+    });
+  }
+  return out;
+}
+
+export function flaggedImagesFromIndexes(
+  images: readonly ExtractedImage[],
+  indexes: readonly number[],
+): readonly ExtractedImage[] {
+  const set = new Set(indexes);
+  return Object.freeze(images.filter((img) => set.has(img.index)));
+}
+
+function asRecord(value: unknown): Record<string, unknown> {
+  return value && typeof value === 'object' && !Array.isArray(value)
+    ? value as Record<string, unknown>
+    : {};
+}

package/src/types.ts

@@ -0,0 +1,143 @@
+export type Severity = 'pass' | 'low' | 'medium' | 'high' | 'critical';
+
+export type Action = 'allow' | 'log' | 'reply' | 'redact' | 'drop' | 'recall';
+
+export type OnErrorPolicy = 'open' | 'closed';
+
+export type Direction = 'inbound' | 'outbound';
+
+export interface TextMatch {
+  readonly start: number;
+  readonly end: number;
+}
+
+export interface ExtractedImage {
+  /** Index among extracted images (for HTTP / flaggedImageIndexes). */
+  readonly index: number;
+  /** Index in the original segment list when applicable. */
+  readonly segmentIndex?: number;
+  readonly url?: string;
+  readonly base64?: string;
+  readonly mime?: string;
+  readonly path?: string;
+}
+
+export interface ScanContext {
+  readonly adapter: string;
+  readonly endpoint: string;
+  readonly conversationKind: string;
+  readonly conversationId: string;
+  readonly sender?: string;
+}
+
+export interface ScanInput {
+  readonly text: string;
+  readonly images: readonly ExtractedImage[];
+  readonly direction: Direction;
+  readonly context: ScanContext;
+}
+
+export interface ProviderResult {
+  readonly sourceId: string;
+  readonly severity: Severity;
+  readonly matches?: readonly TextMatch[];
+  readonly flaggedImageIndexes?: readonly number[];
+  readonly reason?: string;
+  readonly error?: boolean;
+}
+
+export interface MergedResult {
+  readonly severity: Severity;
+  readonly matches: readonly TextMatch[];
+  readonly flaggedImageIndexes: readonly number[];
+  readonly sources: readonly ProviderResult[];
+  readonly reason?: string;
+}
+
+/** Graded lexicon item (config `words` entry). */
+export interface LexiconWordConfig {
+  readonly word: string;
+  /** Per-word severity; falls back to source `defaultSeverity`. */
+  readonly severity: Exclude<Severity, 'pass'>;
+}
+
+export interface LocalSourceConfig {
+  readonly id: string;
+  readonly type: 'local';
+  readonly enabled: boolean;
+  readonly onError: OnErrorPolicy;
+  /** Custom words with per-entry severity. */
+  readonly words: readonly LexiconWordConfig[];
+  readonly wordFiles: readonly string[];
+  /** Merge built-in graded lexicon (default true). */
+  readonly includeBuiltin: boolean;
+  /**
+   * Default severity for plain string words / ungraded file lines.
+   * (Config key may still be written as `severity` for compatibility.)
+   */
+  readonly defaultSeverity: Exclude<Severity, 'pass'>;
+}
+
+export interface HttpSourceConfig {
+  readonly id: string;
+  readonly type: 'http';
+  readonly enabled: boolean;
+  readonly onError: OnErrorPolicy;
+  readonly url: string;
+  readonly headers: Readonly<Record<string, string>>;
+  readonly timeoutMs: number;
+  readonly forceUpload: boolean;
+}
+
+export type SourceConfig = LocalSourceConfig | HttpSourceConfig;
+
+export interface ModerationConfig {
+  readonly enabled: boolean;
+  readonly onError: OnErrorPolicy;
+  readonly maskChar: string;
+  readonly replyTemplate: string;
+  readonly masters: readonly string[];
+  readonly inbound: {
+    readonly enabled: boolean;
+    readonly bypassMasters: boolean;
+    readonly whitelist: {
+      readonly userIds: readonly string[];
+      readonly conversationIds: readonly string[];
+    };
+  };
+  readonly outbound: {
+    readonly enabled: boolean;
+    readonly bypass: boolean;
+  };
+  readonly actions: Readonly<Record<Severity, readonly Action[]>>;
+  readonly sources: readonly SourceConfig[];
+}
+
+export const SEVERITY_RANK: Readonly<Record<Severity, number>> = Object.freeze({
+  pass: 0,
+  low: 1,
+  medium: 2,
+  high: 3,
+  critical: 4,
+});
+
+export function maxSeverity(a: Severity, b: Severity): Severity {
+  return SEVERITY_RANK[a] >= SEVERITY_RANK[b] ? a : b;
+}
+
+export function isSeverity(value: unknown): value is Severity {
+  return value === 'pass'
+    || value === 'low'
+    || value === 'medium'
+    || value === 'high'
+    || value === 'critical';
+}
+
+export function isAction(value: unknown): value is Action {
+  return value === 'allow'
+    || value === 'log'
+    || value === 'reply'
+    || value === 'redact'
+    || value === 'drop'
+    || value === 'recall';
+}