@joliegg/moderation 0.6.0 → 0.9.0

package/src/index.ts

@@ -1,277 +1,3 @@
-import axios from 'axios';
-
-import { Rekognition } from '@aws-sdk/client-rekognition';
-import { LanguageServiceClient } from '@google-cloud/language';
-import { SpeechClient, protos } from '@google-cloud/speech';
-
-import sharp from 'sharp';
-
-import URLBlackList from './url-blacklist.json';
-import URLShortenerList from './url-shorteners.json';
-
-
-import { ModerationCategory, ModerationConfiguration, ModerationResult, ThreatsResponse } from './types';
-
-
-type IRecognitionConfig = protos.google.cloud.speech.v1.IRecognitionConfig;
-
-type ISpeechRecognitionResult = protos.google.cloud.speech.v1.ISpeechRecognitionResult;
-
-const MAX_IMAGE_SIZE = 5 * 1024 * 1024; // 5MB in bytes for Rekognition limit
-
-/**
- * Moderation Client
- *
- * @class ModerationClient
- */
-class ModerationClient {
-
-  private rekognitionClient?: Rekognition;
-  private googleLanguageClient?: LanguageServiceClient;
-  private googleSpeechClient?: SpeechClient;
-  private googleAPIKey?: string;
-  private banList?: string[] = [];
-  private urlBlackList?: string[] = [];
-
-  /**
-   *
-   * @param {ModerationConfiguration} configuration
-   */
-  constructor (configuration: ModerationConfiguration) {
-    if (configuration.aws) {
-      this.rekognitionClient = new Rekognition(configuration.aws);
-    }
-
-    if (typeof configuration.google?.keyFile === 'string') {
-      this.googleLanguageClient = new LanguageServiceClient({ keyFile: configuration.google.keyFile });
-      this.googleSpeechClient = new SpeechClient({ keyFile: configuration.google.keyFile });
-    }
-
-    if (typeof configuration.google?.apiKey === 'string') {
-      this.googleAPIKey = configuration.google.apiKey;
-    }
-
-    if (Array.isArray(configuration.banList)) {
-      this.banList = configuration.banList;
-    }
-
-    if (Array.isArray(configuration.urlBlackList)) {
-      this.urlBlackList = configuration.urlBlackList;
-    }
-  }
-
-  /**
-   * Returns a list of moderation categories detected on a text
-   *
-   * @param {string} text  The text to moderate
-   * @param {number} [minimumConfidence = 50] The minimum confidence required for a category to be considered
-   *
-   * @returns {Promise<ModerationResult>} The list of results that were detected with the minimum confidence specified
-   */
-  async moderateText (text: string, minimumConfidence: number = 50): Promise<ModerationResult> {
-    const categories: ModerationCategory[] = [];
-
-    if (Array.isArray(this.banList)) {
-      const normalizedText = text.toLowerCase();
-      const matches = this.banList.filter(w => normalizedText.indexOf(w) > -1);
-
-      if (matches.length > 0) {
-        const words = normalizedText.split(' ');
-
-        categories.push({
-          category: 'BAN_LIST',
-          confidence: (matches.length / words.length) * 100,
-        });
-      }
-    }
-
-
-    if (typeof this.googleLanguageClient === 'undefined') {
-      return { source: text, moderation: categories };
-    }
-
-    const [ result ] = await this.googleLanguageClient.moderateText({
-      document: {
-        content: text,
-        type: 'PLAIN_TEXT',
-      },
-    });
-
-    if (result && 'moderationCategories' in result) {
-      if (Array.isArray(result.moderationCategories)) {
-        const results = result.moderationCategories.map(c => ({
-          category: c.name ?? 'Unknown',
-          confidence: (c.confidence ?? 0) * 100,
-        })).filter(c => c.confidence >= minimumConfidence);
-        return { source: text, moderation: [...categories, ...results] };
-      }
-    }
-
-    return { source: text, moderation: [] };
-  }
-
-  /**
-   * Returns a list of moderation categories detected on an image
-   *
-   * @param {string} url
-   * @param {number} [minimumConfidence = 50]  The minimum confidence required for a category to be considered
-   *
-   *
-   * @returns {Promise<ModerationResult[]>} The list of results that were detected with the minimum confidence specified
-   */
-  async moderateImage (url: string, minimumConfidence: number = 50): Promise<ModerationResult> {
-    if (typeof this.rekognitionClient === 'undefined') {
-      return { source: url, moderation: [] };
-    }
-
-    const { data } = await axios.get<string>(url, { responseType: 'arraybuffer' });
-
-    let buffer: Buffer;
-
-    // GIFs will be split into frames
-    if (url.toLowerCase().indexOf('.gif') > -1) {
-      buffer = await sharp(data, { pages: -1 }).toFormat('png').toBuffer();
-    } else if (url.toLowerCase().indexOf('.webp') > -1) {
-      buffer = await sharp(data).toFormat('png').toBuffer();
-    } else {
-       // Download image as binary data
-      buffer = Buffer.from(data, 'binary');
-    }
-
-    // Ensure image is not larger than 5MB (Rekognition limit)
-    if (buffer.length > MAX_IMAGE_SIZE) {
-      try {
-        // Calculate new dimensions to reduce size
-        const metadata = await sharp(buffer).metadata();
-
-        const { width, height } = metadata;
-
-        if (typeof width !== 'number' || typeof height !== 'number') {
-          throw new Error('Invalid image metadata');
-        }
-
-        // Calculate the scaling factor
-        const scalingFactor = Math.sqrt(MAX_IMAGE_SIZE / buffer.length);
-
-        // Calculate new dimensions
-        const newWidth = Math.floor(width * scalingFactor);
-        const newHeight = Math.floor(height * scalingFactor);
-
-        const resizedBuffer = await sharp(buffer)
-          .resize(Math.round(newWidth), Math.round(newHeight))
-          .toBuffer();
-
-        buffer = resizedBuffer;
-      } catch {
-        // We can't resize the image. We'll skip the resize and try to process it as is
-      }
-    }
-
-    const { ModerationLabels } = await this.rekognitionClient.detectModerationLabels({
-      Image: {
-        Bytes: buffer
-      },
-      MinConfidence: minimumConfidence
-    });
-
-    if (Array.isArray(ModerationLabels)) {
-      const moderation = ModerationLabels.map(l => ({
-        category: l.Name ?? 'Unknown',
-        confidence: l.Confidence ?? 0,
-      }));
-
-      return { source: url, moderation };
-    }
-
-    return { source: url, moderation: [] };
-  }
-
-  async moderateLink (url: string, allowShorteners = false): Promise<ModerationResult> {
-    try {
-      const domain = new URL(url).hostname;
-
-      const blacklisted = this.urlBlackList?.some(u => u.indexOf(url) > -1);
-
-      if (blacklisted) {
-        return { source: url, moderation: [{ category: 'CUSTOM_BLACK_LIST', confidence: 100 }] };
-      }
-
-      const globallyBlacklisted = URLBlackList.some(u => u === domain);
-
-      if (globallyBlacklisted) {
-        return { source: url, moderation: [{ category: 'BLACK_LIST', confidence: 100 }] };
-      }
-
-      if (!allowShorteners) {
-        const isShortened = URLShortenerList.some(u => u === domain);
-
-        if (isShortened) {
-          return { source: url, moderation: [{ category: 'URL_SHORTENER', confidence: 100 }] };
-        }
-      }
-    } catch {
-      // Invalid URL
-      return { source: url, moderation: [] };
-    }
-
-    if (typeof this.googleAPIKey !== 'string') {
-      return { source: url, moderation: [] };
-    }
-
-    const types = [
-      'MALWARE',
-      'SOCIAL_ENGINEERING',
-      'UNWANTED_SOFTWARE',
-      'SOCIAL_ENGINEERING_EXTENDED_COVERAGE'
-    ];
-
-    const threatTypes = types.join('&threatTypes=');
-    const requestUrl = `https://webrisk.googleapis.com/v1/uris:search?threatTypes=${threatTypes}&key=${this.googleAPIKey}`;
-
-    const { data } = await axios.get<ThreatsResponse>(`${requestUrl}&uri=${encodeURIComponent(url)}`);
-
-    const threats = data?.threat?.threatTypes;
-
-    if (Array.isArray(threats)) {
-      const moderation = threats.map(t => ({
-        category: t,
-        confidence: 100,
-      }));
-
-      return { source: url, moderation };
-    }
-
-    return { source: url, moderation: [] };
-  }
-
-  async moderateAudio (url: string, language: string = 'en-US', minimumConfidence: number = 50): Promise<ModerationResult> {
-    if (typeof this.googleSpeechClient === 'undefined') {
-      return { source: url, moderation: [] };
-    }
-
-    const { data } = await axios.get<string>(url, { responseType: 'arraybuffer' });
-
-
-    const options: IRecognitionConfig = {
-      encoding: 'OGG_OPUS',
-      sampleRateHertz: 48000,
-      languageCode: language,
-    };
-
-    const [ response ] = await this.googleSpeechClient.recognize ({
-      audio: { content: Buffer.from(data, 'binary').toString('base64') },
-      config: options,
-    });
-
-    if (!Array.isArray(response?.results)) {
-      return { source: url, moderation: [] };
-    }
-
-    const transcription = response?.results?.map((result: ISpeechRecognitionResult) => result.alternatives?.at(0)?.transcript ?? '').join(' ');
-
-    return this.moderateText(transcription, minimumConfidence);
-  }
-
-}
-
-export default ModerationClient;
+export { ModerationClient, default } from './client';
+export * from './types';
+export * from './actions';

package/src/providers/aws.ts

@@ -0,0 +1,58 @@
+import { Rekognition } from '@aws-sdk/client-rekognition';
+import axios from 'axios';
+import sharp from 'sharp';
+
+import type { ModerationCategory } from '../types';
+
+// AWS Rekognition 5MB inline-image limit
+const MAX_IMAGE_SIZE = 5 * 1024 * 1024;
+
+/**
+ * AWS Rekognition image moderation adapter. Handles GIF and WEBP
+ * conversion to PNG and downscales images over the 5 MB Rekognition limit.
+ */
+export class RekognitionProvider {
+  constructor(private client: Rekognition) {}
+
+  async moderateImage(url: string, minimumConfidence: number): Promise<ModerationCategory[]> {
+    const { data } = await axios.get<ArrayBuffer>(url, { responseType: 'arraybuffer' });
+    let buffer: Buffer;
+
+    const lowered = url.toLowerCase();
+
+    if (lowered.includes('.gif')) {
+      buffer = await sharp(Buffer.from(data), { pages: -1 }).toFormat('png').toBuffer();
+    } else if (lowered.includes('.webp')) {
+      buffer = await sharp(Buffer.from(data)).toFormat('png').toBuffer();
+    } else {
+      buffer = Buffer.from(data);
+    }
+
+    if (buffer.length > MAX_IMAGE_SIZE) {
+      try {
+        const metadata = await sharp(buffer).metadata();
+        const { width, height } = metadata;
+
+        if (typeof width === 'number' && typeof height === 'number') {
+          const scalingFactor = Math.sqrt(MAX_IMAGE_SIZE / buffer.length);
+          buffer = await sharp(buffer)
+            .resize(Math.round(width * scalingFactor), Math.round(height * scalingFactor))
+            .toBuffer();
+        }
+      } catch {
+        // Resize failed, we will use the original buffer.
+      }
+    }
+
+    const { ModerationLabels } = await this.client.detectModerationLabels({
+      Image: { Bytes: buffer },
+      MinConfidence: minimumConfidence,
+    });
+
+    if (Array.isArray(ModerationLabels)) {
+      return ModerationLabels.map(l => ({ category: l.Name ?? 'Unknown', confidence: l.Confidence ?? 0 }));
+    }
+
+    return [];
+  }
+}

package/src/providers/google.ts

@@ -0,0 +1,63 @@
+import { LanguageServiceClient } from '@google-cloud/language';
+import { SpeechClient, protos } from '@google-cloud/speech';
+import axios from 'axios';
+
+import type { ModerationCategory } from '../types';
+
+type IRecognitionConfig = protos.google.cloud.speech.v1.IRecognitionConfig;
+type ISpeechRecognitionResult = protos.google.cloud.speech.v1.ISpeechRecognitionResult;
+
+/**
+ * Google Cloud Natural Language moderation adapter.
+ */
+export class GoogleLanguageProvider {
+  constructor(private client: LanguageServiceClient) {}
+
+  async moderateText(text: string, minimumConfidence: number): Promise<ModerationCategory[]> {
+    const [result] = await this.client.moderateText({
+      document: { content: text, type: 'PLAIN_TEXT' },
+    });
+
+    if (!result || !('moderationCategories' in result) || !Array.isArray(result.moderationCategories)) {
+      return [];
+    }
+
+    return result.moderationCategories
+      .map(c => ({ category: c.name ?? 'Unknown', confidence: (c.confidence ?? 0) * 100 }))
+      .filter(c => c.confidence >= minimumConfidence);
+  }
+}
+
+/**
+ * Google Cloud Speech-to-Text adapter. Used for audio transcription
+ * before running the transcript through text moderation.
+ */
+export class GoogleSpeechProvider {
+  constructor(private client: SpeechClient) {}
+
+  async transcribe(audioBuffer: Buffer, languageCode: string): Promise<string> {
+    const config: IRecognitionConfig = {
+      encoding: 'OGG_OPUS',
+      sampleRateHertz: 48000,
+      languageCode,
+    };
+
+    const [response] = await this.client.recognize({
+      audio: { content: audioBuffer.toString('base64') },
+      config,
+    });
+
+    if (!Array.isArray(response?.results)) {
+      return '';
+    }
+
+    return response.results
+      .map((r: ISpeechRecognitionResult) => r.alternatives?.at(0)?.transcript ?? '')
+      .join(' ');
+  }
+}
+
+export async function fetchAudio(url: string): Promise<Buffer> {
+  const { data } = await axios.get<ArrayBuffer>(url, { responseType: 'arraybuffer' });
+  return Buffer.from(data);
+}

package/src/providers/openai.ts

@@ -0,0 +1,64 @@
+import type { ModerationCategory } from '../types';
+
+interface OpenAIModerationResponse {
+  results?: Array<{
+    flagged: boolean;
+    categories: Record<string, boolean>;
+    category_scores: Record<string, number>;
+  }>;
+}
+
+/**
+ * OpenAI Moderation API adapter.
+ *
+ * Uses the free `omni-moderation-latest` endpoint (no token cost,
+ * no SDK). Returns `ModerationCategory[]` in the same shape the
+ * Google and AWS providers return, so downstream callers can treat
+ * every provider uniformly.
+ */
+export class OpenAIModerationProvider {
+  constructor(
+    private apiKey: string,
+    private model: string = 'omni-moderation-latest',
+  ) {}
+
+  async moderateText(text: string, minimumConfidence: number = 0): Promise<ModerationCategory[]> {
+    if (!this.apiKey) {
+      return [];
+    }
+
+    let response: Response;
+
+    try {
+      response = await fetch('https://api.openai.com/v1/moderations', {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ model: this.model, input: text }),
+      });
+    } catch {
+      return [];
+    }
+
+    if (!response.ok) {
+      return [];
+    }
+
+    const data = (await response.json()) as OpenAIModerationResponse;
+    const result = data.results?.[0];
+
+    if (!result) {
+      return [];
+    }
+
+    return Object.entries(result.categories)
+      .filter(([, flagged]) => flagged)
+      .map(([category]) => ({
+        category,
+        confidence: (result.category_scores[category] ?? 0) * 100,
+      }))
+      .filter(c => c.confidence >= minimumConfidence);
+  }
+}

package/src/providers/webrisk.ts

@@ -0,0 +1,30 @@
+import axios from 'axios';
+import type { ModerationCategory, ThreatsResponse } from '../types';
+
+const THREAT_TYPES = [
+  'MALWARE',
+  'SOCIAL_ENGINEERING',
+  'UNWANTED_SOFTWARE',
+  'SOCIAL_ENGINEERING_EXTENDED_COVERAGE',
+];
+
+/**
+ * Google Web Risk link moderation adapter.
+ */
+export class WebRiskProvider {
+  constructor(private apiKey: string) {}
+
+  async checkLink(url: string): Promise<ModerationCategory[]> {
+    const threatTypes = THREAT_TYPES.join('&threatTypes=');
+    const requestUrl = `https://webrisk.googleapis.com/v1/uris:search?threatTypes=${threatTypes}&key=${this.apiKey}`;
+    const { data } = await axios.get<ThreatsResponse>(`${requestUrl}&uri=${encodeURIComponent(url)}`);
+
+    const threats = data?.threat?.threatTypes;
+
+    if (Array.isArray(threats)) {
+      return threats.map(t => ({ category: t, confidence: 100 }));
+    }
+
+    return [];
+  }
+}

package/src/raid/age.ts

@@ -0,0 +1,19 @@
+/**
+ * Is a member's account "too new" for the given threshold?
+ *
+ * Returns `false` when both timestamps are missing.
+ */
+export function isAccountTooNew(joinedTimestamp: number | null, createdTimestamp: number | null, minAgeDays: number): boolean {
+  // For raids, we care about join time, not account creation
+  // Altough we can derive some trust from that too.
+  const reference = joinedTimestamp ?? createdTimestamp;
+
+  if (reference === null) {
+    return false;
+  }
+
+  const age = Date.now() - reference;
+  const minAgeMs = minAgeDays * 24 * 60 * 60 * 1000;
+
+  return age < minAgeMs;
+}

package/src/raid/detector.ts

@@ -0,0 +1,133 @@
+export interface RaidDetectorOptions {
+  /** Number of joins within the window that constitutes a raid. Default to 10 (very small community) but this really depends on community size and activity. */
+  joinThreshold?: number;
+  /** Sliding window in seconds. Default to 60 seconds (1 minute). */
+  joinWindow?: number;
+}
+
+export interface MemberJoin {
+  memberId: string;
+  joinedTimestamp: number;
+  createdTimestamp: number;
+}
+
+export interface RaidTrackResult {
+  isRaid: boolean;
+  joinCount: number;
+  windowSeconds: number;
+}
+
+export type EnableResult = 'enabled' | 'already_active' | 'already_enabling';
+
+interface JoinEvent {
+  memberId: string;
+  timestamp: number;
+}
+
+interface GuildState {
+  joins: JoinEvent[];
+  raidActive: boolean;
+}
+
+/**
+ * Platform-agnostic raid detector. Tracks recent joins per guild in a
+ * sliding window and surfaces the "raid" signal when the join count
+ * crosses the configured threshold.
+ *
+ * State transitions (`tryEnable`, `disable`) are guarded by an
+ * in-memory mutex so concurrent `handleMemberJoin` invocations during
+ * a join burst cannot both see "not active" and double-fire the
+ * enable side effects.
+ *
+ * The detector owns the sliding-window state. It does NOT own the
+ * enforcement — callers decide what to do when `isRaid` is true
+ * (timeout, kick, auto-disable timer, mod-channel alert, etc.).
+ */
+export class RaidDetector {
+  readonly joinThreshold: number;
+  readonly joinWindow: number;
+
+  private state = new Map<string, GuildState>();
+  private enabling = new Set<string>();
+
+  constructor(options: RaidDetectorOptions = {}) {
+    this.joinThreshold = options.joinThreshold ?? 10;
+    this.joinWindow = (options.joinWindow ?? 60) * 1000;
+  }
+
+  private getState(guildId: string): GuildState {
+    if (!this.state.has(guildId)) {
+      this.state.set(guildId, { joins: [], raidActive: false });
+    }
+
+    return this.state.get(guildId)!;
+  }
+
+  private cleanupJoins(state: GuildState, now: number): void {
+    state.joins = state.joins.filter(j => now - j.timestamp < this.joinWindow);
+  }
+
+  /**
+   * Record a join and return whether the guild has crossed the raid
+   * threshold inside the window. `tryEnable` is a separate call so the
+   * caller can act on the raid signal atomically.
+   */
+  track(guildId: string, member: MemberJoin): RaidTrackResult {
+    const now = Date.now();
+    const state = this.getState(guildId);
+
+    this.cleanupJoins(state, now);
+    state.joins.push({ memberId: member.memberId, timestamp: now });
+
+    return {
+      isRaid: state.joins.length >= this.joinThreshold,
+      joinCount: state.joins.length,
+      windowSeconds: this.joinWindow / 1000,
+    };
+  }
+
+  isActive(guildId: string): boolean {
+    return this.state.get(guildId)?.raidActive ?? false;
+  }
+
+  /**
+   * Attempt to flip the guild into raid-active state. Returns:
+   *  - 'enabled' if this call performed the transition
+   *  - 'already_active' if raid mode was already on
+   *  - 'already_enabling' if another concurrent call is mid-transition
+   */
+  async tryEnable(guildId: string): Promise<EnableResult> {
+    const state = this.getState(guildId);
+
+    if (state.raidActive) {
+      return 'already_active';
+    }
+
+    if (this.enabling.has(guildId)) {
+      return 'already_enabling';
+    }
+
+    this.enabling.add(guildId);
+
+    try {
+      state.raidActive = true;
+      return 'enabled';
+    } finally {
+      this.enabling.delete(guildId);
+    }
+  }
+
+  disable(guildId: string): void {
+    const state = this.getState(guildId);
+    state.raidActive = false;
+    this.enabling.delete(guildId);
+  }
+
+  getJoinCount(guildId: string, windowSeconds?: number): number {
+    const now = Date.now();
+    const state = this.getState(guildId);
+    const window = (windowSeconds ?? this.joinWindow / 1000) * 1000;
+
+    return state.joins.filter(j => now - j.timestamp < window).length;
+  }
+}

package/src/raid/index.ts

@@ -0,0 +1,2 @@
+export * from './detector';
+export * from './age';

package/src/rubrics/defaults.ts

@@ -0,0 +1,32 @@
+import { ACTION_TYPES } from '../actions';
+import { ScoringRubric } from './rubric';
+
+/**
+ * Aggressive defaults. Flag anything moderately suspicious, delete at
+ * 75%, timeout above 90%. Good for tight-knit communities with low
+ * tolerance for borderline content.
+ */
+export const STRICT_RUBRIC = new ScoringRubric([
+  { match: { minConfidence: 50 }, action: ACTION_TYPES.WARN, severity: 'low' },
+  { match: { minConfidence: 75 }, action: ACTION_TYPES.DELETE, severity: 'medium' },
+  { match: { minConfidence: 90 }, action: ACTION_TYPES.TIMEOUT, severity: 'high' },
+]);
+
+/**
+ * Conservative defaults. Only act on very high confidence. Good for
+ * large communities where false positives are worse than false
+ * negatives.
+ */
+export const PERMISSIVE_RUBRIC = new ScoringRubric([
+  { match: { minConfidence: 95 }, action: ACTION_TYPES.DELETE, severity: 'high' },
+  { match: { minConfidence: 99 }, action: ACTION_TYPES.TIMEOUT, severity: 'critical' },
+]);
+
+/**
+ * NSFW-only rubric. Ignores harassment, hate, violence — only acts on
+ * sexual content. Useful when text moderation is handled out-of-band
+ * but image / link moderation still needs safety filtering.
+ */
+export const NSFW_ONLY_RUBRIC = new ScoringRubric([
+  { match: { category: 'sexual', minConfidence: 70 }, action: ACTION_TYPES.DELETE, severity: 'high' },
+]);

package/src/rubrics/index.ts

@@ -0,0 +1,3 @@
+export * from './types';
+export * from './rubric';
+export * from './defaults';