@llnvd/openclaw-url-guard 0.0.1

package/src/config.ts

@@ -0,0 +1,228 @@
+import type { UrlGuardConfig } from './types';
+
+export const RECOMMENDED_ALLOWLIST = [
+  'docs.python.org',
+  'developer.mozilla.org',
+  'en.wikipedia.org',
+  'github.com',
+  'stackoverflow.com',
+] as const;
+
+const DEFAULT_CONFIG: UrlGuardConfig = {
+  mode: 'allowlist',
+  allowlist: [],
+  blocklist: [],
+  blockPrivateIps: true,
+  threatFeeds: {
+    urlhaus: false,
+    mode: 'fail-open',
+  },
+  scoring: {
+    enabled: false,
+    defaultScore: 0,
+    minScore: -6,
+    rules: [],
+    feedScores: {
+      urlhaus: {
+        online: -10,
+        offline: -7,
+      },
+      spamhaus: {
+        botnet_cc: -10,
+        phishing_domain: -10,
+        spammer_domain: -8,
+        'abused_legit_*': -6,
+        abused_redirector: -4,
+      },
+    },
+  },
+  logging: {
+    enabled: false,
+    logBlocked: true,
+    verboseErrors: false,
+  },
+};
+
+export const configSchema = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['mode'],
+  properties: {
+    mode: {
+      type: 'string',
+      enum: ['allowlist', 'blocklist', 'hybrid'],
+    },
+    allowlist: {
+      type: 'array',
+      items: { type: 'string' },
+    },
+    blocklist: {
+      type: 'array',
+      items: { type: 'string' },
+    },
+    blockPrivateIps: {
+      type: 'boolean',
+    },
+    threatFeeds: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        urlhaus: { type: 'boolean' },
+        mode: {
+          type: 'string',
+          enum: ['fail-open', 'fail-closed'],
+        },
+      },
+    },
+    scoring: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        enabled: { type: 'boolean' },
+        defaultScore: { type: 'number', minimum: -10, maximum: 10 },
+        minScore: { type: 'number', minimum: -10, maximum: 10 },
+        rules: {
+          type: 'array',
+          items: {
+            type: 'object',
+            additionalProperties: false,
+            required: ['domain', 'score'],
+            properties: {
+              domain: { type: 'string' },
+              score: { type: 'number', minimum: -10, maximum: 10 },
+              reason: { type: 'string' },
+            },
+          },
+        },
+        feedScores: {
+          type: 'object',
+          additionalProperties: false,
+          properties: {
+            urlhaus: {
+              type: 'object',
+              additionalProperties: false,
+              properties: {
+                online: { type: 'number', minimum: -10, maximum: 10 },
+                offline: { type: 'number', minimum: -10, maximum: 10 },
+              },
+            },
+            spamhaus: {
+              type: 'object',
+              additionalProperties: false,
+              properties: {
+                botnet_cc: { type: 'number', minimum: -10, maximum: 10 },
+                phishing_domain: { type: 'number', minimum: -10, maximum: 10 },
+                spammer_domain: { type: 'number', minimum: -10, maximum: 10 },
+                abused_redirector: { type: 'number', minimum: -10, maximum: 10 },
+                'abused_legit_*': { type: 'number', minimum: -10, maximum: 10 },
+              },
+            },
+          },
+        },
+      },
+    },
+    logging: {
+      type: 'object',
+      additionalProperties: false,
+      properties: {
+        enabled: { type: 'boolean' },
+        logBlocked: { type: 'boolean' },
+        verboseErrors: { type: 'boolean' },
+      },
+    },
+  },
+} as const;
+
+function clampScore(value: number): number {
+  if (!Number.isFinite(value)) {
+    return 0;
+  }
+
+  return Math.max(-10, Math.min(10, Math.round(value)));
+}
+
+function normalizePatternList(values: string[] | undefined): string[] {
+  if (!values) {
+    return [];
+  }
+
+  return Array.from(
+    new Set(values.map((value) => value.trim().toLowerCase()).filter((value) => value.length > 0))
+  );
+}
+
+export function normalizeConfig(config: UrlGuardConfig | undefined): UrlGuardConfig {
+  const source = config ?? DEFAULT_CONFIG;
+  const defaultScoring = DEFAULT_CONFIG.scoring;
+  const sourceScoring = source.scoring;
+  const sourceRules = sourceScoring?.rules ?? defaultScoring?.rules ?? [];
+
+  return {
+    mode: source.mode ?? DEFAULT_CONFIG.mode,
+    allowlist: normalizePatternList(source.allowlist ?? DEFAULT_CONFIG.allowlist),
+    blocklist: normalizePatternList(source.blocklist ?? DEFAULT_CONFIG.blocklist),
+    blockPrivateIps: source.blockPrivateIps ?? DEFAULT_CONFIG.blockPrivateIps,
+    threatFeeds: {
+      urlhaus: source.threatFeeds?.urlhaus ?? DEFAULT_CONFIG.threatFeeds?.urlhaus,
+      mode: source.threatFeeds?.mode ?? DEFAULT_CONFIG.threatFeeds?.mode,
+    },
+    scoring: {
+      enabled: sourceScoring?.enabled ?? defaultScoring?.enabled,
+      defaultScore: clampScore(sourceScoring?.defaultScore ?? defaultScoring?.defaultScore ?? 0),
+      minScore: clampScore(sourceScoring?.minScore ?? defaultScoring?.minScore ?? -6),
+      rules: sourceRules
+        .map((rule) => ({
+          domain: rule.domain.trim().toLowerCase(),
+          score: clampScore(rule.score),
+          reason: rule.reason,
+        }))
+        .filter((rule) => rule.domain.length > 0),
+      feedScores: {
+        urlhaus: {
+          online: clampScore(
+            sourceScoring?.feedScores?.urlhaus?.online ??
+              defaultScoring?.feedScores?.urlhaus?.online ??
+              -10
+          ),
+          offline: clampScore(
+            sourceScoring?.feedScores?.urlhaus?.offline ??
+              defaultScoring?.feedScores?.urlhaus?.offline ??
+              -7
+          ),
+        },
+        spamhaus: {
+          botnet_cc: clampScore(
+            sourceScoring?.feedScores?.spamhaus?.botnet_cc ??
+              defaultScoring?.feedScores?.spamhaus?.botnet_cc ??
+              -10
+          ),
+          phishing_domain: clampScore(
+            sourceScoring?.feedScores?.spamhaus?.phishing_domain ??
+              defaultScoring?.feedScores?.spamhaus?.phishing_domain ??
+              -10
+          ),
+          spammer_domain: clampScore(
+            sourceScoring?.feedScores?.spamhaus?.spammer_domain ??
+              defaultScoring?.feedScores?.spamhaus?.spammer_domain ??
+              -8
+          ),
+          abused_redirector: clampScore(
+            sourceScoring?.feedScores?.spamhaus?.abused_redirector ??
+              defaultScoring?.feedScores?.spamhaus?.abused_redirector ??
+              -4
+          ),
+          'abused_legit_*': clampScore(
+            sourceScoring?.feedScores?.spamhaus?.['abused_legit_*'] ??
+              defaultScoring?.feedScores?.spamhaus?.['abused_legit_*'] ??
+              -6
+          ),
+        },
+      },
+    },
+    logging: {
+      enabled: source.logging?.enabled ?? DEFAULT_CONFIG.logging?.enabled,
+      logBlocked: source.logging?.logBlocked ?? DEFAULT_CONFIG.logging?.logBlocked,
+      verboseErrors: source.logging?.verboseErrors ?? DEFAULT_CONFIG.logging?.verboseErrors,
+    },
+  };
+}

package/src/filters/matcher.ts

@@ -0,0 +1,126 @@
+import { normalizeConfig } from '../config';
+import type { UrlGuardConfig } from '../types';
+
+const ALLOWED_PROTOCOLS = ['http:', 'https:'];
+const PRIVATE_IP_PATTERNS = [
+  /^127\./,
+  /^10\./,
+  /^172\.(1[6-9]|2\d|3[01])\./,
+  /^192\.168\./,
+  /^169\.254\./,
+  /^0\./,
+  /^\[?::1\]?$/i,
+  /^\[?fc[0-9a-f]{2}:/i,
+  /^\[?fe80:/i,
+  /^\[?::ffff:127\./i,
+  /^\[?::ffff:10\./i,
+  /^\[?::ffff:172\.(1[6-9]|2\d|3[01])\./i,
+  /^\[?::ffff:192\.168\./i,
+];
+
+function normalizeHostname(value: string): string {
+  return value.trim().toLowerCase().replace(/\.$/, '');
+}
+
+export function extractHostname(url: string): string | null {
+  try {
+    const parsed = new URL(url);
+    if (!ALLOWED_PROTOCOLS.includes(parsed.protocol)) {
+      return null;
+    }
+
+    return normalizeHostname(parsed.hostname);
+  } catch {
+    return null;
+  }
+}
+
+export function isPrivateIp(hostname: string): boolean {
+  return PRIVATE_IP_PATTERNS.some((pattern) => pattern.test(normalizeHostname(hostname)));
+}
+
+export function matchesHostnamePattern(hostname: string, pattern: string): boolean {
+  const normalizedHost = normalizeHostname(hostname);
+  const normalizedPattern = normalizeHostname(pattern);
+
+  if (normalizedPattern.startsWith('*.')) {
+    const suffix = normalizedPattern.slice(2);
+    if (!suffix || normalizedHost === suffix) {
+      return false;
+    }
+
+    return normalizedHost.endsWith(`.${suffix}`);
+  }
+
+  return normalizedHost === normalizedPattern;
+}
+
+function matchesAnyPattern(hostname: string, patterns: string[]): boolean {
+  return patterns.some((pattern) => matchesHostnamePattern(hostname, pattern));
+}
+
+export function isAllowed(url: string, config: UrlGuardConfig): boolean {
+  const hostname = extractHostname(url);
+  if (!hostname) {
+    return false;
+  }
+
+  const normalizedConfig = normalizeConfig(config);
+  if (normalizedConfig.blockPrivateIps && isPrivateIp(hostname)) {
+    return false;
+  }
+
+  const allowlist = normalizedConfig.allowlist ?? [];
+  const blocklist = normalizedConfig.blocklist ?? [];
+
+  switch (normalizedConfig.mode) {
+    case 'allowlist': {
+      return matchesAnyPattern(hostname, allowlist);
+    }
+    case 'blocklist': {
+      return !matchesAnyPattern(hostname, blocklist);
+    }
+    case 'hybrid': {
+      const inAllowlist = matchesAnyPattern(hostname, allowlist);
+      const inBlocklist = matchesAnyPattern(hostname, blocklist);
+      return inAllowlist && !inBlocklist;
+    }
+    default:
+      return false;
+  }
+}
+
+export function getBlockReason(url: string, config: UrlGuardConfig): string {
+  const hostname = extractHostname(url);
+  if (!hostname) {
+    return 'invalid URL';
+  }
+
+  const normalizedConfig = normalizeConfig(config);
+  if (normalizedConfig.blockPrivateIps && isPrivateIp(hostname)) {
+    return 'private or internal IP blocked';
+  }
+
+  const allowlist = normalizedConfig.allowlist ?? [];
+  const blocklist = normalizedConfig.blocklist ?? [];
+
+  if (normalizedConfig.mode === 'allowlist' && !matchesAnyPattern(hostname, allowlist)) {
+    return 'not in allowlist';
+  }
+
+  if (normalizedConfig.mode === 'blocklist' && matchesAnyPattern(hostname, blocklist)) {
+    return 'in blocklist';
+  }
+
+  if (normalizedConfig.mode === 'hybrid') {
+    if (!matchesAnyPattern(hostname, allowlist)) {
+      return 'not in allowlist';
+    }
+
+    if (matchesAnyPattern(hostname, blocklist)) {
+      return 'in blocklist';
+    }
+  }
+
+  return 'blocked by policy';
+}

package/src/filters/urlhaus.ts

@@ -0,0 +1,170 @@
+import type { FeedSignal } from '../types';
+import { readFile } from 'node:fs/promises';
+
+export interface UrlhausLookupResult {
+  listed: boolean;
+  unavailable: boolean;
+  reason?: string;
+  feedSignals?: FeedSignal[];
+}
+
+const URLHAUS_LOOKUP_ENDPOINT = 'https://urlhaus-api.abuse.ch/v1/url/';
+const URLHAUS_TIMEOUT_MS = 5000;
+let cachedFixturePath: string | null = null;
+let cachedFixtureSet: Set<string> | null = null;
+
+function normalizeUrl(value: string): string {
+  return value.trim();
+}
+
+async function loadFixtureSet(filePath: string): Promise<Set<string>> {
+  if (cachedFixturePath === filePath && cachedFixtureSet) {
+    return cachedFixtureSet;
+  }
+
+  const content = await readFile(filePath, 'utf8');
+  const lines = content
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+
+  const fixtureUrls = new Set<string>();
+  for (const line of lines.slice(1)) {
+    const columns = line.split(',');
+    const url = columns[2];
+    if (url) {
+      fixtureUrls.add(normalizeUrl(url));
+    }
+  }
+
+  cachedFixturePath = filePath;
+  cachedFixtureSet = fixtureUrls;
+  return fixtureUrls;
+}
+
+function getSpamhausCategory(payload: Record<string, unknown>): string | null {
+  const blacklists = payload.blacklists;
+  if (!blacklists || typeof blacklists !== 'object') {
+    return null;
+  }
+
+  const spamhaus = (blacklists as Record<string, unknown>).spamhaus_dbl;
+  if (!spamhaus) {
+    return null;
+  }
+
+  if (typeof spamhaus === 'string') {
+    const category = spamhaus.trim().toLowerCase();
+    return category.length > 0 ? category : null;
+  }
+
+  if (typeof spamhaus === 'object') {
+    const candidate = (spamhaus as Record<string, unknown>).category;
+    if (typeof candidate === 'string') {
+      const category = candidate.trim().toLowerCase();
+      return category.length > 0 ? category : null;
+    }
+  }
+
+  return null;
+}
+
+export async function lookupUrlhaus(url: string): Promise<UrlhausLookupResult> {
+  const fixtureFile = process.env.URLHAUS_FIXTURE_FILE;
+  if (fixtureFile) {
+    try {
+      const fixtureSet = await loadFixtureSet(fixtureFile);
+      const listed = fixtureSet.has(normalizeUrl(url));
+      return {
+        listed,
+        unavailable: false,
+        reason: listed ? 'listed by URLhaus fixture' : undefined,
+        ...(listed
+          ? {
+              feedSignals: [
+                {
+                  feed: 'urlhaus' as const,
+                  category: 'online',
+                  reason: 'listed by URLhaus fixture',
+                },
+              ],
+            }
+          : {}),
+      };
+    } catch (error) {
+      console.warn('[url-guard] Failed to load URLhaus fixture:', error);
+      return {
+        listed: false,
+        unavailable: true,
+        reason: 'urlhaus fixture unavailable',
+      };
+    }
+  }
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), URLHAUS_TIMEOUT_MS);
+
+  try {
+    const body = new URLSearchParams({ url });
+    const response = await fetch(URLHAUS_LOOKUP_ENDPOINT, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+      },
+      body,
+      signal: controller.signal,
+    });
+
+    if (!response.ok) {
+      return {
+        listed: false,
+        unavailable: true,
+        reason: `urlhaus lookup failed (${response.status})`,
+      };
+    }
+
+    const payload = (await response.json()) as {
+      query_status?: string;
+      url_status?: string;
+      threat?: string;
+      blacklists?: Record<string, unknown>;
+    };
+
+    const urlStatus = payload.url_status?.toLowerCase();
+    const listed =
+      payload.query_status === 'ok' || urlStatus === 'online' || urlStatus === 'offline';
+    const feedSignals: FeedSignal[] = [];
+
+    if (listed && (urlStatus === 'online' || urlStatus === 'offline')) {
+      feedSignals.push({
+        feed: 'urlhaus',
+        category: urlStatus,
+        reason: payload.threat ?? `urlhaus ${urlStatus}`,
+      });
+    }
+
+    const spamhausCategory = getSpamhausCategory(payload as Record<string, unknown>);
+    if (spamhausCategory) {
+      feedSignals.push({
+        feed: 'spamhaus',
+        category: spamhausCategory,
+        reason: `spamhaus_dbl ${spamhausCategory}`,
+      });
+    }
+
+    return {
+      listed,
+      unavailable: false,
+      reason: listed ? (payload.threat ?? 'listed by URLhaus') : undefined,
+      ...(feedSignals.length > 0 ? { feedSignals } : {}),
+    };
+  } catch (error) {
+    if (error instanceof Error && error.name === 'AbortError') {
+      return { listed: false, unavailable: true, reason: 'urlhaus lookup timed out' };
+    }
+
+    return { listed: false, unavailable: true, reason: 'urlhaus lookup unavailable' };
+  } finally {
+    clearTimeout(timeout);
+  }
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+import { configSchema } from './config';
+import { safeWebFetchTool } from './tools/safeFetch';
+import { safeWebSearchTool } from './tools/safeSearch';
+import type { PluginDefinition } from './types';
+import pkg from '../package.json';
+
+const plugin: PluginDefinition = {
+  name: '@llnvd/openclaw-url-guard',
+  version: pkg.version,
+  tools: [safeWebFetchTool, safeWebSearchTool],
+  configSchema,
+};
+
+export default plugin;

package/src/scoring/engine.ts

@@ -0,0 +1,144 @@
+import { normalizeConfig } from '../config';
+import { extractHostname, matchesHostnamePattern } from '../filters/matcher';
+import type { FeedResults, ScoreResult, ScoringRule, UrlGuardConfig } from '../types';
+
+function getRulePriority(rule: ScoringRule): number {
+  const isWildcard = rule.domain.startsWith('*.');
+  const baseLength = isWildcard ? rule.domain.length - 2 : rule.domain.length;
+  return (isWildcard ? 0 : 10_000) + baseLength;
+}
+
+function getBestMatchingRule(hostname: string, rules: ScoringRule[]): ScoringRule | null {
+  let bestRule: ScoringRule | null = null;
+  let bestPriority = -1;
+
+  for (const rule of rules) {
+    if (!matchesHostnamePattern(hostname, rule.domain)) {
+      continue;
+    }
+
+    const priority = getRulePriority(rule);
+    if (priority > bestPriority) {
+      bestPriority = priority;
+      bestRule = rule;
+    }
+  }
+
+  return bestRule;
+}
+
+function mapFeedSignalToScore(
+  signal: { feed: 'urlhaus' | 'spamhaus'; category: string },
+  config: UrlGuardConfig
+): number | null {
+  const category = signal.category.toLowerCase();
+
+  if (signal.feed === 'urlhaus') {
+    if (category === 'online') {
+      return config.scoring?.feedScores?.urlhaus?.online ?? null;
+    }
+
+    if (category === 'offline') {
+      return config.scoring?.feedScores?.urlhaus?.offline ?? null;
+    }
+
+    return null;
+  }
+
+  if (category.startsWith('abused_legit_')) {
+    return config.scoring?.feedScores?.spamhaus?.['abused_legit_*'] ?? null;
+  }
+
+  switch (category) {
+    case 'botnet_cc':
+      return config.scoring?.feedScores?.spamhaus?.botnet_cc ?? null;
+    case 'phishing_domain':
+      return config.scoring?.feedScores?.spamhaus?.phishing_domain ?? null;
+    case 'spammer_domain':
+      return config.scoring?.feedScores?.spamhaus?.spammer_domain ?? null;
+    case 'abused_redirector':
+      return config.scoring?.feedScores?.spamhaus?.abused_redirector ?? null;
+    default:
+      return null;
+  }
+}
+
+export function calculateScore(
+  url: string,
+  config: UrlGuardConfig,
+  feedResults?: FeedResults
+): ScoreResult {
+  const normalizedConfig = normalizeConfig(config);
+  const hostname = extractHostname(url);
+
+  if (!hostname) {
+    return {
+      finalScore: -10,
+      sources: [{ score: -10, source: 'url', reason: 'invalid URL' }],
+      allowed: false,
+    };
+  }
+
+  const minScore = normalizedConfig.scoring?.minScore ?? -6;
+  const defaultScore = normalizedConfig.scoring?.defaultScore ?? 0;
+  const staticRule = getBestMatchingRule(hostname, normalizedConfig.scoring?.rules ?? []);
+
+  if (staticRule) {
+    const finalScore = staticRule.score;
+    return {
+      finalScore,
+      sources: [
+        {
+          score: staticRule.score,
+          source: `static:${staticRule.domain}`,
+          reason: staticRule.reason,
+        },
+      ],
+      allowed: finalScore >= minScore,
+    };
+  }
+
+  const feedSources: Array<{ score: number; source: string; reason?: string }> = [];
+
+  for (const signal of feedResults?.signals ?? []) {
+    const score = mapFeedSignalToScore(signal, normalizedConfig);
+    if (score === null) {
+      continue;
+    }
+
+    feedSources.push({
+      score,
+      source: `${signal.feed}:${signal.category}`,
+      reason: signal.reason,
+    });
+  }
+
+  if (feedSources.length > 0) {
+    const finalScore = Math.min(...feedSources.map((source) => source.score));
+    return {
+      finalScore,
+      sources: feedSources,
+      allowed: finalScore >= minScore,
+    };
+  }
+
+  if (feedResults?.unavailable && normalizedConfig.threatFeeds?.mode === 'fail-closed') {
+    return {
+      finalScore: -10,
+      sources: [
+        {
+          score: -10,
+          source: 'threat-feed-unavailable',
+          reason: feedResults.reason ?? 'threat feed lookup unavailable',
+        },
+      ],
+      allowed: false,
+    };
+  }
+
+  return {
+    finalScore: defaultScore,
+    sources: [{ score: defaultScore, source: 'default-score' }],
+    allowed: defaultScore >= minScore,
+  };
+}