chainlesschain 0.47.8 → 0.47.9

package/src/lib/threat-intel.js

@@ -0,0 +1,268 @@
+/**
+ * Threat Intelligence Store — SQLite-backed IoC (indicator of
+ * compromise) catalog. Imports STIX 2.1 bundles, deduplicates
+ * indicators by (type, value), and supports fast O(1) matching of
+ * an arbitrary observable against the stored feed.
+ *
+ * Designed to pair with `compliance-manager.js`: a compliance scan
+ * can call `matchObservable` to flag blocklisted artefacts during
+ * evidence ingestion.
+ */
+
+import crypto from "crypto";
+import fs from "fs";
+import {
+  extractIndicatorsFromBundle,
+  classifyObservable,
+  IOC_TYPES,
+} from "./stix-parser.js";
+
+/* ── Schema ────────────────────────────────────────────────── */
+
+export function ensureThreatIntelTables(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS threat_intel_indicators (
+      id TEXT PRIMARY KEY,
+      type TEXT NOT NULL,
+      value TEXT NOT NULL,
+      labels TEXT,
+      confidence INTEGER,
+      source_id TEXT,
+      source_name TEXT,
+      valid_from TEXT,
+      valid_until TEXT,
+      first_seen_at TEXT DEFAULT (datetime('now')),
+      last_seen_at TEXT DEFAULT (datetime('now')),
+      UNIQUE(type, value)
+    )
+  `);
+  db.exec(
+    `CREATE INDEX IF NOT EXISTS idx_threat_intel_type ON threat_intel_indicators(type)`,
+  );
+  db.exec(
+    `CREATE INDEX IF NOT EXISTS idx_threat_intel_value ON threat_intel_indicators(value)`,
+  );
+}
+
+/* ── Import ────────────────────────────────────────────────── */
+
+/**
+ * Import a parsed STIX bundle (or a loose array of STIX objects) into
+ * the store. Returns `{imported, updated, skipped, total}`.
+ *
+ * - `imported`: new (type,value) pairs inserted.
+ * - `updated`:  existing pairs whose metadata was refreshed.
+ * - `skipped`:  objects that yielded no usable indicator (unknown
+ *   observable type, non-stix pattern_type, malformed pattern, etc.).
+ */
+export function importStixBundle(db, bundle) {
+  if (!db) throw new Error("Database is required");
+  const iocs = extractIndicatorsFromBundle(bundle);
+  const total = Array.isArray(bundle?.objects)
+    ? bundle.objects.filter((o) => o?.type === "indicator").length
+    : Array.isArray(bundle)
+      ? bundle.filter((o) => o?.type === "indicator").length
+      : 0;
+
+  let imported = 0;
+  let updated = 0;
+
+  const insert = db.prepare(
+    `INSERT INTO threat_intel_indicators
+       (id, type, value, labels, confidence, source_id, source_name,
+        valid_from, valid_until, first_seen_at, last_seen_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))`,
+  );
+  const update = db.prepare(
+    `UPDATE threat_intel_indicators
+        SET labels       = ?,
+            confidence   = ?,
+            source_id    = ?,
+            source_name  = ?,
+            valid_from   = ?,
+            valid_until  = ?,
+            last_seen_at = datetime('now')
+      WHERE type = ? AND value = ?`,
+  );
+  const selectExisting = db.prepare(
+    `SELECT id FROM threat_intel_indicators WHERE type = ? AND value = ?`,
+  );
+
+  for (const rawIoc of iocs) {
+    // File hashes are case-insensitive — normalize on write so that
+    // equality lookups in `matchObservable` don't depend on SQLite's
+    // COLLATE NOCASE (which the mock-db used in tests doesn't emulate).
+    const ioc = rawIoc.type.startsWith("file-")
+      ? { ...rawIoc, value: String(rawIoc.value).toLowerCase() }
+      : rawIoc;
+    const prior = selectExisting.get(ioc.type, ioc.value);
+    const labels = JSON.stringify(ioc.source?.labels || []);
+    const confidence = ioc.source?.confidence ?? null;
+    const sourceId = ioc.source?.indicatorId || null;
+    const sourceName = ioc.source?.name || null;
+    const validFrom = ioc.source?.validFrom || null;
+    const validUntil = ioc.source?.validUntil || null;
+
+    if (prior) {
+      update.run(
+        labels,
+        confidence,
+        sourceId,
+        sourceName,
+        validFrom,
+        validUntil,
+        ioc.type,
+        ioc.value,
+      );
+      updated += 1;
+    } else {
+      insert.run(
+        crypto.randomUUID(),
+        ioc.type,
+        ioc.value,
+        labels,
+        confidence,
+        sourceId,
+        sourceName,
+        validFrom,
+        validUntil,
+      );
+      imported += 1;
+    }
+  }
+  const skipped = Math.max(total - iocs.length, 0);
+  return { imported, updated, skipped, total };
+}
+
+/**
+ * Import a STIX bundle from a file path. Convenience wrapper over
+ * `importStixBundle` — reads the JSON, parses it, defers to the
+ * core importer.
+ */
+export function importStixFile(db, filePath) {
+  const raw = fs.readFileSync(filePath, "utf-8");
+  let bundle;
+  try {
+    bundle = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`Invalid JSON in ${filePath}: ${err.message}`);
+  }
+  return importStixBundle(db, bundle);
+}
+
+/* ── Query ─────────────────────────────────────────────────── */
+
+function _rowToIndicator(r) {
+  if (!r) return null;
+  let labels = [];
+  try {
+    labels = r.labels ? JSON.parse(r.labels) : [];
+  } catch {
+    labels = [];
+  }
+  return {
+    id: r.id,
+    type: r.type,
+    value: r.value,
+    labels,
+    confidence: r.confidence ?? null,
+    sourceId: r.source_id || null,
+    sourceName: r.source_name || null,
+    validFrom: r.valid_from || null,
+    validUntil: r.valid_until || null,
+    firstSeenAt: r.first_seen_at,
+    lastSeenAt: r.last_seen_at,
+  };
+}
+
+/**
+ * List indicators, optionally filtered by type. `limit` caps result
+ * size (default 100, 0 = unlimited).
+ */
+export function listIndicators(db, options = {}) {
+  const { type, limit = 100 } = options;
+  if (type && !IOC_TYPES.includes(type)) {
+    throw new Error(
+      `Unknown IOC type: ${type}. Valid: ${IOC_TYPES.join(", ")}`,
+    );
+  }
+  let sql = `SELECT * FROM threat_intel_indicators`;
+  const params = [];
+  if (type) {
+    sql += ` WHERE type = ?`;
+    params.push(type);
+  }
+  sql += ` ORDER BY last_seen_at DESC`;
+  if (limit && limit > 0) {
+    sql += ` LIMIT ?`;
+    params.push(limit);
+  }
+  return db
+    .prepare(sql)
+    .all(...params)
+    .map(_rowToIndicator);
+}
+
+/**
+ * Match an arbitrary observable against the stored feed. Returns:
+ *   {matched:true, type, indicator} on hit
+ *   {matched:false, type}            on miss (type still classified)
+ *   {matched:false, type:"unknown"}  if we can't classify the input
+ */
+export function matchObservable(db, value) {
+  const type = classifyObservable(value);
+  if (type === "unknown") return { matched: false, type: "unknown" };
+
+  const trimmed = String(value).trim();
+  const normalized = type.startsWith("file-") ? trimmed.toLowerCase() : trimmed;
+
+  const row = db
+    .prepare(
+      `SELECT * FROM threat_intel_indicators
+         WHERE type = ? AND value = ?
+         LIMIT 1`,
+    )
+    .get(type, normalized);
+
+  if (!row) return { matched: false, type };
+  return { matched: true, type, indicator: _rowToIndicator(row) };
+}
+
+/**
+ * Aggregate stats — total indicators + counts per type.
+ */
+export function getStats(db) {
+  const total =
+    db.prepare(`SELECT COUNT(*) AS n FROM threat_intel_indicators`).get()?.n ??
+    0;
+  const rows = db
+    .prepare(
+      `SELECT type, COUNT(*) AS n
+         FROM threat_intel_indicators
+         GROUP BY type
+         ORDER BY n DESC`,
+    )
+    .all();
+  const byType = {};
+  for (const r of rows) byType[r.type] = r.n;
+  return { total, byType };
+}
+
+/**
+ * Remove a single indicator by (type, value). Returns true if a row
+ * was deleted.
+ */
+export function removeIndicator(db, type, value) {
+  const info = db
+    .prepare(`DELETE FROM threat_intel_indicators WHERE type = ? AND value = ?`)
+    .run(type, value);
+  return info.changes > 0;
+}
+
+/**
+ * Drop every stored indicator. Returns the number of rows removed.
+ */
+export function clearAll(db) {
+  const info = db.prepare(`DELETE FROM threat_intel_indicators`).run();
+  return info.changes;
+}

package/src/lib/topic-classifier.js

@@ -0,0 +1,400 @@
+/**
+ * Topic Classifier — language-aware, multilingual.
+ *
+ * Pragmatic alternative to a neural classifier. Improves on boolean keyword
+ * matching by:
+ *   1. Detecting language via Unicode ranges (zh / ja / en / other) so
+ *      CJK text is tokenized per-character instead of being stringified
+ *      as one whitespace-less blob.
+ *   2. Scoring with term frequency (TF) against per-language lexicons,
+ *      then normalizing so scores are comparable across topics.
+ *
+ * Deep-learning-enhanced classification is deferred to the LLM manager
+ * (Desktop main-process); this CLI classifier is offline and fully
+ * deterministic, which is what `cc social analyze` needs.
+ *
+ * Languages: zh (Chinese), ja (Japanese), en (English), other.
+ */
+
+/* ── Unicode ranges ────────────────────────────────────────── */
+
+// Han ideographs (CJK Unified + Extension-A). Shared by zh and ja.
+const RE_HAN = /[\u3400-\u4dbf\u4e00-\u9fff]/;
+// Hiragana + Katakana (distinctive for Japanese).
+const RE_KANA = /[\u3040-\u309f\u30a0-\u30ff]/;
+// Latin alphanumerics.
+const RE_LATIN = /[A-Za-z]/;
+
+/**
+ * Detect the dominant language of a piece of text.
+ *  - Any Hiragana/Katakana → ja (distinctive of Japanese)
+ *  - Else Han ideographs present → zh
+ *  - Else majority Latin letters → en
+ *  - Otherwise 'other'
+ */
+export function detectLanguage(text) {
+  if (!text || typeof text !== "string") return "other";
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return "other";
+  if (RE_KANA.test(trimmed)) return "ja";
+  if (RE_HAN.test(trimmed)) return "zh";
+  const latinCount = (trimmed.match(/[A-Za-z]/g) || []).length;
+  const totalLetters = (trimmed.match(/[^\s\d\p{P}\p{S}]/gu) || []).length;
+  if (totalLetters > 0 && latinCount / totalLetters >= 0.5) return "en";
+  return "other";
+}
+
+/* ── Tokenization ──────────────────────────────────────────── */
+
+/**
+ * Tokenize a piece of text. The strategy depends on language:
+ *  - en / other: split on non-alphanumerics, lowercase.
+ *  - zh: each Han character becomes a token (ideograms carry meaning
+ *    individually, and without a real segmenter this is more useful
+ *    than "one giant token per whitespace-less blob").
+ *  - ja: Kana + Han characters become tokens; Latin words split normally.
+ */
+export function tokenize(text, lang) {
+  if (!text) return [];
+  const resolved = lang || detectLanguage(text);
+  const out = [];
+  if (resolved === "en" || resolved === "other") {
+    for (const word of text.toLowerCase().split(/[^a-z0-9_]+/u)) {
+      if (word) out.push(word);
+    }
+    return out;
+  }
+  // For zh/ja we tokenize char-by-char for CJK and word-by-word for Latin.
+  for (const ch of text) {
+    if (RE_KANA.test(ch) || RE_HAN.test(ch)) {
+      out.push(ch);
+    }
+  }
+  for (const word of text.toLowerCase().split(/[^a-z0-9_]+/u)) {
+    if (word) out.push(word);
+  }
+  return out;
+}
+
+/* ── Lexicons ──────────────────────────────────────────────── */
+
+/**
+ * Default lexicons. Shape:
+ *   DEFAULT_TOPIC_LEXICONS[topic][lang] = Array<string | [string, number]>
+ *
+ * Keywords are matched as exact tokens (language-aware — English keywords
+ * match tokenized English words; CN/JP keywords can be either single CJK
+ * chars or multi-char phrases, which we match via substring inclusion).
+ */
+export const DEFAULT_TOPIC_LEXICONS = Object.freeze({
+  tech: {
+    en: [
+      "tech",
+      "technology",
+      "software",
+      "ai",
+      "algorithm",
+      "code",
+      "developer",
+      "programming",
+      "cloud",
+      "startup",
+    ],
+    zh: [
+      "科技",
+      "技术",
+      "软件",
+      "算法",
+      "编程",
+      "程序",
+      "开发",
+      "云计算",
+      "人工智能",
+      "互联网",
+    ],
+    ja: [
+      "テクノロジー",
+      "技術",
+      "ソフトウェア",
+      "プログラム",
+      "開発",
+      "人工知能",
+      "クラウド",
+      "システム",
+    ],
+  },
+  sports: {
+    en: [
+      "sport",
+      "sports",
+      "game",
+      "match",
+      "football",
+      "basketball",
+      "soccer",
+      "olympic",
+      "tennis",
+      "team",
+    ],
+    zh: [
+      "体育",
+      "运动",
+      "比赛",
+      "足球",
+      "篮球",
+      "奥运",
+      "网球",
+      "联赛",
+      "冠军",
+    ],
+    ja: [
+      "スポーツ",
+      "試合",
+      "サッカー",
+      "野球",
+      "バスケ",
+      "オリンピック",
+      "優勝",
+    ],
+  },
+  health: {
+    en: [
+      "health",
+      "medical",
+      "doctor",
+      "hospital",
+      "disease",
+      "vaccine",
+      "patient",
+      "therapy",
+      "clinic",
+      "surgery",
+    ],
+    zh: [
+      "健康",
+      "医疗",
+      "医生",
+      "医院",
+      "疾病",
+      "疫苗",
+      "治疗",
+      "诊所",
+      "手术",
+    ],
+    ja: ["健康", "医療", "医師", "病院", "病気", "ワクチン", "治療"],
+  },
+  food: {
+    en: [
+      "food",
+      "restaurant",
+      "cuisine",
+      "cooking",
+      "chef",
+      "recipe",
+      "dish",
+      "meal",
+      "dessert",
+    ],
+    zh: ["美食", "餐厅", "料理", "烹饪", "厨师", "菜谱", "甜点", "小吃"],
+    ja: ["料理", "レストラン", "グルメ", "料理人", "デザート", "食事"],
+  },
+  travel: {
+    en: [
+      "travel",
+      "tourism",
+      "hotel",
+      "trip",
+      "flight",
+      "vacation",
+      "tourist",
+      "destination",
+      "cruise",
+    ],
+    zh: ["旅行", "旅游", "酒店", "景点", "机票", "度假", "邮轮"],
+    ja: ["旅行", "ホテル", "観光", "フライト", "休暇", "旅"],
+  },
+  politics: {
+    en: [
+      "politics",
+      "government",
+      "election",
+      "policy",
+      "president",
+      "congress",
+      "senate",
+      "vote",
+      "campaign",
+    ],
+    zh: ["政治", "政府", "选举", "政策", "总统", "议会", "投票", "竞选"],
+    ja: ["政治", "政府", "選挙", "政策", "大統領", "国会", "投票"],
+  },
+  finance: {
+    en: [
+      "finance",
+      "economy",
+      "stock",
+      "investment",
+      "market",
+      "bank",
+      "trade",
+      "currency",
+      "crypto",
+      "inflation",
+    ],
+    zh: [
+      "金融",
+      "经济",
+      "股票",
+      "投资",
+      "市场",
+      "银行",
+      "贸易",
+      "货币",
+      "加密",
+    ],
+    ja: ["金融", "経済", "株式", "投資", "市場", "銀行", "取引", "通貨"],
+  },
+  entertainment: {
+    en: [
+      "entertainment",
+      "movie",
+      "film",
+      "music",
+      "celebrity",
+      "concert",
+      "series",
+      "actor",
+      "singer",
+    ],
+    zh: ["娱乐", "电影", "音乐", "明星", "演唱会", "电视剧", "演员", "歌手"],
+    ja: ["映画", "音楽", "芸能", "コンサート", "ドラマ", "俳優", "歌手"],
+  },
+});
+
+// User-registered lexicon overrides. Indexed by topic → lang → keywords[].
+const _customLexicons = new Map();
+
+/**
+ * Register or override a topic lexicon. Pass `lexicon` as:
+ *   { en: [...], zh: [...], ja: [...] }
+ */
+export function registerTopicLexicon(topic, lexicon) {
+  if (!topic) throw new Error("topic is required");
+  if (!lexicon || typeof lexicon !== "object") {
+    throw new Error("lexicon object is required");
+  }
+  _customLexicons.set(topic, lexicon);
+}
+
+export function unregisterTopicLexicon(topic) {
+  return _customLexicons.delete(topic);
+}
+
+export function listTopicLexicons() {
+  const merged = { ...DEFAULT_TOPIC_LEXICONS };
+  for (const [topic, lexicon] of _customLexicons.entries()) {
+    merged[topic] = lexicon;
+  }
+  return merged;
+}
+
+/* ── Scoring ───────────────────────────────────────────────── */
+
+function _keywordWeight(entry) {
+  if (Array.isArray(entry)) return { keyword: entry[0], weight: entry[1] ?? 1 };
+  return { keyword: entry, weight: 1 };
+}
+
+function _matchCount(text, tokens, keyword, lang) {
+  // For Latin keywords, match as token equality (case-insensitive).
+  // For CJK keywords, match as substring of the text (so multi-char phrases
+  // like "加密货币" match even though tokenize() splits them into chars).
+  if (/^[A-Za-z0-9_]+$/.test(keyword)) {
+    const lower = keyword.toLowerCase();
+    let count = 0;
+    for (const t of tokens) if (t === lower) count += 1;
+    return count;
+  }
+  // CJK multi-char or single-char phrase: substring count.
+  if (!text) return 0;
+  let count = 0;
+  let idx = 0;
+  while (true) {
+    const hit = text.indexOf(keyword, idx);
+    if (hit === -1) break;
+    count += 1;
+    idx = hit + keyword.length;
+  }
+  return count;
+}
+
+/**
+ * Classify `text` into one or more topics.
+ *
+ * @param {string} text
+ * @param {Object} [opts]
+ * @param {Object} [opts.lexicons]  Override per-topic lexicons for this call.
+ * @param {number} [opts.topK=3]    Return the top-K topics.
+ * @param {string} [opts.lang]      Override detected language.
+ * @param {number} [opts.minScore=0] Drop topics with score below this (pre-normalization).
+ * @returns {{
+ *   language: string,
+ *   tokens: string[],
+ *   topics: Array<{ topic: string, score: number, rawScore: number, hits: number }>
+ * }}
+ */
+export function classifyTopic(text, opts = {}) {
+  const {
+    lexicons: overrideLexicons,
+    topK = 3,
+    lang: forceLang,
+    minScore = 0,
+  } = opts;
+
+  const language = forceLang || detectLanguage(text);
+  const tokens = tokenize(text, language);
+  if (!text || tokens.length === 0) {
+    return { language, tokens: [], topics: [] };
+  }
+
+  const lexiconSet = overrideLexicons || listTopicLexicons();
+  const raw = [];
+  for (const [topic, perLang] of Object.entries(lexiconSet)) {
+    if (!perLang || typeof perLang !== "object") continue;
+    const entries = perLang[language] || [];
+    // Fall back to English keywords when the target language has none.
+    const fallback = entries.length === 0 ? perLang.en || [] : [];
+    let rawScore = 0;
+    let hits = 0;
+    for (const entry of [...entries, ...fallback]) {
+      const { keyword, weight } = _keywordWeight(entry);
+      const count = _matchCount(text, tokens, keyword, language);
+      if (count > 0) {
+        rawScore += count * weight;
+        hits += count;
+      }
+    }
+    if (rawScore > minScore) {
+      raw.push({ topic, rawScore, hits });
+    }
+  }
+
+  const total = raw.reduce((s, r) => s + r.rawScore, 0) || 1;
+  const topics = raw
+    .map((r) => ({
+      topic: r.topic,
+      score: r.rawScore / total,
+      rawScore: r.rawScore,
+      hits: r.hits,
+    }))
+    .sort((a, b) => b.rawScore - a.rawScore)
+    .slice(0, topK);
+
+  return { language, tokens, topics };
+}
+
+/* ── Reset (for testing) ───────────────────────────────────── */
+
+export function _resetState() {
+  _customLexicons.clear();
+}