@panguard-ai/threat-cloud 0.2.0 → 0.2.2

package/dist/database.js

@@ -2,11 +2,10 @@
  * SQLite database layer for Threat Cloud
  * 威脅雲 SQLite 資料庫層
  *
- * Stores anonymized threat data, enriched events, IoCs, campaigns, and rules.
+ * Stores anonymized threat data and community rules using better-sqlite3.
  *
  * @module @panguard-ai/threat-cloud/database
  */
-import { createHash } from 'node:crypto';
 import Database from 'better-sqlite3';
 /**
  * Threat Cloud database backed by SQLite
@@ -18,27 +17,9 @@ export class ThreatCloudDB {
         this.db = new Database(dbPath);
         this.db.pragma('journal_mode = WAL');
         this.db.pragma('foreign_keys = ON');
-        this.db.pragma('busy_timeout = 15000');
-        this.db.pragma('synchronous = NORMAL');
-        this.db.pragma('cache_size = -64000');
-        this.db.pragma('temp_store = MEMORY');
-        this.db.pragma('wal_autocheckpoint = 1000');
-        this.db.pragma('journal_size_limit = 104857600');
         this.initialize();
-        this.runMigrations();
     }
-    /** Create a backup of the database / 建立資料庫備份 */
-    backup(destPath) {
-        this.db.backup(destPath);
-    }
-    /** Expose underlying db for sub-modules (IoCStore, etc.) / 暴露底層 DB 給子模組 */
-    getDB() {
-        return this.db;
-    }
-    // -------------------------------------------------------------------------
-    // Schema initialization / 資料表初始化
-    // -------------------------------------------------------------------------
-    /** Create original tables if they don't exist / 建立原始資料表 */
+    /** Create tables if they don't exist / 建立資料表 */
     initialize() {
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS threats (
@@ -58,6 +39,10 @@ export class ThreatCloudDB {
         rule_content TEXT NOT NULL,
         published_at TEXT NOT NULL,
         source TEXT NOT NULL,
+        category TEXT,
+        severity TEXT,
+        mitre_techniques TEXT,
+        tags TEXT,
         created_at TEXT NOT NULL DEFAULT (datetime('now')),
         updated_at TEXT NOT NULL DEFAULT (datetime('now'))
       );
@@ -67,199 +52,96 @@ export class ThreatCloudDB {
       CREATE INDEX IF NOT EXISTS idx_threats_mitre ON threats(mitre_technique);
       CREATE INDEX IF NOT EXISTS idx_rules_published ON rules(published_at);
 
-      CREATE TABLE IF NOT EXISTS schema_migrations (
-        version INTEGER PRIMARY KEY,
-        name TEXT NOT NULL,
-        applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+      CREATE TABLE IF NOT EXISTS atr_proposals (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        pattern_hash TEXT NOT NULL,
+        rule_content TEXT NOT NULL,
+        llm_provider TEXT NOT NULL,
+        llm_model TEXT NOT NULL,
+        self_review_verdict TEXT NOT NULL,
+        client_id TEXT,
+        status TEXT NOT NULL DEFAULT 'pending',
+        confirmations INTEGER NOT NULL DEFAULT 0,
+        llm_review_verdict TEXT,
+        created_at TEXT NOT NULL DEFAULT (datetime('now')),
+        updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+      );
+
+      CREATE TABLE IF NOT EXISTS atr_feedback (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        rule_id TEXT NOT NULL,
+        is_true_positive INTEGER NOT NULL,
+        client_id TEXT,
+        created_at TEXT NOT NULL DEFAULT (datetime('now'))
+      );
+
+      CREATE TABLE IF NOT EXISTS skill_threats (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        skill_hash TEXT NOT NULL,
+        skill_name TEXT NOT NULL,
+        risk_score INTEGER NOT NULL,
+        risk_level TEXT NOT NULL,
+        finding_summaries TEXT,
+        client_id TEXT,
+        created_at TEXT NOT NULL DEFAULT (datetime('now'))
+      );
+
+      CREATE TABLE IF NOT EXISTS ioc_entries (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        type TEXT NOT NULL,
+        value TEXT NOT NULL UNIQUE,
+        reputation INTEGER NOT NULL DEFAULT 50,
+        source TEXT,
+        first_seen TEXT DEFAULT (datetime('now')),
+        last_seen TEXT DEFAULT (datetime('now')),
+        sighting_count INTEGER DEFAULT 1
+      );
+
+      CREATE TABLE IF NOT EXISTS skill_whitelist (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        skill_name TEXT NOT NULL,
+        normalized_name TEXT NOT NULL UNIQUE,
+        fingerprint_hash TEXT,
+        confirmations INTEGER NOT NULL DEFAULT 1,
+        status TEXT NOT NULL DEFAULT 'pending',
+        first_reported TEXT DEFAULT (datetime('now')),
+        last_reported TEXT DEFAULT (datetime('now'))
       );
+
+      -- Migration: add classification columns to existing rules table
+      -- SQLite allows ADD COLUMN on existing tables; IF NOT EXISTS not supported,
+      -- so we catch errors for already-existing columns in migrate().
+    `);
+        this.migrate();
+        this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_rules_category ON rules(category);
+      CREATE INDEX IF NOT EXISTS idx_rules_severity ON rules(severity);
+      CREATE INDEX IF NOT EXISTS idx_rules_source ON rules(source);
+      CREATE INDEX IF NOT EXISTS idx_atr_proposals_status ON atr_proposals(status);
+      CREATE INDEX IF NOT EXISTS idx_atr_proposals_pattern ON atr_proposals(pattern_hash);
+      CREATE INDEX IF NOT EXISTS idx_skill_threats_hash ON skill_threats(skill_hash);
+      CREATE INDEX IF NOT EXISTS idx_atr_feedback_rule ON atr_feedback(rule_id);
+      CREATE INDEX IF NOT EXISTS idx_ioc_entries_type ON ioc_entries(type);
+      CREATE INDEX IF NOT EXISTS idx_ioc_entries_reputation ON ioc_entries(reputation);
+      CREATE INDEX IF NOT EXISTS idx_skill_whitelist_status ON skill_whitelist(status);
+      CREATE INDEX IF NOT EXISTS idx_skill_whitelist_name ON skill_whitelist(normalized_name);
     `);
     }
-    /** Run idempotent schema migrations / 執行冪等 schema 遷移 */
-    runMigrations() {
-        const applied = new Set(this.db.prepare('SELECT version FROM schema_migrations').all().map((r) => r.version));
-        const migrations = [
-            {
-                version: 1,
-                name: 'create_iocs_table',
-                sql: `
-          CREATE TABLE IF NOT EXISTS iocs (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            type TEXT NOT NULL CHECK(type IN ('ip','domain','url','hash_md5','hash_sha1','hash_sha256')),
-            value TEXT NOT NULL,
-            normalized_value TEXT NOT NULL,
-            threat_type TEXT NOT NULL,
-            source TEXT NOT NULL,
-            confidence INTEGER NOT NULL DEFAULT 50 CHECK(confidence BETWEEN 0 AND 100),
-            reputation_score INTEGER NOT NULL DEFAULT 50 CHECK(reputation_score BETWEEN 0 AND 100),
-            first_seen TEXT NOT NULL,
-            last_seen TEXT NOT NULL,
-            sightings INTEGER NOT NULL DEFAULT 1,
-            status TEXT NOT NULL DEFAULT 'active' CHECK(status IN ('active','expired','revoked','under_review')),
-            tags TEXT NOT NULL DEFAULT '[]',
-            metadata TEXT NOT NULL DEFAULT '{}',
-            created_at TEXT NOT NULL DEFAULT (datetime('now')),
-            updated_at TEXT NOT NULL DEFAULT (datetime('now')),
-            UNIQUE(type, normalized_value)
-          );
-          CREATE INDEX IF NOT EXISTS idx_iocs_type ON iocs(type);
-          CREATE INDEX IF NOT EXISTS idx_iocs_normalized ON iocs(normalized_value);
-          CREATE INDEX IF NOT EXISTS idx_iocs_reputation ON iocs(reputation_score DESC);
-          CREATE INDEX IF NOT EXISTS idx_iocs_last_seen ON iocs(last_seen);
-          CREATE INDEX IF NOT EXISTS idx_iocs_status ON iocs(status);
-          CREATE INDEX IF NOT EXISTS idx_iocs_source ON iocs(source);
-        `,
-            },
-            {
-                version: 2,
-                name: 'create_enriched_threats_table',
-                sql: `
-          CREATE TABLE IF NOT EXISTS enriched_threats (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            source_type TEXT NOT NULL CHECK(source_type IN ('guard','trap','external_feed')),
-            attack_source_ip TEXT NOT NULL,
-            attack_type TEXT NOT NULL,
-            mitre_techniques TEXT NOT NULL DEFAULT '[]',
-            sigma_rule_matched TEXT NOT NULL DEFAULT '',
-            timestamp TEXT NOT NULL,
-            industry TEXT,
-            region TEXT NOT NULL DEFAULT 'unknown',
-            confidence INTEGER NOT NULL DEFAULT 50,
-            severity TEXT NOT NULL DEFAULT 'medium',
-            service_type TEXT,
-            skill_level TEXT,
-            intent TEXT,
-            tools TEXT,
-            event_hash TEXT NOT NULL UNIQUE,
-            received_at TEXT NOT NULL DEFAULT (datetime('now')),
-            campaign_id TEXT
-          );
-          CREATE INDEX IF NOT EXISTS idx_enriched_timestamp ON enriched_threats(timestamp);
-          CREATE INDEX IF NOT EXISTS idx_enriched_attack_type ON enriched_threats(attack_type);
-          CREATE INDEX IF NOT EXISTS idx_enriched_ip ON enriched_threats(attack_source_ip);
-          CREATE INDEX IF NOT EXISTS idx_enriched_campaign ON enriched_threats(campaign_id);
-          CREATE INDEX IF NOT EXISTS idx_enriched_source_type ON enriched_threats(source_type);
-          CREATE INDEX IF NOT EXISTS idx_enriched_region ON enriched_threats(region);
-          CREATE INDEX IF NOT EXISTS idx_enriched_severity ON enriched_threats(severity);
-          CREATE INDEX IF NOT EXISTS idx_enriched_received ON enriched_threats(received_at);
-        `,
-            },
-            {
-                version: 3,
-                name: 'create_trap_credentials_table',
-                sql: `
-          CREATE TABLE IF NOT EXISTS trap_credentials (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            enriched_threat_id INTEGER NOT NULL REFERENCES enriched_threats(id) ON DELETE CASCADE,
-            username TEXT NOT NULL,
-            attempt_count INTEGER NOT NULL DEFAULT 1,
-            created_at TEXT NOT NULL DEFAULT (datetime('now'))
-          );
-          CREATE INDEX IF NOT EXISTS idx_trap_creds_threat ON trap_credentials(enriched_threat_id);
-          CREATE INDEX IF NOT EXISTS idx_trap_creds_username ON trap_credentials(username);
-        `,
-            },
-            {
-                version: 4,
-                name: 'create_generated_patterns_table',
-                sql: `
-          CREATE TABLE IF NOT EXISTS generated_patterns (
-            pattern_hash TEXT PRIMARY KEY,
-            attack_type TEXT NOT NULL,
-            mitre_techniques TEXT NOT NULL,
-            rule_id TEXT NOT NULL REFERENCES rules(rule_id) ON DELETE CASCADE,
-            occurrences INTEGER NOT NULL,
-            distinct_ips INTEGER NOT NULL,
-            first_seen TEXT NOT NULL,
-            last_seen TEXT NOT NULL,
-            created_at TEXT NOT NULL DEFAULT (datetime('now')),
-            updated_at TEXT NOT NULL DEFAULT (datetime('now'))
-          );
-          CREATE INDEX IF NOT EXISTS idx_gen_patterns_attack ON generated_patterns(attack_type);
-          CREATE INDEX IF NOT EXISTS idx_gen_patterns_rule ON generated_patterns(rule_id);
-        `,
-            },
-            {
-                version: 5,
-                name: 'create_daily_aggregates_table',
-                sql: `
-          CREATE TABLE IF NOT EXISTS daily_aggregates (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            date TEXT NOT NULL,
-            attack_type TEXT NOT NULL,
-            region TEXT NOT NULL,
-            source_type TEXT NOT NULL,
-            event_count INTEGER NOT NULL,
-            unique_ips INTEGER NOT NULL,
-            avg_confidence REAL NOT NULL,
-            severity_distribution TEXT NOT NULL DEFAULT '{}',
-            created_at TEXT NOT NULL DEFAULT (datetime('now')),
-            UNIQUE(date, attack_type, region, source_type)
-          );
-          CREATE INDEX IF NOT EXISTS idx_daily_agg_date ON daily_aggregates(date);
-          CREATE INDEX IF NOT EXISTS idx_daily_agg_type ON daily_aggregates(attack_type);
-        `,
-            },
-            {
-                version: 6,
-                name: 'create_sightings_table',
-                sql: `
-          CREATE TABLE IF NOT EXISTS sightings (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            ioc_id INTEGER NOT NULL REFERENCES iocs(id) ON DELETE CASCADE,
-            type TEXT NOT NULL CHECK(type IN ('positive','negative','false_positive')),
-            source TEXT NOT NULL,
-            confidence INTEGER NOT NULL DEFAULT 50 CHECK(confidence BETWEEN 0 AND 100),
-            details TEXT NOT NULL DEFAULT '',
-            actor_hash TEXT NOT NULL DEFAULT '',
-            created_at TEXT NOT NULL DEFAULT (datetime('now'))
-          );
-          CREATE INDEX IF NOT EXISTS idx_sightings_ioc ON sightings(ioc_id);
-          CREATE INDEX IF NOT EXISTS idx_sightings_type ON sightings(type);
-          CREATE INDEX IF NOT EXISTS idx_sightings_created ON sightings(created_at);
-        `,
-            },
-            {
-                version: 7,
-                name: 'create_audit_log_table',
-                sql: `
-          CREATE TABLE IF NOT EXISTS audit_log (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            action TEXT NOT NULL,
-            entity_type TEXT NOT NULL,
-            entity_id TEXT NOT NULL,
-            actor_hash TEXT NOT NULL DEFAULT '',
-            ip_address TEXT NOT NULL DEFAULT '',
-            details TEXT NOT NULL DEFAULT '{}',
-            created_at TEXT NOT NULL DEFAULT (datetime('now'))
-          );
-          CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_log(action);
-          CREATE INDEX IF NOT EXISTS idx_audit_entity ON audit_log(entity_type, entity_id);
-          CREATE INDEX IF NOT EXISTS idx_audit_created ON audit_log(created_at);
-          CREATE INDEX IF NOT EXISTS idx_audit_actor ON audit_log(actor_hash);
-        `,
-            },
-            {
-                version: 8,
-                name: 'add_source_reliability_to_iocs',
-                sql: `
-          ALTER TABLE iocs ADD COLUMN source_reliability TEXT NOT NULL DEFAULT 'F'
-            CHECK(source_reliability IN ('A','B','C','D','E','F'));
-        `,
-            },
-        ];
-        const insertMigration = this.db.prepare('INSERT INTO schema_migrations (version, name) VALUES (?, ?)');
-        for (const m of migrations) {
-            if (!applied.has(m.version)) {
-                this.db.transaction(() => {
-                    this.db.exec(m.sql);
-                    insertMigration.run(m.version, m.name);
-                })();
+    /** Run schema migrations for existing databases / 執行既有資料庫的 schema 遷移 */
+    migrate() {
+        const addColumn = (table, column, type) => {
+            try {
+                this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
             }
-        }
+            catch {
+                // Column already exists — safe to ignore
+            }
+        };
+        addColumn('rules', 'category', 'TEXT');
+        addColumn('rules', 'severity', 'TEXT');
+        addColumn('rules', 'mitre_techniques', 'TEXT');
+        addColumn('rules', 'tags', 'TEXT');
     }
-    // -------------------------------------------------------------------------
-    // Legacy threat operations (backward compatible) / 原始威脅操作
-    // -------------------------------------------------------------------------
     /** Insert anonymized threat data / 插入匿名化威脅數據 */
     insertThreat(data) {
         const stmt = this.db.prepare(`
@@ -268,141 +150,293 @@ export class ThreatCloudDB {
     `);
         stmt.run(data.attackSourceIP, data.attackType, data.mitreTechnique, data.sigmaRuleMatched, data.timestamp, data.industry ?? null, data.region);
     }
-    // -------------------------------------------------------------------------
-    // Enriched threat operations / 豐富化威脅操作
-    // -------------------------------------------------------------------------
-    /**
-     * Insert enriched threat event (deduplicates by event_hash).
-     * Returns the row id if inserted, null if duplicate.
-     * 插入豐富化威脅事件（以 event_hash 去重）
-     */
-    insertEnrichedThreat(event) {
-        const stmt = this.db.prepare(`
-      INSERT OR IGNORE INTO enriched_threats
-        (source_type, attack_source_ip, attack_type, mitre_techniques, sigma_rule_matched,
-         timestamp, industry, region, confidence, severity, service_type, skill_level,
-         intent, tools, event_hash)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `);
-        const result = stmt.run(event.sourceType, event.attackSourceIP, event.attackType, JSON.stringify(event.mitreTechniques), event.sigmaRuleMatched, event.timestamp, event.industry ?? null, event.region, event.confidence, event.severity, event.serviceType ?? null, event.skillLevel ?? null, event.intent ?? null, event.tools ? JSON.stringify(event.tools) : null, event.eventHash);
-        return result.changes > 0 ? Number(result.lastInsertRowid) : null;
-    }
-    /**
-     * Insert trap credential records.
-     * Usernames are hashed (SHA-256, truncated to 16 hex chars) before storage
-     * to avoid storing PII from attacker-attempted credentials.
-     * 插入 Trap 憑證記錄（使用者名稱先雜湊化以避免 PII 洩漏）
-     */
-    insertTrapCredentials(enrichedThreatId, credentials) {
-        const stmt = this.db.prepare('INSERT INTO trap_credentials (enriched_threat_id, username, attempt_count) VALUES (?, ?, ?)');
-        const insertAll = this.db.transaction((creds) => {
-            for (const c of creds) {
-                const hashedUsername = createHash('sha256').update(c.username).digest('hex').slice(0, 16);
-                stmt.run(enrichedThreatId, hashedUsername, c.count);
+    /** Extract classification metadata from rule content / 從規則內容提取分類元資料 */
+    extractMetadata(ruleContent, source) {
+        let category = 'unknown';
+        let severity = 'medium';
+        let mitreTechniques = '';
+        let tags = '';
+        try {
+            if (source === 'yara') {
+                // YARA rules: extract from meta section
+                const metaMatch = ruleContent.match(/meta\s*:\s*([\s\S]*?)(?:strings|condition)\s*:/);
+                if (metaMatch) {
+                    const meta = metaMatch[1];
+                    const catMatch = meta.match(/category\s*=\s*"([^"]+)"/);
+                    if (catMatch)
+                        category = catMatch[1].toLowerCase();
+                    const sevMatch = meta.match(/severity\s*=\s*"([^"]+)"/);
+                    if (sevMatch)
+                        severity = sevMatch[1].toLowerCase();
+                    const mitreMatch = meta.match(/mitre_att(?:ack|&ck)\s*=\s*"([^"]+)"/i);
+                    if (mitreMatch)
+                        mitreTechniques = mitreMatch[1];
+                }
+                // Fallback: infer category from rule content keywords
+                if (category === 'unknown') {
+                    if (/malware|trojan|ransom|backdoor|rat_|infostealer/i.test(ruleContent))
+                        category = 'malware';
+                    else if (/exploit|cve-/i.test(ruleContent))
+                        category = 'exploit';
+                    else if (/hack_?tool|offensive|cobalt/i.test(ruleContent))
+                        category = 'hacktool';
+                    else if (/webshell/i.test(ruleContent))
+                        category = 'webshell';
+                    else if (/packer|obfusc|crypter/i.test(ruleContent))
+                        category = 'packer';
+                }
+            }
+            else {
+                // Sigma / ATR: YAML-based extraction
+                // Extract level/severity
+                const sevMatch = ruleContent.match(/(?:^|\n)\s*(?:level|severity)\s*:\s*(\w+)/);
+                if (sevMatch)
+                    severity = sevMatch[1].toLowerCase();
+                // Extract tags list
+                const tagMatch = ruleContent.match(/(?:^|\n)\s*tags\s*:\s*\n((?:\s+-\s*.+\n?)+)/);
+                if (tagMatch) {
+                    const tagLines = tagMatch[1].match(/-\s*(.+)/g) ?? [];
+                    const tagList = tagLines.map((t) => t.replace(/^-\s*/, '').trim());
+                    tags = tagList.join(',');
+                    // Derive MITRE techniques from tags (attack.tXXXX)
+                    const mitreTags = tagList.filter((t) => /^attack\.t\d+/i.test(t));
+                    if (mitreTags.length > 0) {
+                        mitreTechniques = mitreTags
+                            .map((t) => t.replace(/^attack\./i, '').toUpperCase())
+                            .join(',');
+                    }
+                    // Derive category from MITRE ATT&CK tactic tags
+                    const attackTags = tagList.filter((t) => t.startsWith('attack.'));
+                    for (const tag of attackTags) {
+                        if (/initial.access/i.test(tag)) {
+                            category = 'initial-access';
+                            break;
+                        }
+                        if (/execution/i.test(tag)) {
+                            category = 'execution';
+                            break;
+                        }
+                        if (/persistence/i.test(tag)) {
+                            category = 'persistence';
+                            break;
+                        }
+                        if (/privilege.escalation/i.test(tag)) {
+                            category = 'privilege-escalation';
+                            break;
+                        }
+                        if (/defense.evasion/i.test(tag)) {
+                            category = 'defense-evasion';
+                            break;
+                        }
+                        if (/credential.access/i.test(tag)) {
+                            category = 'credential-access';
+                            break;
+                        }
+                        if (/discovery/i.test(tag)) {
+                            category = 'discovery';
+                            break;
+                        }
+                        if (/lateral.movement/i.test(tag)) {
+                            category = 'lateral-movement';
+                            break;
+                        }
+                        if (/collection/i.test(tag)) {
+                            category = 'collection';
+                            break;
+                        }
+                        if (/exfiltration/i.test(tag)) {
+                            category = 'exfiltration';
+                            break;
+                        }
+                        if (/command.and.control|c2/i.test(tag)) {
+                            category = 'command-and-control';
+                            break;
+                        }
+                        if (/impact/i.test(tag)) {
+                            category = 'impact';
+                            break;
+                        }
+                        if (/resource.development/i.test(tag)) {
+                            category = 'resource-development';
+                            break;
+                        }
+                        if (/reconnaissance/i.test(tag)) {
+                            category = 'reconnaissance';
+                            break;
+                        }
+                    }
+                }
+                // Fallback: infer from logsource (Sigma)
+                if (category === 'unknown') {
+                    const lsCatMatch = ruleContent.match(/(?:^|\n)\s*logsource\s*:\s*\n(?:\s+\w+\s*:.+\n)*?\s+category\s*:\s*(\S+)/);
+                    if (lsCatMatch) {
+                        category = lsCatMatch[1].toLowerCase();
+                    }
+                    else {
+                        const lsProdMatch = ruleContent.match(/(?:^|\n)\s*logsource\s*:\s*\n(?:\s+\w+\s*:.+\n)*?\s+product\s*:\s*(\S+)/);
+                        if (lsProdMatch)
+                            category = lsProdMatch[1].toLowerCase();
+                    }
+                }
             }
-        });
-        insertAll(credentials);
-    }
-    /** Get enriched threats count by source type / 依來源類型取得豐富化威脅數量 */
-    getEnrichedThreatCountBySource() {
-        const rows = this.db
-            .prepare('SELECT source_type, COUNT(*) as count FROM enriched_threats GROUP BY source_type')
-            .all();
-        const result = {};
-        for (const r of rows) {
-            result[r.source_type] = r.count;
         }
-        return result;
-    }
-    /** Count related threats for an IP / 計算某 IP 的相關威脅數量 */
-    countRelatedThreats(ip) {
-        return this.db
-            .prepare('SELECT COUNT(*) as count FROM enriched_threats WHERE attack_source_ip = ?')
-            .get(ip).count;
-    }
-    // -------------------------------------------------------------------------
-    // Conversion helpers / 轉換輔助函式
-    // -------------------------------------------------------------------------
-    /** Convert AnonymizedThreatData to EnrichedThreatEvent / 轉換 Guard 資料 */
-    static guardToEnriched(data) {
-        const hashInput = `${data.attackSourceIP}|${data.attackType}|${data.mitreTechnique}|${data.timestamp}`;
-        const eventHash = createHash('sha256').update(hashInput).digest('hex');
-        return {
-            sourceType: 'guard',
-            attackSourceIP: data.attackSourceIP,
-            attackType: data.attackType,
-            mitreTechniques: [data.mitreTechnique],
-            sigmaRuleMatched: data.sigmaRuleMatched,
-            timestamp: data.timestamp,
-            industry: data.industry,
-            region: data.region,
-            confidence: 50,
-            severity: 'medium',
-            eventHash,
-            receivedAt: new Date().toISOString(),
-        };
-    }
-    /** Convert TrapIntelligencePayload to EnrichedThreatEvent / 轉換 Trap 資料 */
-    static trapToEnriched(data) {
-        const techniques = data.mitreTechniques ?? [];
-        const hashInput = `${data.sourceIP}|${data.attackType}|${techniques.join(',')}|${data.timestamp}`;
-        const eventHash = createHash('sha256').update(hashInput).digest('hex');
-        return {
-            sourceType: 'trap',
-            attackSourceIP: data.sourceIP,
-            attackType: data.attackType,
-            mitreTechniques: techniques,
-            sigmaRuleMatched: '',
-            timestamp: data.timestamp,
-            region: data.region ?? 'unknown',
-            confidence: 60,
-            severity: data.skillLevel === 'apt' || data.skillLevel === 'advanced' ? 'high' : 'medium',
-            serviceType: data.serviceType,
-            skillLevel: data.skillLevel,
-            intent: data.intent,
-            tools: data.tools,
-            eventHash,
-            receivedAt: new Date().toISOString(),
-        };
+        catch {
+            // Extraction failed — keep defaults
+        }
+        // Normalize category to lowercase
+        category = category.toLowerCase();
+        severity = severity.toLowerCase();
+        return { category, severity, mitreTechniques, tags };
     }
-    // -------------------------------------------------------------------------
-    // Rules / 規則
-    // -------------------------------------------------------------------------
     /** Insert or update a community rule / 插入或更新社群規則 */
     upsertRule(rule) {
+        // Extract classification from content if not provided
+        const meta = this.extractMetadata(rule.ruleContent, rule.source);
+        const category = rule.category ?? meta.category;
+        const severity = rule.severity ?? meta.severity;
+        const mitreTechniques = rule.mitreTechniques ?? meta.mitreTechniques;
+        const tags = rule.tags ?? meta.tags;
         const stmt = this.db.prepare(`
-      INSERT INTO rules (rule_id, rule_content, published_at, source)
-      VALUES (?, ?, ?, ?)
+      INSERT INTO rules (rule_id, rule_content, published_at, source, category, severity, mitre_techniques, tags)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(rule_id) DO UPDATE SET
         rule_content = excluded.rule_content,
         published_at = excluded.published_at,
         source = excluded.source,
+        category = excluded.category,
+        severity = excluded.severity,
+        mitre_techniques = excluded.mitre_techniques,
+        tags = excluded.tags,
         updated_at = datetime('now')
     `);
-        stmt.run(rule.ruleId, rule.ruleContent, rule.publishedAt, rule.source);
+        stmt.run(rule.ruleId, rule.ruleContent, rule.publishedAt, rule.source, category, severity, mitreTechniques, tags);
     }
     /** Fetch rules published after a given timestamp / 取得指定時間後發佈的規則 */
-    getRulesSince(since) {
+    getRulesSince(since, filters) {
+        let sql = `SELECT rule_id as ruleId, rule_content as ruleContent, published_at as publishedAt, source,
+      category, severity, mitre_techniques as mitreTechniques, tags
+      FROM rules WHERE published_at > ?`;
+        const params = [since];
+        if (filters?.category) {
+            sql += ' AND category = ?';
+            params.push(filters.category);
+        }
+        if (filters?.severity) {
+            sql += ' AND severity = ?';
+            params.push(filters.severity);
+        }
+        if (filters?.source) {
+            sql += ' AND source = ?';
+            params.push(filters.source);
+        }
+        sql += ' ORDER BY published_at ASC';
+        return this.db.prepare(sql).all(...params);
+    }
+    /** Fetch all rules with limit / 取得所有規則（含限制） */
+    getAllRules(limit = 5000, filters) {
+        let sql = `SELECT rule_id as ruleId, rule_content as ruleContent, published_at as publishedAt, source,
+      category, severity, mitre_techniques as mitreTechniques, tags
+      FROM rules WHERE 1=1`;
+        const params = [];
+        if (filters?.category) {
+            sql += ' AND category = ?';
+            params.push(filters.category);
+        }
+        if (filters?.severity) {
+            sql += ' AND severity = ?';
+            params.push(filters.severity);
+        }
+        if (filters?.source) {
+            sql += ' AND source = ?';
+            params.push(filters.source);
+        }
+        sql += ' ORDER BY published_at DESC LIMIT ?';
+        params.push(limit);
+        return this.db.prepare(sql).all(...params);
+    }
+    /** Insert ATR rule proposal / 插入 ATR 規則提案 */
+    insertATRProposal(proposal) {
         const stmt = this.db.prepare(`
-      SELECT rule_id as ruleId, rule_content as ruleContent, published_at as publishedAt, source
-      FROM rules
-      WHERE published_at > ?
-      ORDER BY published_at ASC
+      INSERT INTO atr_proposals (pattern_hash, rule_content, llm_provider, llm_model, self_review_verdict, client_id)
+      VALUES (?, ?, ?, ?, ?, ?)
     `);
-        return stmt.all(since);
+        stmt.run(proposal.patternHash, proposal.ruleContent, proposal.llmProvider, proposal.llmModel, proposal.selfReviewVerdict, proposal.clientId ?? null);
+    }
+    /** Get ATR proposals, optionally filtered by status / 取得 ATR 提案 */
+    getATRProposals(status) {
+        if (status) {
+            return this.db
+                .prepare('SELECT * FROM atr_proposals WHERE status = ? ORDER BY created_at DESC')
+                .all(status);
+        }
+        return this.db.prepare('SELECT * FROM atr_proposals ORDER BY created_at DESC').all();
+    }
+    /** Increment confirmations for a proposal; auto-confirm at >= 3 / 增加提案確認數 */
+    confirmATRProposal(patternHash) {
+        this.db
+            .prepare(`
+      UPDATE atr_proposals
+      SET confirmations = confirmations + 1,
+          status = CASE WHEN confirmations + 1 >= 3 THEN 'confirmed' ELSE status END,
+          updated_at = datetime('now')
+      WHERE pattern_hash = ?
+    `)
+            .run(patternHash);
+    }
+    /** Update LLM review verdict for a proposal / 更新 LLM 審查結果 */
+    updateATRProposalLLMReview(patternHash, verdict) {
+        this.db
+            .prepare(`
+      UPDATE atr_proposals SET llm_review_verdict = ?, updated_at = datetime('now') WHERE pattern_hash = ?
+    `)
+            .run(verdict, patternHash);
     }
-    /** Fetch all rules / 取得所有規則 */
-    getAllRules() {
+    /** Insert ATR feedback / 插入 ATR 回饋 */
+    insertATRFeedback(ruleId, isTruePositive, clientId) {
+        this.db
+            .prepare(`
+      INSERT INTO atr_feedback (rule_id, is_true_positive, client_id) VALUES (?, ?, ?)
+    `)
+            .run(ruleId, isTruePositive ? 1 : 0, clientId ?? null);
+    }
+    /** Get feedback stats for a rule / 取得規則回饋統計 */
+    getATRFeedbackStats(ruleId) {
+        const tp = this.db
+            .prepare('SELECT COUNT(*) as count FROM atr_feedback WHERE rule_id = ? AND is_true_positive = 1')
+            .get(ruleId).count;
+        const fp = this.db
+            .prepare('SELECT COUNT(*) as count FROM atr_feedback WHERE rule_id = ? AND is_true_positive = 0')
+            .get(ruleId).count;
+        return { truePositives: tp, falsePositives: fp };
+    }
+    /** Insert skill threat submission / 插入技能威脅提交 */
+    insertSkillThreat(submission) {
         const stmt = this.db.prepare(`
-      SELECT rule_id as ruleId, rule_content as ruleContent, published_at as publishedAt, source
-      FROM rules
-      ORDER BY published_at DESC
+      INSERT INTO skill_threats (skill_hash, skill_name, risk_score, risk_level, finding_summaries, client_id)
+      VALUES (?, ?, ?, ?, ?, ?)
     `);
-        return stmt.all();
+        stmt.run(submission.skillHash, submission.skillName, submission.riskScore, submission.riskLevel, submission.findingSummaries ? JSON.stringify(submission.findingSummaries) : null, submission.clientId ?? null);
+    }
+    /** Get recent skill threats / 取得最近技能威脅 */
+    getSkillThreats(limit = 50) {
+        return this.db
+            .prepare('SELECT * FROM skill_threats ORDER BY created_at DESC LIMIT ?')
+            .all(limit);
+    }
+    /** Get proposal statistics / 取得提案統計 */
+    getProposalStats() {
+        const pending = this.db
+            .prepare("SELECT COUNT(*) as count FROM atr_proposals WHERE status = 'pending'")
+            .get().count;
+        const confirmed = this.db
+            .prepare("SELECT COUNT(*) as count FROM atr_proposals WHERE status = 'confirmed'")
+            .get().count;
+        const rejected = this.db
+            .prepare("SELECT COUNT(*) as count FROM atr_proposals WHERE status = 'rejected'")
+            .get().count;
+        const total = this.db.prepare('SELECT COUNT(*) as count FROM atr_proposals').get().count;
+        return { pending, confirmed, rejected, total };
     }
-    // -------------------------------------------------------------------------
-    // Statistics / 統計
-    // -------------------------------------------------------------------------
     /** Get threat statistics / 取得威脅統計 */
     getStats() {
         const totalThreats = this.db.prepare('SELECT COUNT(*) as count FROM threats').get().count;
@@ -426,6 +460,27 @@ export class ThreatCloudDB {
       GROUP BY mitre_technique
       ORDER BY count DESC
       LIMIT 10
+    `)
+            .all();
+        const proposalStats = this.getProposalStats();
+        const skillThreatsTotal = this.db.prepare('SELECT COUNT(*) as count FROM skill_threats').get().count;
+        const skillBlacklistTotal = this.getSkillBlacklist().length;
+        const rulesByCategory = this.db
+            .prepare(`
+      SELECT COALESCE(category, 'unknown') as category, COUNT(*) as count
+      FROM rules GROUP BY category ORDER BY count DESC LIMIT 20
+    `)
+            .all();
+        const rulesBySeverity = this.db
+            .prepare(`
+      SELECT COALESCE(severity, 'unknown') as severity, COUNT(*) as count
+      FROM rules GROUP BY severity ORDER BY count DESC
+    `)
+            .all();
+        const rulesBySource = this.db
+            .prepare(`
+      SELECT source, COUNT(*) as count
+      FROM rules GROUP BY source ORDER BY count DESC
     `)
             .all();
         return {
@@ -434,8 +489,219 @@ export class ThreatCloudDB {
             topAttackTypes,
             topMitreTechniques,
             last24hThreats: last24h,
+            proposalStats,
+            skillThreatsTotal,
+            skillBlacklistTotal,
+            rulesByCategory,
+            rulesBySeverity,
+            rulesBySource,
         };
     }
+    /** Get confirmed/promoted ATR rules, optionally filtered by date / 取得已確認 ATR 規則 */
+    getConfirmedATRRules(since) {
+        if (since) {
+            return this.db
+                .prepare(`
+        SELECT pattern_hash as ruleId, rule_content as ruleContent, updated_at as publishedAt, 'atr-community' as source
+        FROM atr_proposals
+        WHERE (status = 'confirmed' OR status = 'promoted') AND updated_at > ?
+        ORDER BY updated_at ASC
+      `)
+                .all(since);
+        }
+        return this.db
+            .prepare(`
+      SELECT pattern_hash as ruleId, rule_content as ruleContent, updated_at as publishedAt, 'atr-community' as source
+      FROM atr_proposals
+      WHERE status = 'confirmed' OR status = 'promoted'
+      ORDER BY updated_at ASC
+    `)
+            .all();
+    }
+    /** Get IP blocklist from IoC entries and aggregated threat data / 取得 IP 封鎖清單 */
+    getIPBlocklist(minReputation) {
+        // IoC entries with sufficient reputation
+        const iocIPs = this.db
+            .prepare(`
+      SELECT value FROM ioc_entries
+      WHERE type = 'ip' AND reputation >= ?
+      ORDER BY reputation DESC
+    `)
+            .all(minReputation);
+        // Aggregate from threats table: distinct IPs with >= 3 occurrences
+        const threatIPs = this.db
+            .prepare(`
+      SELECT attack_source_ip as value
+      FROM threats
+      GROUP BY attack_source_ip
+      HAVING COUNT(*) >= 3
+    `)
+            .all();
+        // Merge and deduplicate
+        const ipSet = new Set();
+        for (const row of iocIPs)
+            ipSet.add(row.value);
+        for (const row of threatIPs)
+            ipSet.add(row.value);
+        return Array.from(ipSet);
+    }
+    /** Get domain blocklist from IoC entries / 取得域名封鎖清單 */
+    getDomainBlocklist(minReputation) {
+        const iocDomains = this.db
+            .prepare(`
+      SELECT value FROM ioc_entries
+      WHERE type = 'domain' AND reputation >= ?
+      ORDER BY reputation DESC
+    `)
+            .all(minReputation);
+        return iocDomains.map((row) => row.value);
+    }
+    /** Upsert an IoC entry / 插入或更新 IoC 條目 */
+    upsertIoC(type, value, reputation, source) {
+        this.db
+            .prepare(`
+      INSERT INTO ioc_entries (type, value, reputation, source)
+      VALUES (?, ?, ?, ?)
+      ON CONFLICT(value) DO UPDATE SET
+        reputation = excluded.reputation,
+        source = excluded.source,
+        last_seen = datetime('now'),
+        sighting_count = sighting_count + 1
+    `)
+            .run(type, value, reputation, source);
+    }
+    /** Promote confirmed proposals with approved LLM review to rules / 推廣已確認提案為規則 */
+    promoteConfirmedProposals() {
+        const proposals = this.db
+            .prepare(`
+      SELECT pattern_hash, rule_content, llm_review_verdict
+      FROM atr_proposals
+      WHERE status = 'confirmed' AND llm_review_verdict IS NOT NULL
+    `)
+            .all();
+        let promoted = 0;
+        for (const proposal of proposals) {
+            try {
+                const verdict = JSON.parse(proposal.llm_review_verdict);
+                if (verdict.approved !== true)
+                    continue;
+                this.upsertRule({
+                    ruleId: proposal.pattern_hash,
+                    ruleContent: proposal.rule_content,
+                    publishedAt: new Date().toISOString(),
+                    source: 'atr-community',
+                });
+                this.db
+                    .prepare(`
+          UPDATE atr_proposals SET status = 'promoted', updated_at = datetime('now')
+          WHERE pattern_hash = ?
+        `)
+                    .run(proposal.pattern_hash);
+                promoted++;
+            }
+            catch {
+                // Skip proposals with unparseable verdicts
+            }
+        }
+        return promoted;
+    }
+    /** Reject an ATR proposal / 拒絕 ATR 提案 */
+    rejectATRProposal(patternHash) {
+        this.db
+            .prepare(`
+      UPDATE atr_proposals SET status = 'rejected', updated_at = datetime('now')
+      WHERE pattern_hash = ?
+    `)
+            .run(patternHash);
+    }
+    /** Get rules by source type, optionally filtered by date / 依來源取得規則 */
+    getRulesBySource(source, since) {
+        if (since) {
+            return this.db
+                .prepare(`
+        SELECT rule_id as ruleId, rule_content as ruleContent, published_at as publishedAt, source,
+          category, severity, mitre_techniques as mitreTechniques, tags
+        FROM rules
+        WHERE source = ? AND published_at > ?
+        ORDER BY published_at ASC
+      `)
+                .all(source, since);
+        }
+        return this.db
+            .prepare(`
+      SELECT rule_id as ruleId, rule_content as ruleContent, published_at as publishedAt, source,
+        category, severity, mitre_techniques as mitreTechniques, tags
+      FROM rules
+      WHERE source = ?
+      ORDER BY published_at ASC
+    `)
+            .all(source);
+    }
+    /** Report a safe skill (increment confirmations, auto-confirm at 3+) / 回報安全 skill */
+    reportSafeSkill(skillName, fingerprintHash) {
+        const normalized = skillName.toLowerCase().trim().replace(/\s+/g, '-');
+        this.db
+            .prepare(`
+      INSERT INTO skill_whitelist (skill_name, normalized_name, fingerprint_hash)
+      VALUES (?, ?, ?)
+      ON CONFLICT(normalized_name) DO UPDATE SET
+        confirmations = confirmations + 1,
+        status = CASE WHEN confirmations + 1 >= 3 THEN 'confirmed' ELSE status END,
+        fingerprint_hash = COALESCE(excluded.fingerprint_hash, fingerprint_hash),
+        last_reported = datetime('now')
+    `)
+            .run(skillName, normalized, fingerprintHash ?? null);
+    }
+    /** Get confirmed community whitelist / 取得社群白名單 */
+    getSkillWhitelist() {
+        return this.db
+            .prepare(`
+      SELECT skill_name as name, fingerprint_hash as hash, confirmations
+      FROM skill_whitelist
+      WHERE status = 'confirmed'
+      ORDER BY confirmations DESC
+    `)
+            .all();
+    }
+    /**
+     * Get skill blacklist: skills reported by 3+ distinct clients with avg risk >= 70
+     * 取得技能黑名單：3+ 不同客戶端回報且平均風險 >= 70 的技能
+     */
+    getSkillBlacklist(minReports = 3, minAvgRisk = 70) {
+        return this.db
+            .prepare(`
+      SELECT
+        skill_hash as skillHash,
+        skill_name as skillName,
+        ROUND(AVG(risk_score)) as avgRiskScore,
+        MAX(risk_level) as maxRiskLevel,
+        COUNT(DISTINCT COALESCE(client_id, 'anonymous')) as reportCount,
+        MIN(created_at) as firstReported,
+        MAX(created_at) as lastReported
+      FROM skill_threats
+      GROUP BY skill_hash
+      HAVING reportCount >= ? AND AVG(risk_score) >= ?
+      ORDER BY avgRiskScore DESC
+    `)
+            .all(minReports, minAvgRisk);
+    }
+    /** Backfill classification for rules with NULL category / 回填缺少分類的規則 */
+    backfillClassification() {
+        const unclassified = this.db
+            .prepare(`SELECT rule_id, rule_content, source FROM rules WHERE category IS NULL`)
+            .all();
+        let updated = 0;
+        const stmt = this.db.prepare(`
+      UPDATE rules SET category = ?, severity = ?, mitre_techniques = ?, tags = ?, updated_at = datetime('now')
+      WHERE rule_id = ?
+    `);
+        for (const row of unclassified) {
+            const meta = this.extractMetadata(row.rule_content, row.source);
+            stmt.run(meta.category, meta.severity, meta.mitreTechniques, meta.tags, row.rule_id);
+            updated++;
+        }
+        return updated;
+    }
     /** Close the database / 關閉資料庫 */
     close() {
         this.db.close();