@panguard-ai/threat-cloud 1.4.1 → 1.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/admin-dashboard.js +5 -5
- package/dist/audit-logger.d.ts +1 -1
- package/dist/audit-logger.d.ts.map +1 -1
- package/dist/audit-logger.js.map +1 -1
- package/dist/badge-api.d.ts +58 -0
- package/dist/badge-api.d.ts.map +1 -0
- package/dist/badge-api.js +248 -0
- package/dist/badge-api.js.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/database.d.ts +254 -2
- package/dist/database.d.ts.map +1 -1
- package/dist/database.js +769 -72
- package/dist/database.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/llm-reviewer-tools.d.ts +110 -0
- package/dist/llm-reviewer-tools.d.ts.map +1 -0
- package/dist/llm-reviewer-tools.js +446 -0
- package/dist/llm-reviewer-tools.js.map +1 -0
- package/dist/llm-reviewer.d.ts +54 -0
- package/dist/llm-reviewer.d.ts.map +1 -1
- package/dist/llm-reviewer.js +708 -64
- package/dist/llm-reviewer.js.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/migrations.js +215 -0
- package/dist/migrations.js.map +1 -1
- package/dist/migrator-crystallization.d.ts +80 -0
- package/dist/migrator-crystallization.d.ts.map +1 -0
- package/dist/migrator-crystallization.js +108 -0
- package/dist/migrator-crystallization.js.map +1 -0
- package/dist/server.d.ts +75 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +1249 -130
- package/dist/server.js.map +1 -1
- package/dist/types.d.ts +33 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +15 -12
package/dist/database.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* @module @panguard-ai/threat-cloud/database
|
|
8
8
|
*/
|
|
9
9
|
import Database from 'better-sqlite3';
|
|
10
|
-
import { createHmac, createHash } from 'node:crypto';
|
|
10
|
+
import { createHmac, createHash, randomUUID } from 'node:crypto';
|
|
11
11
|
import { runMigrations } from './migrations.js';
|
|
12
12
|
import { AuditLogger } from './audit-logger.js';
|
|
13
13
|
/**
|
|
@@ -24,6 +24,14 @@ export class ThreatCloudDB {
|
|
|
24
24
|
this.initialize();
|
|
25
25
|
this.audit = new AuditLogger(this.db);
|
|
26
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Escape hatch for callers that need direct DB access (e.g. modules
|
|
29
|
+
* defined outside this class that perform table-specific queries).
|
|
30
|
+
* Prefer adding a typed method here when the access pattern stabilises.
|
|
31
|
+
*/
|
|
32
|
+
getRawDb() {
|
|
33
|
+
return this.db;
|
|
34
|
+
}
|
|
27
35
|
/** Create tables if they don't exist / 建立資料表 */
|
|
28
36
|
initialize() {
|
|
29
37
|
this.db.exec(`
|
|
@@ -123,13 +131,83 @@ export class ThreatCloudDB {
|
|
|
123
131
|
CREATE INDEX IF NOT EXISTS idx_usage_type ON usage_events(event_type);
|
|
124
132
|
CREATE INDEX IF NOT EXISTS idx_usage_created ON usage_events(created_at);
|
|
125
133
|
|
|
134
|
+
CREATE TABLE IF NOT EXISTS telemetry_events (
|
|
135
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
136
|
+
event_type TEXT NOT NULL,
|
|
137
|
+
platform TEXT NOT NULL DEFAULT 'unknown',
|
|
138
|
+
skill_count INTEGER NOT NULL DEFAULT 0,
|
|
139
|
+
finding_count INTEGER NOT NULL DEFAULT 0,
|
|
140
|
+
severity TEXT NOT NULL DEFAULT 'LOW',
|
|
141
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
142
|
+
);
|
|
143
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_type ON telemetry_events(event_type);
|
|
144
|
+
CREATE INDEX IF NOT EXISTS idx_telemetry_created ON telemetry_events(created_at);
|
|
145
|
+
|
|
146
|
+
CREATE TABLE IF NOT EXISTS telemetry_hourly_aggregates (
|
|
147
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
148
|
+
hour_bucket TEXT NOT NULL,
|
|
149
|
+
event_type TEXT NOT NULL,
|
|
150
|
+
platform TEXT NOT NULL DEFAULT 'unknown',
|
|
151
|
+
event_count INTEGER NOT NULL DEFAULT 0,
|
|
152
|
+
total_findings INTEGER NOT NULL DEFAULT 0,
|
|
153
|
+
total_skills INTEGER NOT NULL DEFAULT 0,
|
|
154
|
+
avg_severity TEXT NOT NULL DEFAULT 'LOW',
|
|
155
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
156
|
+
);
|
|
157
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_telemetry_hourly_unique
|
|
158
|
+
ON telemetry_hourly_aggregates(hour_bucket, event_type, platform);
|
|
159
|
+
|
|
160
|
+
CREATE TABLE IF NOT EXISTS activations (
|
|
161
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
162
|
+
client_id TEXT NOT NULL UNIQUE,
|
|
163
|
+
platform TEXT NOT NULL DEFAULT 'unknown',
|
|
164
|
+
os_type TEXT NOT NULL DEFAULT 'unknown',
|
|
165
|
+
panguard_version TEXT NOT NULL DEFAULT 'unknown',
|
|
166
|
+
node_version TEXT NOT NULL DEFAULT 'unknown',
|
|
167
|
+
activated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
168
|
+
);
|
|
169
|
+
|
|
126
170
|
-- Migrations are handled by the numbered migration system in migrations.ts
|
|
171
|
+
|
|
172
|
+
-- Org/Device/Policy tables (migration v8, duplicated here as safety net
|
|
173
|
+
-- because CREATE TABLE IF NOT EXISTS is idempotent)
|
|
174
|
+
CREATE TABLE IF NOT EXISTS orgs (
|
|
175
|
+
id TEXT PRIMARY KEY,
|
|
176
|
+
name TEXT NOT NULL,
|
|
177
|
+
api_key_hash TEXT,
|
|
178
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
179
|
+
);
|
|
180
|
+
CREATE TABLE IF NOT EXISTS devices (
|
|
181
|
+
id TEXT PRIMARY KEY,
|
|
182
|
+
org_id TEXT NOT NULL REFERENCES orgs(id),
|
|
183
|
+
hostname TEXT,
|
|
184
|
+
os_type TEXT,
|
|
185
|
+
agent_count INTEGER NOT NULL DEFAULT 0,
|
|
186
|
+
guard_version TEXT,
|
|
187
|
+
last_seen TEXT NOT NULL DEFAULT (datetime('now')),
|
|
188
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
189
|
+
);
|
|
190
|
+
CREATE INDEX IF NOT EXISTS idx_devices_org ON devices(org_id);
|
|
191
|
+
CREATE INDEX IF NOT EXISTS idx_devices_last_seen ON devices(last_seen);
|
|
192
|
+
CREATE TABLE IF NOT EXISTS org_policies (
|
|
193
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
194
|
+
org_id TEXT NOT NULL REFERENCES orgs(id),
|
|
195
|
+
category TEXT NOT NULL,
|
|
196
|
+
action TEXT NOT NULL CHECK(action IN ('allow', 'block')),
|
|
197
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
198
|
+
UNIQUE(org_id, category)
|
|
199
|
+
);
|
|
127
200
|
`);
|
|
128
201
|
try {
|
|
129
|
-
runMigrations(this.db);
|
|
202
|
+
const applied = runMigrations(this.db);
|
|
203
|
+
if (applied > 0) {
|
|
204
|
+
console.log(`[threat-cloud] ${applied} migration(s) applied`);
|
|
205
|
+
}
|
|
130
206
|
}
|
|
131
207
|
catch (err) {
|
|
132
|
-
console.error('[threat-cloud]
|
|
208
|
+
console.error('[threat-cloud] MIGRATION FAILED:', err instanceof Error ? err.message : String(err));
|
|
209
|
+
// Re-throw — migrations failing means the DB is in a broken state
|
|
210
|
+
throw err;
|
|
133
211
|
}
|
|
134
212
|
this.db.exec(`
|
|
135
213
|
CREATE INDEX IF NOT EXISTS idx_rules_category ON rules(category);
|
|
@@ -325,6 +403,19 @@ export class ThreatCloudDB {
|
|
|
325
403
|
`);
|
|
326
404
|
stmt.run(rule.ruleId, rule.ruleContent, rule.publishedAt, rule.source, category, severity, mitreTechniques, tags);
|
|
327
405
|
}
|
|
406
|
+
/** Delete all rules with a given source. Returns number of deleted rows. */
|
|
407
|
+
deleteRulesBySource(source) {
|
|
408
|
+
const stmt = this.db.prepare('DELETE FROM rules WHERE source = ?');
|
|
409
|
+
const result = stmt.run(source);
|
|
410
|
+
return result.changes;
|
|
411
|
+
}
|
|
412
|
+
/** Delete rules by a list of rule IDs. Returns number of deleted rows. */
|
|
413
|
+
deleteRulesByIds(ruleIds) {
|
|
414
|
+
const placeholders = ruleIds.map(() => '?').join(',');
|
|
415
|
+
const stmt = this.db.prepare(`DELETE FROM rules WHERE rule_id IN (${placeholders})`);
|
|
416
|
+
const result = stmt.run(...ruleIds);
|
|
417
|
+
return result.changes;
|
|
418
|
+
}
|
|
328
419
|
/** Fetch rules published after a given timestamp / 取得指定時間後發佈的規則 */
|
|
329
420
|
getRulesSince(since, filters) {
|
|
330
421
|
let sql = `SELECT rule_id as ruleId, rule_content as ruleContent, published_at as publishedAt, source,
|
|
@@ -440,9 +531,10 @@ export class ThreatCloudDB {
|
|
|
440
531
|
const findingRows = this.db
|
|
441
532
|
.prepare(`SELECT finding_summaries FROM skill_threats
|
|
442
533
|
WHERE skill_name = ? AND finding_summaries IS NOT NULL
|
|
443
|
-
ORDER BY created_at DESC LIMIT
|
|
534
|
+
ORDER BY created_at DESC LIMIT 10`)
|
|
444
535
|
.all(skillName);
|
|
445
536
|
const findings = [];
|
|
537
|
+
const ruleIdSet = new Set();
|
|
446
538
|
for (const fr of findingRows) {
|
|
447
539
|
try {
|
|
448
540
|
const parsed = JSON.parse(fr.finding_summaries);
|
|
@@ -450,6 +542,9 @@ export class ThreatCloudDB {
|
|
|
450
542
|
if (f.title && !findings.includes(f.title)) {
|
|
451
543
|
findings.push(f.title);
|
|
452
544
|
}
|
|
545
|
+
if (f.id && f.id.startsWith('ATR-')) {
|
|
546
|
+
ruleIdSet.add(f.id);
|
|
547
|
+
}
|
|
453
548
|
}
|
|
454
549
|
}
|
|
455
550
|
catch {
|
|
@@ -461,8 +556,19 @@ export class ThreatCloudDB {
|
|
|
461
556
|
avgRiskScore: row?.avg_score ?? 0,
|
|
462
557
|
maxRiskLevel: row?.max_level ?? 'LOW',
|
|
463
558
|
findings: findings.slice(0, 10),
|
|
559
|
+
atrRuleIds: [...ruleIdSet],
|
|
464
560
|
};
|
|
465
561
|
}
|
|
562
|
+
/** Fetch rule content by IDs. Returns the YAML detection patterns for crystallization. */
|
|
563
|
+
getRuleContentByIds(ruleIds) {
|
|
564
|
+
if (ruleIds.length === 0)
|
|
565
|
+
return [];
|
|
566
|
+
const placeholders = ruleIds.map(() => '?').join(',');
|
|
567
|
+
const rows = this.db
|
|
568
|
+
.prepare(`SELECT rule_id, rule_content FROM rules WHERE rule_id IN (${placeholders})`)
|
|
569
|
+
.all(...ruleIds);
|
|
570
|
+
return rows.map((r) => ({ ruleId: r.rule_id, ruleContent: r.rule_content }));
|
|
571
|
+
}
|
|
466
572
|
/** Check if an ATR proposal already exists for a pattern hash */
|
|
467
573
|
hasATRProposal(patternHash) {
|
|
468
574
|
const row = this.db
|
|
@@ -476,6 +582,75 @@ export class ThreatCloudDB {
|
|
|
476
582
|
.prepare('SELECT client_id, confirmations FROM atr_proposals WHERE pattern_hash = ? LIMIT 1')
|
|
477
583
|
.get(patternHash);
|
|
478
584
|
}
|
|
585
|
+
// -------------------------------------------------------------------------
|
|
586
|
+
// Payload fingerprint dedup — avoids paying Anthropic API for duplicate
|
|
587
|
+
// garak prompts. Typical hit rate is 90%+ on public corpora.
|
|
588
|
+
// 提示詞指紋去重 — 避免為重複的 garak 提示付 Anthropic API 錢。
|
|
589
|
+
// -------------------------------------------------------------------------
|
|
590
|
+
/**
|
|
591
|
+
* Look up a previously-judged payload fingerprint. Caller should call this
|
|
592
|
+
* BEFORE invoking the LLM drafter; on hit, skip the Anthropic call entirely
|
|
593
|
+
* and reuse the cached verdict.
|
|
594
|
+
*
|
|
595
|
+
* 查詢先前已判斷過的 payload fingerprint。呼叫端應在叫用 LLM drafter 之前查
|
|
596
|
+
* 這個表;命中時直接跳過 Anthropic 呼叫,回覆之前的結果。
|
|
597
|
+
*/
|
|
598
|
+
getPayloadFingerprint(fingerprint) {
|
|
599
|
+
const row = this.db
|
|
600
|
+
.prepare(`SELECT result, rule_id, pattern_hash, hit_count
|
|
601
|
+
FROM payload_fingerprints WHERE fingerprint = ? LIMIT 1`)
|
|
602
|
+
.get(fingerprint);
|
|
603
|
+
if (!row)
|
|
604
|
+
return null;
|
|
605
|
+
const result = row.result;
|
|
606
|
+
return {
|
|
607
|
+
result,
|
|
608
|
+
ruleId: row.rule_id,
|
|
609
|
+
patternHash: row.pattern_hash,
|
|
610
|
+
hitCount: row.hit_count,
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
/**
|
|
614
|
+
* Insert or bump a payload fingerprint verdict. UPSERT by fingerprint:
|
|
615
|
+
* - First time: insert with hit_count=1
|
|
616
|
+
* - Repeat: increment hit_count, refresh last_seen_at
|
|
617
|
+
*
|
|
618
|
+
* 新增或累計 payload fingerprint 判斷結果。
|
|
619
|
+
*/
|
|
620
|
+
recordPayloadFingerprint(fingerprint, result, details) {
|
|
621
|
+
this.db
|
|
622
|
+
.prepare(`INSERT INTO payload_fingerprints
|
|
623
|
+
(fingerprint, result, rule_id, pattern_hash, first_seen_at, last_seen_at, hit_count)
|
|
624
|
+
VALUES (?, ?, ?, ?, datetime('now'), datetime('now'), 1)
|
|
625
|
+
ON CONFLICT(fingerprint) DO UPDATE SET
|
|
626
|
+
hit_count = hit_count + 1,
|
|
627
|
+
last_seen_at = datetime('now')`)
|
|
628
|
+
.run(fingerprint, result, details?.ruleId ?? null, details?.patternHash ?? null);
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* Stats for the fingerprint cache. Surfaced in admin dashboard so you can
|
|
632
|
+
* see the hit rate and confirm the cost-saving is real.
|
|
633
|
+
* 指紋快取統計。供 admin dashboard 顯示命中率,確認省錢有效。
|
|
634
|
+
*/
|
|
635
|
+
getPayloadFingerprintStats() {
|
|
636
|
+
const totals = this.db
|
|
637
|
+
.prepare(`SELECT
|
|
638
|
+
COUNT(*) as total,
|
|
639
|
+
COALESCE(SUM(CASE WHEN result='novel' THEN 1 ELSE 0 END), 0) as novel,
|
|
640
|
+
COALESCE(SUM(CASE WHEN result='duplicate' THEN 1 ELSE 0 END), 0) as duplicate,
|
|
641
|
+
COALESCE(SUM(CASE WHEN result='rejected' THEN 1 ELSE 0 END), 0) as rejected,
|
|
642
|
+
COALESCE(SUM(hit_count), 0) as total_hits
|
|
643
|
+
FROM payload_fingerprints`)
|
|
644
|
+
.get();
|
|
645
|
+
return {
|
|
646
|
+
total: totals.total,
|
|
647
|
+
novel: totals.novel,
|
|
648
|
+
duplicate: totals.duplicate,
|
|
649
|
+
rejected: totals.rejected,
|
|
650
|
+
totalHits: totals.total_hits,
|
|
651
|
+
cacheHits: Math.max(0, totals.total_hits - totals.total),
|
|
652
|
+
};
|
|
653
|
+
}
|
|
479
654
|
/** Get recent skill threats / 取得最近技能威脅 */
|
|
480
655
|
getSkillThreats(limit = 50) {
|
|
481
656
|
return this.db
|
|
@@ -484,42 +659,156 @@ export class ThreatCloudDB {
|
|
|
484
659
|
}
|
|
485
660
|
/** Get proposal statistics / 取得提案統計 */
|
|
486
661
|
getProposalStats() {
|
|
487
|
-
const
|
|
488
|
-
.prepare(
|
|
489
|
-
.get().count;
|
|
490
|
-
const confirmed = this.db
|
|
491
|
-
.prepare("SELECT COUNT(*) as count FROM atr_proposals WHERE status = 'confirmed'")
|
|
492
|
-
.get().count;
|
|
493
|
-
const rejected = this.db
|
|
494
|
-
.prepare("SELECT COUNT(*) as count FROM atr_proposals WHERE status = 'rejected'")
|
|
495
|
-
.get().count;
|
|
662
|
+
const countByStatus = (status) => this.db
|
|
663
|
+
.prepare('SELECT COUNT(*) as count FROM atr_proposals WHERE status = ?')
|
|
664
|
+
.get(status).count;
|
|
496
665
|
const total = this.db.prepare('SELECT COUNT(*) as count FROM atr_proposals').get().count;
|
|
497
|
-
return {
|
|
666
|
+
return {
|
|
667
|
+
pending: countByStatus('pending'),
|
|
668
|
+
confirmed: countByStatus('confirmed'),
|
|
669
|
+
canary: countByStatus('canary'),
|
|
670
|
+
quarantined: countByStatus('quarantined'),
|
|
671
|
+
rejected: countByStatus('rejected'),
|
|
672
|
+
total,
|
|
673
|
+
};
|
|
498
674
|
}
|
|
499
675
|
/** Get threat statistics / 取得威脅統計 */
|
|
500
676
|
// ---------------------------------------------------------------------------
|
|
501
677
|
// Usage Events
|
|
502
678
|
// ---------------------------------------------------------------------------
|
|
503
679
|
recordUsageEvent(eventType, source, metadata) {
|
|
504
|
-
this.db
|
|
680
|
+
this.db
|
|
681
|
+
.prepare('INSERT INTO usage_events (event_type, source, metadata) VALUES (?, ?, ?)')
|
|
682
|
+
.run(eventType, source, metadata ? JSON.stringify(metadata) : null);
|
|
505
683
|
}
|
|
506
684
|
getUsageStats() {
|
|
507
|
-
const totalScans = this.db
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
const
|
|
685
|
+
const totalScans = this.db
|
|
686
|
+
.prepare("SELECT COUNT(*) as count FROM usage_events WHERE event_type IN ('scan', 'cli_scan')")
|
|
687
|
+
.get().count;
|
|
688
|
+
const scansToday = this.db
|
|
689
|
+
.prepare("SELECT COUNT(*) as count FROM usage_events WHERE event_type = 'scan' AND created_at > datetime('now', '-1 day')")
|
|
690
|
+
.get().count;
|
|
691
|
+
const scansThisWeek = this.db
|
|
692
|
+
.prepare("SELECT COUNT(*) as count FROM usage_events WHERE event_type = 'scan' AND created_at > datetime('now', '-7 days')")
|
|
693
|
+
.get().count;
|
|
694
|
+
const sourceRows = this.db
|
|
695
|
+
.prepare("SELECT source, COUNT(*) as count FROM usage_events WHERE event_type = 'scan' GROUP BY source")
|
|
696
|
+
.all();
|
|
511
697
|
const scansBySource = {};
|
|
512
698
|
for (const r of sourceRows)
|
|
513
699
|
scansBySource[r.source] = r.count;
|
|
514
|
-
const cliInstalls = this.db
|
|
515
|
-
|
|
700
|
+
const cliInstalls = this.db
|
|
701
|
+
.prepare("SELECT COUNT(*) as count FROM usage_events WHERE event_type = 'cli_install'")
|
|
702
|
+
.get().count;
|
|
703
|
+
const dailyTrend = this.db
|
|
704
|
+
.prepare("SELECT date(created_at) as date, COUNT(*) as count FROM usage_events WHERE event_type = 'scan' AND created_at > datetime('now', '-30 days') GROUP BY date(created_at) ORDER BY date")
|
|
705
|
+
.all();
|
|
516
706
|
return { totalScans, scansToday, scansThisWeek, scansBySource, cliInstalls, dailyTrend };
|
|
517
707
|
}
|
|
708
|
+
// ---------------------------------------------------------------------------
|
|
709
|
+
// Telemetry Events
|
|
710
|
+
// ---------------------------------------------------------------------------
|
|
711
|
+
/** Record a telemetry event / 記錄遙測事件 */
|
|
712
|
+
recordTelemetryEvent(event) {
|
|
713
|
+
this.db
|
|
714
|
+
.prepare('INSERT INTO telemetry_events (event_type, platform, skill_count, finding_count, severity) VALUES (?, ?, ?, ?, ?)')
|
|
715
|
+
.run(event.eventType, event.platform, event.skillCount, event.findingCount, event.severity);
|
|
716
|
+
}
|
|
717
|
+
/** Get telemetry stats merging raw events + hourly aggregates / 取得遙測統計 */
|
|
718
|
+
getTelemetryStats() {
|
|
719
|
+
// Raw events (last 24h — anything older is aggregated)
|
|
720
|
+
const rawTotal = this.db.prepare('SELECT COUNT(*) as count FROM telemetry_events').get().count;
|
|
721
|
+
const aggTotal = this.db
|
|
722
|
+
.prepare('SELECT COALESCE(SUM(event_count), 0) as count FROM telemetry_hourly_aggregates')
|
|
723
|
+
.get().count;
|
|
724
|
+
const eventsToday = this.db
|
|
725
|
+
.prepare("SELECT COUNT(*) as count FROM telemetry_events WHERE created_at > datetime('now', '-1 day')")
|
|
726
|
+
.get().count;
|
|
727
|
+
// By event type (raw)
|
|
728
|
+
const rawByType = this.db
|
|
729
|
+
.prepare('SELECT event_type, COUNT(*) as count FROM telemetry_events GROUP BY event_type')
|
|
730
|
+
.all();
|
|
731
|
+
const aggByType = this.db
|
|
732
|
+
.prepare('SELECT event_type, SUM(event_count) as count FROM telemetry_hourly_aggregates GROUP BY event_type')
|
|
733
|
+
.all();
|
|
734
|
+
const byEventType = {};
|
|
735
|
+
for (const r of rawByType)
|
|
736
|
+
byEventType[r.event_type] = (byEventType[r.event_type] ?? 0) + r.count;
|
|
737
|
+
for (const r of aggByType)
|
|
738
|
+
byEventType[r.event_type] = (byEventType[r.event_type] ?? 0) + r.count;
|
|
739
|
+
// By platform (raw)
|
|
740
|
+
const rawByPlatform = this.db
|
|
741
|
+
.prepare('SELECT platform, COUNT(*) as count FROM telemetry_events GROUP BY platform')
|
|
742
|
+
.all();
|
|
743
|
+
const aggByPlatform = this.db
|
|
744
|
+
.prepare('SELECT platform, SUM(event_count) as count FROM telemetry_hourly_aggregates GROUP BY platform')
|
|
745
|
+
.all();
|
|
746
|
+
const byPlatform = {};
|
|
747
|
+
for (const r of rawByPlatform)
|
|
748
|
+
byPlatform[r.platform] = (byPlatform[r.platform] ?? 0) + r.count;
|
|
749
|
+
for (const r of aggByPlatform)
|
|
750
|
+
byPlatform[r.platform] = (byPlatform[r.platform] ?? 0) + r.count;
|
|
751
|
+
// Avg finding count (raw only — aggregates don't track individual)
|
|
752
|
+
const avgRow = this.db
|
|
753
|
+
.prepare('SELECT AVG(finding_count) as avg FROM telemetry_events')
|
|
754
|
+
.get();
|
|
755
|
+
return {
|
|
756
|
+
totalEvents: rawTotal + aggTotal,
|
|
757
|
+
eventsToday,
|
|
758
|
+
byEventType,
|
|
759
|
+
byPlatform,
|
|
760
|
+
avgFindingCount: avgRow.avg ?? 0,
|
|
761
|
+
};
|
|
762
|
+
}
|
|
763
|
+
/** Aggregate raw telemetry events into hourly buckets, then delete old raw events / 彙總遙測事件 */
|
|
764
|
+
cleanupTelemetryEvents() {
|
|
765
|
+
// Step 1: Aggregate events older than 24h into hourly buckets
|
|
766
|
+
this.db.exec(`
|
|
767
|
+
INSERT INTO telemetry_hourly_aggregates (hour_bucket, event_type, platform, event_count, total_findings, total_skills, avg_severity)
|
|
768
|
+
SELECT
|
|
769
|
+
strftime('%Y-%m-%dT%H:00:00', created_at) as hour_bucket,
|
|
770
|
+
event_type,
|
|
771
|
+
platform,
|
|
772
|
+
COUNT(*) as event_count,
|
|
773
|
+
SUM(finding_count) as total_findings,
|
|
774
|
+
SUM(skill_count) as total_skills,
|
|
775
|
+
CASE
|
|
776
|
+
WHEN MAX(CASE severity WHEN 'CRITICAL' THEN 4 WHEN 'HIGH' THEN 3 WHEN 'MEDIUM' THEN 2 WHEN 'LOW' THEN 1 ELSE 0 END) >= 4 THEN 'CRITICAL'
|
|
777
|
+
WHEN MAX(CASE severity WHEN 'CRITICAL' THEN 4 WHEN 'HIGH' THEN 3 WHEN 'MEDIUM' THEN 2 WHEN 'LOW' THEN 1 ELSE 0 END) >= 3 THEN 'HIGH'
|
|
778
|
+
WHEN MAX(CASE severity WHEN 'CRITICAL' THEN 4 WHEN 'HIGH' THEN 3 WHEN 'MEDIUM' THEN 2 WHEN 'LOW' THEN 1 ELSE 0 END) >= 2 THEN 'MEDIUM'
|
|
779
|
+
ELSE 'LOW'
|
|
780
|
+
END as avg_severity
|
|
781
|
+
FROM telemetry_events
|
|
782
|
+
WHERE created_at < datetime('now', '-24 hours')
|
|
783
|
+
GROUP BY hour_bucket, event_type, platform
|
|
784
|
+
ON CONFLICT(hour_bucket, event_type, platform) DO UPDATE SET
|
|
785
|
+
event_count = event_count + excluded.event_count,
|
|
786
|
+
total_findings = total_findings + excluded.total_findings,
|
|
787
|
+
total_skills = total_skills + excluded.total_skills
|
|
788
|
+
`);
|
|
789
|
+
// Step 2: Delete raw events older than 24h
|
|
790
|
+
const result = this.db
|
|
791
|
+
.prepare("DELETE FROM telemetry_events WHERE created_at < datetime('now', '-24 hours')")
|
|
792
|
+
.run();
|
|
793
|
+
return result.changes;
|
|
794
|
+
}
|
|
518
795
|
clearAllRules() {
|
|
519
|
-
const rulesDeleted =
|
|
520
|
-
const proposalsDeleted =
|
|
796
|
+
const rulesDeleted = this.db.prepare('DELETE FROM rules').run().changes;
|
|
797
|
+
const proposalsDeleted = this.db.prepare('DELETE FROM atr_proposals').run().changes;
|
|
521
798
|
return rulesDeleted + proposalsDeleted;
|
|
522
799
|
}
|
|
800
|
+
/** Get current schema version for health diagnostics */
|
|
801
|
+
getSchemaVersion() {
|
|
802
|
+
try {
|
|
803
|
+
const row = this.db
|
|
804
|
+
.prepare('SELECT version FROM schema_version ORDER BY version DESC LIMIT 1')
|
|
805
|
+
.get();
|
|
806
|
+
return row?.version ?? -1;
|
|
807
|
+
}
|
|
808
|
+
catch {
|
|
809
|
+
return -1;
|
|
810
|
+
}
|
|
811
|
+
}
|
|
523
812
|
getStats() {
|
|
524
813
|
const totalThreats = this.db.prepare('SELECT COUNT(*) as count FROM threats').get().count;
|
|
525
814
|
const totalRules = this.db.prepare('SELECT COUNT(*) as count FROM rules').get().count;
|
|
@@ -589,7 +878,7 @@ export class ThreatCloudDB {
|
|
|
589
878
|
.prepare(`
|
|
590
879
|
SELECT pattern_hash as ruleId, rule_content as ruleContent, updated_at as publishedAt, 'atr-community' as source
|
|
591
880
|
FROM atr_proposals
|
|
592
|
-
WHERE
|
|
881
|
+
WHERE status = 'promoted' ${since ? 'AND updated_at > ?' : ''}
|
|
593
882
|
ORDER BY updated_at ASC
|
|
594
883
|
`)
|
|
595
884
|
.all(...sinceParams);
|
|
@@ -657,15 +946,17 @@ export class ThreatCloudDB {
|
|
|
657
946
|
}
|
|
658
947
|
/** Promote proposals to rules based on community consensus and/or LLM approval / 推廣提案為規則 */
|
|
659
948
|
promoteConfirmedProposals() {
|
|
660
|
-
// Path 1: LLM approved (
|
|
661
|
-
// Path 2: Community consensus (3+ confirmations,
|
|
949
|
+
// Path 1: LLM approved via self-review (JSON verdict with "approved":true)
|
|
950
|
+
// Path 2: Community consensus (3+ confirmations, no LLM required)
|
|
662
951
|
// — This ensures the flywheel works even without ANTHROPIC_API_KEY
|
|
663
|
-
// Path 3:
|
|
952
|
+
// Path 3: Admin manually approved via dashboard
|
|
953
|
+
// (status='approved' set by admin-approve endpoint; verdict stored as
|
|
954
|
+
// literal 'admin-approved' or a similar opaque marker — not JSON)
|
|
664
955
|
const proposals = this.db
|
|
665
956
|
.prepare(`
|
|
666
957
|
SELECT pattern_hash, rule_content, llm_review_verdict, confirmations, status
|
|
667
958
|
FROM atr_proposals
|
|
668
|
-
WHERE status IN ('confirmed', 'pending')
|
|
959
|
+
WHERE status IN ('confirmed', 'pending', 'approved')
|
|
669
960
|
AND status != 'rejected'
|
|
670
961
|
AND (
|
|
671
962
|
-- Path 1: LLM approved
|
|
@@ -673,10 +964,16 @@ export class ThreatCloudDB {
|
|
|
673
964
|
OR
|
|
674
965
|
-- Path 2: Community consensus (3+ confirmations, even without LLM)
|
|
675
966
|
(confirmations >= 3)
|
|
967
|
+
OR
|
|
968
|
+
-- Path 3: Admin dashboard approval (status promoted to 'approved'
|
|
969
|
+
-- by a human reviewer; the verdict field may be a plain marker
|
|
970
|
+
-- like 'admin-approved' rather than a JSON blob, so trust the
|
|
971
|
+
-- status column)
|
|
972
|
+
(status = 'approved')
|
|
676
973
|
)
|
|
677
974
|
`)
|
|
678
975
|
.all();
|
|
679
|
-
let
|
|
976
|
+
let moved = 0;
|
|
680
977
|
for (const proposal of proposals) {
|
|
681
978
|
// If LLM reviewed, check it's approved (don't promote LLM-rejected proposals)
|
|
682
979
|
if (proposal.llm_review_verdict) {
|
|
@@ -689,6 +986,83 @@ export class ThreatCloudDB {
|
|
|
689
986
|
// Unparseable verdict — allow community consensus to override
|
|
690
987
|
}
|
|
691
988
|
}
|
|
989
|
+
// Move to canary staging instead of direct promotion
|
|
990
|
+
this.db
|
|
991
|
+
.prepare(`
|
|
992
|
+
UPDATE atr_proposals
|
|
993
|
+
SET status = 'canary', canary_started_at = datetime('now'), updated_at = datetime('now')
|
|
994
|
+
WHERE pattern_hash = ?
|
|
995
|
+
`)
|
|
996
|
+
.run(proposal.pattern_hash);
|
|
997
|
+
moved++;
|
|
998
|
+
}
|
|
999
|
+
return moved;
|
|
1000
|
+
}
|
|
1001
|
+
/**
|
|
1002
|
+
* Self-heal: find proposals that are status='promoted' but have no
|
|
1003
|
+
* corresponding rule in the rules table, and re-upsert them. This
|
|
1004
|
+
* handles historical bugs where promoteCanaryRules set the status but
|
|
1005
|
+
* the upsertRule call failed silently (e.g., during a prior schema or
|
|
1006
|
+
* constraint mismatch). Idempotent — safe to run every promotion cycle.
|
|
1007
|
+
* Returns the number of rules restored.
|
|
1008
|
+
*/
|
|
1009
|
+
healOrphanedPromotedProposals() {
|
|
1010
|
+
const orphans = this.db
|
|
1011
|
+
.prepare(`
|
|
1012
|
+
SELECT p.pattern_hash, p.rule_content, p.updated_at
|
|
1013
|
+
FROM atr_proposals p
|
|
1014
|
+
LEFT JOIN rules r ON r.rule_id = p.pattern_hash
|
|
1015
|
+
WHERE p.status = 'promoted' AND r.rule_id IS NULL
|
|
1016
|
+
`)
|
|
1017
|
+
.all();
|
|
1018
|
+
let healed = 0;
|
|
1019
|
+
for (const o of orphans) {
|
|
1020
|
+
try {
|
|
1021
|
+
this.upsertRule({
|
|
1022
|
+
ruleId: o.pattern_hash,
|
|
1023
|
+
ruleContent: o.rule_content,
|
|
1024
|
+
publishedAt: o.updated_at || new Date().toISOString(),
|
|
1025
|
+
source: 'atr-community',
|
|
1026
|
+
});
|
|
1027
|
+
healed++;
|
|
1028
|
+
}
|
|
1029
|
+
catch {
|
|
1030
|
+
/* skip individual errors — next cycle will retry */
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
return healed;
|
|
1034
|
+
}
|
|
1035
|
+
/** Canary period in milliseconds (24 hours) / Canary 觀察期(24 小時) */
|
|
1036
|
+
static CANARY_PERIOD_MS = 24 * 60 * 60 * 1000;
|
|
1037
|
+
/**
|
|
1038
|
+
* Promote canary rules that have survived the 24-hour observation period
|
|
1039
|
+
* without negative feedback. Quarantine rules with negative feedback.
|
|
1040
|
+
* 推廣存活 24 小時且無負面回饋的 canary 規則。隔離有負面回饋的規則。
|
|
1041
|
+
*/
|
|
1042
|
+
promoteCanaryRules() {
|
|
1043
|
+
const canaryProposals = this.db
|
|
1044
|
+
.prepare(`
|
|
1045
|
+
SELECT pattern_hash, rule_content, canary_started_at
|
|
1046
|
+
FROM atr_proposals
|
|
1047
|
+
WHERE status = 'canary'
|
|
1048
|
+
AND canary_started_at IS NOT NULL
|
|
1049
|
+
`)
|
|
1050
|
+
.all();
|
|
1051
|
+
let promoted = 0;
|
|
1052
|
+
let quarantined = 0;
|
|
1053
|
+
for (const proposal of canaryProposals) {
|
|
1054
|
+
const negativeFeedback = this.getNegativeFeedbackCount(proposal.pattern_hash);
|
|
1055
|
+
if (negativeFeedback >= 3) {
|
|
1056
|
+
// Too much negative feedback — quarantine
|
|
1057
|
+
this.quarantineProposal(proposal.pattern_hash);
|
|
1058
|
+
quarantined++;
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
const elapsed = Date.now() - new Date(proposal.canary_started_at + 'Z').getTime();
|
|
1062
|
+
if (elapsed < ThreatCloudDB.CANARY_PERIOD_MS) {
|
|
1063
|
+
continue; // Still in canary period
|
|
1064
|
+
}
|
|
1065
|
+
// Survived canary period — promote to rule
|
|
692
1066
|
this.upsertRule({
|
|
693
1067
|
ruleId: proposal.pattern_hash,
|
|
694
1068
|
ruleContent: proposal.rule_content,
|
|
@@ -696,14 +1070,34 @@ export class ThreatCloudDB {
|
|
|
696
1070
|
source: 'atr-community',
|
|
697
1071
|
});
|
|
698
1072
|
this.db
|
|
699
|
-
.prepare(`
|
|
700
|
-
|
|
701
|
-
WHERE pattern_hash = ?
|
|
702
|
-
`)
|
|
1073
|
+
.prepare(`UPDATE atr_proposals SET status = 'promoted', updated_at = datetime('now')
|
|
1074
|
+
WHERE pattern_hash = ?`)
|
|
703
1075
|
.run(proposal.pattern_hash);
|
|
704
1076
|
promoted++;
|
|
705
1077
|
}
|
|
706
|
-
return promoted;
|
|
1078
|
+
return { promoted, quarantined };
|
|
1079
|
+
}
|
|
1080
|
+
/** Get count of negative feedback for a rule / 取得規則負面回饋數 */
|
|
1081
|
+
getNegativeFeedbackCount(ruleId) {
|
|
1082
|
+
return this.db
|
|
1083
|
+
.prepare('SELECT COUNT(*) as count FROM atr_feedback WHERE rule_id = ? AND is_true_positive = 0')
|
|
1084
|
+
.get(ruleId).count;
|
|
1085
|
+
}
|
|
1086
|
+
/** Quarantine a proposal / 隔離提案 */
|
|
1087
|
+
quarantineProposal(patternHash) {
|
|
1088
|
+
this.db
|
|
1089
|
+
.prepare(`UPDATE atr_proposals SET status = 'quarantined', updated_at = datetime('now')
|
|
1090
|
+
WHERE pattern_hash = ?`)
|
|
1091
|
+
.run(patternHash);
|
|
1092
|
+
}
|
|
1093
|
+
/** Get canary rules (for 10% client sampling) / 取得 canary 規則(供 10% 客戶端抽樣) */
|
|
1094
|
+
getCanaryATRRules() {
|
|
1095
|
+
return this.db
|
|
1096
|
+
.prepare(`SELECT pattern_hash as ruleId, rule_content as ruleContent, canary_started_at as publishedAt, 'atr-canary' as source
|
|
1097
|
+
FROM atr_proposals
|
|
1098
|
+
WHERE status = 'canary'
|
|
1099
|
+
ORDER BY canary_started_at ASC`)
|
|
1100
|
+
.all();
|
|
707
1101
|
}
|
|
708
1102
|
/** Reject an ATR proposal / 拒絕 ATR 提案 */
|
|
709
1103
|
rejectATRProposal(patternHash) {
|
|
@@ -722,17 +1116,21 @@ export class ThreatCloudDB {
|
|
|
722
1116
|
if (!proposal)
|
|
723
1117
|
return false;
|
|
724
1118
|
// Update proposal status
|
|
725
|
-
this.db
|
|
1119
|
+
this.db
|
|
1120
|
+
.prepare(`
|
|
726
1121
|
UPDATE atr_proposals SET status = 'approved', llm_review_verdict = 'admin-approved', updated_at = datetime('now')
|
|
727
1122
|
WHERE pattern_hash = ?
|
|
728
|
-
`)
|
|
1123
|
+
`)
|
|
1124
|
+
.run(patternHash);
|
|
729
1125
|
// Insert as confirmed rule
|
|
730
1126
|
const ruleId = `ATR-2026-DRAFT-${patternHash.slice(0, 8)}`;
|
|
731
1127
|
try {
|
|
732
|
-
this.db
|
|
1128
|
+
this.db
|
|
1129
|
+
.prepare(`
|
|
733
1130
|
INSERT OR IGNORE INTO rules (rule_id, rule_content, source, category, severity)
|
|
734
1131
|
VALUES (?, ?, 'atr-community', 'skill-compromise', 'high')
|
|
735
|
-
`)
|
|
1132
|
+
`)
|
|
1133
|
+
.run(ruleId, proposal.rule_content);
|
|
736
1134
|
}
|
|
737
1135
|
catch {
|
|
738
1136
|
// Rule may already exist
|
|
@@ -742,9 +1140,11 @@ export class ThreatCloudDB {
|
|
|
742
1140
|
/** Admin: remove a skill from whitelist */
|
|
743
1141
|
removeFromWhitelist(skillName) {
|
|
744
1142
|
const normalized = skillName.toLowerCase().trim().replace(/\s+/g, '-');
|
|
745
|
-
const result = this.db
|
|
1143
|
+
const result = this.db
|
|
1144
|
+
.prepare(`
|
|
746
1145
|
DELETE FROM skill_whitelist WHERE normalized_name = ?
|
|
747
|
-
`)
|
|
1146
|
+
`)
|
|
1147
|
+
.run(normalized);
|
|
748
1148
|
return result.changes > 0;
|
|
749
1149
|
}
|
|
750
1150
|
/** Admin: manually add a skill to blacklist via skill_threats */
|
|
@@ -752,27 +1152,33 @@ export class ThreatCloudDB {
|
|
|
752
1152
|
const hash = createHash('sha256').update(skillName).digest('hex').slice(0, 16);
|
|
753
1153
|
// Insert 3 threat reports to trigger blacklist threshold
|
|
754
1154
|
for (let i = 0; i < 3; i++) {
|
|
755
|
-
this.db
|
|
1155
|
+
this.db
|
|
1156
|
+
.prepare(`
|
|
756
1157
|
INSERT INTO skill_threats (skill_hash, skill_name, risk_score, risk_level, finding_summaries, client_id)
|
|
757
1158
|
VALUES (?, ?, 100, 'CRITICAL', ?, ?)
|
|
758
|
-
`)
|
|
1159
|
+
`)
|
|
1160
|
+
.run(hash, skillName, reason, `admin-${i}`);
|
|
759
1161
|
}
|
|
760
1162
|
}
|
|
761
1163
|
/** Admin: remove a skill from blacklist by clearing its threat reports */
|
|
762
1164
|
removeFromBlacklist(skillHash) {
|
|
763
|
-
const result = this.db
|
|
1165
|
+
const result = this.db
|
|
1166
|
+
.prepare(`
|
|
764
1167
|
DELETE FROM skill_threats WHERE skill_hash = ?
|
|
765
|
-
`)
|
|
1168
|
+
`)
|
|
1169
|
+
.run(skillHash);
|
|
766
1170
|
return result.changes > 0;
|
|
767
1171
|
}
|
|
768
1172
|
/** Get all whitelist entries (including unconfirmed, for admin) */
|
|
769
1173
|
getAllWhitelistEntries() {
|
|
770
|
-
return this.db
|
|
1174
|
+
return this.db
|
|
1175
|
+
.prepare(`
|
|
771
1176
|
SELECT skill_name, normalized_name, fingerprint_hash, confirmations, status,
|
|
772
1177
|
first_reported, last_reported
|
|
773
1178
|
FROM skill_whitelist
|
|
774
1179
|
ORDER BY first_reported DESC
|
|
775
|
-
`)
|
|
1180
|
+
`)
|
|
1181
|
+
.all();
|
|
776
1182
|
}
|
|
777
1183
|
/** Get rules by source type, optionally filtered by date / 依來源取得規則 */
|
|
778
1184
|
getRulesBySource(source, since) {
|
|
@@ -813,36 +1219,57 @@ export class ThreatCloudDB {
|
|
|
813
1219
|
.run(skillName, normalized, fingerprintHash ?? null);
|
|
814
1220
|
}
|
|
815
1221
|
/** Get confirmed community whitelist / 取得社群白名單 */
|
|
816
|
-
getSkillWhitelist() {
|
|
1222
|
+
getSkillWhitelist(since) {
|
|
1223
|
+
if (since) {
|
|
1224
|
+
return this.db
|
|
1225
|
+
.prepare(`SELECT skill_name as name, fingerprint_hash as hash, confirmations
|
|
1226
|
+
FROM skill_whitelist
|
|
1227
|
+
WHERE status = 'confirmed' AND last_reported > ?
|
|
1228
|
+
ORDER BY last_reported DESC`)
|
|
1229
|
+
.all(since);
|
|
1230
|
+
}
|
|
817
1231
|
return this.db
|
|
818
|
-
.prepare(`
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
ORDER BY confirmations DESC
|
|
823
|
-
`)
|
|
1232
|
+
.prepare(`SELECT skill_name as name, fingerprint_hash as hash, confirmations
|
|
1233
|
+
FROM skill_whitelist
|
|
1234
|
+
WHERE status = 'confirmed'
|
|
1235
|
+
ORDER BY confirmations DESC`)
|
|
824
1236
|
.all();
|
|
825
1237
|
}
|
|
826
1238
|
/**
|
|
827
1239
|
* Get skill blacklist: skills reported by 3+ distinct clients with avg risk >= 70
|
|
828
1240
|
* 取得技能黑名單:3+ 不同客戶端回報且平均風險 >= 70 的技能
|
|
829
1241
|
*/
|
|
830
|
-
getSkillBlacklist(minReports = 3, minAvgRisk = 70) {
|
|
1242
|
+
getSkillBlacklist(minReports = 3, minAvgRisk = 70, since) {
|
|
1243
|
+
if (since) {
|
|
1244
|
+
// Incremental: only entries with new reports since the given timestamp
|
|
1245
|
+
return this.db
|
|
1246
|
+
.prepare(`SELECT
|
|
1247
|
+
skill_hash as skillHash,
|
|
1248
|
+
skill_name as skillName,
|
|
1249
|
+
ROUND(AVG(risk_score)) as avgRiskScore,
|
|
1250
|
+
MAX(risk_level) as maxRiskLevel,
|
|
1251
|
+
COUNT(DISTINCT COALESCE(client_id, 'anonymous')) as reportCount,
|
|
1252
|
+
MIN(created_at) as firstReported,
|
|
1253
|
+
MAX(created_at) as lastReported
|
|
1254
|
+
FROM skill_threats
|
|
1255
|
+
GROUP BY skill_hash
|
|
1256
|
+
HAVING reportCount >= ? AND AVG(risk_score) >= ? AND MAX(created_at) > ?
|
|
1257
|
+
ORDER BY lastReported DESC`)
|
|
1258
|
+
.all(minReports, minAvgRisk, since);
|
|
1259
|
+
}
|
|
831
1260
|
return this.db
|
|
832
|
-
.prepare(`
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
ORDER BY avgRiskScore DESC
|
|
845
|
-
`)
|
|
1261
|
+
.prepare(`SELECT
|
|
1262
|
+
skill_hash as skillHash,
|
|
1263
|
+
skill_name as skillName,
|
|
1264
|
+
ROUND(AVG(risk_score)) as avgRiskScore,
|
|
1265
|
+
MAX(risk_level) as maxRiskLevel,
|
|
1266
|
+
COUNT(DISTINCT COALESCE(client_id, 'anonymous')) as reportCount,
|
|
1267
|
+
MIN(created_at) as firstReported,
|
|
1268
|
+
MAX(created_at) as lastReported
|
|
1269
|
+
FROM skill_threats
|
|
1270
|
+
GROUP BY skill_hash
|
|
1271
|
+
HAVING reportCount >= ? AND AVG(risk_score) >= ?
|
|
1272
|
+
ORDER BY avgRiskScore DESC`)
|
|
846
1273
|
.all(minReports, minAvgRisk);
|
|
847
1274
|
}
|
|
848
1275
|
/** Backfill classification for rules with NULL category / 回填缺少分類的規則 */
|
|
@@ -868,17 +1295,21 @@ export class ThreatCloudDB {
|
|
|
868
1295
|
* 取得尚未被 LLM 審查的待處理提案(用於重試)
|
|
869
1296
|
*/
|
|
870
1297
|
getUnreviewedProposals(limit = 5) {
|
|
1298
|
+
// Retry pending proposals that either (a) have never been reviewed, or
|
|
1299
|
+
// (b) previously hit a transient error (rate limit, timeout, network).
|
|
1300
|
+
// A legitimate `approved: false` verdict is NOT retried — that's a terminal
|
|
1301
|
+
// rejection and handled by rejectATRProposal() in the reviewer; leaving it
|
|
1302
|
+
// in the retry pool would loop forever and waste LLM API quota.
|
|
871
1303
|
return this.db
|
|
872
1304
|
.prepare(`SELECT pattern_hash as patternHash, rule_content as ruleContent
|
|
873
1305
|
FROM atr_proposals
|
|
874
1306
|
WHERE status = 'pending'
|
|
875
1307
|
AND (llm_review_verdict IS NULL
|
|
876
|
-
OR llm_review_verdict LIKE '%
|
|
877
|
-
OR llm_review_verdict LIKE '%failed%'
|
|
878
|
-
OR llm_review_verdict LIKE '%error%'
|
|
1308
|
+
OR llm_review_verdict LIKE '%LLM review failed%'
|
|
879
1309
|
OR llm_review_verdict LIKE '%rate_limit%'
|
|
880
1310
|
OR llm_review_verdict LIKE '%429%'
|
|
881
|
-
OR llm_review_verdict LIKE '%timed out%'
|
|
1311
|
+
OR llm_review_verdict LIKE '%timed out%'
|
|
1312
|
+
OR llm_review_verdict LIKE '%503%')
|
|
882
1313
|
ORDER BY created_at ASC
|
|
883
1314
|
LIMIT ?`)
|
|
884
1315
|
.all(limit);
|
|
@@ -890,6 +1321,34 @@ export class ThreatCloudDB {
|
|
|
890
1321
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
891
1322
|
.run(event.source, event.skillsScanned, event.findingsCount, event.confirmedMalicious, event.highlySuspicious, event.generalSuspicious, event.cleanCount, event.deviceHash ?? null);
|
|
892
1323
|
}
|
|
1324
|
+
/** Record a one-time activation event / 記錄一次性啟動事件 */
|
|
1325
|
+
recordActivation(activation) {
|
|
1326
|
+
try {
|
|
1327
|
+
this.db
|
|
1328
|
+
.prepare(`INSERT OR IGNORE INTO activations (client_id, platform, os_type, panguard_version, node_version)
|
|
1329
|
+
VALUES (?, ?, ?, ?, ?)`)
|
|
1330
|
+
.run(activation.clientId, activation.platform, activation.osType, activation.panguardVersion, activation.nodeVersion);
|
|
1331
|
+
return true;
|
|
1332
|
+
}
|
|
1333
|
+
catch {
|
|
1334
|
+
return false;
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
/** Get activation stats / 取得啟動統計 */
|
|
1338
|
+
getActivationStats() {
|
|
1339
|
+
const total = this.db.prepare(`SELECT COUNT(*) as count FROM activations`).get().count;
|
|
1340
|
+
const byPlatform = this.db
|
|
1341
|
+
.prepare(`SELECT platform, COUNT(*) as count FROM activations GROUP BY platform ORDER BY count DESC`)
|
|
1342
|
+
.all();
|
|
1343
|
+
const byOs = this.db
|
|
1344
|
+
.prepare(`SELECT os_type as osType, COUNT(*) as count FROM activations GROUP BY os_type ORDER BY count DESC`)
|
|
1345
|
+
.all();
|
|
1346
|
+
const recent = this.db
|
|
1347
|
+
.prepare(`SELECT activated_at as activatedAt, platform, os_type as osType, panguard_version as panguardVersion
|
|
1348
|
+
FROM activations ORDER BY activated_at DESC LIMIT 20`)
|
|
1349
|
+
.all();
|
|
1350
|
+
return { total, byPlatform, byOs, recent };
|
|
1351
|
+
}
|
|
893
1352
|
/** Get contributor leaderboard (hashed IDs, no PII) / 取得貢獻者排行榜 */
|
|
894
1353
|
getContributorLeaderboard(limit = 20) {
|
|
895
1354
|
// Hash client_id with SHA-256 for privacy
|
|
@@ -973,11 +1432,17 @@ export class ThreatCloudDB {
|
|
|
973
1432
|
const atrRulesCount = this.db
|
|
974
1433
|
.prepare(`SELECT COUNT(*) as count FROM rules WHERE source IN ('atr', 'atr-community')`)
|
|
975
1434
|
.get().count;
|
|
1435
|
+
// Whitelist count
|
|
1436
|
+
const whitelistCount = this.db.prepare(`SELECT COUNT(*) as count FROM skill_whitelist`).get().count;
|
|
1437
|
+
// Blacklist count (uses aggregated skill_threats view, same as getSkillBlacklist)
|
|
1438
|
+
const blacklistCount = this.getSkillBlacklist().length;
|
|
976
1439
|
return {
|
|
977
1440
|
totalSkillsScanned: totals.totalSkills,
|
|
978
1441
|
totalAgentsProtected: agentsProtected,
|
|
979
1442
|
totalThreatsDetected: totals.totalFindings,
|
|
980
1443
|
totalAtrRules: atrRulesCount,
|
|
1444
|
+
whitelistedSkills: whitelistCount,
|
|
1445
|
+
blacklistedSkills: blacklistCount,
|
|
981
1446
|
sources: {
|
|
982
1447
|
bulk: { skills: bulk.skills, findings: bulk.findings },
|
|
983
1448
|
cli: { skills: cli.skills, findings: cli.findings, devices: cli.devices },
|
|
@@ -986,6 +1451,238 @@ export class ThreatCloudDB {
|
|
|
986
1451
|
lastUpdated: new Date().toISOString(),
|
|
987
1452
|
};
|
|
988
1453
|
}
|
|
1454
|
+
// ── Verdict Cache / 判定快取 ──────────────────────────────────
|
|
1455
|
+
/** Look up a cached verdict by content hash. Returns null if miss or expired. */
|
|
1456
|
+
getVerdictCache(contentHash) {
|
|
1457
|
+
const row = this.db
|
|
1458
|
+
.prepare(`SELECT content_hash, skill_name, verdict, scanned_at, scan_count
|
|
1459
|
+
FROM verdict_cache
|
|
1460
|
+
WHERE content_hash = ? AND expires_at > datetime('now')`)
|
|
1461
|
+
.get(contentHash);
|
|
1462
|
+
if (!row)
|
|
1463
|
+
return null;
|
|
1464
|
+
// Increment scan_count on cache hit
|
|
1465
|
+
this.db
|
|
1466
|
+
.prepare(`UPDATE verdict_cache SET scan_count = scan_count + 1 WHERE content_hash = ?`)
|
|
1467
|
+
.run(contentHash);
|
|
1468
|
+
return {
|
|
1469
|
+
contentHash: row.content_hash,
|
|
1470
|
+
skillName: row.skill_name,
|
|
1471
|
+
verdict: row.verdict,
|
|
1472
|
+
scannedAt: row.scanned_at,
|
|
1473
|
+
scanCount: row.scan_count + 1,
|
|
1474
|
+
};
|
|
1475
|
+
}
|
|
1476
|
+
/** Insert or replace a verdict cache entry with 30-day TTL. */
|
|
1477
|
+
insertVerdictCache(entry) {
|
|
1478
|
+
this.db
|
|
1479
|
+
.prepare(`INSERT OR REPLACE INTO verdict_cache (content_hash, skill_name, verdict, scanned_at, expires_at, scan_count)
|
|
1480
|
+
VALUES (?, ?, ?, datetime('now'), datetime('now', '+30 days'), 1)`)
|
|
1481
|
+
.run(entry.contentHash, entry.skillName, entry.verdict);
|
|
1482
|
+
}
|
|
1483
|
+
/** Delete a specific cache entry. Returns true if a row was deleted. */
|
|
1484
|
+
invalidateVerdictCache(contentHash) {
|
|
1485
|
+
const result = this.db
|
|
1486
|
+
.prepare(`DELETE FROM verdict_cache WHERE content_hash = ?`)
|
|
1487
|
+
.run(contentHash);
|
|
1488
|
+
return result.changes > 0;
|
|
1489
|
+
}
|
|
1490
|
+
/** Purge all expired cache entries. Returns count deleted. */
|
|
1491
|
+
purgeExpiredVerdictCache() {
|
|
1492
|
+
const result = this.db
|
|
1493
|
+
.prepare(`DELETE FROM verdict_cache WHERE expires_at <= datetime('now')`)
|
|
1494
|
+
.run();
|
|
1495
|
+
return result.changes;
|
|
1496
|
+
}
|
|
1497
|
+
// ── Skill Hash History / 技能雜湊歷史 ────────────────────────
|
|
1498
|
+
/** Get the latest (non-superseded) hash entry for a skill. */
|
|
1499
|
+
getLatestSkillHash(skillName) {
|
|
1500
|
+
const row = this.db
|
|
1501
|
+
.prepare(`SELECT content_hash, scan_verdict
|
|
1502
|
+
FROM skill_hash_history
|
|
1503
|
+
WHERE skill_name = ? AND superseded_by IS NULL
|
|
1504
|
+
ORDER BY last_seen DESC
|
|
1505
|
+
LIMIT 1`)
|
|
1506
|
+
.get(skillName);
|
|
1507
|
+
if (!row)
|
|
1508
|
+
return null;
|
|
1509
|
+
return { contentHash: row.content_hash, scanVerdict: row.scan_verdict };
|
|
1510
|
+
}
|
|
1511
|
+
/** Record a skill hash observation. Creates or updates last_seen. */
|
|
1512
|
+
recordSkillHash(entry) {
|
|
1513
|
+
this.db
|
|
1514
|
+
.prepare(`INSERT INTO skill_hash_history (skill_name, content_hash, scan_verdict, rug_pull_flag)
|
|
1515
|
+
VALUES (?, ?, ?, ?)
|
|
1516
|
+
ON CONFLICT(skill_name, content_hash) DO UPDATE SET
|
|
1517
|
+
last_seen = datetime('now'),
|
|
1518
|
+
scan_verdict = COALESCE(excluded.scan_verdict, scan_verdict)`)
|
|
1519
|
+
.run(entry.skillName, entry.contentHash, entry.scanVerdict ?? null, entry.rugPullFlag ? 1 : 0);
|
|
1520
|
+
}
|
|
1521
|
+
/** Mark a previous hash as superseded by a new hash. */
|
|
1522
|
+
markSkillHashSuperseded(skillName, oldHash, newHash) {
|
|
1523
|
+
this.db
|
|
1524
|
+
.prepare(`UPDATE skill_hash_history
|
|
1525
|
+
SET superseded_by = ?
|
|
1526
|
+
WHERE skill_name = ? AND content_hash = ?`)
|
|
1527
|
+
.run(newHash, skillName, oldHash);
|
|
1528
|
+
}
|
|
1529
|
+
/** Get full hash history for a skill (audit trail). */
|
|
1530
|
+
getSkillHashHistory(skillName) {
|
|
1531
|
+
const rows = this.db
|
|
1532
|
+
.prepare(`SELECT content_hash, first_seen, last_seen, scan_verdict, superseded_by, rug_pull_flag
|
|
1533
|
+
FROM skill_hash_history
|
|
1534
|
+
WHERE skill_name = ?
|
|
1535
|
+
ORDER BY first_seen ASC`)
|
|
1536
|
+
.all(skillName);
|
|
1537
|
+
return rows.map((r) => ({
|
|
1538
|
+
contentHash: r.content_hash,
|
|
1539
|
+
firstSeen: r.first_seen,
|
|
1540
|
+
lastSeen: r.last_seen,
|
|
1541
|
+
scanVerdict: r.scan_verdict,
|
|
1542
|
+
supersededBy: r.superseded_by,
|
|
1543
|
+
rugPullFlag: r.rug_pull_flag === 1,
|
|
1544
|
+
}));
|
|
1545
|
+
}
|
|
1546
|
+
// ─── Org / Device / Policy (Threat Model #1, #6) ───────────────
|
|
1547
|
+
/** Create an organization / 建立組織 */
|
|
1548
|
+
createOrg(id, name) {
|
|
1549
|
+
this.db.prepare('INSERT OR IGNORE INTO orgs (id, name) VALUES (?, ?)').run(id, name);
|
|
1550
|
+
}
|
|
1551
|
+
/** Get org by ID / 取得組織 */
|
|
1552
|
+
getOrg(id) {
|
|
1553
|
+
return this.db.prepare('SELECT * FROM orgs WHERE id = ?').get(id);
|
|
1554
|
+
}
|
|
1555
|
+
/** Upsert device heartbeat / 更新裝置心跳
|
|
1556
|
+
* Threat Model: Fleet view (#6 Scope Escalation — org-level visibility) */
|
|
1557
|
+
upsertDevice(data) {
|
|
1558
|
+
this.db
|
|
1559
|
+
.prepare(`INSERT INTO devices (id, org_id, hostname, os_type, agent_count, guard_version, last_seen)
|
|
1560
|
+
VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
|
|
1561
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1562
|
+
org_id = excluded.org_id,
|
|
1563
|
+
hostname = COALESCE(excluded.hostname, devices.hostname),
|
|
1564
|
+
os_type = COALESCE(excluded.os_type, devices.os_type),
|
|
1565
|
+
agent_count = excluded.agent_count,
|
|
1566
|
+
guard_version = COALESCE(excluded.guard_version, devices.guard_version),
|
|
1567
|
+
last_seen = datetime('now')`)
|
|
1568
|
+
.run(data.deviceId, data.orgId, data.hostname ?? null, data.osType ?? null, data.agentCount ?? 0, data.guardVersion ?? null);
|
|
1569
|
+
}
|
|
1570
|
+
/** List devices for an org / 列出組織下的裝置
|
|
1571
|
+
* Threat Model: Fleet view (#6) */
|
|
1572
|
+
getDevicesByOrg(orgId, limit = 100, offset = 0) {
|
|
1573
|
+
return this.db
|
|
1574
|
+
.prepare('SELECT id, hostname, os_type, agent_count, guard_version, last_seen, created_at FROM devices WHERE org_id = ? ORDER BY last_seen DESC LIMIT ? OFFSET ?')
|
|
1575
|
+
.all(orgId, limit, offset);
|
|
1576
|
+
}
|
|
1577
|
+
/** Get device count for an org / 取得組織裝置數 */
|
|
1578
|
+
getDeviceCount(orgId) {
|
|
1579
|
+
return this.db.prepare('SELECT COUNT(*) as count FROM devices WHERE org_id = ?').get(orgId).count;
|
|
1580
|
+
}
|
|
1581
|
+
/** Set org policy / 設定組織策略
|
|
1582
|
+
* Threat Model: Policy engine (#1 Supply Chain, #6 Scope Escalation) */
|
|
1583
|
+
setOrgPolicy(orgId, category, action) {
|
|
1584
|
+
this.db
|
|
1585
|
+
.prepare(`INSERT INTO org_policies (org_id, category, action)
|
|
1586
|
+
VALUES (?, ?, ?)
|
|
1587
|
+
ON CONFLICT(org_id, category) DO UPDATE SET action = excluded.action`)
|
|
1588
|
+
.run(orgId, category, action);
|
|
1589
|
+
}
|
|
1590
|
+
/** Get org policies / 取得組織策略 */
|
|
1591
|
+
getOrgPolicies(orgId) {
|
|
1592
|
+
return this.db
|
|
1593
|
+
.prepare('SELECT category, action, created_at FROM org_policies WHERE org_id = ? ORDER BY category')
|
|
1594
|
+
.all(orgId);
|
|
1595
|
+
}
|
|
1596
|
+
/** Delete org policy / 刪除組織策略 */
|
|
1597
|
+
deleteOrgPolicy(orgId, category) {
|
|
1598
|
+
const result = this.db
|
|
1599
|
+
.prepare('DELETE FROM org_policies WHERE org_id = ? AND category = ?')
|
|
1600
|
+
.run(orgId, category);
|
|
1601
|
+
return result.changes > 0;
|
|
1602
|
+
}
|
|
1603
|
+
// ---------------------------------------------------------------------------
|
|
1604
|
+
// Client Keys / 客戶端金鑰
|
|
1605
|
+
// ---------------------------------------------------------------------------
|
|
1606
|
+
/** Register a new client key. Returns the raw key (only time it's visible). */
|
|
1607
|
+
registerClientKey(clientId, ipAddress) {
|
|
1608
|
+
// Limit to 5 active keys per clientId to prevent DB flooding
|
|
1609
|
+
const existing = this.db
|
|
1610
|
+
.prepare('SELECT COUNT(*) as cnt FROM client_keys WHERE client_id = ? AND revoked = 0')
|
|
1611
|
+
.get(clientId);
|
|
1612
|
+
if (existing.cnt >= 5) {
|
|
1613
|
+
// Revoke oldest key to make room
|
|
1614
|
+
this.db
|
|
1615
|
+
.prepare(`UPDATE client_keys SET revoked = 1, revoked_at = datetime('now')
|
|
1616
|
+
WHERE id = (SELECT id FROM client_keys WHERE client_id = ? AND revoked = 0 ORDER BY created_at ASC LIMIT 1)`)
|
|
1617
|
+
.run(clientId);
|
|
1618
|
+
}
|
|
1619
|
+
const clientKey = randomUUID();
|
|
1620
|
+
const hash = createHash('sha256').update(clientKey).digest('hex');
|
|
1621
|
+
this.db
|
|
1622
|
+
.prepare('INSERT INTO client_keys (client_id, client_key_hash, ip_address) VALUES (?, ?, ?)')
|
|
1623
|
+
.run(clientId, hash, ipAddress);
|
|
1624
|
+
return { clientKey };
|
|
1625
|
+
}
|
|
1626
|
+
/** Validate a raw client key. Updates last_used_at on success. */
|
|
1627
|
+
validateClientKey(rawKey) {
|
|
1628
|
+
const hash = createHash('sha256').update(rawKey).digest('hex');
|
|
1629
|
+
const row = this.db
|
|
1630
|
+
.prepare('SELECT id FROM client_keys WHERE client_key_hash = ? AND revoked = 0')
|
|
1631
|
+
.get(hash);
|
|
1632
|
+
if (!row)
|
|
1633
|
+
return false;
|
|
1634
|
+
this.db
|
|
1635
|
+
.prepare("UPDATE client_keys SET last_used_at = datetime('now') WHERE id = ?")
|
|
1636
|
+
.run(row.id);
|
|
1637
|
+
return true;
|
|
1638
|
+
}
|
|
1639
|
+
/**
|
|
1640
|
+
* Look up role + clientId for a raw client key. Returns null if invalid/revoked.
|
|
1641
|
+
* Does NOT update last_used_at (caller should use validateClientKey for that).
|
|
1642
|
+
* Used by L5 partner-sync endpoints to enforce role-based access.
|
|
1643
|
+
*/
|
|
1644
|
+
getClientKeyInfo(rawKey) {
|
|
1645
|
+
const hash = createHash('sha256').update(rawKey).digest('hex');
|
|
1646
|
+
const row = this.db
|
|
1647
|
+
.prepare('SELECT client_id as clientId, COALESCE(key_role, ?) as role FROM client_keys WHERE client_key_hash = ? AND revoked = 0')
|
|
1648
|
+
.get('guard', hash);
|
|
1649
|
+
return row ?? null;
|
|
1650
|
+
}
|
|
1651
|
+
/**
|
|
1652
|
+
* Issue a partner key. Partner keys are manually issued by admin and can
|
|
1653
|
+
* access L5 live-sync endpoints (e.g. /api/atr-rules/live). Returns the
|
|
1654
|
+
* raw key once — caller must store it securely.
|
|
1655
|
+
*/
|
|
1656
|
+
registerPartnerKey(partnerName, issuedBy) {
|
|
1657
|
+
const clientId = `partner:${partnerName}`;
|
|
1658
|
+
const clientKey = randomUUID();
|
|
1659
|
+
const hash = createHash('sha256').update(clientKey).digest('hex');
|
|
1660
|
+
this.db
|
|
1661
|
+
.prepare('INSERT INTO client_keys (client_id, client_key_hash, ip_address, key_role) VALUES (?, ?, ?, ?)')
|
|
1662
|
+
.run(clientId, hash, `issued-by:${issuedBy}`, 'partner');
|
|
1663
|
+
return { clientKey };
|
|
1664
|
+
}
|
|
1665
|
+
/** Revoke all keys for a client. Returns number of keys revoked. */
|
|
1666
|
+
revokeClientKey(clientId) {
|
|
1667
|
+
const result = this.db
|
|
1668
|
+
.prepare("UPDATE client_keys SET revoked = 1, revoked_at = datetime('now') WHERE client_id = ? AND revoked = 0")
|
|
1669
|
+
.run(clientId);
|
|
1670
|
+
return result.changes;
|
|
1671
|
+
}
|
|
1672
|
+
/** List client keys (admin). Never returns raw keys. */
|
|
1673
|
+
listClientKeys(limit = 50, offset = 0) {
|
|
1674
|
+
const rows = this.db
|
|
1675
|
+
.prepare('SELECT id, client_id, created_at, last_used_at, revoked, ip_address FROM client_keys ORDER BY created_at DESC LIMIT ? OFFSET ?')
|
|
1676
|
+
.all(limit, offset);
|
|
1677
|
+
return rows.map((r) => ({
|
|
1678
|
+
id: r.id,
|
|
1679
|
+
clientId: r.client_id,
|
|
1680
|
+
createdAt: r.created_at,
|
|
1681
|
+
lastUsedAt: r.last_used_at,
|
|
1682
|
+
revoked: r.revoked === 1,
|
|
1683
|
+
ipAddress: r.ip_address,
|
|
1684
|
+
}));
|
|
1685
|
+
}
|
|
989
1686
|
/** Close the database / 關閉資料庫 */
|
|
990
1687
|
close() {
|
|
991
1688
|
this.db.close();
|