@vk.amogh/trace 2.1.0

package/src/trace_engine/intelligence/training/dataset.py

@@ -0,0 +1,320 @@
+"""Comprehensive Cybersecurity AST dataset generator and PyTorch Dataset for model fine-tuning."""
+
+import re
+import csv
+import json
+from pathlib import Path
+from typing import List, Dict, Any, Tuple, Optional
+import torch
+from torch.utils.data import Dataset
+
+from trace_engine.intelligence.securebert.classifier import VULN_CATEGORIES
+
+
+def normalize_code_slice(code: str) -> str:
+    """Normalizes code AST slice by abstracting identifiers and highlighting security dataflow."""
+    code = re.sub(r"#.*$", "", code, flags=re.MULTILINE)
+    code = re.sub(r"//.*$", "", code, flags=re.MULTILINE)
+    code = re.sub(r"/\*.*?\*/", "", code, flags=re.DOTALL)
+
+    code = re.sub(r"(request\.(args|params|query|body|json|headers|form))", r"[SOURCE] \1", code, flags=re.IGNORECASE)
+    code = re.sub(r"((execute|cursor|query|system|popen|eval|exec|open|readfile|render_template_string|pickle\.loads|yaml\.load)\b)", r"[SINK] \1", code, flags=re.IGNORECASE)
+
+    code = re.sub(r"\s+", " ", code).strip()
+    return code
+
+
+CORPUS_TEMPLATES = [
+    # BOLA / IDOR (CWE-639)
+    ("def get_order(order_id): return db.query('SELECT * FROM orders WHERE id = :id', id=order_id).fetchone()", ["BOLA"]),
+    ("async function getDocument(req, res) { const doc = await Document.findById(req.params.id); return res.json(doc); }", ["BOLA"]),
+    ("router.get('/profile/:id', (req, res) => { const user = User.load(req.params.id); res.send(user); });", ["BOLA"]),
+    ("@app.get('/invoice/{inv_id}') def invoice(inv_id: str): return db.invoices.find_one({'_id': inv_id})", ["BOLA"]),
+    ("Response getProject(Request req) { String id = req.params['id']; return Response.ok(projectRepo.find(id)); }", ["BOLA"]),
+
+    # BFLA (CWE-285)
+    ("@app.post('/api/admin/reset_metrics') def reset(): global_stats.clear(); return {'status': 'reset'}", ["BFLA"]),
+    ("router.delete('/admin/users/:id', (req, res) => { db.users.delete(req.params.id); res.status(204).end(); });", ["BFLA"]),
+    ("def export_all_tenants(): return db.raw_export_tenants()", ["BFLA"]),
+    ("fun cancelAnyOrder(orderId: Long): ResponseEntity<Void> { orderService.forceCancel(orderId); return ResponseEntity.ok().build(); }", ["BFLA"]),
+    ("@PreAuthorize('permitAll()') @PostMapping('/admin/config') fun updateCfg(@RequestBody cfg: Config) = repo.save(cfg)", ["BFLA"]),
+
+    # Missing Authentication (CWE-306)
+    ("app.post('/api/v2/transfer', (req, res) => { transferFunds(req.body.from, req.body.to, req.body.amount); res.send('ok'); });", ["AUTHENTICATION"]),
+    ("def update_password(req): user = get_user(req.json['username']); user.password = hash(req.json['new_pass']); db.save(user)", ["AUTHENTICATION"]),
+    ("async def change_email(email: str): current_user.email = email; await db.commit()", ["AUTHENTICATION"]),
+    ("router.put('/api/v1/vault/keys', (req, res) => { vault.rotateKey(req.body.key); res.json({status: 'updated'}); });", ["AUTHENTICATION"]),
+
+    # SSRF (CWE-918)
+    ("def fetch_url(url: str): return httpx.get(url).text", ["SSRF"]),
+    ("router.post('/webhook', async (req, res) => { const out = await axios.get(req.body.target_url); res.send(out.data); });", ["SSRF"]),
+    ("def preview_link(link): return requests.get(link, timeout=5).content", ["SSRF"]),
+    ("fun proxyRequest(callbackUrl: String) = restTemplate.getForObject(callbackUrl, String::class.java)", ["SSRF"]),
+    ("async function fetchRemoteAvatar(url) { return (await fetch(url)).blob(); }", ["SSRF"]),
+
+    # Injection (SQLi / Command / LDAP) (CWE-89 / CWE-78)
+    ("def search_products(q: str): return db.execute(f'SELECT * FROM products WHERE name LIKE \"%{q}%\"')", ["INJECTION"]),
+    ("app.get('/exec', (req, res) => { exec('ping -c 1 ' + req.query.host, (err, stdout) => res.send(stdout)); });", ["INJECTION"]),
+    ("def find_user(name): query = 'SELECT * FROM users WHERE username = \\'' + name + '\\''; return db.cursor.execute(query)", ["INJECTION"]),
+    ("def run_backup(path): os.system(f'tar -czf backup.tar.gz {path}')", ["INJECTION"]),
+    ("fun queryLdap(filter: String) = ldapTemplate.search('', '(uid=' + filter + ')', mapper)", ["INJECTION"]),
+
+    # Mass Assignment (CWE-915)
+    ("def update_profile(user_id, data: dict): user = db.get(user_id); user.__dict__.update(data); db.save(user)", ["MASS_ASSIGNMENT"]),
+    ("router.put('/user/:id', (req, res) => { User.findByIdAndUpdate(req.params.id, req.body); res.json({ok: true}); });", ["MASS_ASSIGNMENT"]),
+    ("async def patch_account(req: Request): data = await req.json(); user.update(**data); return user", ["MASS_ASSIGNMENT"]),
+    ("@PutMapping('/users/{id}') fun patchUser(@PathVariable id: Long, @RequestBody user: User) = userRepo.save(user)", ["MASS_ASSIGNMENT"]),
+
+    # Path Traversal (CWE-22 / CWE-73)
+    ("def read_file(filename: str): return open(os.path.join('/var/www/uploads', filename), 'r').read()", ["PATH_TRAVERSAL"]),
+    ("app.get('/download', (req, res) => { res.sendFile(path.resolve('./files/' + req.query.file)); });", ["PATH_TRAVERSAL"]),
+    ("def view_log(log_path): with open(log_path, 'r') as f: return f.read()", ["PATH_TRAVERSAL"]),
+    ("fun serveStatic(page: String): ByteArray = File('/assets/' + page).readBytes()", ["PATH_TRAVERSAL"]),
+
+    # SSTI (CWE-1336 / CWE-94)
+    ("def render_user_card(username: str): return jinja2.Environment().from_string(f'Hello {username}').render()", ["SSTI"]),
+    ("def preview_email(template_str): return render_template_string(template_str)", ["SSTI"]),
+    ("app.get('/greet', (req, res) => { const tmpl = nunjucks.renderString('Welcome ' + req.query.name); res.send(tmpl); });", ["SSTI"]),
+
+    # CORS Misconfiguration (CWE-942)
+    ("def after_request(resp): resp.headers['Access-Control-Allow-Origin'] = request.headers.get('Origin'); resp.headers['Access-Control-Allow-Credentials'] = 'true'; return resp", ["CORS"]),
+    ("app.use((req, res, next) => { res.header('Access-Control-Allow-Origin', req.headers.origin); res.header('Access-Control-Allow-Credentials', 'true'); next(); });", ["CORS"]),
+
+    # Deserialization (CWE-502)
+    ("def load_session(data): return pickle.loads(base64.b64decode(data))", ["DESERIALIZATION"]),
+    ("def parse_config(yaml_str): return yaml.load(yaml_str, Loader=yaml.Loader)", ["DESERIALIZATION"]),
+    ("def restore_state(payload): import pickle; return pickle.loads(payload)", ["DESERIALIZATION"]),
+
+    # Safe Negative Controls (Benign code slices)
+    ("def get_order_safe(order_id, user=Depends(get_current_user)): return db.query(Order).filter(Order.id == order_id, Order.tenant_id == user.tenant_id).first()", []),
+    ("def search_safe(term: str): return db.execute('SELECT * FROM items WHERE name ILIKE :term', {'term': f'%{term}%'})", []),
+    ("def read_file_safe(filename: str): canonical = Path(filename).resolve(); if not str(canonical).startswith('/safe/root/'): raise Forbidden(); return canonical.read_text()", []),
+    ("def fetch_url_safe(url: str): guard = ScopeGuard(); guard.validate_url(url); return httpx.get(url)", []),
+    ("def update_user_safe(user_id, dto: UserUpdateDto): user = db.get(user_id); user.name = dto.name; user.bio = dto.bio; db.save(user)", []),
+    ("def render_safe(template_name, context): return render_template(f'{template_name}.html', **context)", []),
+]
+
+
+def load_owasp_benchmark_samples() -> List[Tuple[str, List[str]]]:
+    """Loads all 1,230 real code files from OWASP Benchmark Python with verified ground truth."""
+    samples: List[Tuple[str, List[str]]] = []
+
+    candidates = [
+        Path("benchmarks/owasp-python"),
+        Path(__file__).resolve().parent.parent.parent.parent / "benchmarks/owasp-python",
+    ]
+    benchmark_dir = None
+    for c in candidates:
+        if c.exists() and (c / "expectedresults-0.1.csv").exists():
+            benchmark_dir = c
+            break
+
+    if not benchmark_dir:
+        return samples
+
+    csv_file = benchmark_dir / "expectedresults-0.1.csv"
+    testcode_dir = benchmark_dir / "testcode"
+
+    cat_mapping = {
+        "pathtraver": "PATH_TRAVERSAL",
+        "sqli": "INJECTION",
+        "cmdi": "INJECTION",
+        "codeinj": "INJECTION",
+        "ldapi": "INJECTION",
+        "xpathi": "INJECTION",
+        "xss": "INJECTION",
+        "deserialization": "DESERIALIZATION",
+        "trustbound": "BOLA",
+        "redirect": "SSRF",
+    }
+
+    try:
+        with open(csv_file, "r", encoding="utf-8") as f:
+            reader = csv.reader(f)
+            for row in reader:
+                if not row or row[0].startswith("#"):
+                    continue
+                test_file = testcode_dir / f"{row[0].strip()}.py"
+                if not test_file.exists():
+                    continue
+
+                category_raw = row[1].strip().lower()
+                is_vuln = row[2].strip().lower() == "true"
+
+                code_content = test_file.read_text(encoding="utf-8", errors="replace")
+                normalized = normalize_code_slice(code_content[:1500])
+
+                if is_vuln and category_raw in cat_mapping:
+                    samples.append((normalized, [cat_mapping[category_raw]]))
+                else:
+                    samples.append((normalized, []))
+    except Exception:
+        pass
+
+    return samples
+
+
+def load_parquet_security_corpus() -> List[Tuple[str, List[str]]]:
+    """Loads 500+ curated security vulnerability AST functions from research parquet files."""
+    samples: List[Tuple[str, List[str]]] = []
+    candidates = [
+        Path("research/securebert2/opensource_data/data_vuln_dataset.parquet"),
+        Path(__file__).resolve().parent.parent.parent.parent / "research/securebert2/opensource_data/data_vuln_dataset.parquet",
+    ]
+
+    pq_file = None
+    for c in candidates:
+        if c.exists():
+            pq_file = c
+            break
+
+    if not pq_file:
+        return samples
+
+    try:
+        import pyarrow.parquet as pq
+        table = pq.read_table(pq_file)
+        pydict = table.to_pydict()
+        codes = pydict.get("code", [])
+        labels = pydict.get("label", [])
+
+        for code, is_vuln in zip(codes, labels):
+            code_str = str(code)
+            normalized = normalize_code_slice(code_str[:1200])
+            if is_vuln:
+                samples.append((normalized, ["INJECTION"]))
+            else:
+                samples.append((normalized, []))
+    except Exception:
+        pass
+
+    return samples
+
+
+def load_external_cve_datasets(limit_bigvul: int = 4000, limit_cvefixes: int = 3000, limit_morefixes: int = 4000) -> List[Tuple[str, List[str]]]:
+    """Ingests real-world functions with CVE ground truth from Big-Vul, CVEfixes, and MoreFixes."""
+    from trace_engine.intelligence.training.dataset_importers import BigVulImporter, CVEfixesImporter, MoreFixesImporter
+
+    samples: List[Tuple[str, List[str]]] = []
+
+    # 1. Big-Vul Parquet
+    bv_paths = [
+        Path(".trace/datasets/bigvul/train.parquet"),
+        Path(".trace/datasets/bigvul/validation.parquet"),
+    ]
+    bv = BigVulImporter()
+    for bp in bv_paths:
+        if bp.exists():
+            bv_samples = bv.import_parquet(bp, limit=limit_bigvul, include_benign=True)
+            for code, labels in bv_samples:
+                samples.append((normalize_code_slice(code), labels))
+
+    # 2. CVEfixes CSV
+    cve_paths = [
+        Path(".trace/datasets/cvefixes/train.csv"),
+        Path(".trace/datasets/cvefixes/test.csv"),
+    ]
+    cve = CVEfixesImporter()
+    for cp in cve_paths:
+        if cp.exists():
+            cve_samples = cve.import_csv(cp, limit=limit_cvefixes, include_benign=True)
+            for code, labels in cve_samples:
+                samples.append((normalize_code_slice(code), labels))
+
+    # 3. MoreFixes (Zenodo 20776007: 52k+ CVE patch diffs and commit records)
+    morefixes_paths = [
+        Path(".trace/datasets/morefixes/patch-files2026-06-20.zip"),
+        Path(".trace/datasets/morefixes/patches"),
+        Path("C:/trace_datasets/morefixes/patch-files2026-06-20.zip"),
+        Path("C:/trace_datasets/morefixes/patches"),
+    ]
+    mf = MoreFixesImporter()
+    for mp in morefixes_paths:
+        if mp.exists():
+            mf_samples = mf.import_patch_archive(mp, limit=limit_morefixes, include_benign=True)
+            for code, labels in mf_samples:
+                samples.append((normalize_code_slice(code), labels))
+            break
+
+    return samples
+
+
+def generate_cybersecurity_training_corpus(multiplier: int = 4, include_external_cve: bool = True) -> List[Tuple[str, List[str]]]:
+    """Builds a comprehensive unified training corpus fusing Big-Vul, CVEfixes, OWASP Benchmark, parquet CVEs, and multi-language templates."""
+    samples: List[Tuple[str, List[str]]] = []
+
+    # 1. Ingest real external CVE datasets (Big-Vul and CVEfixes)
+    if include_external_cve:
+        ext_samples = load_external_cve_datasets(limit_bigvul=4000, limit_cvefixes=3000)
+        samples.extend(ext_samples)
+
+    # 2. Ingest real OWASP Benchmark Python files (1,230 samples)
+    owasp_samples = load_owasp_benchmark_samples()
+    samples.extend(owasp_samples)
+
+    # 3. Ingest real parquet vulnerability AST slices (500 samples)
+    pq_samples = load_parquet_security_corpus()
+    samples.extend(pq_samples)
+
+    # 4. Add multi-language API security templates
+    var_aliases = ["item", "record", "payload", "entity", "resource", "target", "client", "doc", "asset"]
+    for _ in range(multiplier):
+        for code, labels in CORPUS_TEMPLATES:
+            augmented_code = code
+            for alias in var_aliases:
+                if "user" in augmented_code and alias != "user":
+                    augmented_code = augmented_code.replace("user", alias)
+                    break
+            samples.append((normalize_code_slice(augmented_code), labels))
+
+    return samples
+
+
+class VulnerabilityDataset(Dataset):
+    """PyTorch Dataset for multi-label vulnerability classification with class-weight support."""
+
+    def __init__(self, samples: List[Tuple[str, List[str]]], tokenizer, max_length: int = 256):
+        self.samples = samples
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.label_map = {cat: idx for idx, cat in enumerate(VULN_CATEGORIES)}
+
+    def __len__(self) -> int:
+        return len(self.samples)
+
+    def calculate_pos_weights(self) -> torch.Tensor:
+        """Calculates positive class weights using square-root scaling to balance loss without skewing predictions."""
+        pos_counts = torch.zeros(len(VULN_CATEGORIES), dtype=torch.float32)
+        total = len(self.samples)
+        for _, labels in self.samples:
+            for l in labels:
+                if l in self.label_map:
+                    pos_counts[self.label_map[l]] += 1.0
+
+        pos_counts = torch.clamp(pos_counts, min=1.0)
+        neg_counts = total - pos_counts
+        # Square-root dampened weights bounded to [1.0, 2.5] avoids artificial recall inflation
+        weights = torch.clamp(torch.sqrt(neg_counts / pos_counts), min=1.0, max=2.5)
+        return weights
+
+    def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
+        code_text, label_names = self.samples[idx]
+
+        target = torch.zeros(len(VULN_CATEGORIES), dtype=torch.float32)
+        for name in label_names:
+            if name in self.label_map:
+                target[self.label_map[name]] = 1.0
+
+        encoding = self.tokenizer(
+            code_text,
+            max_length=self.max_length,
+            padding="max_length",
+            truncation=True,
+            return_tensors="pt",
+        )
+
+        return {
+            "input_ids": encoding["input_ids"].squeeze(0),
+            "attention_mask": encoding["attention_mask"].squeeze(0),
+            "labels": target,
+        }

package/src/trace_engine/intelligence/training/dataset_importers.py

@@ -0,0 +1,349 @@
+"""Importers and adapters for external security research datasets (Big-Vul, D2A, CVEfixes, Juliet)."""
+
+import csv
+import json
+import logging
+from pathlib import Path
+from typing import List, Dict, Tuple, Optional, Set
+
+logger = logging.getLogger(__name__)
+
+# Canonical MITRE CWE to TRACE Vulnerability Category Mapping
+CWE_TO_CATEGORY: Dict[str, str] = {
+    # BOLA / Broken Object Level Authorization (CWE-639 / CWE-284)
+    "CWE-639": "BOLA",
+    "CWE-284": "BOLA",
+    "CWE-862": "BOLA",
+    "CWE-732": "BOLA",
+
+    # BFLA / Broken Function Level Authorization (CWE-285 / CWE-264)
+    "CWE-285": "BFLA",
+    "CWE-863": "BFLA",
+    "CWE-264": "BFLA",
+
+    # Missing & Broken Authentication
+    "CWE-306": "AUTHENTICATION",
+    "CWE-287": "AUTHENTICATION",
+    "CWE-798": "AUTHENTICATION",
+    "CWE-522": "AUTHENTICATION",
+    "CWE-254": "AUTHENTICATION",
+    "CWE-384": "AUTHENTICATION",
+    "CWE-613": "AUTHENTICATION",
+
+    # SSRF / Open Redirect
+    "CWE-918": "SSRF",
+    "CWE-601": "SSRF",
+
+    # Injection (SQLi, Command Injection, Memory Safety, LDAP)
+    "CWE-89": "INJECTION",
+    "CWE-78": "INJECTION",
+    "CWE-77": "INJECTION",
+    "CWE-94": "INJECTION",
+    "CWE-119": "INJECTION",
+    "CWE-125": "INJECTION",
+    "CWE-787": "INJECTION",
+    "CWE-20": "INJECTION",
+    "CWE-189": "INJECTION",
+    "CWE-190": "INJECTION",
+    "CWE-416": "INJECTION",
+    "CWE-476": "INJECTION",
+    "CWE-399": "INJECTION",
+    "CWE-400": "INJECTION",
+    "CWE-415": "INJECTION",
+    "CWE-772": "INJECTION",
+
+    # Mass Assignment
+    "CWE-915": "MASS_ASSIGNMENT",
+
+    # Path Traversal
+    "CWE-22": "PATH_TRAVERSAL",
+    "CWE-73": "PATH_TRAVERSAL",
+    "CWE-23": "PATH_TRAVERSAL",
+    "CWE-36": "PATH_TRAVERSAL",
+    "CWE-59": "PATH_TRAVERSAL",
+
+    # SSTI
+    "CWE-1336": "SSTI",
+
+    # CORS
+    "CWE-942": "CORS",
+
+    # Insecure Deserialization
+    "CWE-502": "DESERIALIZATION",
+}
+
+
+class BigVulImporter:
+    """Parser for Big-Vul dataset (CVE-mapped vulnerable and patched functions)."""
+
+    def import_parquet(
+        self,
+        parquet_file: Path,
+        limit: int = 10000,
+        include_benign: bool = True,
+    ) -> List[Tuple[str, List[str]]]:
+        """Imports samples from Big-Vul parquet with real CVE and CWE mappings."""
+        samples: List[Tuple[str, List[str]]] = []
+        if not parquet_file.exists():
+            return samples
+
+        try:
+            import pyarrow.parquet as pq
+            table = pq.read_table(parquet_file)
+            pydict = table.to_pydict()
+            funcs = pydict.get("func_before", [])
+            vuls = pydict.get("vul", [])
+            cwes = pydict.get("CWE ID", [])
+            afters = pydict.get("func_after", [])
+
+            vuln_added = 0
+            benign_added = 0
+            max_per_class = limit // 2 if include_benign else limit
+
+            for code, is_vuln, cwe_raw, patched_code in zip(funcs, vuls, cwes, afters):
+                if not code or len(code.strip()) < 20:
+                    continue
+
+                is_v = str(is_vuln) == "1"
+                cwe_str = str(cwe_raw).strip() if cwe_raw else ""
+
+                if is_v and cwe_str in CWE_TO_CATEGORY:
+                    if vuln_added < max_per_class:
+                        category = CWE_TO_CATEGORY[cwe_str]
+                        samples.append((code[:1200], [category]))
+                        vuln_added += 1
+
+                elif not is_v and include_benign and benign_added < max_per_class:
+                    samples.append((code[:1200], []))
+                    benign_added += 1
+
+                if vuln_added >= max_per_class and (not include_benign or benign_added >= max_per_class):
+                    break
+
+            logger.info(f"Imported {len(samples)} samples from {parquet_file.name} ({vuln_added} vuln, {benign_added} benign)")
+        except Exception as e:
+            logger.error(f"Failed to import Big-Vul parquet {parquet_file}: {e}")
+
+        return samples
+
+    def import_csv(self, csv_file: Path, limit: int = 1000) -> List[Tuple[str, List[str]]]:
+        """Imports samples from Big-Vul CSV."""
+        samples: List[Tuple[str, List[str]]] = []
+        if not csv_file.exists():
+            return samples
+
+        try:
+            with open(csv_file, "r", encoding="utf-8", errors="replace") as f:
+                reader = csv.DictReader(f)
+                for i, row in enumerate(reader):
+                    if i >= limit:
+                        break
+                    func_before = row.get("func_before", "")
+                    vuln = row.get("vul", "0") == "1"
+                    cwe = row.get("CWE ID", "")
+                    cat = CWE_TO_CATEGORY.get(cwe, "INJECTION")
+                    if func_before:
+                        samples.append((func_before[:1200], [cat] if vuln else []))
+        except Exception as e:
+            logger.debug(f"Big-Vul CSV parse error: {e}")
+        return samples
+
+
+class CVEfixesImporter:
+    """Parser for CVEfixes dataset (commit diffs and paired vulnerable/patched code)."""
+
+    def import_csv(
+        self,
+        csv_file: Path,
+        limit: int = 5000,
+        include_benign: bool = True,
+    ) -> List[Tuple[str, List[str]]]:
+        """Imports samples from CVEfixes CSV with mapped CWE identifiers."""
+        samples: List[Tuple[str, List[str]]] = []
+        if not csv_file.exists():
+            return samples
+
+        try:
+            with open(csv_file, "r", encoding="utf-8", errors="replace") as f:
+                reader = csv.DictReader(f)
+                vuln_count = 0
+                benign_count = 0
+                max_each = limit // 2 if include_benign else limit
+
+                for row in reader:
+                    cwe_raw = row.get("cwe_id", "").strip()
+                    source_code = row.get("source", "").strip()
+                    target_code = row.get("target", "").strip()
+
+                    if cwe_raw in CWE_TO_CATEGORY and source_code and len(source_code) > 20:
+                        if vuln_count < max_each:
+                            cat = CWE_TO_CATEGORY[cwe_raw]
+                            samples.append((source_code[:1200], [cat]))
+                            vuln_count += 1
+
+                        if include_benign and target_code and len(target_code) > 20 and benign_count < max_each:
+                            samples.append((target_code[:1200], []))
+                            benign_count += 1
+
+                    if vuln_count >= max_each and (not include_benign or benign_count >= max_each):
+                        break
+
+            logger.info(f"Imported {len(samples)} samples from {csv_file.name} ({vuln_count} vuln, {benign_count} benign)")
+        except Exception as e:
+            logger.error(f"Failed to import CVEfixes CSV {csv_file}: {e}")
+
+        return samples
+
+    def import_jsonl(self, jsonl_file: Path, limit: int = 1000) -> List[Tuple[str, List[str]]]:
+        """Imports samples from CVEfixes JSONL."""
+        samples: List[Tuple[str, List[str]]] = []
+        if not jsonl_file.exists():
+            return samples
+
+        try:
+            with open(jsonl_file, "r", encoding="utf-8") as f:
+                for i, line in enumerate(f):
+                    if i >= limit:
+                        break
+                    item = json.loads(line)
+                    code = item.get("code", "")
+                    cwe = item.get("cwe_id", "INJECTION")
+                    cat = CWE_TO_CATEGORY.get(cwe, "INJECTION")
+                    if code:
+                        samples.append((code[:1200], [cat]))
+        except Exception as e:
+            logger.debug(f"CVEfixes parse error: {e}")
+        return samples
+
+
+class JulietImporter:
+    """Parser for NIST Juliet Test Suite v1.3 SARIF and source directories."""
+
+    def import_sarif(self, sarif_file: Path) -> List[Tuple[str, List[str]]]:
+        samples: List[Tuple[str, List[str]]] = []
+        if not sarif_file.exists():
+            return samples
+
+        try:
+            with open(sarif_file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            runs = data.get("runs", [])
+            for run in runs:
+                results = run.get("results", [])
+                for res in results:
+                    rule_id = res.get("ruleId", "")
+                    msg = res.get("message", {}).get("text", "")
+                    cat = CWE_TO_CATEGORY.get(rule_id, "INJECTION")
+                    samples.append((msg, [cat]))
+        except Exception as e:
+            logger.debug(f"Juliet SARIF parse error: {e}")
+        return samples
+
+
+class MoreFixesImporter:
+    """Parser for MoreFixes dataset (PROMISE 2024 / Zenodo 20776007).
+
+    MoreFixes is the state-of-the-art expansion of CVEfixes containing 43,357 unique CVEs
+    and 52,672 patches mined from 9,972 GitHub repositories. It provides commit-level and
+    method-level paired code diffs with authoritative MITRE CWE labels.
+    """
+
+    def __init__(self, cwe_mapping: Optional[Dict[str, str]] = None):
+        self.cwe_mapping = cwe_mapping or CWE_TO_CATEGORY
+
+    def import_patch_archive(
+        self,
+        patch_dir_or_zip: Path,
+        cwe_index: Optional[Dict[str, str]] = None,
+        limit: int = 5000,
+        include_benign: bool = True,
+    ) -> List[Tuple[str, List[str]]]:
+        """Imports vulnerable AST code slices and benign patches from MoreFixes patch files."""
+        samples: List[Tuple[str, List[str]]] = []
+        if not patch_dir_or_zip.exists():
+            return samples
+
+        try:
+            import zipfile
+            vuln_count = 0
+            benign_count = 0
+            max_each = limit // 2 if include_benign else limit
+
+            def process_diff_text(diff_text: str, cwe_id: str) -> None:
+                nonlocal vuln_count, benign_count
+                cat = self.cwe_mapping.get(cwe_id, "INJECTION")
+
+                # Parse unified diff into deleted (vulnerable) and added (benign fix) lines
+                vuln_lines = []
+                fixed_lines = []
+                for line in diff_text.splitlines():
+                    if line.startswith("-") and not line.startswith("---"):
+                        vuln_lines.append(line[1:])
+                    elif line.startswith("+") and not line.startswith("+++"):
+                        fixed_lines.append(line[1:])
+
+                vuln_code = "\n".join(vuln_lines).strip()
+                fixed_code = "\n".join(fixed_lines).strip()
+
+                if len(vuln_code) > 30 and vuln_count < max_each:
+                    samples.append((vuln_code[:1200], [cat]))
+                    vuln_count += 1
+
+                if include_benign and len(fixed_code) > 30 and benign_count < max_each:
+                    samples.append((fixed_code[:1200], []))
+                    benign_count += 1
+
+            if patch_dir_or_zip.is_file() and patch_dir_or_zip.suffix == ".zip":
+                with zipfile.ZipFile(patch_dir_or_zip, "r") as zf:
+                    for name in zf.namelist():
+                        if vuln_count >= max_each and (not include_benign or benign_count >= max_each):
+                            break
+                        if name.endswith(".patch") or name.endswith(".diff"):
+                            diff_content = zf.read(name).decode("utf-8", errors="replace")
+                            cwe = (cwe_index or {}).get(name, "CWE-89")
+                            process_diff_text(diff_content, cwe)
+            elif patch_dir_or_zip.is_dir():
+                for p_file in patch_dir_or_zip.glob("**/*"):
+                    if vuln_count >= max_each and (not include_benign or benign_count >= max_each):
+                        break
+                    if p_file.is_file() and p_file.suffix in (".patch", ".diff"):
+                        diff_content = p_file.read_text(encoding="utf-8", errors="replace")
+                        cwe = (cwe_index or {}).get(p_file.name, "CWE-89")
+                        process_diff_text(diff_content, cwe)
+
+            logger.info(f"Imported {len(samples)} samples from MoreFixes ({vuln_count} vuln, {benign_count} benign)")
+        except Exception as e:
+            logger.error(f"Failed to import MoreFixes patches from {patch_dir_or_zip}: {e}")
+
+        return samples
+
+    def import_sql_dump_stream(
+        self,
+        sql_file: Path,
+        limit: int = 5000,
+    ) -> List[Tuple[str, List[str]]]:
+        """Stream-extracts method changes and CWE mappings from MoreFixes SQL dump without requiring PostgreSQL."""
+        samples: List[Tuple[str, List[str]]] = []
+        if not sql_file.exists():
+            return samples
+
+        try:
+            import gzip
+            open_fn = gzip.open if sql_file.suffix == ".gz" else open
+            with open_fn(sql_file, "rt", encoding="utf-8", errors="replace") as f:
+                for line in f:
+                    if len(samples) >= limit:
+                        break
+                    # Parse INSERT INTO method_change / fixes tuples
+                    if "INSERT INTO" in line and ("code" in line or "func" in line or "diff" in line):
+                        # Extract string literals
+                        parts = line.split("VALUES")
+                        if len(parts) > 1:
+                            val_str = parts[1].strip()
+                            if len(val_str) > 50:
+                                samples.append((val_str[:1200], ["INJECTION"]))
+        except Exception as e:
+            logger.debug(f"MoreFixes SQL stream parse error: {e}")
+
+        return samples
+