eduevidence 5.2.0

package/engine/citation_check.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+"""citation_check — per-citation DOI verification against registries (plan E6).
+
+Turns the v5.2.0 integrity audit into a first-class engine capability:
+
+- every Source/Evidence citation can carry `doi_verified` and `retracted`
+  fields, produced by this module against Crossref (most DOIs) and DataCite
+  (arXiv 10.48550/*);
+- verification is evidence-grade: it never guesses. A DOI that does not
+  resolve is `not_found`; a resolved record whose title clearly differs from
+  the local title is `mismatch`; a registered retraction is surfaced as
+  `retracted=True`.
+
+Security: identical discipline to scripts/audit_dois.py — strict DOI shape,
+two allow-listed HTTPS registry hosts, redirects confined to those hosts.
+
+Stdlib only. Network calls happen only in `verify()`/`check_doi`;
+`classify()` is pure and fully unit-testable offline.
+"""
+
+from __future__ import annotations
+
+import http.client
+import json
+import re
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any
+
+CROSSREF_API = "https://api.crossref.org/works/{doi}"
+DATACITE_API = "https://api.datacite.org/dois/{doi}"
+ALLOWED_HOSTS = {"api.crossref.org", "api.datacite.org"}
+DATACITE_PREFIXES = {"10.48550"}
+MAILTO = "citation-check@eduevidence.local"
+
+STRICT_DOI_RE = re.compile(r"^10\.\d{4,9}/[A-Za-z0-9._()/:-]+$")
+DOI_CLEAN_RE = re.compile(r"\b(10\.\d{4,9}/[A-Za-z0-9._()/:;-]+)")
+STRIP_SUFFIX = ".,;)"
+
+TITLE_OVERLAP_THRESHOLD = 0.5
+
+# Classification vocabulary used by callers and schemas.
+STATUS_OK = "ok"
+STATUS_MISMATCH = "mismatch"
+STATUS_NOT_FOUND = "not_found"
+STATUS_ERROR = "error"
+
+
+def clean_doi(doi: str) -> str:
+    return doi.strip().rstrip(STRIP_SUFFIX)
+
+
+def token_set(text: str) -> set[str]:
+    out: set[str] = set()
+    for tok in re.split(r"[^a-z0-9]+", text.lower()):
+        if len(tok) > 2:
+            out.add(tok)
+            if tok.endswith("s") and len(tok) > 3:
+                out.add(tok[:-1])
+    return out
+
+
+def title_overlap(a: str | None, b: str | None) -> float | None:
+    if not a or not b:
+        return None
+    ta, tb = token_set(a), token_set(b)
+    if not ta or not tb:
+        return None
+    return len(ta & tb) / min(len(ta), len(tb))
+
+
+def classify(doi: str, registry_status: str, registry_msg: dict[str, Any] | None,
+             local_title: str | None) -> dict[str, Any]:
+    """Pure classifier: registry outcome + optional local title → verdict dict."""
+    cr_title = None
+    if isinstance(registry_msg, dict):
+        titles = registry_msg.get("title") or []
+        cr_title = titles[0] if titles else None
+    status = registry_status
+    similarity = title_overlap(local_title, cr_title)
+    if status == STATUS_OK and similarity is not None and similarity < TITLE_OVERLAP_THRESHOLD:
+        status = STATUS_MISMATCH
+
+    retracted = False
+    if isinstance(registry_msg, dict) and status == STATUS_OK:
+        updates = registry_msg.get("update-to") or []
+        if isinstance(updates, list):
+            retracted = any(
+                isinstance(u, dict) and str(u.get("type", "")).lower() == "retraction"
+                for u in updates)
+        if not retracted:
+            # Some deposits mark retraction in the title itself ("Retracted: ...").
+            t = (cr_title or "").strip().lower()
+            retracted = t.startswith("retracted") or "(retracted" in t
+
+    result: dict[str, Any] = {
+        "doi": doi,
+        "status": status,
+        "doi_verified": status == STATUS_OK,
+        "retracted": retracted,
+        "registry_title": cr_title,
+    }
+    if similarity is not None:
+        result["title_overlap"] = round(similarity, 3)
+    return result
+
+
+class _SameHostRedirectHandler(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        if urllib.parse.urlparse(newurl).hostname not in ALLOWED_HOSTS:
+            return None
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+class RegistryClient:
+    """Small cached HTTP client bound to the two registry hosts."""
+
+    def __init__(self, timeout_s: int = 20, sleep_s: float = 0.25):
+        self.timeout_s = timeout_s
+        self.sleep_s = sleep_s
+        self.cache: dict[str, tuple[str, dict | None]] = {}
+        self.opener = urllib.request.build_opener(_SameHostRedirectHandler)
+
+    def fetch_raw(self, doi: str) -> tuple[str, dict | None]:
+        """Return (registry_status, payload_message) for a cleaned DOI."""
+        if not STRICT_DOI_RE.match(doi):
+            return (STATUS_ERROR, {"detail": f"strict shape check failed: {doi!r}"})
+        template = DATACITE_API if doi.split("/", 1)[0] in DATACITE_PREFIXES else CROSSREF_API
+        url = template.format(doi=urllib.request.quote(doi, safe=""))
+        req = urllib.request.Request(url, headers={
+            "User-Agent": f"EduEvidence-citation-check/1.0 (mailto:{MAILTO})"})
+        result: tuple[str, dict | None] = (STATUS_ERROR, None)
+        for attempt in range(2):
+            try:
+                with self.opener.open(req, timeout=self.timeout_s) as resp:
+                    payload = json.loads(resp.read().decode("utf-8"))
+                message = payload.get("message", payload.get("data", {}))
+                if isinstance(message, dict) and "attributes" in message:
+                    attrs = message["attributes"]
+                    titles = [t.get("title") for t in attrs.get("titles", []) if t.get("title")]
+                    message = {"title": titles}
+                result = (STATUS_OK, message)
+                break
+            except urllib.error.HTTPError as exc:
+                result = ((STATUS_NOT_FOUND, {"status": exc.code}) if exc.code == 404
+                          else (STATUS_ERROR, {"status": exc.code}))
+                break
+            except (urllib.error.URLError, http.client.RemoteDisconnected, ConnectionError,
+                    TimeoutError, OSError, json.JSONDecodeError) as exc:
+                result = (STATUS_ERROR, {"detail": str(exc)})
+                if attempt == 0:
+                    time.sleep(1.5)
+        time.sleep(self.sleep_s)
+        return result
+
+    def check(self, doi: str, local_title: str | None = None) -> dict[str, Any]:
+        doi = clean_doi(doi)
+        cache_key = (doi, local_title or "")
+        if cache_key not in self.cache:
+            status, msg = self.fetch_raw(doi)
+            self.cache[cache_key] = classify(doi, status, msg, local_title)
+        return self.cache[cache_key]
+
+
+def extract_dois(record: dict[str, Any]) -> list[tuple[str, str | None]]:
+    """Pull (doi, paired_title) pairs from a flat artifact record."""
+    pairs: list[tuple[str, str | None]] = []
+    doi_val = None
+    title_val = None
+    for key, val in record.items():
+        if not isinstance(val, str):
+            continue
+        k = key.strip().lower()
+        if k.endswith("_doi") or k == "doi":
+            d = clean_doi(val)
+            if STRICT_DOI_RE.match(d):
+                doi_val = d
+        elif k == "title" or k == "source_title":
+            title_val = val
+    if doi_val:
+        pairs.append((doi_val, title_val))
+    else:
+        for key, val in record.items():
+            if isinstance(val, str):
+                for raw in DOI_CLEAN_RE.findall(val):
+                    d = clean_doi(raw)
+                    if STRICT_DOI_RE.match(d):
+                        pairs.append((d, title_val))
+                        break
+    return pairs

package/engine/contracts.py

@@ -0,0 +1,117 @@
+"""V2 schema registry and validation helpers.
+
+Reuses the repository's existing zero-dependency Draft-07 validator
+(`scripts.validate_schema`) — the engine layer never prints; errors are
+returned as stable strings.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable
+
+from scripts.validate_schema import SchemaError, validate
+
+_REPO_SCHEMA_DIR = Path(__file__).resolve().parent.parent / "schemas" / "v2"
+
+
+def _resolve_schema_dir() -> Path:
+    """Repository layout first; wheel-installed share/ layout as fallback."""
+    if _REPO_SCHEMA_DIR.is_dir():
+        return _REPO_SCHEMA_DIR
+    import sys
+    share = Path(sys.prefix) / "share" / "eduevidence" / "schemas" / "v2"
+    if share.is_dir():
+        return share
+    return _REPO_SCHEMA_DIR
+
+
+SCHEMA_DIR = _resolve_schema_dir()
+
+# name -> schema file name (frozen registry)
+_REGISTRY: dict[str, str] = {
+    "research-intent": "research-intent.schema.json",
+    "project": "project.schema.json",
+    "run": "run.schema.json",
+    "source": "source.schema.json",
+    "study": "study.schema.json",
+    "finding": "finding.schema.json",
+    "outcome": "outcome.schema.json",
+    "claim": "claim.schema.json",
+    "evidence-link": "evidence-link.schema.json",
+    "methodology-audit": "methodology-audit.schema.json",
+    "graph-revision": "graph-revision.schema.json",
+    "knowledge-gap": "knowledge-gap.schema.json",
+    "study-design": "study-design.schema.json",
+    "dataset-asset": "dataset-asset.schema.json",
+    "analysis-plan": "analysis-plan.schema.json",
+    "analysis-run": "analysis-run.schema.json",
+    "decision-snapshot": "decision-snapshot.schema.json",
+}
+
+_cache: dict[str, dict] = {}
+
+
+def schema_path(name: str) -> Path:
+    """Resolve a registered schema name to its file; raise FileNotFoundError."""
+    if name not in _REGISTRY:
+        raise FileNotFoundError(f"unknown V2 schema name {name!r}")
+    return SCHEMA_DIR / _REGISTRY[name]
+
+
+def load_schema(name: str) -> dict:
+    """Load (cached) the schema document for a registered name."""
+    if name not in _cache:
+        _cache[name] = __import__("json").loads(schema_path(name).read_text(encoding="utf-8"))
+    return _cache[name]
+
+
+def reload_schemas() -> None:
+    """Drop the schema cache so later loads re-read schema files.
+
+    Needed after schema files change at runtime (e.g. tests that mutate a
+    schema to prove briefs are schema-derived). Use this instead of reaching
+    into the private `_cache`.
+    """
+    _cache.clear()
+
+
+def validate_record(name: str, record: dict) -> list[str]:
+    """Validate `record` against the named V2 schema.
+
+    Returns a list of stable error strings (empty == valid). Errors are
+    collected by re-validating per-property so a single record yields all
+    violations, not just the first.
+    """
+    if name not in _REGISTRY:
+        raise FileNotFoundError(f"unknown V2 schema name {name!r}")
+    schema = load_schema(name)
+    errors: list[str] = []
+    try:
+        validate(record, schema)
+    except SchemaError as exc:
+        errors.append(str(exc))
+        _collect_property_errors(record, schema, errors)
+    return errors
+
+
+def _collect_property_errors(record: dict, schema: dict, errors: list[str]) -> None:
+    """Best-effort per-property error collection after a whole-record failure.
+
+    Individual-property validation keeps errors stable and granular without
+    changing the validator's semantics.
+    """
+    props = schema.get("properties", {})
+    for key, prop_schema in props.items():
+        if key in record:
+            try:
+                validate(record[key], prop_schema, f"$.{key}")
+            except SchemaError as exc:
+                if str(exc) not in errors:
+                    errors.append(str(exc))
+    required = schema.get("required", [])
+    for key in required:
+        if key not in record:
+            msg = f"$: required property {key!r} is missing"
+            if msg not in errors:
+                errors.append(msg)

package/engine/datasets.py

@@ -0,0 +1,165 @@
+"""Immutable DatasetAsset ingest + privacy gate.
+
+Raw bytes are copied once into datasets/raw/<dataset_id>/ and SHA-256 hashed;
+re-ingesting identical bytes deduplicates by hash inside the same Project.
+Raw user data never enters the Shared Research Library. Missing privacy
+classification fails the gate; deidentification_required=True with
+deidentification_status=not_done blocks analysis. Raw files are never
+mutated after ingest.
+"""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import json
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+
+from engine.contracts import validate_record
+from engine.ids import new_local_id
+from engine.project import ProjectWorkspace
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def derive_csv_profile(path: Path) -> dict:
+    """Row/column count + per-column missingness without pandas."""
+    columns: list[str] | None = None
+    row_count = 0
+    missing: dict[str, int] = {}
+    with path.open("r", encoding="utf-8-sig", newline="") as f:
+        reader = csv.reader(f)
+        for i, row in enumerate(reader):
+            if i == 0:
+                columns = row
+                for c in columns:
+                    missing[c] = 0
+                continue
+            row_count += 1
+            if columns is not None:
+                for idx, c in enumerate(columns):
+                    if idx >= len(row) or row[idx].strip() == "":
+                        missing[c] += 1
+    if columns is None:
+        return {"row_count": 0, "column_count": 0, "missingness": {}}
+    return {
+        "row_count": row_count,
+        "column_count": len(columns),
+        "missingness": {c: missing[c] for c in columns},
+    }
+
+
+def validate_dataset_asset(project: ProjectWorkspace, asset: dict) -> list[str]:
+    """Privacy gate + schema + project locality."""
+    errors: list[str] = []
+    schema_errors = validate_record("dataset-asset", asset)
+    if schema_errors:
+        errors.extend(schema_errors)
+        return errors
+    if asset.get("project_id") != project.project_id:
+        errors.append(
+            f"asset project_id {asset.get('project_id')} != workspace "
+            f"{project.project_id}")
+    return errors
+
+
+def ingest_dataset(project: ProjectWorkspace, *, design_id: str,
+                   source_path: Path, privacy: dict,
+                   variable_dictionary: dict[str, str] | None = None) -> dict:
+    """Ingest raw dataset bytes immutably; returns the DatasetAsset dict."""
+    source_path = Path(source_path).expanduser().resolve()
+    if not source_path.is_file():
+        raise FileNotFoundError(f"dataset source missing: {source_path}")
+
+    # privacy gate: classification required
+    classification = privacy.get("classification")
+    if classification not in ("public", "internal", "confidential", "restricted"):
+        raise ValueError(
+            f"missing/invalid privacy classification {classification!r}; "
+            f"dataset ingest blocked")
+
+    content_hash = _sha256(source_path)
+
+    # dedupe by hash inside this project; a stricter classification request
+    # must never silently rebind to a weaker stored classification
+    raw_dir = project.path / "datasets" / "raw"
+    for existing in sorted(raw_dir.glob("DAT-*")):
+        manifest = existing / "manifest.json"
+        if manifest.is_file():
+            rec = json.loads(manifest.read_text(encoding="utf-8"))
+            if rec.get("content_hash") == content_hash:
+                stored = rec.get("privacy_classification")
+                order = ("public", "internal", "confidential", "restricted")
+                if order.index(classification) > order.index(stored):
+                    raise ValueError(
+                        f"dataset {content_hash[:10]}… already stored with "
+                        f"weaker classification {stored!r}; re-ingesting as "
+                        f"{classification!r} is refused"
+                    )
+                return rec
+
+    dataset_id = new_local_id("DAT", {
+        p.name for p in raw_dir.iterdir() if p.is_dir()})
+    dest_dir = raw_dir / dataset_id
+    dest_dir.mkdir(parents=True, exist_ok=True)
+
+    # copy bytes once
+    dest_raw = dest_dir / source_path.name
+    shutil.copy2(source_path, dest_raw)
+
+    profile = derive_csv_profile(dest_raw)
+    asset = {
+        "dataset_id": dataset_id,
+        "project_id": project.project_id,
+        "design_id": design_id,
+        "source_type": source_path.suffix.lstrip(".").lower() or "unknown",
+        "path": str(dest_raw),
+        "content_hash": content_hash,
+        "schema_summary": {"columns": profile["column_count"]},
+        "row_count": profile["row_count"],
+        "column_count": profile["column_count"],
+        "variable_dictionary": variable_dictionary,
+        "privacy_classification": classification,
+        "consent_metadata": privacy.get("consent_metadata"),
+        "deidentification_status": privacy.get("deidentification_status", "not_done"),
+        "created_at": _now_iso(),
+        "extensions": {
+            "missingness": profile["missingness"],
+            "deidentification_required": bool(
+                privacy.get("deidentification_required", False)),
+        },
+    }
+    errors = validate_dataset_asset(project, asset)
+    if errors:
+        shutil.rmtree(dest_dir, ignore_errors=True)
+        raise ValueError("dataset asset invalid:\n- " + "\n- ".join(errors))
+
+    # atomic manifest write
+    tmp = dest_dir / "manifest.json.tmp"
+    tmp.write_text(json.dumps(asset, ensure_ascii=False, indent=2) + "\n",
+                   encoding="utf-8")
+    tmp.replace(dest_dir / "manifest.json")
+    return asset
+
+
+def analysis_blocked_by_privacy(asset: dict) -> list[str]:
+    """Return blocking reasons if analysis must not proceed."""
+    reasons: list[str] = []
+    if asset.get("deidentification_status") == "not_done" and (
+            asset.get("extensions") or {}).get("deidentification_required"):
+        reasons.append(
+            "deidentification_required but deidentification_status=not_done; "
+            "analysis blocked until deidentified")
+    return reasons

package/engine/events.py

@@ -0,0 +1,67 @@
+"""engine/events.py — Real-time thread-safe EventBus for EduEvidence.
+
+Provides event publishing and subscription across orchestrator, engine modules,
+and dashboard SSE streams with bounded ring-buffer history.
+"""
+from __future__ import annotations
+
+import collections
+import json
+import threading
+import time
+from typing import Any, Callable, Deque, Dict, List, Optional
+
+
+class EventBus:
+    _instance: Optional[EventBus] = None
+    _lock = threading.Lock()
+
+    def __new__(cls) -> EventBus:
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = super(EventBus, cls).__new__(cls)
+                cls._instance._subscribers = []
+                cls._instance._history = collections.deque(maxlen=500)
+                cls._instance._bus_lock = threading.RLock()
+            return cls._instance
+
+    def subscribe(self, callback: Callable[[Dict[str, Any]], None]) -> None:
+        with self._bus_lock:
+            if callback not in self._subscribers:
+                self._subscribers.append(callback)
+
+    def unsubscribe(self, callback: Callable[[Dict[str, Any]], None]) -> None:
+        with self._bus_lock:
+            if callback in self._subscribers:
+                self._subscribers.remove(callback)
+
+    def publish(self, event_type: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+        event = {
+            "type": event_type,
+            "payload": payload,
+            "timestamp": time.time(),
+            "iso_time": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        }
+        with self._bus_lock:
+            self._history.append(event)
+            subs = list(self._subscribers)
+            
+        for sub in subs:
+            try:
+                sub(event)
+            except Exception:
+                pass
+        return event
+
+    def get_history(self, event_type: Optional[str] = None) -> List[Dict[str, Any]]:
+        with self._bus_lock:
+            if event_type:
+                return [e for e in self._history if e["type"] == event_type]
+            return list(self._history)
+
+    def clear(self) -> None:
+        with self._bus_lock:
+            self._history.clear()
+
+
+event_bus = EventBus()