mednotes-opencode 0.1.0

package/.opencode/mednotes/src/mednotes/domains/wiki/capabilities/notes/raw_chats.py

@@ -0,0 +1,273 @@
+"""Raw chat frontmatter and filesystem helpers."""
+from __future__ import annotations
+
+import errno
+import json
+import os
+import re
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+from pydantic import ConfigDict
+
+from mednotes.domains.wiki.capabilities.notes.note_plan import note_plan_summary, parse_triage_note_plan
+from mednotes.domains.wiki.capabilities.notes.note_style.frontmatter import (
+    FrontmatterYamlUnavailable,
+    load_frontmatter_yaml,
+)
+from mednotes.domains.wiki.common import FileWriteError, MissingPathError, ValidationError
+from mednotes.kernel.base import ContractModel
+from mednotes.platform.backup_policy import (
+    DEFAULT_BACKUP_POLICY,
+    BackupPolicy,
+    cleanup_backups,
+    policy_from_values,
+)
+
+_FRONTMATTER_DELIM = "---"
+_KEY_RE = re.compile(r"^([A-Za-z0-9_-]+)\s*:\s*(.*)$")
+_ATOMIC_WRITE_RETRY_DELAYS = (0.05, 0.1, 0.2, 0.4, 0.8)
+_WINDOWS_LOCK_WINERRORS = {5, 32, 33}
+
+
+class _RawSummaryDecisionFields(ContractModel):
+    """Typed subset of raw chat summary fields used for status filtering."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True, validate_assignment=True)
+
+    status: str = ""
+    tipo: str = ""
+    chat_id: str = ""
+
+
+def split_frontmatter(text: str) -> tuple[list[str] | None, str]:
+    lines = text.splitlines(keepends=True)
+    if not lines or lines[0].strip() != _FRONTMATTER_DELIM:
+        return None, text
+    for idx in range(1, len(lines)):
+        if lines[idx].strip() == _FRONTMATTER_DELIM:
+            return lines[1:idx], "".join(lines[idx + 1 :])
+    return None, text
+
+
+def _strip_quotes(value: str) -> str:
+    value = value.strip()
+    if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+        try:
+            parsed = json.loads(value)
+            if isinstance(parsed, str):
+                return parsed
+        except json.JSONDecodeError:
+            pass
+        return value[1:-1]
+    return value
+
+
+def parse_frontmatter(text: str) -> dict[str, str]:
+    frontmatter, _body = split_frontmatter(text)
+    if frontmatter is None:
+        return {}
+    parsed: dict[str, str] = {}
+    for line in frontmatter:
+        match = _KEY_RE.match(line.strip())
+        if match:
+            parsed[match.group(1)] = _strip_quotes(match.group(2))
+    return parsed
+
+
+def _format_yaml_value(value: str) -> str:
+    if value == "":
+        return '""'
+    if re.match(r"^[A-Za-z0-9_./@+-]+$", value):
+        return value
+    return json.dumps(value, ensure_ascii=False)
+
+
+def update_frontmatter(text: str, updates: dict[str, str]) -> str:
+    frontmatter, body = split_frontmatter(text)
+    formatted = {key: f"{key}: {_format_yaml_value(value)}\n" for key, value in updates.items()}
+    if frontmatter is None:
+        return "---\n" + "".join(formatted.values()) + "---\n" + text
+
+    seen: set[str] = set()
+    out: list[str] = []
+    for line in frontmatter:
+        match = _KEY_RE.match(line.strip())
+        if match and match.group(1) in formatted:
+            key = match.group(1)
+            out.append(formatted[key])
+            seen.add(key)
+        else:
+            out.append(line)
+    for key, line in formatted.items():
+        if key not in seen:
+            out.append(line)
+    return "---\n" + "".join(out) + "---\n" + body
+
+
+def read_note_meta(path: Path) -> dict[str, str]:
+    try:
+        return parse_frontmatter(path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise MissingPathError(f"File not found: {path}") from exc
+
+
+def create_backup(path: Path, *, policy: BackupPolicy = DEFAULT_BACKUP_POLICY) -> Path:
+    policy.validate()
+    if not path.exists():
+        raise MissingPathError(f"File not found: {path}")
+    raise RuntimeError("Markdown .bak backups are retired; use vault version control restore points.")
+
+
+def prune_backup_files(
+    root: Path,
+    *,
+    max_per_file: int | None = None,
+    retention_days: int | None = None,
+    policy: BackupPolicy = DEFAULT_BACKUP_POLICY,
+) -> dict[str, Any]:
+    resolved_policy = policy_from_values(
+        max_per_file=max_per_file,
+        retention_days=retention_days,
+        policy=policy,
+    )
+    try:
+        return cleanup_backups(root, policy=resolved_policy)
+    except FileNotFoundError as exc:
+        raise MissingPathError(str(exc)) from exc
+    except ValueError as exc:
+        raise ValidationError(str(exc)) from exc
+
+
+def _is_retryable_replace_error(exc: OSError) -> bool:
+    if isinstance(exc, PermissionError):
+        return True
+    if getattr(exc, "winerror", None) in _WINDOWS_LOCK_WINERRORS:
+        return True
+    return getattr(exc, "errno", None) in {errno.EACCES, errno.EPERM, errno.EBUSY}
+
+
+def _replace_with_retries(path: Path, tmp: Path, retry_delays: tuple[float, ...]) -> None:
+    attempts = len(retry_delays) + 1
+    last_error: OSError | None = None
+    for attempt_idx in range(attempts):
+        try:
+            os.replace(tmp, path)
+            return
+        except OSError as exc:
+            last_error = exc
+            if attempt_idx >= len(retry_delays) or not _is_retryable_replace_error(exc):
+                break
+            time.sleep(retry_delays[attempt_idx])
+
+    raise FileWriteError(
+        f"Could not atomically replace {path} after {attempts} attempts. "
+        f"The file may be locked by Obsidian, iCloud Drive, antivirus, or another process. "
+        f"Original error: {last_error}"
+    ) from last_error
+
+
+def atomic_write_text(path: Path, text: str, *, retry_delays: tuple[float, ...] | None = None) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent))
+    tmp = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8", newline="") as fh:
+            fh.write(text)
+        _replace_with_retries(path, tmp, _ATOMIC_WRITE_RETRY_DELAYS if retry_delays is None else retry_delays)
+    finally:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except OSError:
+                pass
+
+
+def mutate_raw_frontmatter(raw_file: Path, updates: dict[str, str], dry_run: bool = False, backup: bool = False) -> dict[str, Any]:
+    if not raw_file.exists():
+        raise MissingPathError(f"Raw file not found: {raw_file}")
+    original = raw_file.read_text(encoding="utf-8")
+    updated = update_frontmatter(original, updates)
+    if dry_run:
+        return {"raw_file": str(raw_file), "backup": None, "updated": False, "updates": updates}
+    atomic_write_text(raw_file, updated)
+    return {"raw_file": str(raw_file), "backup": None, "updated": True, "updates": updates}
+
+
+def list_raw_files(raw_dir: Path) -> list[Path]:
+    if not raw_dir.exists():
+        raise MissingPathError(f"Raw dir not found: {raw_dir}")
+    return sorted(path for path in raw_dir.glob("*.md") if path.is_file())
+
+
+def raw_summary(path: Path) -> dict[str, str]:
+    meta = read_note_meta(path)
+    chat_id = meta.get("fonte_id", "").strip()
+    result = {
+        "path": str(path),
+        "status": meta.get("status", ""),
+        "tipo": meta.get("tipo", ""),
+        "titulo_triagem": meta.get("titulo_triagem", ""),
+        "fonte_id": meta.get("fonte_id", ""),
+        "chat_id": chat_id,
+        "title": meta.get("titulo_triagem", ""),
+        "url": f"https://gemini.google.com/app/{chat_id}" if chat_id else "",
+        "date_created": meta.get("date_created", ""),
+        "exported_at": meta.get("exported_at", ""),
+    }
+    raw_plan = meta.get("note_plan", "")
+    if raw_plan:
+        try:
+            result.update({key: str(value) for key, value in note_plan_summary(parse_triage_note_plan(raw_plan, path)).items()})
+        except ValidationError as exc:
+            result["note_plan_error"] = str(exc)
+    return result
+
+
+def covered_raw_chat_index(wiki_dir: Path) -> dict[str, list[str]]:
+    if not wiki_dir.exists():
+        return {}
+    index: dict[str, list[str]] = {}
+    for note_path in sorted(wiki_dir.rglob("*.md")):
+        if not note_path.is_file():
+            continue
+        try:
+            frontmatter = load_frontmatter_yaml(note_path.read_text(encoding="utf-8"))
+        except (OSError, FrontmatterYamlUnavailable):
+            continue
+        raw_chats = frontmatter.get("chats")
+        if not isinstance(raw_chats, list):
+            continue
+        for chat in raw_chats:
+            if not isinstance(chat, dict):
+                continue
+            chat_id = str(chat.get("id") or "").strip()
+            if not chat_id:
+                continue
+            index.setdefault(chat_id, []).append(str(note_path))
+    return index
+
+
+def list_by_status(
+    raw_dir: Path,
+    mode: str,
+    *,
+    covered_raw_chat_ids: set[str] | None = None,
+) -> list[dict[str, str]]:
+    covered_ids = covered_raw_chat_ids or set()
+    rows: list[dict[str, str]] = []
+    for path in list_raw_files(raw_dir):
+        item = raw_summary(path)
+        fields = _RawSummaryDecisionFields.model_validate(item)
+        status = fields.status.lower()
+        tipo = fields.tipo.lower()
+        if mode == "pending" and status in {"", "pendente"}:
+            raw_id = fields.chat_id or path.stem
+            if raw_id in covered_ids:
+                continue
+            rows.append(item)
+        elif mode == "triados" and status == "triado" and tipo == "medicina":
+            rows.append(item)
+    return rows

package/.opencode/mednotes/src/mednotes/domains/wiki/capabilities/notes/sources_backfill.py

@@ -0,0 +1,235 @@
+"""Backfill canonical chat provenance for existing Wiki notes."""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from pydantic import ConfigDict, Field
+
+from mednotes.domains.wiki.capabilities.markdown.markdown_query import MarkdownDbChatMetadataProvider
+from mednotes.domains.wiki.capabilities.notes.note_iter import iter_notes
+from mednotes.domains.wiki.capabilities.notes.note_style.frontmatter import FrontmatterYamlUnavailable
+from mednotes.domains.wiki.capabilities.notes.provenance import (
+    ChatProvenance,
+    apply_note_provenance,
+    audit_note_provenance,
+    classify_note_provenance,
+)
+from mednotes.domains.wiki.capabilities.notes.raw_chats import atomic_write_text
+from mednotes.domains.wiki.capabilities.vocabulary.link_terms import is_index_note as _is_index_note
+from mednotes.domains.wiki.config import MedConfig
+from mednotes.kernel.base import ContractModel, JsonObject
+
+SOURCES_BACKFILL_AUDIT_SCHEMA = "medical-notes-workbench.chats-backfill-audit.v1"
+SOURCES_BACKFILL_RECEIPT_SCHEMA = "medical-notes-workbench.chats-backfill-receipt.v1"
+
+
+class _FallbackChatMetadataProvider:
+    def lookup_chat(self, chat_id: str) -> None:
+        return None
+
+
+class _ChatMetadataProvider(Protocol):
+    """Small provider contract needed by provenance backfill."""
+
+    def lookup_chat(self, chat_id: str) -> object | None: ...
+
+
+class _ProvenanceAuditFields(ContractModel):
+    """Typed view over provenance audit output used by this backfill flow."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True, validate_assignment=True)
+
+    errors: list[JsonObject] = Field(default_factory=list)
+    warnings: list[JsonObject] = Field(default_factory=list)
+    blocked_reason: str = ""
+
+
+class _BackfillReportFields(ContractModel):
+    """Closed operational fields for one note provenance backfill report."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True, validate_assignment=True)
+
+    status: str = ""
+    chat_ids: list[str] = Field(default_factory=list)
+    legacy_urls: list[str] = Field(default_factory=list)
+    warnings: list[JsonObject] = Field(default_factory=list)
+    wrote: bool = Field(default=False, strict=True)
+
+
+class _MetadataLookupFields(ContractModel):
+    """Typed metadata lookup mode used to summarize backfill execution."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True, validate_assignment=True)
+
+    mode: str = ""
+
+
+def audit_sources_backfill(config: MedConfig, *, node_modules_path: Path | None = None) -> dict[str, object]:
+    provider = MarkdownDbChatMetadataProvider(
+        wiki_dir=config.wiki_dir,
+        raw_dir=config.raw_dir,
+        node_modules_path=node_modules_path,
+    )
+    reports = [_note_backfill_report(path, provider) for path in _backfillable_notes(config.wiki_dir)]
+    return _summary_payload(
+        schema=SOURCES_BACKFILL_AUDIT_SCHEMA,
+        config=config,
+        reports=reports,
+        dry_run=True,
+        backup=False,
+        backup_paths=[],
+        write_errors=[],
+        metadata_lookup={"mode": "not_required_for_audit", "skipped_reason": "", "warning": None},
+    )
+
+
+def apply_sources_backfill(
+    config: MedConfig,
+    *,
+    backup: bool,
+    node_modules_path: Path | None = None,
+    metadata_fallback_reason: str = "",
+    metadata_warning: JsonObject | None = None,
+) -> dict[str, object]:
+    backup = False
+    if metadata_fallback_reason:
+        provider: _ChatMetadataProvider = _FallbackChatMetadataProvider()
+        metadata_lookup = {
+            "mode": "fallback",
+            "skipped_reason": metadata_fallback_reason,
+            "warning": metadata_warning,
+        }
+    else:
+        provider = MarkdownDbChatMetadataProvider(
+            wiki_dir=config.wiki_dir,
+            raw_dir=config.raw_dir,
+            node_modules_path=node_modules_path,
+        )
+        metadata_lookup = {"mode": "markdown_query", "skipped_reason": "", "warning": None}
+    reports: list[JsonObject] = []
+    backup_paths: list[str] = []
+    write_errors: list[dict[str, str]] = []
+    for path in _backfillable_notes(config.wiki_dir):
+        report = _note_backfill_report(path, provider)
+        report_fields = _BackfillReportFields.model_validate(report)
+        if report_fields.status != "planned":
+            reports.append(report)
+            continue
+        original = path.read_text(encoding="utf-8")
+        try:
+            updated = _apply_text(original, report, provider)
+            if updated != original:
+                atomic_write_text(path, updated)
+                report = {**report, "status": "written", "wrote": True, "changed": True}
+            else:
+                report = {**report, "status": "already_canonical", "wrote": False, "changed": False}
+        except (FrontmatterYamlUnavailable, OSError) as exc:
+            write_errors.append({"path": str(path), "error": str(exc)})
+            report = {**report, "status": "write_error", "wrote": False, "changed": False, "error": str(exc)}
+        reports.append(report)
+    return _summary_payload(
+        schema=SOURCES_BACKFILL_RECEIPT_SCHEMA,
+        config=config,
+        reports=reports,
+        dry_run=False,
+        backup=backup,
+        backup_paths=backup_paths,
+        write_errors=write_errors,
+        metadata_lookup=metadata_lookup,
+    )
+
+
+def _backfillable_notes(wiki_dir: Path) -> list[Path]:
+    notes: list[Path] = []
+    for path in iter_notes(wiki_dir):
+        try:
+            text = path.read_text(encoding="utf-8")
+        except OSError:
+            notes.append(path)
+            continue
+        if _is_index_note(path, text):
+            continue
+        notes.append(path)
+    return notes
+
+
+def _note_backfill_report(path: Path, provider: _ChatMetadataProvider) -> JsonObject:
+    text = path.read_text(encoding="utf-8")
+    audit = audit_note_provenance(text, chat_lookup=provider)
+    audit_fields = _ProvenanceAuditFields.model_validate(audit)
+    state = classify_note_provenance(text)
+    base = {
+        "path": str(path),
+        "chat_ids": list(state.chat_ids),
+        "legacy_urls": list(state.legacy_urls),
+        "errors": audit_fields.errors,
+        "warnings": audit_fields.warnings,
+        "wrote": False,
+        "changed": False,
+    }
+    if audit_fields.errors and audit_fields.blocked_reason == "chats.shape_invalid":
+        return {**base, "status": "blocked", "blocked_reason": "chats.shape_invalid"}
+    if state.status == "already_canonical":
+        return {**base, "status": "already_canonical", "blocked_reason": ""}
+    if state.status == "migratable" or state.chat_ids:
+        return {**base, "status": "planned", "blocked_reason": "", "would_write": True}
+    return {**base, "status": "warning", "blocked_reason": "", "would_write": False}
+
+
+def _apply_text(text: str, report: JsonObject, provider: _ChatMetadataProvider) -> str:
+    report_fields = _BackfillReportFields.model_validate(report)
+    chats = [ChatProvenance(chat_id) for chat_id in report_fields.chat_ids]
+    if not chats:
+        chats = [ChatProvenance(url) for url in report_fields.legacy_urls]
+    result = apply_note_provenance(text, chats=chats, chat_lookup=provider)
+    return str(result["text"])
+
+
+def _summary_payload(
+    *,
+    schema: str,
+    config: MedConfig,
+    reports: list[JsonObject],
+    dry_run: bool,
+    backup: bool,
+    backup_paths: list[str],
+    write_errors: list[dict[str, str]],
+    metadata_lookup: JsonObject,
+) -> dict[str, object]:
+    report_fields = [_BackfillReportFields.model_validate(report) for report in reports]
+    metadata_fields = _MetadataLookupFields.model_validate(metadata_lookup)
+    written_count = sum(1 for report in report_fields if report.wrote)
+    planned_count = sum(1 for report in report_fields if report.status == "planned")
+    if not dry_run and written_count == 0 and planned_count == 0 and metadata_fields.mode == "markdown_query":
+        metadata_lookup = {"mode": "not_required", "skipped_reason": "no_recoverable_sources", "warning": None}
+    warning_items = [
+        warning
+        for report in report_fields
+        for warning in report.warnings
+    ]
+    blocked_count = sum(1 for report in report_fields if report.status in {"blocked", "write_error"})
+    status = "blocked" if blocked_count else "planned" if dry_run and planned_count else "completed"
+    return {
+        "schema": schema,
+        "phase": "provenance_backfill",
+        "status": status,
+        "dry_run": dry_run,
+        "backup": backup,
+        "wiki_dir": str(config.wiki_dir),
+        "raw_dir": str(config.raw_dir),
+        "scanned_count": len(reports),
+        "recoverable_count": sum(1 for report in report_fields if report.status == "planned"),
+        "would_write_count": planned_count if dry_run else 0,
+        "written_count": written_count,
+        "already_canonical_count": sum(1 for report in report_fields if report.status == "already_canonical"),
+        "unrecoverable_count": sum(1 for report in report_fields if report.status == "warning"),
+        "warning_count": len(warning_items),
+        "warnings": warning_items,
+        "blocked_count": blocked_count,
+        "write_error_count": len(write_errors),
+        "write_errors": write_errors,
+        "backup_paths": backup_paths,
+        "reports": reports,
+        "metadata_lookup": metadata_lookup,
+    }

package/.opencode/mednotes/src/mednotes/domains/wiki/capabilities/pdf/__init__.py

@@ -0,0 +1,10 @@
+"""Local PDF image library for Medical Notes Workbench."""
+
+__all__ = [
+    "config",
+    "db",
+    "doctor",
+    "ingest",
+    "search",
+    "insert",
+]

package/.opencode/mednotes/src/mednotes/domains/wiki/capabilities/pdf/anchors.py

@@ -0,0 +1,16 @@
+"""PDF-library anchor adapter."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from mednotes.domains.wiki.capabilities.illustrate.anchors import AnchorProviderConfig, build_or_load_anchors
+
+
+def anchors_for_note(note_path: Path, *, cache_db: Path, max_anchors: int = 5, preferred_language: str = "pt-br"):
+    return build_or_load_anchors(
+        note_path,
+        cache_db=cache_db,
+        max_anchors=max_anchors,
+        preferred_language=preferred_language,
+        provider_config=AnchorProviderConfig(),
+    )

package/.opencode/mednotes/src/mednotes/domains/wiki/capabilities/pdf/captions.py

@@ -0,0 +1,47 @@
+"""Caption extraction for PDF text."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from mednotes.domains.wiki.capabilities.pdf import figure_ids
+
+
+@dataclass(frozen=True)
+class Caption:
+    page_number: int
+    figure_id: str
+    text: str
+    bbox: tuple[float, float, float, float] | None = None
+    section_path_guess: list[str] = field(default_factory=list)
+
+
+_CAPTION_RE = re.compile(
+    r"(?P<label>\b(?:fig(?:ure|ura)?|table|tabela|box|plate|algorithm|algoritmo)\.?\s*[0-9]+(?:[.\-][0-9]+)?\s*[- ]?\s*[a-zA-Z]?)\.?\s*(?P<body>[^\n]{0,240})",
+    re.IGNORECASE,
+)
+
+
+def extract_captions(text: str, *, page_number: int, section_path_guess: list[str] | None = None) -> list[Caption]:
+    captions: list[Caption] = []
+    seen: set[tuple[str, str]] = set()
+    for match in _CAPTION_RE.finditer(text):
+        label = match.group("label")
+        body = match.group("body").strip()
+        sentence = f"{label}. {body}".strip()
+        if len(sentence) < len(label) + 2:
+            continue
+        figure_id = figure_ids.normalize(label)
+        key = (figure_id, sentence)
+        if key in seen:
+            continue
+        seen.add(key)
+        captions.append(
+            Caption(
+                page_number=page_number,
+                figure_id=figure_id,
+                text=sentence,
+                section_path_guess=list(section_path_guess or []),
+            )
+        )
+    return captions