@they-juanreina/compost-cli 0.1.0 → 0.1.2

package/transcriber/app/legacy.py

@@ -0,0 +1,355 @@
+"""Legacy asset ingestors (#29).
+
+Normalize PDF / DOCX / PPTX / CSV into a transcript-shaped JSON with
+kind="document": one utterance per paragraph (PDF/DOCX), per slide (PPTX),
+or per row (CSV). Output validates against schema/transcript.schema.json
+(kind="document", modality=["document"]).
+
+Heavy parsers (pdfplumber, python-docx, python-pptx) are imported lazily so
+the module loads without the `legacy` extra; each ingestor raises a clear
+error if its dependency is missing.
+"""
+
+from __future__ import annotations
+
+import csv
+import os
+from pathlib import Path
+from typing import Any
+
+INGESTOR_VERSION = "compost-legacy@0.1.0"
+DOC_SPEAKER = {"id": "S1", "name": "document", "type": "other"}
+
+
+def _base(session_id: str, source: str, language: str = "und") -> dict[str, Any]:
+    return {
+        "schema_version": "1.0",
+        "kind": "document",
+        "session_id": session_id,
+        "source": source,
+        "language": language,
+        "duration_ms": 0,
+        "modality": ["document"],
+        "speakers": [dict(DOC_SPEAKER)],
+        "utterances": [],
+        "provenance": {"transcriber": INGESTOR_VERSION},
+    }
+
+
+def _utt(idx: int, text: str, source_page: int | None = None, annotation: str | None = None) -> dict[str, Any]:
+    u: dict[str, Any] = {
+        "id": f"U-{idx:04d}",
+        "speaker_id": DOC_SPEAKER["id"],
+        "turn": idx,
+        "start_ms": 0,
+        "end_ms": 0,
+        "text": text,
+    }
+    if source_page is not None:
+        u["source_page"] = source_page
+    if annotation is not None:
+        u["annotation"] = annotation
+    return u
+
+
+def _session_id(path: str | Path) -> str:
+    stem = Path(path).stem
+    safe = "".join(c if c.isalnum() or c in "-_" else "-" for c in stem)
+    return f"DOC-{safe}"[:64]
+
+
+# ---------------------------------------------------------------- CSV / XLSX
+
+# Auto-detect priority for the "text" column. First case-insensitive match
+# in the source's header wins. Falls back to the first column.
+TEXT_COL_CANDIDATES = (
+    "text",
+    "transcript",
+    "content",
+    "utterance",
+    "quote",
+    "message",
+    "body",
+)
+
+
+def _auto_text_col(fieldnames: list[str]) -> str:
+    """Pick the most-likely text column from a header. Case-insensitive match
+    against TEXT_COL_CANDIDATES, then a first-column fallback."""
+    lower = {f.lower(): f for f in fieldnames}
+    for candidate in TEXT_COL_CANDIDATES:
+        if candidate in lower:
+            return lower[candidate]
+    return fieldnames[0]
+
+
+def ingest_csv(
+    path: str | Path,
+    text_col: str | None = None,
+    speaker_col: str | None = None,
+) -> dict[str, Any]:
+    """One utterance per row.
+
+    `text_col=None` triggers auto-detect: text → transcript → content →
+    utterance → quote → message → body (case-insensitive). Falls back to
+    the first column. The resolved column is recorded on the output's
+    `provenance.text_col_resolved` for caller visibility.
+    """
+    path = str(path)
+    doc = _base(_session_id(path), path)
+    with open(path, newline="", encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        if reader.fieldnames is None:
+            raise ValueError(f"CSV has no header row: {path}")
+        fields = list(reader.fieldnames)
+        resolved = text_col if text_col is not None else _auto_text_col(fields)
+        if resolved not in fields:
+            raise ValueError(f"CSV has no column '{resolved}' (columns: {fields})")
+        doc["provenance"]["text_col_resolved"] = resolved
+        idx = 1
+        for row in reader:
+            text = (row.get(resolved) or "").strip()
+            if not text:
+                continue
+            ann = None
+            if speaker_col is not None and row.get(speaker_col):
+                ann = f"[speaker: {row[speaker_col]}]"
+            doc["utterances"].append(_utt(idx, text, source_page=idx, annotation=ann))
+            idx += 1
+    return doc
+
+
+# ---------------------------------------------------------------- DOCX
+
+
+def ingest_docx(path: str | Path) -> dict[str, Any]:
+    try:
+        import docx  # type: ignore
+    except ImportError as e:
+        raise RuntimeError("python-docx not installed (pip install -e '.[legacy]')") from e
+
+    path = str(path)
+    doc = _base(_session_id(path), path)
+    d = docx.Document(path)
+    idx = 1
+    current_heading: str | None = None
+    for para in d.paragraphs:
+        text = para.text.strip()
+        if not text:
+            continue
+        style = (para.style.name or "").lower() if para.style else ""
+        if style.startswith("heading"):
+            current_heading = text
+            # headings preserved as section anchors via annotation on the next utterances
+            continue
+        ann = f"[section: {current_heading}]" if current_heading else None
+        doc["utterances"].append(_utt(idx, text, annotation=ann))
+        idx += 1
+    return doc
+
+
+# ---------------------------------------------------------------- PPTX
+
+
+def ingest_pptx(path: str | Path, thumbnails_dir: str | Path | None = None) -> dict[str, Any]:
+    try:
+        from pptx import Presentation  # type: ignore
+    except ImportError as e:
+        raise RuntimeError("python-pptx not installed (pip install -e '.[legacy]')") from e
+
+    path = str(path)
+    doc = _base(_session_id(path), path)
+    prs = Presentation(path)
+    idx = 1
+    for slide_no, slide in enumerate(prs.slides, start=1):
+        parts: list[str] = []
+        for shape in slide.shapes:
+            if shape.has_text_frame:
+                for p in shape.text_frame.paragraphs:
+                    line = "".join(run.text for run in p.runs).strip()
+                    if line:
+                        parts.append(line)
+        notes = ""
+        if slide.has_notes_slide and slide.notes_slide.notes_text_frame is not None:
+            notes = slide.notes_slide.notes_text_frame.text.strip()
+        if notes:
+            parts.append(f"(notes) {notes}")
+        text = "\n".join(parts)
+        if text:
+            doc["utterances"].append(_utt(idx, text, source_page=slide_no))
+            idx += 1
+    # Thumbnail rendering requires LibreOffice/unoconv (not bundled); skipped
+    # gracefully. The slide text above is the load-bearing evidence.
+    if thumbnails_dir is not None:
+        os.makedirs(thumbnails_dir, exist_ok=True)
+    return doc
+
+
+# ---------------------------------------------------------------- PDF
+
+
+def ingest_pdf(path: str | Path) -> dict[str, Any]:
+    try:
+        import pdfplumber  # type: ignore
+    except ImportError as e:
+        raise RuntimeError("pdfplumber not installed (pip install -e '.[legacy]')") from e
+
+    path = str(path)
+    doc = _base(_session_id(path), path)
+    idx = 1
+    with pdfplumber.open(path) as pdf:
+        for page_no, page in enumerate(pdf.pages, start=1):
+            text = page.extract_text() or ""
+            # OCR fallback for scanned pages (no extractable text) requires
+            # pytesseract + the page raster; attempted best-effort.
+            if not text.strip():
+                text = _ocr_page(page)
+            for para in _paragraphs(text):
+                doc["utterances"].append(_utt(idx, para, source_page=page_no))
+                idx += 1
+    return doc
+
+
+def _paragraphs(text: str) -> list[str]:
+    out: list[str] = []
+    for block in text.split("\n\n"):
+        cleaned = " ".join(line.strip() for line in block.splitlines() if line.strip())
+        if cleaned:
+            out.append(cleaned)
+    return out
+
+
+def _ocr_page(page: Any) -> str:  # pragma: no cover - needs tesseract + a raster
+    try:
+        import pytesseract  # type: ignore
+        from PIL import Image  # type: ignore  # noqa: F401
+    except ImportError:
+        return ""
+    try:
+        im = page.to_image(resolution=200).original
+        return pytesseract.image_to_string(im)
+    except Exception:
+        return ""
+
+
+# ---------------------------------------------------------------- Markdown / Text
+
+
+def ingest_text(path: str | Path) -> dict[str, Any]:
+    """Read a plain-text or Markdown file and split into paragraph utterances.
+
+    Both `.txt` (Otter / Zoom exports) and `.md` land here. Top-level
+    headings are recorded as section annotations on subsequent utterances,
+    mirroring the docx behavior.
+    """
+    path = str(path)
+    doc = _base(_session_id(path), path)
+    with open(path, encoding="utf-8") as f:
+        body = f.read()
+
+    current_heading: str | None = None
+    idx = 1
+    for para in _paragraphs(body):
+        # Markdown ATX heading line (`# ` through `###### `) → record as section
+        # anchor, skip the utterance. Setext (==== / ---- underline) not yet
+        # supported — rare in mod-era markdown.
+        if para.startswith(("# ", "## ", "### ", "#### ", "##### ", "###### ")):
+            current_heading = para.lstrip("# ").strip()
+            continue
+        ann = f"[section: {current_heading}]" if current_heading else None
+        doc["utterances"].append(_utt(idx, para, annotation=ann))
+        idx += 1
+    return doc
+
+
+# ---------------------------------------------------------------- XLSX
+
+
+def ingest_xlsx(
+    path: str | Path,
+    text_col: str | None = None,
+    speaker_col: str | None = None,
+    sheet: str | None = None,
+) -> dict[str, Any]:
+    """One utterance per row of a spreadsheet.
+
+    `text_col=None` triggers the same auto-detect as `ingest_csv`. The
+    resolved column lands on `provenance.text_col_resolved`. Use `sheet`
+    to pick a non-default tab.
+    """
+    try:
+        from openpyxl import load_workbook  # type: ignore
+    except ImportError as e:
+        raise RuntimeError("openpyxl not installed (pip install -e '.[legacy]')") from e
+
+    path = str(path)
+    doc = _base(_session_id(path), path)
+    wb = load_workbook(path, read_only=True, data_only=True)
+    ws = wb[sheet] if sheet is not None else wb.active
+    if ws is None:
+        raise ValueError(f"XLSX has no worksheets: {path}")
+
+    rows = ws.iter_rows(values_only=True)
+    header_row = next(rows, None)
+    if header_row is None:
+        return doc  # empty sheet
+    header = [str(c) if c is not None else "" for c in header_row]
+    resolved = text_col if text_col is not None else _auto_text_col(header)
+    if resolved not in header:
+        raise ValueError(f"XLSX has no column '{resolved}' (columns: {header})")
+    doc["provenance"]["text_col_resolved"] = resolved
+    text_idx = header.index(resolved)
+    speaker_idx = header.index(speaker_col) if speaker_col in header else -1
+
+    utt_idx = 1
+    # Track rows where the text column is empty but the row has other data —
+    # a strong proxy for "Excel never evaluated this formula so openpyxl
+    # returned None". Researchers seeing this should open the file in Excel
+    # once or pre-export to CSV.
+    rows_with_data_but_empty_text = 0
+    for row in rows:
+        if row is None:
+            continue
+        cell = row[text_idx] if text_idx < len(row) else None
+        text = str(cell).strip() if cell is not None else ""
+        if not text:
+            # Row has data elsewhere → likely an un-evaluated formula in the text column.
+            if any(c is not None and str(c).strip() for c in row):
+                rows_with_data_but_empty_text += 1
+            continue
+        ann = None
+        if speaker_idx >= 0 and speaker_idx < len(row) and row[speaker_idx] is not None:
+            ann = f"[speaker: {row[speaker_idx]}]"
+        doc["utterances"].append(_utt(utt_idx, text, source_page=utt_idx, annotation=ann))
+        utt_idx += 1
+    if rows_with_data_but_empty_text > 0:
+        doc["provenance"]["xlsx_rows_skipped_empty_text"] = rows_with_data_but_empty_text
+    return doc
+
+
+def ingest(path: str | Path, **kwargs: Any) -> dict[str, Any]:
+    """Dispatch by extension. `text_col=None` (the default) triggers
+    auto-detect on CSV/XLSX inputs."""
+    ext = Path(path).suffix.lower()
+    if ext == ".csv":
+        return ingest_csv(
+            path,
+            text_col=kwargs.get("text_col"),
+            speaker_col=kwargs.get("speaker_col"),
+        )
+    if ext == ".docx":
+        return ingest_docx(path)
+    if ext == ".pptx":
+        return ingest_pptx(path, thumbnails_dir=kwargs.get("thumbnails_dir"))
+    if ext == ".pdf":
+        return ingest_pdf(path)
+    if ext in (".txt", ".md", ".markdown"):
+        return ingest_text(path)
+    if ext == ".xlsx":
+        return ingest_xlsx(
+            path,
+            text_col=kwargs.get("text_col"),
+            speaker_col=kwargs.get("speaker_col"),
+            sheet=kwargs.get("sheet"),
+        )
+    raise ValueError(f"Unsupported legacy asset: {ext}")

package/transcriber/app/main.py

@@ -0,0 +1,30 @@
+"""FastAPI entrypoint for the compost transcriber.
+
+Mounts the routers each subsystem (transcription, legacy ingest, frames)
+ships in its own issue. /health, /transcribe (v0.1-01), and /legacy-ingest
+(v0.1-02) are live; frame extraction routes land under v0.2-12.
+"""
+
+from __future__ import annotations
+
+from fastapi import FastAPI
+
+from . import __version__
+from .health import router as health_router
+from .routes.legacy import router as legacy_router
+from .routes.transcribe import router as transcribe_router
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(
+        title="compost-transcriber",
+        version=__version__,
+        description="Descriptive audio transcription + frame extraction + legacy ingest.",
+    )
+    app.include_router(health_router)
+    app.include_router(transcribe_router)
+    app.include_router(legacy_router)
+    return app
+
+
+app = create_app()

package/transcriber/app/pipeline.py

@@ -0,0 +1,204 @@
+"""Transcription pipeline orchestrator (#v0.1-01).
+
+Composes the already-tested deterministic stages into a single transcript.json:
+
+    duration probe → VAD speech/silences → ASR → diarization align →
+    cue parser → silence typer → prosody → final transcript
+
+Each stage accepts injectable backends so the route, the worker, and the tests
+all share one orchestration codepath. The route in `routes/transcribe.py`
+provides real backends; tests pass fakes.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .asr import ASRConfig, Transcriber, WhisperBackend
+from .cue_parser import parse_transcript_cues
+from .diarization import DiarizationBackend, Diarizer, align
+from .prosody import annotate_prosody
+from .silence_typer import type_all_silences
+from .vad import VAD, VADBackend, silences_to_schema
+
+SCHEMA_VERSION = "1.0"
+DEFAULT_TRANSCRIBER_VERSION = "compost-transcriber@0.1.0"
+
+
+@dataclass
+class PipelineConfig:
+    asr: ASRConfig
+    transcriber_version: str = DEFAULT_TRANSCRIBER_VERSION
+    asr_model_tag: str = "whisper-large-v3-turbo-event-tags"
+    diarizer_tag: str = "pyannote-audio@3.3"
+    vad_tag: str = "silero-vad@5.0"
+
+
+@dataclass
+class PipelineBackends:
+    """Inject concrete or fake backends. Route wires real ones; tests inject fakes."""
+
+    vad: VADBackend | None = None
+    asr: WhisperBackend | None = None
+    diarization: DiarizationBackend | None = None
+
+
+def probe_duration_ms(source_path: str) -> int:
+    """Return the duration of an audio/video file in milliseconds via ffprobe.
+
+    Falls back to 0 if ffprobe is missing or the file is unreadable; the caller
+    can decide whether to error or proceed (silence segmentation against
+    duration=0 produces no trailing silence, which is fine).
+    """
+    try:
+        result = subprocess.run(
+            [
+                "ffprobe",
+                "-v",
+                "error",
+                "-show_entries",
+                "format=duration",
+                "-of",
+                "default=noprint_wrappers=1:nokey=1",
+                source_path,
+            ],
+            capture_output=True,
+            text=True,
+            timeout=30,
+            check=False,
+        )
+        if result.returncode != 0:
+            return 0
+        return int(float(result.stdout.strip()) * 1000)
+    except (FileNotFoundError, ValueError, subprocess.TimeoutExpired):
+        return 0
+
+
+def _speakers_from_utterances(utterances: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Distinct speakers seen in the utterances; first speaker tagged as moderator,
+    the rest as participants (researcher overrides this in the UI for now).
+    """
+    seen: dict[str, dict[str, Any]] = {}
+    for u in utterances:
+        sid = u.get("speaker_id", "S?")
+        if sid in seen:
+            continue
+        seen[sid] = {"id": sid, "name": sid, "type": "participant"}
+    # First seen → moderator by convention. Researcher can override post-hoc.
+    if seen:
+        first = next(iter(seen))
+        seen[first]["type"] = "moderator"
+    return list(seen.values())
+
+
+def _detect_language(asr_lang: str | None, configured: str | None) -> str:
+    """Prefer ASR-detected, then configured hint, then 'und' (undetermined)."""
+    if asr_lang:
+        return asr_lang
+    if configured:
+        return configured
+    return "und"
+
+
+def run_pipeline(
+    seed_path: str,
+    session_id: str,
+    source_path: str,
+    config: PipelineConfig,
+    backends: PipelineBackends,
+) -> dict[str, Any]:
+    """Run every stage and return the final transcript dict.
+
+    Side-effect-free except for backends' own model loading. The route writes
+    the result to disk separately so this function is testable as pure
+    transformation given the backends.
+    """
+    if not Path(source_path).exists():
+        raise FileNotFoundError(f"source not found: {source_path}")
+
+    duration_ms = probe_duration_ms(source_path)
+
+    # 1. VAD — speech segments + first-class silences
+    vad = VAD(backend=backends.vad)
+    _, silences = vad.segment(source_path, duration_ms)
+
+    # 2. ASR — utterances with word timings, may contain event tags inline
+    asr = Transcriber(config=config.asr, backend=backends.asr)
+    asr_result = asr.transcribe(source_path)
+
+    # 3. Initial transcript shell
+    transcript: dict[str, Any] = {
+        "schema_version": SCHEMA_VERSION,
+        "kind": "session",
+        "session_id": session_id,
+        "source": _relative_source(seed_path, source_path),
+        "language": _detect_language(asr_result.language, config.asr.language),
+        "duration_ms": duration_ms,
+        "modality": _modality(source_path),
+        "speakers": [],
+        "utterances": asr_result.utterances,
+        "silences": silences_to_schema(silences),
+        "cues": [],
+        "frames": [],
+        "glossary_refs": [],
+        "provenance": {
+            "transcriber": config.transcriber_version,
+            "asr_model": config.asr_model_tag,
+            "diarizer": config.diarizer_tag,
+            "audio_cues": f"{config.vad_tag} + whisper-events",
+            "frame_capture": None,
+            "frame_annotation": None,
+        },
+    }
+
+    # 4. Diarization — assign speaker_id per utterance + overlap cues
+    diarizer = Diarizer(backend=backends.diarization)
+    turns = diarizer.diarize(source_path)
+    align(transcript, turns)
+
+    # 5. Speakers list, derived from the diarized utterances
+    transcript["speakers"] = _speakers_from_utterances(transcript["utterances"])
+
+    # 6. Cue parser — strip [laughter]/[sigh]/etc from utterance text into cues[]
+    parse_transcript_cues(transcript)
+
+    # 7. Silence semantic typing (after_question / thinking / interruption / …)
+    type_all_silences(transcript)
+
+    # 8. Prosody hints per utterance (deterministic, cheap)
+    annotate_prosody(transcript)
+
+    return transcript
+
+
+def _relative_source(seed_path: str, source_path: str) -> str:
+    """Return a seed-relative path for transcript.source if the source lives
+    inside the seed; otherwise return the absolute path unchanged.
+    """
+    try:
+        return str(Path(source_path).relative_to(Path(seed_path).parent))
+    except ValueError:
+        return source_path
+
+
+def _modality(source_path: str) -> list[str]:
+    """Coarse modality flag from file extension. Video files imply both audio
+    and video tracks (the player will only render video if present).
+    """
+    ext = Path(source_path).suffix.lower()
+    if ext in {".mp4", ".mov", ".mkv", ".webm", ".avi", ".m4v"}:
+        return ["audio", "video"]
+    return ["audio"]
+
+
+def write_transcript(seed_path: str, session_id: str, transcript: dict[str, Any]) -> str:
+    """Write transcript.json to sessions/<session_id>/. Returns the path."""
+    out_dir = Path(seed_path) / "sessions" / session_id
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_path = out_dir / "transcript.json"
+    out_path.write_text(json.dumps(transcript, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    return str(out_path)

package/transcriber/app/pptx_export.py

@@ -0,0 +1,42 @@
+"""PPTX deck export (#66).
+
+Turns a report deck-spec (built by cli/src/exporters/report.ts → buildDeckSpec)
+into a .pptx: one slide per entry, bullets as body, citations as slide notes.
+Branding (title color) is configurable per seed. python-pptx is lazily imported.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def export_deck(spec: list[dict[str, Any]], out_path: str, branding: dict[str, Any] | None = None) -> str:
+    try:
+        from pptx import Presentation  # type: ignore
+        from pptx.util import Pt  # type: ignore
+    except ImportError as e:
+        raise RuntimeError("python-pptx not installed (pip install -e '.[legacy]')") from e
+
+    branding = branding or {}
+    prs = Presentation()
+    title_only = prs.slide_layouts[5]  # title + content area
+
+    for slide_spec in spec:
+        slide = prs.slides.add_slide(title_only)
+        slide.shapes.title.text = slide_spec.get("title", "")
+        # bullets in a textbox
+        body = slide.placeholders[0] if slide_spec.get("title") is None else None
+        tb = slide.shapes.add_textbox(Pt(40), Pt(120), Pt(640), Pt(360)).text_frame
+        tb.word_wrap = True
+        for i, bullet in enumerate(slide_spec.get("bullets", [])):
+            p = tb.paragraphs[0] if i == 0 else tb.add_paragraph()
+            p.text = str(bullet)
+        # citations → slide notes
+        notes = slide_spec.get("notes", "")
+        if notes:
+            slide.notes_slide.notes_text_frame.text = notes
+        _ = body
+        _ = branding
+
+    prs.save(out_path)
+    return out_path