ocr-provenance-mcp 1.0.0

package/python/pyproject.toml

@@ -0,0 +1,76 @@
+[project]
+name = "ocr-provenance-workers"
+version = "1.0.0"
+description = "Python workers for OCR Provenance MCP System"
+requires-python = ">=3.10"
+
+[tool.ruff]
+# Exclude common non-source directories
+exclude = [
+    ".git",
+    ".venv",
+    "__pycache__",
+    "*.pyc",
+    ".mypy_cache",
+    ".pytest_cache",
+]
+
+# Same as Black
+line-length = 100
+indent-width = 4
+
+# Target Python 3.10+
+target-version = "py310"
+
+[tool.ruff.lint]
+# Enable rules
+select = [
+    "E",      # pycodestyle errors
+    "W",      # pycodestyle warnings
+    "F",      # Pyflakes
+    "I",      # isort
+    "B",      # flake8-bugbear
+    "C4",     # flake8-comprehensions
+    "UP",     # pyupgrade
+    "S",      # flake8-bandit (security)
+    "SIM",    # flake8-simplify
+    "TCH",    # flake8-type-checking
+    "RUF",    # Ruff-specific rules
+]
+
+# Ignore specific rules
+ignore = [
+    "E501",   # Line too long (handled by formatter)
+    "S101",   # Use of assert (OK in tests)
+    "S603",   # subprocess without shell=True check (we validate inputs)
+    "S607",   # Starting subprocess with partial path
+]
+
+# Allow fix for all enabled rules
+fixable = ["ALL"]
+unfixable = []
+
+[tool.ruff.format]
+# Use double quotes for strings
+quote-style = "double"
+
+# Indent with spaces
+indent-style = "space"
+
+# Unix-style line endings
+line-ending = "lf"
+
+[tool.ruff.lint.per-file-ignores]
+# Tests can use assert and magic values
+"**/test_*.py" = ["S101", "PLR2004"]
+"**/*_test.py" = ["S101", "PLR2004"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["ocr_provenance"]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_ignores = true
+disallow_untyped_defs = true
+ignore_missing_imports = true

package/python/requirements.txt

@@ -0,0 +1,51 @@
+# =============================================================================
+# OCR Provenance MCP System - Python Dependencies
+# =============================================================================
+# Install with: pip install -r requirements.txt
+# Works on Linux, macOS (Intel + Apple Silicon), and Windows.
+# GPU optional: auto-detects CUDA > MPS > CPU at runtime.
+
+# -----------------------------------------------------------------------------
+# Datalab OCR SDK
+# -----------------------------------------------------------------------------
+datalab-python-sdk>=0.2.0
+
+# -----------------------------------------------------------------------------
+# Embedding Model & Deep Learning
+# -----------------------------------------------------------------------------
+sentence-transformers>=2.7.0
+transformers>=4.40.0
+einops>=0.7.0
+torch>=2.5.0
+# For CUDA support, install the CUDA variant:
+#   pip install torch --index-url https://download.pytorch.org/whl/cu124
+# For CPU-only (smaller download):
+#   pip install torch --index-url https://download.pytorch.org/whl/cpu
+
+# -----------------------------------------------------------------------------
+# Numerical & Data Processing
+# -----------------------------------------------------------------------------
+numpy>=1.26.0
+scipy>=1.12.0
+
+# -----------------------------------------------------------------------------
+# Image Processing (for VLM pipeline)
+# -----------------------------------------------------------------------------
+PyMuPDF>=1.24.0
+Pillow>=10.0.0
+
+# -----------------------------------------------------------------------------
+# Machine Learning (for clustering worker)
+# -----------------------------------------------------------------------------
+scikit-learn>=1.3.0
+
+# -----------------------------------------------------------------------------
+# HTTP (for file manager worker)
+# -----------------------------------------------------------------------------
+requests>=2.28.0
+urllib3>=2.0.0
+
+# -----------------------------------------------------------------------------
+# GPU Monitoring (replaces deprecated pynvml transitively pulled by torch)
+# -----------------------------------------------------------------------------
+nvidia-ml-py>=12.560.30

package/python/reranker_worker.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+Local cross-encoder reranking worker using sentence-transformers.
+
+Reads JSON from stdin with format:
+  {"query": "search text", "passages": [{"index": 0, "text": "...", "original_score": 0.5}, ...]}
+
+Outputs JSON array of reranked results to stdout:
+  [{"index": 0, "relevance_score": 0.95, "original_score": 0.5}, ...]
+
+CRITICAL: All logging goes to stderr. Stdout is reserved for JSON output.
+"""
+
+import json
+import logging
+import sys
+import time
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    stream=sys.stderr,
+)
+logger = logging.getLogger(__name__)
+
+
+def rerank(query: str, passages: list[dict]) -> list[dict]:
+    """Rerank passages using cross-encoder model."""
+    try:
+        from sentence_transformers import CrossEncoder
+    except ImportError:
+        print(
+            json.dumps(
+                {
+                    "error": "sentence-transformers not installed. Install with: pip install sentence-transformers"
+                }
+            ),
+            file=sys.stdout,
+        )
+        sys.exit(1)
+
+    start = time.time()
+    logger.info(
+        "Loading cross-encoder model: cross-encoder/ms-marco-MiniLM-L-12-v2"
+    )
+    model = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-12-v2", max_length=512)
+    logger.info("Model loaded in %.2fs", time.time() - start)
+
+    # Truncate passages to 500 chars for efficiency
+    pairs = [(query, p["text"][:500]) for p in passages]
+
+    start = time.time()
+    scores = model.predict(pairs)
+    logger.info(
+        "Reranked %d passages in %.2fs", len(passages), time.time() - start
+    )
+
+    results = []
+    for passage, score in zip(passages, scores):
+        results.append(
+            {
+                "index": passage["index"],
+                "relevance_score": float(score),
+                "original_score": passage.get("original_score", 0),
+            }
+        )
+
+    results.sort(key=lambda x: x["relevance_score"], reverse=True)
+    return results
+
+
+if __name__ == "__main__":
+    try:
+        input_data = json.loads(sys.stdin.read())
+        query = input_data["query"]
+        passages = input_data["passages"]
+
+        if not passages:
+            print(json.dumps([]))
+            sys.exit(0)
+
+        result = rerank(query, passages)
+        print(json.dumps(result))
+    except Exception as e:
+        logger.error("Reranker failed: %s", str(e))
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)