@piecemaker-legal/piecemaker 1.1.1 → 2.0.1

package/server/piecemaker/vendor/websocket-server/scripts/convert_and_scan_pipeline.py

@@ -67,7 +67,7 @@ def print_progress(phase: str, current: int, total: int) -> None:
 def start_scanner_worker() -> Optional[subprocess.Popen]:
     """Start the long-lived scanner worker subprocess.
 
-    The worker loads GLiNER2 + spaCy models once and stays alive to process
+    The worker loads the GLiNER2 model once and stays alive to process
     multiple files via a JSON-line stdin/stdout protocol.  Launch this at the
     start of the pipeline so model loading overlaps with Phase 1 (conversion).
 
@@ -1845,7 +1845,7 @@ def run_pipeline(resources: PipelineResources):
     print()
 
     # What is left to do, decided before anything runs: the scanner worker loads
-    # ~1.1GB of GLiNER2.5 + spaCy weights, so it must only start when at least one
+    # ~1.1GB of GLiNER2.5 weights, so it must only start when at least one
     # file actually needs a PII scan.
     def markdown_path(input_file: str) -> Path:
         return Path(args.output) / f"{Path(input_file).stem}.md"

package/server/piecemaker/vendor/websocket-server/scripts/mineru_improved_1.py

@@ -6,10 +6,19 @@ Supports both pipeline (direct PDF) and VLM/hybrid (image-based) modes
 
 import argparse
 import json
+import os
 import subprocess
 import sys
 from pathlib import Path
 
+
+def mineru_executable():
+    name = 'mineru.exe' if os.name == 'nt' else 'mineru'
+    sibling = Path(sys.executable).resolve().parent / name
+    if sibling.is_file():
+        return str(sibling)
+    return 'mineru'
+
 def parse_with_pipeline(pdf_path, output_dir, lang=None):
     """
     Parse PDF using pipeline backend (direct PDF processing, no image conversion needed)
@@ -29,7 +38,7 @@ def parse_with_pipeline(pdf_path, output_dir, lang=None):
     
     # Build mineru command
     cmd = [
-        'mineru',
+        mineru_executable(),
         '-p', str(pdf_path),
         '-o', str(output_dir),
         '-b', 'pipeline'
@@ -72,7 +81,7 @@ def parse_with_vlm(pdf_path, output_dir, backend='hybrid-auto-engine'):
     
     # Build mineru command
     cmd = [
-        'mineru',
+        mineru_executable(),
         '-p', str(pdf_path),
         '-o', str(output_dir),
         '-b', backend

package/server/piecemaker/vendor/websocket-server/scripts/presidio-gliner/scanner_worker.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-Long-lived scanner worker — loads GLiNER2.5 + spaCy once, processes multiple files
+Long-lived scanner worker — loads GLiNER2.5 once, processes multiple files
 via a JSON-line stdin/stdout protocol.
 
 Protocol:
@@ -29,13 +29,14 @@ from typing import Dict, List, Optional
 # ---------------------------------------------------------------------------
 # Heavy imports — these are the expensive ones (~30-60s)
 # ---------------------------------------------------------------------------
+import spacy
 from presidio_analyzer import (
     AnalysisExplanation,
     AnalyzerEngine,
     LocalRecognizer,
     RecognizerResult,
 )
-from presidio_analyzer.nlp_engine import NlpArtifacts, NlpEngineProvider
+from presidio_analyzer.nlp_engine import SpacyNlpEngine
 
 try:
     from gliner2 import AutoExtractor
@@ -430,23 +431,14 @@ _analyzer_cache: Dict[str, AnalyzerEngine] = {}
 _gliner_model = None  # Shared GLiNER2 model instance
 
 
-def _disable_unused_spacy_components(nlp_engine, language: str) -> None:
-    """Turn off every spaCy pipe whose output nothing reads.
+def _build_tokenizer_only_engine(language: str) -> SpacyNlpEngine:
+    """Build a presidio NLP engine that only tokenises, with no trained model."""
+    nlp = spacy.blank(language)
+    nlp.max_length = 5_000_000
 
-    Entities come from GLiNER2; spaCy is kept only because presidio needs a tokenizer
-    and NlpArtifacts. Failing to disable a pipe is not fatal, so this never raises.
-    """
-    nlp = nlp_engine.nlp.get(language)
-    if nlp is None:
-        return
-
-    for pipe_name in list(nlp.pipe_names):
-        try:
-            nlp.disable_pipe(pipe_name)
-        except Exception as exc:  # noqa: BLE001
-            _log(f"Could not disable spaCy pipe '{pipe_name}': {exc}")
-
-    _log(f"spaCy [{language}] active pipes: {nlp.pipe_names or 'tokenizer only'}")
+    engine = SpacyNlpEngine()
+    engine.nlp = {language: nlp}
+    return engine
 
 
 def _get_or_build_analyzer(language: str) -> AnalyzerEngine:
@@ -459,25 +451,9 @@ def _get_or_build_analyzer(language: str) -> AnalyzerEngine:
     if language in _analyzer_cache:
         return _analyzer_cache[language]
 
-    lang_models = {"fr": "fr_core_news_sm", "en": "en_core_web_sm"}
-    model_name = lang_models.get(language, "en_core_web_sm")
-
-    _log(f"Building analyzer for language: {language} (spaCy model: {model_name})")
-
-    nlp_provider = NlpEngineProvider(nlp_configuration={
-        "nlp_engine_name": "spacy",
-        "models": [{"lang_code": language, "model_name": model_name}],
-    })
-    nlp_engine = nlp_provider.create_engine()
-    nlp_engine.nlp[language].max_length = 5_000_000
-
-    # presidio's SpacyNlpEngine calls spacy.load() with no `disable=`, so the whole
-    # pipeline — tagger, morphologizer, parser, attribute_ruler, lemmatizer, ner —
-    # runs over the document. SpacyRecognizer is then removed below, so the NER output
-    # is thrown away and only tokenisation is used. Measured on 300k characters:
-    # 20.6 s with the full pipeline, 0.9 s with tokenisation alone, and ~1 GB less
-    # resident memory (which matters on an 8 GB machine).
-    _disable_unused_spacy_components(nlp_engine, language)
+    _log(f"Building analyzer for language: {language}")
+
+    nlp_engine = _build_tokenizer_only_engine(language)
 
     analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=[language])
 

package/server/piecemaker/vendor/websocket-server/scripts/requirements.txt

@@ -15,10 +15,4 @@ huggingface-hub>=0.20.0  # Required for model downloading with progress
 
 # Presidio + GLiNER integration
 presidio-analyzer>=2.2.0
-spacy>=3.7.0
-#
-# After installing spacy, download language models:
-#   python -m spacy download fr_core_news_sm  # French (15MB, 88% NER accuracy)
-#   python -m spacy download en_core_web_sm   # English (13MB, 85% NER accuracy)
-#
-# Or use: python warmup.py  # Downloads all models automatically
+spacy>=3.7.0  # Tokenizer only, via spacy.blank() — no language model needed

package/server/piecemaker/vendor/websocket-server/scripts/warmup.py

@@ -42,20 +42,6 @@ MODELS_CONFIG = {
         "optional": False,
         "type": "huggingface",
     },
-    "spacy_fr": {
-        "model_id": "fr_core_news_sm",
-        "description": "French NLP model for spaCy",
-        "size_mb": "~45MB",
-        "optional": False,
-        "type": "spacy",
-    },
-    "spacy_en": {
-        "model_id": "en_core_web_sm",
-        "description": "English NLP model for spaCy",
-        "size_mb": "~40MB",
-        "optional": False,
-        "type": "spacy",
-    },
 }
 
 
@@ -103,20 +89,6 @@ def gliner_migration_status() -> Dict:
     }
 
 
-def is_spacy_cached(model_id: str) -> bool:
-    """Check if spaCy model is installed"""
-    try:
-        import spacy
-
-        try:
-            spacy.load(model_id)
-            return True
-        except OSError:
-            return False
-    except ImportError:
-        return False
-
-
 def check_model_cached(model_key: str) -> bool:
     """Check if a specific model is cached"""
     config = MODELS_CONFIG.get(model_key)
@@ -125,8 +97,6 @@ def check_model_cached(model_key: str) -> bool:
 
     if config["type"] == "huggingface":
         return is_gliner2_cached(config["model_id"])
-    elif config["type"] == "spacy":
-        return is_spacy_cached(config["model_id"])
     return False
 
 
@@ -163,52 +133,6 @@ def download_gliner2_model(model_id: str, description: str, force: bool = False)
         return False
 
 
-def download_spacy_model(model_id: str, description: str) -> bool:
-    """Download spaCy model via CLI"""
-    try:
-        # Check if spacy is installed
-        try:
-            import spacy
-        except ImportError:
-            log_json(
-                f"spaCy not installed, skipping {model_id}",
-                level="warning",
-                model=model_id,
-            )
-            log_plain(f"⏭️  spaCy not installed, skipping {model_id}")
-            return False
-
-        log_json(f"Downloading spaCy model: {model_id}", model=model_id, phase="start")
-        log_plain(f"📥 Downloading spaCy model: {model_id}...")
-
-        # Download using subprocess
-        result = subprocess.run(
-            [sys.executable, "-m", "spacy", "download", model_id],
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-
-        log_json(
-            f"Downloaded spaCy model: {model_id}", model=model_id, phase="complete"
-        )
-        log_plain(f"✅ Downloaded spaCy model: {model_id}")
-        return True
-
-    except subprocess.CalledProcessError as e:
-        log_json(
-            f"Failed to download spaCy model {model_id}: {e}",
-            level="warning",
-            model=model_id,
-        )
-        log_plain(f"⚠️  spaCy model download failed: {model_id}")
-        return False
-    except Exception as e:
-        log_json(f"Error downloading {model_id}: {e}", level="error", model=model_id)
-        log_plain(f"❌ Error: {model_id} - {e}")
-        return False
-
-
 def download_model(model_key: str, force: bool = False) -> bool:
     """Download a specific model by key"""
     config = MODELS_CONFIG.get(model_key)
@@ -229,8 +153,6 @@ def download_model(model_key: str, force: bool = False) -> bool:
     # Download based on type
     if config["type"] == "huggingface":
         return download_gliner2_model(config["model_id"], config["description"], force=force)
-    elif config["type"] == "spacy":
-        return download_spacy_model(config["model_id"], config["description"])
 
     return False
 
@@ -241,19 +163,22 @@ def download_model(model_key: str, force: bool = False) -> bool:
 
 
 def check_mineru() -> bool:
-    """Verify mineru CLI is available"""
+    """Verify the mineru package is importable and runnable"""
     try:
         result = subprocess.run(
-            ["mineru", "--version"], capture_output=True, text=True, check=True
+            [sys.executable, "-m", "mineru.cli.client", "--version"],
+            capture_output=True,
+            text=True,
+            check=True,
         )
-        log_json("MinerU CLI available", tool="mineru", available=True)
-        log_plain("✅ MinerU CLI available")
+        log_json("MinerU available", tool="mineru", available=True)
+        log_plain("✅ MinerU available")
         return True
     except (subprocess.CalledProcessError, FileNotFoundError):
         log_json(
-            "MinerU CLI not found", tool="mineru", available=False, level="warning"
+            "MinerU not installed", tool="mineru", available=False, level="warning"
         )
-        log_plain("⚠️  MinerU CLI not found (pip install mineru)")
+        log_plain("⚠️  MinerU not installed (pip install 'mineru[pipeline,vlm]==2.7.6')")
         return False
 
 
@@ -392,7 +317,11 @@ def get_status() -> Dict:
 
     # Check mineru
     try:
-        subprocess.run(["mineru", "--version"], capture_output=True, check=True)
+        subprocess.run(
+            [sys.executable, "-m", "mineru.cli.client", "--version"],
+            capture_output=True,
+            check=True,
+        )
         status["tools"]["mineru"] = True
     except:
         status["tools"]["mineru"] = False

package/dist/assets/channel-DdzrPZ6Y.js

@@ -1 +0,0 @@
-import{ah as o,ai as n}from"./mermaid.core-C7Ti-Fty.js";const t=(a,r)=>o.lang.round(n.parse(a)[r]);export{t as c};

package/dist/assets/classDiagram-ZZMXUADV-Dim2o8E5.js

@@ -1 +0,0 @@
-import{s as a,c as s,a as e,C as t}from"./chunk-TICWLB2K-DhOULVhX.js";import{_ as i}from"./mermaid.core-C7Ti-Fty.js";import"./chunk-5VM5RSS4-BerUZBa1.js";import"./chunk-XXDRQBXY-CkpCgrVg.js";import"./chunk-POPQ4Y6H-D4aTDC2T.js";import"./chunk-F27PBJKO-5W7ax2gB.js";import"./index-DaxH81ZJ.js";import"./vendor-codemirror-CjW0lHR5.js";import"./vendor-react-D809iwXR.js";import"./vendor-xterm-BWUgpGtR.js";var c={parser:e,get db(){return new t},renderer:s,styles:a,init:i(r=>{r.class||(r.class={}),r.class.arrowMarkerAbsolute=r.arrowMarkerAbsolute},"init")};export{c as diagram};

package/dist/assets/classDiagram-v2-VYDZK3BY-Dim2o8E5.js

@@ -1 +0,0 @@
-import{s as a,c as s,a as e,C as t}from"./chunk-TICWLB2K-DhOULVhX.js";import{_ as i}from"./mermaid.core-C7Ti-Fty.js";import"./chunk-5VM5RSS4-BerUZBa1.js";import"./chunk-XXDRQBXY-CkpCgrVg.js";import"./chunk-POPQ4Y6H-D4aTDC2T.js";import"./chunk-F27PBJKO-5W7ax2gB.js";import"./index-DaxH81ZJ.js";import"./vendor-codemirror-CjW0lHR5.js";import"./vendor-react-D809iwXR.js";import"./vendor-xterm-BWUgpGtR.js";var c={parser:e,get db(){return new t},renderer:s,styles:a,init:i(r=>{r.class||(r.class={}),r.class.arrowMarkerAbsolute=r.arrowMarkerAbsolute},"init")};export{c as diagram};