@piecemaker-legal/piecemaker 1.0.0

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/nlp_engine/nlp_artifacts.py

@@ -0,0 +1,88 @@
+import json
+from typing import List, Optional
+
+from spacy.tokens import Doc, Span
+
+
+class NlpArtifacts:
+    """
+    NlpArtifacts is an abstraction layer over the results of an NLP pipeline.
+
+    processing over a given text, it holds attributes such as entities,
+    tokens and lemmas which can be used by any recognizer
+
+    :param entities: Identified entities
+    :param tokens: Tokenized text
+    :param tokens_indices: Indices of tokens
+    :param lemmas: List of lemmas in text
+    :param nlp_engine: NlpEngine object
+    :param language: Text language
+    :param scores: Entity confidence scores
+    """
+
+    def __init__(
+        self,
+        entities: List[Span],
+        tokens: Doc,
+        tokens_indices: List[int],
+        lemmas: List[str],
+        nlp_engine: "NlpEngine",  # noqa: F821
+        language: str,
+        scores: Optional[List[float]] = None,
+    ):
+        self.entities = entities
+        self.tokens = tokens
+        self.lemmas = lemmas
+        self.tokens_indices = tokens_indices
+        self.keywords = self.set_keywords(nlp_engine, lemmas, language)
+        self.nlp_engine = nlp_engine
+        self.scores = scores if scores else [0.85] * len(entities)
+
+    @staticmethod
+    def set_keywords(
+        nlp_engine,
+        lemmas: List[str],
+        language: str,
+    ) -> List[str]:
+        """
+        Return keywords fpr text.
+
+        Extracts lemmas with certain conditions as keywords.
+        """
+        if not nlp_engine:
+            return []
+        keywords = [
+            k.lower()
+            for k in lemmas
+            if not nlp_engine.is_stopword(k, language)
+            and not nlp_engine.is_punct(k, language)
+            and k != "-PRON-"
+            and k != "be"
+        ]
+
+        # best effort, try even further to break tokens into sub tokens,
+        # this can result in reducing false negatives
+        keywords = [i.split(":") for i in keywords]
+
+        # splitting the list can, if happened, will result in list of lists,
+        # we flatten the list
+        keywords = [item for sublist in keywords for item in sublist]
+        return keywords
+
+    def to_json(self) -> str:
+        """Convert nlp artifacts to json."""
+
+        return_dict = self.__dict__.copy()
+
+        # Ignore NLP engine as it's not serializable currently
+        del return_dict["nlp_engine"]
+
+        # Converting spaCy tokens and spans to string as they are not serializable
+        if "tokens" in return_dict:
+            return_dict["tokens"] = [token.text for token in self.tokens]
+        if "entities" in return_dict:
+            return_dict["entities"] = [entity.text for entity in self.entities]
+        if "scores" in return_dict:
+            return_dict["scores"] = [float(score) for score in self.scores]
+
+        return json.dumps(return_dict)

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/nlp_engine/nlp_engine.py

@@ -0,0 +1,65 @@
+from abc import ABC, abstractmethod
+from typing import Iterable, Iterator, List, Tuple
+
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+
+class NlpEngine(ABC):
+    """
+    NlpEngine is an abstraction layer over the nlp module.
+
+    It provides NLP preprocessing functionality as well as other queries
+    on tokens.
+    """
+
+    @abstractmethod
+    def load(self) -> None:
+        """Load the NLP model."""
+
+    @abstractmethod
+    def is_loaded(self) -> bool:
+        """Return True if the model is already loaded."""
+
+    @abstractmethod
+    def process_text(self, text: str, language: str) -> NlpArtifacts:
+        """Execute the NLP pipeline on the given text and language."""
+
+    @abstractmethod
+    def process_batch(
+        self,
+        texts: Iterable[str],
+        language: str,
+        batch_size: int = 1,
+        n_process: int = 1,
+        **kwargs,
+    ) -> Iterator[Tuple[str, NlpArtifacts]]:
+        """Execute the NLP pipeline on a batch of texts.
+
+        Returns a tuple of (text, NlpArtifacts)
+        """
+
+    @abstractmethod
+    def is_stopword(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a stop word.
+
+        (within the given language)
+        """
+
+    @abstractmethod
+    def is_punct(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a punctuation word.
+
+        (within the given language)
+        """
+
+    @abstractmethod
+    def get_supported_entities(self) -> List[str]:
+        """Return the supported entities for this NLP engine."""
+        pass
+
+    @abstractmethod
+    def get_supported_languages(self) -> List[str]:
+        """Return the supported languages for this NLP engine."""
+        pass

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/nlp_engine/nlp_engine_provider.py

@@ -0,0 +1,114 @@
+import logging
+from pathlib import Path
+from typing import Dict, Optional, Tuple, Union
+
+import yaml
+
+from presidio_analyzer.input_validation import ConfigurationValidator
+from presidio_analyzer.nlp_engine import (
+    NerModelConfiguration,
+    NlpEngine,
+    SpacyNlpEngine,
+    StanzaNlpEngine,
+    TransformersNlpEngine,
+)
+
+logger = logging.getLogger("presidio-analyzer")
+
+
+class NlpEngineProvider:
+    """Create different NLP engines from configuration.
+
+    :param nlp_engines: List of available NLP engines.
+    Default: (SpacyNlpEngine, StanzaNlpEngine)
+    :param nlp_configuration: Dict containing nlp configuration
+    :example: configuration:
+            {
+                "nlp_engine_name": "spacy",
+                "models": [{"lang_code": "en",
+                            "model_name": "en_core_web_lg"
+                          }]
+            }
+    Nlp engine names available by default: spacy, stanza.
+    :param conf_file: Path to yaml file containing nlp engine configuration.
+    """
+
+    def __init__(
+        self,
+        nlp_engines: Optional[Tuple] = None,
+        conf_file: Optional[Union[Path, str]] = None,
+        nlp_configuration: Optional[Dict] = None,
+    ):
+        if nlp_engines is None:
+            nlp_engines = (SpacyNlpEngine, StanzaNlpEngine, TransformersNlpEngine)
+
+        self.nlp_engines = {
+            engine.engine_name: engine for engine in nlp_engines if engine.is_available
+        }
+        logger.debug(
+            f"Loaded these available nlp engines: {list(self.nlp_engines.keys())}"
+        )
+
+        if conf_file and nlp_configuration:
+            raise ValueError(
+                "Either conf_file or nlp_configuration should be provided, not both."
+            )
+
+        if nlp_configuration:
+            ConfigurationValidator.validate_nlp_configuration(nlp_configuration)
+            self.nlp_configuration = nlp_configuration
+
+        if conf_file or conf_file == "":
+            if conf_file == "":
+                raise ValueError("conf_file is empty")
+            ConfigurationValidator.validate_file_path(conf_file)
+            self.nlp_configuration = self._read_nlp_conf(conf_file)
+
+        if conf_file is None and nlp_configuration is None:
+            conf_file = self._get_full_conf_path()
+            logger.debug(f"Reading default conf file from {conf_file}")
+            self.nlp_configuration = self._read_nlp_conf(conf_file)
+            ConfigurationValidator.validate_nlp_configuration(self.nlp_configuration)
+
+    @staticmethod
+    def _read_nlp_conf(conf_file: Union[Path, str]) -> Dict:
+        """Read NLP configuration from a YAML file."""
+        with open(conf_file) as file:
+            return yaml.safe_load(file)
+
+
+    @staticmethod
+    def _get_full_conf_path(
+        default_conf_file: Union[Path, str] = "default.yaml"
+    ) -> Path:
+        """Return a Path to the default conf file."""
+        return Path(Path(__file__).parent, "../conf", default_conf_file)
+
+    def create_engine(self) -> NlpEngine:
+        """Create an NLP engine instance."""
+        # Configuration is already validated by Pydantic in __init__
+        nlp_engine_name = self.nlp_configuration["nlp_engine_name"]
+        if nlp_engine_name not in self.nlp_engines:
+            raise ValueError(
+                f"NLP engine '{nlp_engine_name}' is not available. "
+                "Make sure you have all required packages installed"
+            )
+
+        nlp_engine_class = self.nlp_engines[nlp_engine_name]
+        nlp_models = self.nlp_configuration["models"]
+
+        ner_model_configuration = self.nlp_configuration.get("ner_model_configuration")
+        if ner_model_configuration:
+            ner_model_configuration = NerModelConfiguration.from_dict(
+                ner_model_configuration
+            )
+
+        engine = nlp_engine_class(
+            models=nlp_models, ner_model_configuration=ner_model_configuration
+        )
+        engine.load()
+        logger.info(
+            f"Created NLP engine: {engine.engine_name}. "
+            f"Loaded models: {list(engine.nlp.keys())}"
+        )
+        return engine

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/nlp_engine/spacy_nlp_engine.py

@@ -0,0 +1,290 @@
+import logging
+from pathlib import Path
+from typing import Any, Dict, Generator, List, Optional, Tuple, Union
+
+import spacy
+from spacy.language import Language
+from spacy.tokens import Doc, Span
+
+from presidio_analyzer.nlp_engine import (
+    NerModelConfiguration,
+    NlpArtifacts,
+    NlpEngine,
+    device_detector,
+)
+
+logger = logging.getLogger("presidio-analyzer")
+
+
+class SpacyNlpEngine(NlpEngine):
+    """
+    SpacyNlpEngine is an abstraction layer over the nlp module.
+
+    It provides processing functionality as well as other queries
+    on tokens.
+    The SpacyNlpEngine uses SpaCy as its NLP module
+    """
+
+    engine_name = "spacy"
+    is_available = bool(spacy)
+
+    def __init__(
+        self,
+        models: Optional[List[Dict[str, str]]] = None,
+        ner_model_configuration: Optional[NerModelConfiguration] = None,
+    ):
+        """
+        Initialize a wrapper on spaCy functionality.
+
+        :param models: Dictionary with the name of the spaCy model per language.
+        For example: models = [{"lang_code": "en", "model_name": "en_core_web_lg"}]
+        :param ner_model_configuration: Parameters for the NER model.
+        See conf/spacy.yaml for an example
+        """
+        if not models:
+            models = [{"lang_code": "en", "model_name": "en_core_web_lg"}]
+        self.models = models
+
+        if not ner_model_configuration:
+            ner_model_configuration = NerModelConfiguration()
+        self.ner_model_configuration = ner_model_configuration
+
+        self.nlp = None
+
+    def _enable_gpu(self) -> None:
+        """Enable GPU support for spaCy/transformers if available."""
+        device = device_detector.get_device()
+        if device != "cpu":
+            try:
+                spacy.require_gpu()
+            except Exception as e:
+                logger.warning(
+                    f"Failed to enable GPU ({device}), falling back to CPU: {e}"
+                )
+
+    def load(self) -> None:
+        """Load the spaCy NLP model."""
+        logger.debug(f"Loading SpaCy models: {self.models}")
+
+        self._enable_gpu()
+
+        self.nlp = {}
+        for model in self.models:
+            self._validate_model_params(model)
+            self._download_spacy_model_if_needed(model["model_name"])
+            self.nlp[model["lang_code"]] = spacy.load(model["model_name"])
+
+    @staticmethod
+    def _download_spacy_model_if_needed(model_name: str) -> None:
+        if not (spacy.util.is_package(model_name) or Path(model_name).exists()):
+            logger.warning(f"Model {model_name} is not installed. Downloading...")
+            spacy.cli.download(model_name)
+            logger.info(f"Finished downloading model {model_name}")
+
+    @staticmethod
+    def _validate_model_params(model: Dict) -> None:
+        if "lang_code" not in model:
+            raise ValueError("lang_code is missing from model configuration")
+        if "model_name" not in model:
+            raise ValueError("model_name is missing from model configuration")
+        if not isinstance(model["model_name"], str):
+            raise ValueError("model_name must be a string")
+
+    def get_supported_entities(self) -> List[str]:
+        """Return the supported entities for this NLP engine."""
+        if not self.ner_model_configuration.model_to_presidio_entity_mapping:
+            raise ValueError(
+                "model_to_presidio_entity_mapping is missing from model configuration"
+            )
+        entities_from_mapping = list(
+            set(self.ner_model_configuration.model_to_presidio_entity_mapping.values())
+        )
+        entities = [
+            ent
+            for ent in entities_from_mapping
+            if ent not in self.ner_model_configuration.labels_to_ignore
+        ]
+        return entities
+
+    def get_supported_languages(self) -> List[str]:
+        """Return the supported languages for this NLP engine."""
+        if not self.nlp:
+            raise ValueError("NLP engine is not loaded. Consider calling .load()")
+        return list(self.nlp.keys())
+
+    def is_loaded(self) -> bool:
+        """Return True if the model is already loaded."""
+        return self.nlp is not None
+
+    def process_text(self, text: str, language: str) -> NlpArtifacts:
+        """Execute the SpaCy NLP pipeline on the given text and language."""
+        if not self.nlp:
+            raise ValueError("NLP engine is not loaded. Consider calling .load()")
+
+        doc = self.nlp[language](text)
+        return self._doc_to_nlp_artifact(doc, language)
+
+    def process_batch(
+        self,
+        texts: Union[List[str], List[Tuple[str, object]]],
+        language: str,
+        batch_size: int = 1,
+        n_process: int = 1,
+        as_tuples: bool = False,
+    ) -> Generator[
+        Union[Tuple[Any, NlpArtifacts, Any], Tuple[Any, NlpArtifacts]], Any, None
+    ]:
+        """Execute the NLP pipeline on a batch of texts using spacy pipe.
+
+        :param texts: A list of texts to process. if as_tuples is set to True,
+            texts should be a list of tuples (text, context).
+        :param language: The language of the texts.
+        :param batch_size: Default batch size for pipe and evaluate.
+        :param n_process: Number of processors to process texts.
+        :param as_tuples: If set to True, inputs should be a sequence of
+            (text, context) tuples. Output will then be a sequence of
+            (doc, context) tuples. Defaults to False.
+
+        :return: A generator of tuples (text, NlpArtifacts, context) or
+            (text, NlpArtifacts) depending on the value of as_tuples.
+        """
+
+        if not self.nlp:
+            raise ValueError("NLP engine is not loaded. Consider calling .load()")
+
+        if as_tuples:
+            if not all(isinstance(item, tuple) and len(item) == 2 for item in texts):
+                raise ValueError(
+                    "When 'as_tuples' is True, "
+                    "'texts' must be a list of tuples (text, context)."
+                )
+            texts = ((str(text), context) for text, context in texts)
+        else:
+            texts = (str(text) for text in texts)
+        batch_output = self.nlp[language].pipe(
+            texts, as_tuples=as_tuples, batch_size=batch_size, n_process=n_process
+        )
+        for output in batch_output:
+            if as_tuples:
+                doc, context = output
+                yield doc.text, self._doc_to_nlp_artifact(doc, language), context
+            else:
+                doc = output
+                yield doc.text, self._doc_to_nlp_artifact(doc, language)
+
+    def is_stopword(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a stop word.
+
+        (within the given language)
+        """
+        return self.nlp[language].vocab[word].is_stop
+
+    def is_punct(self, word: str, language: str) -> bool:
+        """
+        Return true if the given word is a punctuation word.
+
+        (within the given language).
+        """
+        return self.nlp[language].vocab[word].is_punct
+
+    def get_nlp(self, language: str) -> Language:
+        """
+        Return the language model loaded for a language.
+
+        :param language: Language
+        :return: Model from spaCy
+        """
+        return self.nlp[language]
+
+    def _doc_to_nlp_artifact(self, doc: Doc, language: str) -> NlpArtifacts:
+        lemmas = [token.lemma_ for token in doc]
+        tokens_indices = [token.idx for token in doc]
+
+        entities = self._get_entities(doc)
+        scores = self._get_scores_for_entities(doc)
+
+        entities, scores = self._get_updated_entities(entities, scores)
+
+        return NlpArtifacts(
+            entities=entities,
+            tokens=doc,
+            tokens_indices=tokens_indices,
+            lemmas=lemmas,
+            nlp_engine=self,
+            language=language,
+            scores=scores,
+        )
+
+    def _get_entities(self, doc: Doc) -> List[Span]:
+        """
+        Extract entities out of a spaCy pipeline, depending on the type of pipeline.
+
+        For normal spaCy, this would be doc.ents
+        :param doc: the output spaCy doc.
+        :return: List of entities
+        """
+
+        return doc.ents
+
+    def _get_scores_for_entities(self, doc: Doc) -> List[float]:
+        """Extract scores for entities from the doc.
+
+        Since spaCy does not provide confidence scores for entities by default,
+        we use the default score from the ner model configuration.
+        :param doc: SpaCy doc
+        """
+
+        entities = doc.ents
+        scores = [self.ner_model_configuration.default_score] * len(entities)
+        return scores
+
+    def _get_updated_entities(
+        self, entities: List[Span], scores: List[float]
+    ) -> Tuple[List[Span], List[float]]:
+        """
+        Get an updated list of entities based on the ner model configuration.
+
+        Remove entities that are in labels_to_ignore,
+        update entity names based on model_to_presidio_entity_mapping
+
+        :param entities: Entities that were extracted from a spaCy pipeline
+        :param scores: Original confidence scores for the entities extracted
+        :return: Tuple holding the entities and confidence scores
+        """
+        if len(entities) != len(scores):
+            raise ValueError("Entities and scores must be the same length")
+
+        new_entities = []
+        new_scores = []
+
+        mapping = self.ner_model_configuration.model_to_presidio_entity_mapping
+        to_ignore = self.ner_model_configuration.labels_to_ignore
+        for ent, score in zip(entities, scores):
+            # Remove model labels in the ignore list
+            if ent.label_ in to_ignore:
+                continue
+
+            # Update entity label based on mapping
+            if ent.label_ in mapping:
+                ent.label_ = mapping[ent.label_]
+            else:
+                logger.warning(
+                    f"Entity {ent.label_} is not mapped to a Presidio entity, "
+                    f"but keeping anyway. "
+                    f"Add to `NerModelConfiguration.labels_to_ignore` to remove."
+                )
+
+            # Remove presidio entities in the ignore list
+            if ent.label_ in to_ignore:
+                continue
+
+            new_entities.append(ent)
+
+            # Update score if entity is in low score entity names
+            if ent.label_ in self.ner_model_configuration.low_score_entity_names:
+                score *= self.ner_model_configuration.low_confidence_score_multiplier
+
+            new_scores.append(score)
+
+        return new_entities, new_scores

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/pattern.py

@@ -0,0 +1,66 @@
+import json
+from typing import Dict
+
+import regex as re
+
+
+class Pattern:
+    """
+    A class that represents a regex pattern.
+
+    :param name: the name of the pattern
+    :param regex: the regex pattern to detect
+    :param score: the pattern's strength (values varies 0-1)
+    """
+
+    def __init__(self, name: str, regex: str, score: float):
+        self.name = name
+        self.regex = regex
+        self.score = score
+        self.compiled_regex = None
+        self.compiled_with_flags = None
+
+        self.__validate_regex(self.regex)
+        self.__validate_score(self.score)
+
+    @staticmethod
+    def __validate_regex(pattern: str) -> None:
+        """Validate that the regex pattern is valid."""
+        try:
+            re.compile(pattern)
+        except re.error as e:
+            raise ValueError(f"Invalid regex pattern: {e}")
+
+    @staticmethod
+    def __validate_score(score: float) -> None:
+        if score < 0 or score > 1:
+            raise ValueError(
+                f"Invalid score: {score}. " "Score should be between 0 and 1"
+            )
+
+    def to_dict(self) -> Dict:
+        """
+        Turn this instance into a dictionary.
+
+        :return: a dictionary
+        """
+        return_dict = {"name": self.name, "score": self.score, "regex": self.regex}
+        return return_dict
+
+    @classmethod
+    def from_dict(cls, pattern_dict: Dict) -> "Pattern":
+        """
+        Load an instance from a dictionary.
+
+        :param pattern_dict: a dictionary holding the pattern's parameters
+        :return: a Pattern instance
+        """
+        return cls(**pattern_dict)
+
+    def __repr__(self):
+        """Return string representation of instance."""
+        return json.dumps(self.to_dict())
+
+    def __str__(self):
+        """Return string representation of instance."""
+        return json.dumps(self.to_dict())