@piecemaker-legal/piecemaker 1.0.0

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/analysis_explanation.py

@@ -0,0 +1,65 @@
+from typing import Dict
+
+
+class AnalysisExplanation:
+    """
+    Hold tracing information to explain why PII entities were identified as such.
+
+    :param recognizer: name of recognizer that made the decision
+    :param original_score: recognizer's confidence in result
+    :param pattern_name: name of pattern
+            (if decision was made by a PatternRecognizer)
+    :param pattern: regex pattern that was applied (if PatternRecognizer)
+    :param validation_result: result of a validation (e.g. checksum)
+    :param textual_explanation: Free text for describing
+            a decision of a logic or model
+    """
+
+    def __init__(
+        self,
+        recognizer: str,
+        original_score: float,
+        pattern_name: str = None,
+        pattern: str = None,
+        validation_result: float = None,
+        textual_explanation: str = None,
+        regex_flags: int = None,
+    ):
+        self.recognizer = recognizer
+        self.pattern_name = pattern_name
+        self.pattern = pattern
+        self.original_score = original_score
+        self.score = original_score
+        self.textual_explanation = textual_explanation
+        self.score_context_improvement = 0
+        self.supportive_context_word = ""
+        self.validation_result = validation_result
+        self.regex_flags = regex_flags
+
+    def __repr__(self):
+        """Create string representation of the object."""
+        return str(self.__dict__)
+
+    def set_improved_score(self, score: float) -> None:
+        """Update the score and calculate the difference from the original score."""
+        self.score = score
+        self.score_context_improvement = self.score - self.original_score
+
+    def set_supportive_context_word(self, word: str) -> None:
+        """Set the context word which helped increase the score."""
+        self.supportive_context_word = word
+
+    def append_textual_explanation_line(self, text: str) -> None:
+        """Append a new line to textual_explanation field."""
+        if self.textual_explanation is None:
+            self.textual_explanation = text
+        else:
+            self.textual_explanation = f"{self.textual_explanation}\n{text}"
+
+    def to_dict(self) -> Dict:
+        """
+        Serialize self to dictionary.
+
+        :return: a dictionary
+        """
+        return self.__dict__

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/entity_recognizer.py

@@ -0,0 +1,213 @@
+import logging
+from abc import abstractmethod
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+
+from presidio_analyzer import RecognizerResult
+
+if TYPE_CHECKING:
+    from presidio_analyzer.nlp_engine import NlpArtifacts
+
+logger = logging.getLogger("presidio-analyzer")
+
+
+class EntityRecognizer:
+    """
+    A class representing an abstract PII entity recognizer.
+
+    EntityRecognizer is an abstract class to be inherited by
+    Recognizers which hold the logic for recognizing specific PII entities.
+
+    EntityRecognizer exposes a method called enhance_using_context which
+    can be overridden in case a custom context aware enhancement is needed
+    in derived class of a recognizer.
+
+    :param supported_entities: the entities supported by this recognizer
+    (for example, phone number, address, etc.)
+    :param supported_language: the language supported by this recognizer.
+    The supported language code is iso6391Name
+    :param name: the name of this recognizer (optional)
+    :param version: the recognizer current version
+    :param context: a list of words which can help boost confidence score
+    when they appear in context of the matched entity
+    """
+
+    MIN_SCORE = 0
+    MAX_SCORE = 1.0
+
+    def __init__(
+        self,
+        supported_entities: List[str],
+        name: str = None,
+        supported_language: str = "en",
+        version: str = "0.0.1",
+        context: Optional[List[str]] = None,
+    ):
+        self.supported_entities = supported_entities
+
+        if name is None:
+            self.name = self.__class__.__name__  # assign class name as name
+        else:
+            self.name = name
+
+        self._id = f"{self.name}_{id(self)}"
+
+        self.supported_language = supported_language
+        self.version = version
+        self.is_loaded = False
+        self.context = context if context else []
+
+        self.load()
+        logger.info("Loaded recognizer: %s", self.name)
+        self.is_loaded = True
+
+    @property
+    def id(self):
+        """Return a unique identifier of this recognizer."""
+
+        return self._id
+
+    @abstractmethod
+    def load(self) -> None:
+        """
+        Initialize the recognizer assets if needed.
+
+        (e.g. machine learning models)
+        """
+
+    @abstractmethod
+    def analyze(
+        self, text: str, entities: List[str], nlp_artifacts: "NlpArtifacts"
+    ) -> List[RecognizerResult]:
+        """
+        Analyze text to identify entities.
+
+        :param text: The text to be analyzed
+        :param entities: The list of entities this recognizer is able to detect
+        :param nlp_artifacts: A group of attributes which are the result of
+        an NLP process over the input text.
+        :return: List of results detected by this recognizer.
+        """
+        return None
+
+    def enhance_using_context(
+        self,
+        text: str,
+        raw_recognizer_results: List[RecognizerResult],
+        other_raw_recognizer_results: List[RecognizerResult],
+        nlp_artifacts: "NlpArtifacts",
+        context: Optional[List[str]] = None,
+    ) -> List[RecognizerResult]:
+        """Enhance confidence score using context of the entity.
+
+        Override this method in derived class in case a custom logic
+        is needed, otherwise return value will be equal to
+        raw_results.
+
+        in case a result score is boosted, derived class need to update
+        result.recognition_metadata[RecognizerResult.IS_SCORE_ENHANCED_BY_CONTEXT_KEY]
+
+        :param text: The actual text that was analyzed
+        :param raw_recognizer_results: This recognizer's results, to be updated
+        based on recognizer specific context.
+        :param other_raw_recognizer_results: Other recognizer results matched in
+        the given text to allow related entity context enhancement
+        :param nlp_artifacts: The nlp artifacts contains elements
+                              such as lemmatized tokens for better
+                              accuracy of the context enhancement process
+        :param context: list of context words
+        """
+        return raw_recognizer_results
+
+    def get_supported_entities(self) -> List[str]:
+        """
+        Return the list of entities this recognizer can identify.
+
+        :return: A list of the supported entities by this recognizer
+        """
+        return self.supported_entities
+
+    def get_supported_language(self) -> str:
+        """
+        Return the language this recognizer can support.
+
+        :return: A list of the supported language by this recognizer
+        """
+        return self.supported_language
+
+    def get_version(self) -> str:
+        """
+        Return the version of this recognizer.
+
+        :return: The current version of this recognizer
+        """
+        return self.version
+
+    def to_dict(self) -> Dict:
+        """
+        Serialize self to dictionary.
+
+        :return: a dictionary
+        """
+        return_dict = {
+            "supported_entities": self.supported_entities,
+            "supported_language": self.supported_language,
+            "name": self.name,
+            "version": self.version,
+        }
+        return return_dict
+
+    @classmethod
+    def from_dict(cls, entity_recognizer_dict: Dict) -> "EntityRecognizer":
+        """
+        Create EntityRecognizer from a dict input.
+
+        :param entity_recognizer_dict: Dict containing keys and values for instantiation
+        """
+        return cls(**entity_recognizer_dict)
+
+    @staticmethod
+    def remove_duplicates(results: List[RecognizerResult]) -> List[RecognizerResult]:
+        """
+        Remove duplicate results.
+
+        Remove duplicates in case the two results
+        have identical start and ends and types.
+        :param results: List[RecognizerResult]
+        :return: List[RecognizerResult]
+        """
+        results = list(set(results))
+        results = sorted(results, key=lambda x: (-x.score, x.start, -(x.end - x.start)))
+        filtered_results = []
+
+        for result in results:
+            if result.score == 0:
+                continue
+
+            to_keep = result not in filtered_results  # equals based comparison
+            if to_keep:
+                for filtered in filtered_results:
+                    # If result is contained in one of the other results
+                    if (
+                        result.contained_in(filtered)
+                        and result.entity_type == filtered.entity_type
+                    ):
+                        to_keep = False
+                        break
+
+            if to_keep:
+                filtered_results.append(result)
+
+        return filtered_results
+
+    @staticmethod
+    def sanitize_value(text: str, replacement_pairs: List[Tuple[str, str]]) -> str:
+        """
+        Cleanse the input string of the replacement pairs specified as argument.
+
+        :param text: input string
+        :param replacement_pairs: pairs of what has to be replaced with which value
+        :return: cleansed string
+        """
+        for search_string, replacement_string in replacement_pairs:
+            text = text.replace(search_string, replacement_string)
+        return text

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/input_validation/__init__.py

@@ -0,0 +1,21 @@
+"""Configuration validation module for Presidio."""
+
+from .language_validation import validate_language_codes
+from .schemas import ConfigurationValidator
+from .yaml_recognizer_models import (
+    BaseRecognizerConfig,
+    CustomRecognizerConfig,
+    LanguageContextConfig,
+    PredefinedRecognizerConfig,
+    RecognizerRegistryConfig,
+)
+
+__all__ = [
+    "validate_language_codes",
+    "ConfigurationValidator",
+    "BaseRecognizerConfig",
+    "CustomRecognizerConfig",
+    "LanguageContextConfig",
+    "PredefinedRecognizerConfig",
+    "RecognizerRegistryConfig",
+]

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/input_validation/language_validation.py

@@ -0,0 +1,18 @@
+from typing import List
+
+import regex as re
+
+
+def validate_language_codes(languages: List[str]) -> None:
+    """Validate language codes format.
+
+    :param languages: List of languages to validate.
+    """
+    language_code_regex = re.compile(r"^[a-z]{2}(-[A-Z]{2})?$")
+
+    for lang in languages:
+        if not re.match(language_code_regex, lang):
+            raise ValueError(
+                f"Invalid language code format: {lang}. "
+                f"Expected format: 'en' or 'en-US'"
+            )

package/server/piecemaker/vendor/websocket-server/scripts/presidio_analyzer/input_validation/schemas.py

@@ -0,0 +1,132 @@
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from pydantic import ValidationError
+
+from . import validate_language_codes
+from .yaml_recognizer_models import RecognizerRegistryConfig
+
+
+class ConfigurationValidator:
+    """Class for validating configurations using Pydantic-enabled classes."""
+
+    @staticmethod
+    def validate_language_codes(languages: List[str]) -> List[str]:
+        """Validate language codes format.
+
+        :param languages: List of languages to validate.
+        """
+        validate_language_codes(languages)
+        return languages
+
+    @staticmethod
+    def validate_file_path(file_path: Union[str, Path]) -> Path:
+        """Validate file path exists and is readable.
+
+        :param file_path: Path to validate.
+        """
+        path = Path(file_path)
+        if not path.exists():
+            raise ValueError(f"Configuration file does not exist: {path}")
+        if not path.is_file():
+            raise ValueError(f"Path is not a file: {path}")
+        return path
+
+    @staticmethod
+    def validate_score_threshold(threshold: float) -> float:
+        """Validate score threshold is within valid range.
+
+        :param threshold: score threshold to validate.
+        """
+        if not 0.0 <= threshold <= 1.0:
+            raise ValueError(
+                f"Score threshold must be between 0.0 and 1.0, got: {threshold}"
+            )
+        return threshold
+
+    @staticmethod
+    def validate_nlp_configuration(config: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate NLP configuration structure.
+
+        :param config: NLP Configuration to validate.
+        """
+        if not isinstance(config, dict):
+            raise ValueError("NLP configuration must be a dictionary")
+
+        required_fields = ["nlp_engine_name", "models"]
+        missing_fields = [field for field in required_fields if field not in config]
+        if missing_fields:
+            raise ValueError(
+                f"NLP configuration missing required fields: {missing_fields}"
+            )
+
+        # Validate models structure
+        if not isinstance(config["models"], list) or not config["models"]:
+            raise ValueError("Models must be a non-empty list")
+
+        for model in config["models"]:
+            if not isinstance(model, dict):
+                raise ValueError("Each model must be a dictionary")
+            if "lang_code" not in model or "model_name" not in model:
+                raise ValueError("Each model must have 'lang_code' and 'model_name'")
+
+        return config
+
+    @staticmethod
+    def validate_recognizer_registry_configuration(
+        config: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """Validate recognizer registry configuration using Pydantic models."""
+        try:
+            # Use Pydantic model for validation
+            validated_config = RecognizerRegistryConfig(**config)
+            # Use model_dump() without exclude_unset to include default values
+            return validated_config.model_dump(exclude_unset=False)
+        except ValidationError as e:
+            raise ValueError("Invalid recognizer registry configuration") from e
+
+    @staticmethod
+    def validate_analyzer_configuration(config: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate analyzer engine configuration."""
+        if not isinstance(config, dict):
+            raise ValueError("Analyzer configuration must be a dictionary")
+
+        # Define valid top-level keys for analyzer configuration
+        valid_keys = {
+            "supported_languages",
+            "default_score_threshold",
+            "nlp_configuration",
+            "recognizer_registry",
+        }
+
+        # Check for unknown keys
+        unknown_keys = set(config.keys()) - valid_keys
+        if unknown_keys:
+            raise ValueError(
+                f"Unknown configuration key(s) in "
+                f"analyzer configuration: {sorted(unknown_keys)}. "
+                f"Valid keys are: {sorted(valid_keys)}"
+            )
+
+        # Validate supported languages if present
+        if "supported_languages" in config:
+            validate_language_codes(config["supported_languages"])
+
+        # Validate score threshold if present
+        if "default_score_threshold" in config:
+            ConfigurationValidator.validate_score_threshold(
+                config["default_score_threshold"]
+            )
+
+        # Validate nested configurations
+        if "nlp_configuration" in config:
+            ConfigurationValidator.validate_nlp_configuration(
+                config["nlp_configuration"]
+            )
+
+        if "recognizer_registry" in config:
+            ConfigurationValidator.validate_recognizer_registry_configuration(
+                config["recognizer_registry"]
+            )
+
+        return config