@devilsdev/rag-pipeline-utils 2.2.1 → 2.3.0

package/src/config/redaction-patterns.json

@@ -0,0 +1,150 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Redaction Patterns Configuration",
+  "description": "Configuration for secure logger redaction patterns. Define custom patterns to automatically redact sensitive data from logs.",
+  "type": "object",
+  "properties": {
+    "enabled": {
+      "type": "boolean",
+      "description": "Enable or disable secure logging globally",
+      "default": true
+    },
+    "trackStats": {
+      "type": "boolean",
+      "description": "Enable statistics tracking for redactions",
+      "default": true
+    },
+    "redactionMarker": {
+      "type": "string",
+      "description": "Default marker to use for redacted values",
+      "default": "[REDACTED]"
+    },
+    "enabledPatterns": {
+      "type": "array",
+      "description": "List of pattern names to enable (leave empty to use all)",
+      "items": {
+        "type": "string"
+      },
+      "default": []
+    },
+    "disabledPatterns": {
+      "type": "array",
+      "description": "List of pattern names to disable",
+      "items": {
+        "type": "string"
+      },
+      "default": []
+    },
+    "redactEmails": {
+      "type": "boolean",
+      "description": "Enable email address redaction",
+      "default": false
+    },
+    "redactIPs": {
+      "type": "boolean",
+      "description": "Enable IP address redaction",
+      "default": false
+    },
+    "customPatterns": {
+      "type": "object",
+      "description": "Custom redaction patterns",
+      "patternProperties": {
+        "^[a-zA-Z0-9_]+$": {
+          "type": "object",
+          "properties": {
+            "regex": {
+              "type": "string",
+              "description": "Regular expression pattern (as string)"
+            },
+            "flags": {
+              "type": "string",
+              "description": "Regex flags (e.g., 'gi')",
+              "default": "g"
+            },
+            "replacement": {
+              "type": "string",
+              "description": "Replacement string"
+            },
+            "description": {
+              "type": "string",
+              "description": "Pattern description"
+            },
+            "enabled": {
+              "type": "boolean",
+              "description": "Enable this pattern",
+              "default": true
+            }
+          },
+          "required": ["regex", "replacement"]
+        }
+      },
+      "default": {}
+    },
+    "sensitiveFields": {
+      "type": "array",
+      "description": "Additional field names to treat as sensitive",
+      "items": {
+        "type": "string"
+      },
+      "default": []
+    },
+    "environments": {
+      "type": "object",
+      "description": "Environment-specific configurations",
+      "properties": {
+        "development": {
+          "type": "object",
+          "description": "Configuration for development environment"
+        },
+        "production": {
+          "type": "object",
+          "description": "Configuration for production environment"
+        },
+        "test": {
+          "type": "object",
+          "description": "Configuration for test environment"
+        }
+      },
+      "default": {}
+    }
+  },
+  "examples": [
+    {
+      "enabled": true,
+      "trackStats": true,
+      "redactionMarker": "***REDACTED***",
+      "redactEmails": true,
+      "redactIPs": false,
+      "customPatterns": {
+        "internalToken": {
+          "regex": "\\bINT-[A-Z0-9]{32}\\b",
+          "flags": "g",
+          "replacement": "[INTERNAL_TOKEN]",
+          "description": "Internal application tokens",
+          "enabled": true
+        },
+        "sessionId": {
+          "regex": "\\bsession_id=([a-f0-9]{64})\\b",
+          "flags": "gi",
+          "replacement": "session_id=[REDACTED_SESSION]",
+          "description": "Session identifiers",
+          "enabled": true
+        }
+      },
+      "sensitiveFields": ["ssn", "taxId", "bankAccount", "routingNumber"],
+      "disabledPatterns": ["creditCard"],
+      "environments": {
+        "development": {
+          "enabled": false,
+          "trackStats": false
+        },
+        "production": {
+          "enabled": true,
+          "trackStats": true,
+          "redactEmails": true,
+          "redactIPs": true
+        }
+      }
+    }
+  ]
+}

package/src/config/sanitization-rules.json

@@ -0,0 +1,323 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Input Sanitization Rules Configuration",
+  "description": "Configuration for input validation and sanitization rules across the application",
+  "type": "object",
+  "properties": {
+    "enabled": {
+      "type": "boolean",
+      "description": "Enable or disable input sanitization globally",
+      "default": true
+    },
+    "enableCache": {
+      "type": "boolean",
+      "description": "Enable result caching for performance",
+      "default": true
+    },
+    "cacheSize": {
+      "type": "integer",
+      "description": "Maximum number of cached sanitization results",
+      "default": 1000,
+      "minimum": 0
+    },
+    "cacheTTL": {
+      "type": "integer",
+      "description": "Cache time-to-live in milliseconds",
+      "default": 60000,
+      "minimum": 1000
+    },
+    "throwOnInvalid": {
+      "type": "boolean",
+      "description": "Throw error on invalid input instead of sanitizing",
+      "default": false
+    },
+    "stripInvalid": {
+      "type": "boolean",
+      "description": "Strip invalid content instead of rejecting",
+      "default": true
+    },
+    "trackStats": {
+      "type": "boolean",
+      "description": "Track sanitization statistics",
+      "default": true
+    },
+    "defaultObjectDepth": {
+      "type": "integer",
+      "description": "Maximum depth for recursive object sanitization",
+      "default": 10,
+      "minimum": 1,
+      "maximum": 50
+    },
+    "fieldRules": {
+      "type": "object",
+      "description": "Field-specific sanitization rules",
+      "properties": {
+        "cli": {
+          "type": "object",
+          "description": "CLI input rules",
+          "properties": {
+            "command": {
+              "type": "string",
+              "enum": ["command", "alphanumeric"],
+              "default": "command"
+            },
+            "path": {
+              "type": "string",
+              "enum": ["path"],
+              "default": "path"
+            },
+            "file": {
+              "type": "string",
+              "enum": ["path"],
+              "default": "path"
+            },
+            "url": {
+              "type": "string",
+              "enum": ["url"],
+              "default": "url"
+            },
+            "email": {
+              "type": "string",
+              "enum": ["email"],
+              "default": "email"
+            },
+            "string": {
+              "type": "string",
+              "enum": ["html", "xss"],
+              "default": "html"
+            }
+          }
+        },
+        "api": {
+          "type": "object",
+          "description": "API input rules",
+          "properties": {
+            "query": {
+              "type": "string",
+              "enum": ["html", "xss"],
+              "default": "html"
+            },
+            "body": {
+              "type": "string",
+              "enum": ["html", "xss"],
+              "default": "xss"
+            },
+            "headers": {
+              "type": "string",
+              "enum": ["html"],
+              "default": "html"
+            },
+            "url": {
+              "type": "string",
+              "enum": ["url"],
+              "default": "url"
+            }
+          }
+        },
+        "database": {
+          "type": "object",
+          "description": "Database input rules",
+          "properties": {
+            "query": {
+              "type": "string",
+              "enum": ["sql"],
+              "default": "sql"
+            },
+            "text": {
+              "type": "string",
+              "enum": ["html", "xss"],
+              "default": "html"
+            }
+          }
+        },
+        "filesystem": {
+          "type": "object",
+          "description": "Filesystem operation rules",
+          "properties": {
+            "path": {
+              "type": "string",
+              "enum": ["path"],
+              "default": "path"
+            },
+            "filename": {
+              "type": "string",
+              "enum": ["alphanumeric", "slug"],
+              "default": "slug"
+            }
+          }
+        },
+        "common": {
+          "type": "object",
+          "description": "Common field rules",
+          "properties": {
+            "id": {
+              "type": "string",
+              "enum": ["uuid", "alphanumeric"],
+              "default": "alphanumeric"
+            },
+            "uuid": {
+              "type": "string",
+              "enum": ["uuid"],
+              "default": "uuid"
+            },
+            "email": {
+              "type": "string",
+              "enum": ["email"],
+              "default": "email"
+            },
+            "url": {
+              "type": "string",
+              "enum": ["url"],
+              "default": "url"
+            },
+            "slug": {
+              "type": "string",
+              "enum": ["slug"],
+              "default": "slug"
+            },
+            "domain": {
+              "type": "string",
+              "enum": ["domain"],
+              "default": "domain"
+            },
+            "ipv4": {
+              "type": "string",
+              "enum": ["ipv4"],
+              "default": "ipv4"
+            },
+            "ipv6": {
+              "type": "string",
+              "enum": ["ipv6"],
+              "default": "ipv6"
+            }
+          }
+        }
+      }
+    },
+    "environments": {
+      "type": "object",
+      "description": "Environment-specific configurations",
+      "properties": {
+        "development": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": true
+            },
+            "throwOnInvalid": {
+              "type": "boolean",
+              "default": true
+            },
+            "trackStats": {
+              "type": "boolean",
+              "default": true
+            }
+          }
+        },
+        "production": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": true
+            },
+            "throwOnInvalid": {
+              "type": "boolean",
+              "default": false
+            },
+            "stripInvalid": {
+              "type": "boolean",
+              "default": true
+            },
+            "trackStats": {
+              "type": "boolean",
+              "default": true
+            }
+          }
+        },
+        "test": {
+          "type": "boolean",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": true
+            },
+            "enableCache": {
+              "type": "boolean",
+              "default": false
+            },
+            "throwOnInvalid": {
+              "type": "boolean",
+              "default": true
+            }
+          }
+        }
+      }
+    }
+  },
+  "examples": [
+    {
+      "enabled": true,
+      "enableCache": true,
+      "cacheSize": 2000,
+      "cacheTTL": 120000,
+      "throwOnInvalid": false,
+      "stripInvalid": true,
+      "trackStats": true,
+      "defaultObjectDepth": 10,
+      "fieldRules": {
+        "cli": {
+          "command": "command",
+          "path": "path",
+          "file": "path",
+          "url": "url",
+          "email": "email",
+          "string": "xss"
+        },
+        "api": {
+          "query": "xss",
+          "body": "xss",
+          "headers": "html",
+          "url": "url"
+        },
+        "database": {
+          "query": "sql",
+          "text": "xss"
+        },
+        "filesystem": {
+          "path": "path",
+          "filename": "slug"
+        },
+        "common": {
+          "id": "alphanumeric",
+          "uuid": "uuid",
+          "email": "email",
+          "url": "url",
+          "slug": "slug",
+          "domain": "domain",
+          "ipv4": "ipv4",
+          "ipv6": "ipv6"
+        }
+      },
+      "environments": {
+        "development": {
+          "enabled": true,
+          "throwOnInvalid": true,
+          "trackStats": true
+        },
+        "production": {
+          "enabled": true,
+          "throwOnInvalid": false,
+          "stripInvalid": true,
+          "trackStats": true
+        },
+        "test": {
+          "enabled": true,
+          "enableCache": false,
+          "throwOnInvalid": true
+        }
+      }
+    }
+  ]
+}