@privacyscrubber/sdk 2.2.3 → 2.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -1
- package/cli.js +4 -4
- package/index.d.ts +123 -0
- package/index.js +648 -0
- package/package.json +1 -1
- package/ps-license-manager.js +1 -1
- package/ps-pii-engine.cjs +1 -1
- package/scrubber-core.cjs +1 -1
package/index.js
CHANGED
|
@@ -12,6 +12,7 @@ import { createRequire } from 'module';
|
|
|
12
12
|
const require = createRequire(import.meta.url);
|
|
13
13
|
import fs from 'fs';
|
|
14
14
|
import path from 'path';
|
|
15
|
+
import os from 'os';
|
|
15
16
|
import { execSync } from 'child_process';
|
|
16
17
|
import { Transform } from 'stream';
|
|
17
18
|
|
|
@@ -859,6 +860,364 @@ export function createLangChainTransform(options = {}) {
|
|
|
859
860
|
};
|
|
860
861
|
}
|
|
861
862
|
|
|
863
|
+
/**
|
|
864
|
+
* LangChain Document Transformer for RAG pre-ingestion pipelines.
|
|
865
|
+
* Compliant with LangChain BaseDocumentTransformer and Runnable interfaces.
|
|
866
|
+
*
|
|
867
|
+
* Intercepts LangChain Document objects ({ pageContent, metadata }) before
|
|
868
|
+
* embedding or vector storage, scrubbing PII in pageContent and optional sensitive metadata keys.
|
|
869
|
+
*
|
|
870
|
+
* @param {Object} [options={}]
|
|
871
|
+
* @param {string} [options.profile='General'] Detection profile ('General', 'Medical', 'Finance', 'Dev', etc.)
|
|
872
|
+
* @param {string[]} [options.sensitiveMetadataKeys=[]] Metadata keys to sanitize (e.g. ['author', 'user_id', 'contact'])
|
|
873
|
+
* @param {boolean} [options.attachTelemetry=true] Whether to attach _ztds_telemetry to document metadata
|
|
874
|
+
* @param {boolean} [options.attachTokenMap=false] Whether to attach _ztds_token_map to document metadata
|
|
875
|
+
* @returns {Object} Document transformer with transformDocuments and invoke methods
|
|
876
|
+
*/
|
|
877
|
+
export function createDocumentTransformer(options = {}) {
|
|
878
|
+
const {
|
|
879
|
+
profile = 'General',
|
|
880
|
+
sensitiveMetadataKeys = [],
|
|
881
|
+
attachTelemetry = true,
|
|
882
|
+
attachTokenMap = false,
|
|
883
|
+
...engineOptions
|
|
884
|
+
} = options;
|
|
885
|
+
|
|
886
|
+
const engine = new PrivacyScrubberEngine({ profile, ...engineOptions });
|
|
887
|
+
|
|
888
|
+
async function transformDocuments(documents) {
|
|
889
|
+
if (!Array.isArray(documents)) {
|
|
890
|
+
throw new TypeError("documents must be an array of LangChain Document objects");
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
const transformed = [];
|
|
894
|
+
for (const doc of documents) {
|
|
895
|
+
const pageContent = doc.pageContent || doc.text || "";
|
|
896
|
+
const metadata = { ...(doc.metadata || {}) };
|
|
897
|
+
|
|
898
|
+
// Sanitize page content
|
|
899
|
+
const scrubResult = engine.sanitize(pageContent, { profile });
|
|
900
|
+
|
|
901
|
+
// Sanitize designated sensitive metadata fields if present
|
|
902
|
+
if (Array.isArray(sensitiveMetadataKeys) && sensitiveMetadataKeys.length > 0) {
|
|
903
|
+
for (const key of sensitiveMetadataKeys) {
|
|
904
|
+
if (metadata[key] && typeof metadata[key] === 'string') {
|
|
905
|
+
const metaScrub = engine.sanitize(metadata[key], { profile });
|
|
906
|
+
metadata[key] = metaScrub.scrubbedText;
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
if (attachTelemetry) {
|
|
912
|
+
metadata._ztds_telemetry = {
|
|
913
|
+
riskLevel: scrubResult.telemetry?.riskLevel || 'CLEAN (ZERO PII)',
|
|
914
|
+
totalCount: scrubResult.count || 0,
|
|
915
|
+
timestamp: new Date().toISOString()
|
|
916
|
+
};
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
if (attachTokenMap && scrubResult.tokenMap) {
|
|
920
|
+
metadata._ztds_token_map = { ...scrubResult.tokenMap };
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
transformed.push({
|
|
924
|
+
...doc,
|
|
925
|
+
pageContent: scrubResult.scrubbedText,
|
|
926
|
+
metadata
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
return transformed;
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
return {
|
|
934
|
+
transformDocuments,
|
|
935
|
+
invoke: async (documents) => transformDocuments(documents),
|
|
936
|
+
sessionMap: engine.sessionMap,
|
|
937
|
+
restore: (output, tokenMap) => engine.restore(output, tokenMap || engine.sessionMap),
|
|
938
|
+
resetSession: () => engine.resetSession()
|
|
939
|
+
};
|
|
940
|
+
}
|
|
941
|
+
export const createLangChainDocumentTransformer = createDocumentTransformer;
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* LlamaIndex Node/Document Transform component for RAG ingestion.
|
|
945
|
+
*
|
|
946
|
+
* Compatible with LlamaIndex ingestion pipelines and custom transformations.
|
|
947
|
+
* Sanitizes node text while preserving node IDs, relationships, and metadata.
|
|
948
|
+
*
|
|
949
|
+
* @param {Object} [options={}]
|
|
950
|
+
* @returns {Object} LlamaIndex transform with transform method
|
|
951
|
+
*/
|
|
952
|
+
export function createLlamaIndexTransform(options = {}) {
|
|
953
|
+
const { profile = 'General', ...engineOptions } = options;
|
|
954
|
+
const engine = new PrivacyScrubberEngine({ profile, ...engineOptions });
|
|
955
|
+
|
|
956
|
+
return {
|
|
957
|
+
transform: (nodes) => {
|
|
958
|
+
if (!nodes) return nodes;
|
|
959
|
+
const isArray = Array.isArray(nodes);
|
|
960
|
+
const list = isArray ? nodes : [nodes];
|
|
961
|
+
|
|
962
|
+
const transformed = list.map((node) => {
|
|
963
|
+
const text = typeof node.getContent === 'function'
|
|
964
|
+
? node.getContent()
|
|
965
|
+
: (node.text || node.pageContent || '');
|
|
966
|
+
const result = engine.sanitize(text, { profile });
|
|
967
|
+
|
|
968
|
+
if (typeof node.setContent === 'function') {
|
|
969
|
+
node.setContent(result.scrubbedText);
|
|
970
|
+
} else {
|
|
971
|
+
node.text = result.scrubbedText;
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
if (node.metadata) {
|
|
975
|
+
node.metadata._ztds_sanitized = true;
|
|
976
|
+
node.metadata._ztds_pii_count = result.count;
|
|
977
|
+
}
|
|
978
|
+
return node;
|
|
979
|
+
});
|
|
980
|
+
|
|
981
|
+
return isArray ? transformed : transformed[0];
|
|
982
|
+
},
|
|
983
|
+
sessionMap: engine.sessionMap,
|
|
984
|
+
restore: (text, map) => engine.restore(text, map || engine.sessionMap),
|
|
985
|
+
resetSession: () => engine.resetSession()
|
|
986
|
+
};
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
/**
|
|
990
|
+
* Universal Vector Database Ingestion Guard (Chroma, Pinecone, Qdrant, Weaviate, pgvector, Milvus).
|
|
991
|
+
*
|
|
992
|
+
* Guarantees zero-PII vector embedding and storage in local/cloud vector databases.
|
|
993
|
+
* Provides pre-ingestion record sanitization, query alignment, search result re-hydration,
|
|
994
|
+
* and transparent proxy wrapping of vector store clients.
|
|
995
|
+
*
|
|
996
|
+
* @param {Object} [options={}]
|
|
997
|
+
* @param {string} [options.profile='General']
|
|
998
|
+
* @param {string[]} [options.sensitiveMetadataKeys=[]]
|
|
999
|
+
* @param {boolean} [options.autoRestoreResults=true]
|
|
1000
|
+
* @returns {Object} Vector ingestion guard utilities
|
|
1001
|
+
*/
|
|
1002
|
+
export function createVectorIngestionGuard(options = {}) {
|
|
1003
|
+
const {
|
|
1004
|
+
profile = 'General',
|
|
1005
|
+
sensitiveMetadataKeys = [],
|
|
1006
|
+
autoRestoreResults = true,
|
|
1007
|
+
...engineOptions
|
|
1008
|
+
} = options;
|
|
1009
|
+
|
|
1010
|
+
const engine = new PrivacyScrubberEngine({ profile, ...engineOptions });
|
|
1011
|
+
|
|
1012
|
+
/**
|
|
1013
|
+
* Sanitizes raw database records prior to vector store upsert / add.
|
|
1014
|
+
* Supports standard formats:
|
|
1015
|
+
* - Chroma: { ids: [], documents: [], metadatas: [] }
|
|
1016
|
+
* - Pinecone: [{ id, values, metadata }]
|
|
1017
|
+
* - Qdrant: [{ id, vector, payload: { text, ... } }]
|
|
1018
|
+
* - Universal: [{ id, text, metadata }]
|
|
1019
|
+
*/
|
|
1020
|
+
function sanitizeRecords(records) {
|
|
1021
|
+
if (!records) return records;
|
|
1022
|
+
|
|
1023
|
+
// Chroma-style columnar batch: { ids, documents, metadatas }
|
|
1024
|
+
if (records.documents && Array.isArray(records.documents)) {
|
|
1025
|
+
const sanitizedDocs = [];
|
|
1026
|
+
const sanitizedMetas = records.metadatas ? [...records.metadatas] : [];
|
|
1027
|
+
let totalTokens = 0;
|
|
1028
|
+
|
|
1029
|
+
records.documents.forEach((doc, idx) => {
|
|
1030
|
+
const res = engine.sanitize(doc || '', { profile });
|
|
1031
|
+
sanitizedDocs.push(res.scrubbedText);
|
|
1032
|
+
totalTokens += res.count;
|
|
1033
|
+
|
|
1034
|
+
if (sanitizedMetas[idx]) {
|
|
1035
|
+
const meta = { ...sanitizedMetas[idx] };
|
|
1036
|
+
if (sensitiveMetadataKeys.length > 0) {
|
|
1037
|
+
for (const k of sensitiveMetadataKeys) {
|
|
1038
|
+
if (typeof meta[k] === 'string') {
|
|
1039
|
+
meta[k] = engine.sanitize(meta[k], { profile }).scrubbedText;
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
meta._ztds_sanitized = true;
|
|
1044
|
+
sanitizedMetas[idx] = meta;
|
|
1045
|
+
}
|
|
1046
|
+
});
|
|
1047
|
+
|
|
1048
|
+
return {
|
|
1049
|
+
...records,
|
|
1050
|
+
documents: sanitizedDocs,
|
|
1051
|
+
metadatas: sanitizedMetas,
|
|
1052
|
+
_ztds_total_tokens: totalTokens
|
|
1053
|
+
};
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
// Array of record objects (Pinecone, Qdrant, Weaviate, LangChain vectorstores)
|
|
1057
|
+
if (Array.isArray(records)) {
|
|
1058
|
+
let totalTokens = 0;
|
|
1059
|
+
const sanitized = records.map((record) => {
|
|
1060
|
+
const copy = { ...record };
|
|
1061
|
+
// Handle text / document / pageContent / payload.text
|
|
1062
|
+
const textField = copy.text !== undefined ? 'text'
|
|
1063
|
+
: copy.document !== undefined ? 'document'
|
|
1064
|
+
: copy.pageContent !== undefined ? 'pageContent'
|
|
1065
|
+
: null;
|
|
1066
|
+
|
|
1067
|
+
if (textField && typeof copy[textField] === 'string') {
|
|
1068
|
+
const res = engine.sanitize(copy[textField], { profile });
|
|
1069
|
+
copy[textField] = res.scrubbedText;
|
|
1070
|
+
totalTokens += res.count;
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
// Handle Qdrant payload
|
|
1074
|
+
if (copy.payload && typeof copy.payload === 'object') {
|
|
1075
|
+
copy.payload = { ...copy.payload };
|
|
1076
|
+
if (typeof copy.payload.text === 'string') {
|
|
1077
|
+
const res = engine.sanitize(copy.payload.text, { profile });
|
|
1078
|
+
copy.payload.text = res.scrubbedText;
|
|
1079
|
+
totalTokens += res.count;
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
// Handle metadata
|
|
1084
|
+
if (copy.metadata && typeof copy.metadata === 'object') {
|
|
1085
|
+
copy.metadata = { ...copy.metadata };
|
|
1086
|
+
for (const k of sensitiveMetadataKeys) {
|
|
1087
|
+
if (typeof copy.metadata[k] === 'string') {
|
|
1088
|
+
copy.metadata[k] = engine.sanitize(copy.metadata[k], { profile }).scrubbedText;
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
copy.metadata._ztds_sanitized = true;
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
return copy;
|
|
1095
|
+
});
|
|
1096
|
+
|
|
1097
|
+
return sanitized;
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
return records;
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
/**
|
|
1104
|
+
* Sanitizes user queries before generating search embeddings.
|
|
1105
|
+
* Ensures search vectors align with sanitized tokenized corpus representations.
|
|
1106
|
+
*/
|
|
1107
|
+
function sanitizeQuery(query) {
|
|
1108
|
+
if (typeof query !== 'string') return query;
|
|
1109
|
+
const res = engine.sanitize(query, { profile });
|
|
1110
|
+
const matches = res.scrubbedText.match(/\[[A-Z_]+_\d+\]/g);
|
|
1111
|
+
const tokensDetected = matches ? matches.length : (res.count || 0);
|
|
1112
|
+
return {
|
|
1113
|
+
query: res.scrubbedText,
|
|
1114
|
+
rawQuery: query,
|
|
1115
|
+
tokensDetected,
|
|
1116
|
+
tokenMap: res.tokenMap
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/**
|
|
1121
|
+
* Restores search results retrieved from vector DB using the volatile session map.
|
|
1122
|
+
*/
|
|
1123
|
+
function restoreSearchResults(results, customTokenMap = null) {
|
|
1124
|
+
const map = customTokenMap || engine.sessionMap;
|
|
1125
|
+
if (!results || Object.keys(map).length === 0) return results;
|
|
1126
|
+
|
|
1127
|
+
function restoreText(t) {
|
|
1128
|
+
if (typeof t !== 'string') return t;
|
|
1129
|
+
return engine.restore(t, map).restoredText;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
if (Array.isArray(results)) {
|
|
1133
|
+
return results.map(item => {
|
|
1134
|
+
const copy = { ...item };
|
|
1135
|
+
if (typeof copy.pageContent === 'string') copy.pageContent = restoreText(copy.pageContent);
|
|
1136
|
+
if (typeof copy.text === 'string') copy.text = restoreText(copy.text);
|
|
1137
|
+
if (typeof copy.document === 'string') copy.document = restoreText(copy.document);
|
|
1138
|
+
if (copy.payload && typeof copy.payload === 'object') {
|
|
1139
|
+
copy.payload = { ...copy.payload };
|
|
1140
|
+
if (typeof copy.payload.text === 'string') {
|
|
1141
|
+
copy.payload.text = restoreText(copy.payload.text);
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
return copy;
|
|
1145
|
+
});
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
return results;
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
/**
|
|
1152
|
+
* Wraps an existing vector store client instance with zero-trust hooks.
|
|
1153
|
+
*/
|
|
1154
|
+
function wrapVectorStore(vectorStore) {
|
|
1155
|
+
const proxy = new Proxy(vectorStore, {
|
|
1156
|
+
get(target, prop, receiver) {
|
|
1157
|
+
const orig = Reflect.get(target, prop, receiver);
|
|
1158
|
+
if (typeof orig !== 'function') return orig;
|
|
1159
|
+
|
|
1160
|
+
// Intercept ingestion methods: addDocuments, upsert, add, insert
|
|
1161
|
+
if (prop === 'addDocuments') {
|
|
1162
|
+
return async function(docs, ...rest) {
|
|
1163
|
+
if (!Array.isArray(docs)) return orig.apply(target, [docs, ...rest]);
|
|
1164
|
+
const safeDocs = docs.map(doc => {
|
|
1165
|
+
const pageContent = doc.pageContent || doc.text || "";
|
|
1166
|
+
const metadata = { ...(doc.metadata || {}) };
|
|
1167
|
+
const scrubResult = engine.sanitize(pageContent, { profile });
|
|
1168
|
+
for (const key of sensitiveMetadataKeys) {
|
|
1169
|
+
if (metadata[key] && typeof metadata[key] === 'string') {
|
|
1170
|
+
metadata[key] = engine.sanitize(metadata[key], { profile }).scrubbedText;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
metadata._ztds_telemetry = {
|
|
1174
|
+
riskLevel: scrubResult.telemetry?.riskLevel || 'CLEAN (ZERO PII)',
|
|
1175
|
+
totalCount: scrubResult.count || 0,
|
|
1176
|
+
timestamp: new Date().toISOString()
|
|
1177
|
+
};
|
|
1178
|
+
return { ...doc, pageContent: scrubResult.scrubbedText, metadata };
|
|
1179
|
+
});
|
|
1180
|
+
return orig.apply(target, [safeDocs, ...rest]);
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
if (prop === 'upsert' || prop === 'add' || prop === 'insert') {
|
|
1185
|
+
return async function(records, ...rest) {
|
|
1186
|
+
const safeRecords = sanitizeRecords(records);
|
|
1187
|
+
return orig.apply(target, [safeRecords, ...rest]);
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
// Intercept query methods: similaritySearch, query, search
|
|
1192
|
+
if (prop === 'similaritySearch' || prop === 'query' || prop === 'search') {
|
|
1193
|
+
return async function(queryArg, ...rest) {
|
|
1194
|
+
const safeQ = typeof queryArg === 'string' ? sanitizeQuery(queryArg).query : queryArg;
|
|
1195
|
+
const rawResults = await orig.apply(target, [safeQ, ...rest]);
|
|
1196
|
+
if (autoRestoreResults) {
|
|
1197
|
+
return restoreSearchResults(rawResults);
|
|
1198
|
+
}
|
|
1199
|
+
return rawResults;
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
return orig.bind(target);
|
|
1204
|
+
}
|
|
1205
|
+
});
|
|
1206
|
+
|
|
1207
|
+
return proxy;
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
return {
|
|
1211
|
+
sanitizeRecords,
|
|
1212
|
+
sanitizeQuery,
|
|
1213
|
+
restoreSearchResults,
|
|
1214
|
+
wrapVectorStore,
|
|
1215
|
+
sessionMap: engine.sessionMap,
|
|
1216
|
+
restore: (text, map) => engine.restore(text, map || engine.sessionMap),
|
|
1217
|
+
resetSession: () => engine.resetSession()
|
|
1218
|
+
};
|
|
1219
|
+
}
|
|
1220
|
+
|
|
862
1221
|
/**
|
|
863
1222
|
* Helper to escape special regex characters.
|
|
864
1223
|
*/
|
|
@@ -1229,6 +1588,248 @@ export function createExpressMiddleware(options = {}) {
|
|
|
1229
1588
|
* @param {object} [options] - Options: companyName, department, format ('markdown'|'json'|'summary').
|
|
1230
1589
|
* @returns {object|string}
|
|
1231
1590
|
*/
|
|
1591
|
+
export function formatCefEvent(eventData = {}) {
|
|
1592
|
+
const {
|
|
1593
|
+
vendor = 'PrivacyScrubber',
|
|
1594
|
+
product = 'ZTDS',
|
|
1595
|
+
version = '2.2.5',
|
|
1596
|
+
eventClassId = 'PII_REDACTION',
|
|
1597
|
+
eventName = 'Zero-Trust PII Sanitization Executed',
|
|
1598
|
+
riskLevel = 'LOW EXPOSURE',
|
|
1599
|
+
tokensMasked = 0,
|
|
1600
|
+
entities = {},
|
|
1601
|
+
frameworks = ['ZTDS Standard'],
|
|
1602
|
+
sessionHash = '',
|
|
1603
|
+
source = 'local_ram',
|
|
1604
|
+
action = 'redact_and_mask',
|
|
1605
|
+
organization = '',
|
|
1606
|
+
department = ''
|
|
1607
|
+
} = eventData;
|
|
1608
|
+
|
|
1609
|
+
let severity = 1;
|
|
1610
|
+
if (riskLevel.includes('CRITICAL')) severity = 9;
|
|
1611
|
+
else if (riskLevel.includes('MODERATE')) severity = 6;
|
|
1612
|
+
else if (riskLevel.includes('LOW')) severity = 3;
|
|
1613
|
+
else severity = 0;
|
|
1614
|
+
|
|
1615
|
+
const entitiesStr = Object.entries(entities).map(([k, v]) => `${k}:${v}`).join(',') || 'none';
|
|
1616
|
+
const frameworksStr = Array.isArray(frameworks) ? frameworks.join(',') : String(frameworks);
|
|
1617
|
+
const safeMsg = `Sanitized ${tokensMasked} entity tokens in volatile RAM. 0 bytes network egress.`;
|
|
1618
|
+
|
|
1619
|
+
const extension = [
|
|
1620
|
+
`src=${source}`,
|
|
1621
|
+
`act=${action}`,
|
|
1622
|
+
`cs1Label=RiskLevel cs1=${riskLevel.replace(/\s+/g, '_')}`,
|
|
1623
|
+
`cn1Label=TokensMasked cn1=${tokensMasked}`,
|
|
1624
|
+
`cs2Label=Frameworks cs2=${frameworksStr.replace(/\s+/g, '_')}`,
|
|
1625
|
+
`cs3Label=Entities cs3=${entitiesStr}`,
|
|
1626
|
+
sessionHash ? `cs4Label=SessionHash cs4=${sessionHash.substring(0, 16)}` : '',
|
|
1627
|
+
organization ? `suser=${organization.replace(/\s+/g, '_')}` : '',
|
|
1628
|
+
department ? `cs5Label=Department cs5=${department.replace(/\s+/g, '_')}` : '',
|
|
1629
|
+
`msg=${safeMsg}`
|
|
1630
|
+
].filter(Boolean).join(' ');
|
|
1631
|
+
|
|
1632
|
+
return `CEF:0|${vendor}|${product}|${version}|${eventClassId}|${eventName}|${severity}|${extension}`;
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
export function formatSyslogEvent(eventData = {}) {
|
|
1636
|
+
const {
|
|
1637
|
+
timestamp = new Date().toISOString(),
|
|
1638
|
+
hostname = (typeof os !== 'undefined' && typeof os.hostname === 'function' ? os.hostname() : 'localhost'),
|
|
1639
|
+
pid = (typeof process !== 'undefined' && process.pid ? process.pid : 1),
|
|
1640
|
+
riskLevel = 'LOW EXPOSURE',
|
|
1641
|
+
tokensMasked = 0,
|
|
1642
|
+
frameworks = ['ZTDS Standard'],
|
|
1643
|
+
sessionHash = '',
|
|
1644
|
+
organization = ''
|
|
1645
|
+
} = eventData;
|
|
1646
|
+
|
|
1647
|
+
let pri = 134; // local0.info
|
|
1648
|
+
if (riskLevel.includes('CRITICAL')) pri = 132; // local0.warning
|
|
1649
|
+
else if (riskLevel.includes('MODERATE')) pri = 133; // local0.notice
|
|
1650
|
+
|
|
1651
|
+
const frameworksStr = Array.isArray(frameworks) ? frameworks.join(';') : String(frameworks);
|
|
1652
|
+
const hashPrefix = sessionHash.substring(0, 16) || 'none';
|
|
1653
|
+
const orgStr = organization ? ` org="${organization.replace(/"/g, '')}"` : '';
|
|
1654
|
+
|
|
1655
|
+
const sd = `[ztds@49152 risk="${riskLevel}" tokens="${tokensMasked}" frameworks="${frameworksStr}" session="${hashPrefix}"${orgStr}]`;
|
|
1656
|
+
const msg = `Zero-Trust Data Sanitization executed in RAM. ${tokensMasked} token(s) masked. 0 bytes network egress.`;
|
|
1657
|
+
|
|
1658
|
+
return `<${pri}>1 ${timestamp} ${hostname} privacyscrubber ${pid} ZTDS - ${sd} ${msg}`;
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
export function formatJsonlEvent(eventData = {}) {
|
|
1662
|
+
const {
|
|
1663
|
+
timestamp = new Date().toISOString(),
|
|
1664
|
+
version = '2.2.5',
|
|
1665
|
+
protocol = 'ZTDS',
|
|
1666
|
+
event = 'sanitization',
|
|
1667
|
+
riskLevel = 'LOW EXPOSURE',
|
|
1668
|
+
tokensMasked = 0,
|
|
1669
|
+
entities = {},
|
|
1670
|
+
frameworks = ['ZTDS Standard'],
|
|
1671
|
+
sessionHash = '',
|
|
1672
|
+
organization = '',
|
|
1673
|
+
department = '',
|
|
1674
|
+
executionMs = 0,
|
|
1675
|
+
metadata = {}
|
|
1676
|
+
} = eventData;
|
|
1677
|
+
|
|
1678
|
+
const record = {
|
|
1679
|
+
timestamp,
|
|
1680
|
+
version,
|
|
1681
|
+
protocol,
|
|
1682
|
+
event,
|
|
1683
|
+
risk_level: riskLevel,
|
|
1684
|
+
tokens_masked: tokensMasked,
|
|
1685
|
+
entities,
|
|
1686
|
+
frameworks,
|
|
1687
|
+
session_hash: sessionHash,
|
|
1688
|
+
zero_egress: true,
|
|
1689
|
+
verification_mode: "100% Offline (In-Memory RAM)",
|
|
1690
|
+
...(organization && { organization }),
|
|
1691
|
+
...(department && { department }),
|
|
1692
|
+
...(executionMs && { execution_ms: executionMs }),
|
|
1693
|
+
...(Object.keys(metadata).length > 0 && { metadata })
|
|
1694
|
+
};
|
|
1695
|
+
|
|
1696
|
+
return JSON.stringify(record);
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
export function createAuditLogger(options = {}) {
|
|
1700
|
+
const {
|
|
1701
|
+
format = 'jsonl',
|
|
1702
|
+
destination = null,
|
|
1703
|
+
organization = 'PrivacyScrubber Enterprise Client',
|
|
1704
|
+
department = 'SecOps'
|
|
1705
|
+
} = options;
|
|
1706
|
+
|
|
1707
|
+
let fileHandle = null;
|
|
1708
|
+
let streamHandle = null;
|
|
1709
|
+
const buffer = [];
|
|
1710
|
+
|
|
1711
|
+
if (typeof destination === 'string' && destination.trim()) {
|
|
1712
|
+
try {
|
|
1713
|
+
const dir = path.dirname(destination);
|
|
1714
|
+
if (!fs.existsSync(dir)) {
|
|
1715
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
1716
|
+
}
|
|
1717
|
+
fileHandle = destination;
|
|
1718
|
+
} catch {}
|
|
1719
|
+
} else if (destination && typeof destination.write === 'function') {
|
|
1720
|
+
streamHandle = destination;
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1723
|
+
function formatEvent(eventData) {
|
|
1724
|
+
const enriched = {
|
|
1725
|
+
organization,
|
|
1726
|
+
department,
|
|
1727
|
+
timestamp: new Date().toISOString(),
|
|
1728
|
+
...eventData
|
|
1729
|
+
};
|
|
1730
|
+
|
|
1731
|
+
if (format === 'cef') {
|
|
1732
|
+
return formatCefEvent(enriched);
|
|
1733
|
+
}
|
|
1734
|
+
if (format === 'syslog') {
|
|
1735
|
+
return formatSyslogEvent(enriched);
|
|
1736
|
+
}
|
|
1737
|
+
if (format === 'json') {
|
|
1738
|
+
return JSON.stringify(enriched, null, 2);
|
|
1739
|
+
}
|
|
1740
|
+
return formatJsonlEvent(enriched);
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
function writeEntry(line) {
|
|
1744
|
+
buffer.push(line);
|
|
1745
|
+
if (fileHandle) {
|
|
1746
|
+
try {
|
|
1747
|
+
fs.appendFileSync(fileHandle, line + '\n', 'utf8');
|
|
1748
|
+
} catch (e) {
|
|
1749
|
+
console.error('[PrivacyScrubber AuditLogger] File write error:', e);
|
|
1750
|
+
}
|
|
1751
|
+
} else if (streamHandle) {
|
|
1752
|
+
try {
|
|
1753
|
+
streamHandle.write(line + '\n');
|
|
1754
|
+
} catch (e) {
|
|
1755
|
+
console.error('[PrivacyScrubber AuditLogger] Stream write error:', e);
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
function logSanitization(result, context = {}) {
|
|
1761
|
+
if (!result) return '';
|
|
1762
|
+
let tokenMap = {};
|
|
1763
|
+
let telemetry = null;
|
|
1764
|
+
let count = 0;
|
|
1765
|
+
let execMs = 0;
|
|
1766
|
+
|
|
1767
|
+
if (result.tokenMap && typeof result.tokenMap === 'object') {
|
|
1768
|
+
tokenMap = result.tokenMap;
|
|
1769
|
+
telemetry = result.telemetry;
|
|
1770
|
+
count = result.count !== undefined ? result.count : Object.keys(tokenMap).length;
|
|
1771
|
+
execMs = result.executionMs || 0;
|
|
1772
|
+
} else if (typeof result === 'object') {
|
|
1773
|
+
tokenMap = result;
|
|
1774
|
+
count = result.count !== undefined ? result.count : (result.tokensMasked !== undefined ? result.tokensMasked : Object.keys(tokenMap).length);
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
if (!telemetry) {
|
|
1778
|
+
telemetry = buildAuditTelemetry(tokenMap);
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
let sessionHash = '';
|
|
1782
|
+
try {
|
|
1783
|
+
const cryptoMod = require('crypto');
|
|
1784
|
+
sessionHash = cryptoMod.createHash('sha256').update(JSON.stringify(tokenMap) + Date.now()).digest('hex');
|
|
1785
|
+
} catch {
|
|
1786
|
+
sessionHash = 'ztds_' + Math.random().toString(16).substring(2, 10);
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
const eventData = {
|
|
1790
|
+
event: 'pii_sanitization',
|
|
1791
|
+
riskLevel: telemetry.riskLevel || 'CLEAN (ZERO PII)',
|
|
1792
|
+
tokensMasked: count,
|
|
1793
|
+
entities: telemetry.entities || {},
|
|
1794
|
+
frameworks: telemetry.frameworksList || ['ZTDS Standard'],
|
|
1795
|
+
sessionHash,
|
|
1796
|
+
executionMs: execMs,
|
|
1797
|
+
metadata: context.metadata || {}
|
|
1798
|
+
};
|
|
1799
|
+
|
|
1800
|
+
const formatted = formatEvent(eventData);
|
|
1801
|
+
writeEntry(formatted);
|
|
1802
|
+
return formatted;
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
function logEvent(eventData) {
|
|
1806
|
+
const formatted = formatEvent(eventData);
|
|
1807
|
+
writeEntry(formatted);
|
|
1808
|
+
return formatted;
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
function getEntries() {
|
|
1812
|
+
return [...buffer];
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
function clear() {
|
|
1816
|
+
buffer.length = 0;
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
return {
|
|
1820
|
+
formatEvent,
|
|
1821
|
+
logSanitization,
|
|
1822
|
+
logEvent,
|
|
1823
|
+
log: logEvent,
|
|
1824
|
+
getEntries,
|
|
1825
|
+
getBuffer: getEntries,
|
|
1826
|
+
clear,
|
|
1827
|
+
clearBuffer: clear,
|
|
1828
|
+
format,
|
|
1829
|
+
destination
|
|
1830
|
+
};
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1232
1833
|
export function generateAuditReport(sessionMap = {}, options = {}) {
|
|
1233
1834
|
const { companyName = 'PrivacyScrubber Client', department = 'SecOps / Compliance', format = 'markdown' } = options;
|
|
1234
1835
|
const telemetry = buildAuditTelemetry(sessionMap);
|
|
@@ -1244,6 +1845,45 @@ export function generateAuditReport(sessionMap = {}, options = {}) {
|
|
|
1244
1845
|
const timestamp = new Date().toISOString();
|
|
1245
1846
|
const entitySummary = Object.entries(telemetry.entities).map(([t, count]) => `[${t}]: ${count}`).join(', ') || 'None (Clean)';
|
|
1246
1847
|
|
|
1848
|
+
if (format === 'cef') {
|
|
1849
|
+
return formatCefEvent({
|
|
1850
|
+
organization: companyName,
|
|
1851
|
+
department: department,
|
|
1852
|
+
timestamp,
|
|
1853
|
+
sessionHash,
|
|
1854
|
+
riskLevel: telemetry.riskLevel,
|
|
1855
|
+
tokensMasked: telemetry.totalCount,
|
|
1856
|
+
entities: telemetry.entities,
|
|
1857
|
+
frameworks: telemetry.frameworksList
|
|
1858
|
+
});
|
|
1859
|
+
}
|
|
1860
|
+
|
|
1861
|
+
if (format === 'syslog') {
|
|
1862
|
+
return formatSyslogEvent({
|
|
1863
|
+
organization: companyName,
|
|
1864
|
+
department: department,
|
|
1865
|
+
timestamp,
|
|
1866
|
+
sessionHash,
|
|
1867
|
+
riskLevel: telemetry.riskLevel,
|
|
1868
|
+
tokensMasked: telemetry.totalCount,
|
|
1869
|
+
entities: telemetry.entities,
|
|
1870
|
+
frameworks: telemetry.frameworksList
|
|
1871
|
+
});
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
if (format === 'jsonl') {
|
|
1875
|
+
return formatJsonlEvent({
|
|
1876
|
+
organization: companyName,
|
|
1877
|
+
department: department,
|
|
1878
|
+
timestamp,
|
|
1879
|
+
sessionHash,
|
|
1880
|
+
riskLevel: telemetry.riskLevel,
|
|
1881
|
+
tokensMasked: telemetry.totalCount,
|
|
1882
|
+
entities: telemetry.entities,
|
|
1883
|
+
frameworks: telemetry.frameworksList
|
|
1884
|
+
});
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1247
1887
|
if (format === 'json') {
|
|
1248
1888
|
return {
|
|
1249
1889
|
protocol: "Zero-Trust Data Sanitization (ZTDS)",
|
|
@@ -1676,11 +2316,19 @@ export default {
|
|
|
1676
2316
|
buildAuditTelemetry,
|
|
1677
2317
|
formatAuditReceipt,
|
|
1678
2318
|
generateAuditReport,
|
|
2319
|
+
formatCefEvent,
|
|
2320
|
+
formatSyslogEvent,
|
|
2321
|
+
formatJsonlEvent,
|
|
2322
|
+
createAuditLogger,
|
|
1679
2323
|
generateProcurementMemo,
|
|
1680
2324
|
PrivacyScrubberEngine,
|
|
1681
2325
|
PrivacyScrubberLicenseError,
|
|
1682
2326
|
wrapOpenAI,
|
|
1683
2327
|
createLangChainTransform,
|
|
2328
|
+
createDocumentTransformer,
|
|
2329
|
+
createLangChainDocumentTransformer,
|
|
2330
|
+
createLlamaIndexTransform,
|
|
2331
|
+
createVectorIngestionGuard,
|
|
1684
2332
|
StreamingDetokenizer,
|
|
1685
2333
|
createStreamTokenRestorer,
|
|
1686
2334
|
createNodeStreamTokenRestorer,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@privacyscrubber/sdk",
|
|
3
|
-
"version": "2.2.
|
|
3
|
+
"version": "2.2.5",
|
|
4
4
|
"description": "In-process consumer privacy defense and pre-emptive PII interception SDK for Node.js, Next.js, and AI pipelines. Zero data loss, zero third parties, sub-millisecond in-memory execution.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"browser": "browser.js",
|