@flexorch/audit 0.6.0 → 0.8.1

package/dist/index.cjs

@@ -26,10 +26,12 @@ __export(index_exports, {
   auditStream: () => auditStream,
   complianceReport: () => complianceReport,
   detectPii: () => detectPii,
+  estimateTokens: () => estimateTokens,
   mask: () => mask,
   noiseMetrics: () => noiseMetrics,
   noiseRatio: () => noiseRatio,
   qualityMetrics: () => qualityMetrics,
+  redactForLlm: () => redactForLlm,
   version: () => version
 });
 module.exports = __toCommonJS(index_exports);
@@ -57,6 +59,7 @@ var COMPANY_NAME_TR_RE = new RegExp(
   "gu"
 );
 var MERSIS_RE = /\b([1-9]\d{15})\b/g;
+var SGK_RE = /(?:SGK\s*(?:Sicil\s*No(?:su)?|No(?:su)?|Numara(?:s[ıi])?)?|Sigortal[ıi]\s*(?:Sicil\s*)?(?:No|Numara(?:s[ıi])?)|SSK\s*(?:No|Numara(?:s[ıi])?|Sicil))\s*[:#]*\s*(\d{10,11})\b/giu;
 var POSTAL_CODE_TR_RE = /\b((?:0[1-9]|[1-7]\d|80|81)\d{3})\b/g;
 var _TR_PROVINCES_SORTED = [
   "Afyonkarahisar",
@@ -336,12 +339,24 @@ function validEinUs(s) {
   return !_EIN_INVALID_PREFIXES.has(s.slice(0, 2));
 }
 var PESEL_PL_RE = /\b(\d{11})\b/g;
+var NIP_PL_RE = /(?:NIP|Numer\s+NIP|Numer\s+Identyfikacji\s+Podatkowej)\s*[:#]*\s*(\d{10})\b/gi;
 function validPeselPl(s) {
   if (s.length !== 11 || !/^\d+$/.test(s)) return false;
   const weights = [1, 3, 7, 9, 1, 3, 7, 9, 1, 3];
   const total = weights.reduce((sum, w, i) => sum + w * parseInt(s[i]), 0);
   return (10 - total % 10) % 10 === parseInt(s[10]);
 }
+function validNifPt(s) {
+  if (s.length !== 9 || !/^\d+$/.test(s) || s[0] === "0") return false;
+  let total = 0;
+  for (let i = 0; i < 8; i++) total += (9 - i) * parseInt(s[i]);
+  const check = (11 - total % 11) % 11;
+  return (check >= 10 ? 0 : check) === parseInt(s[8]);
+}
+var NIF_PT_RE = /(?:NIF|N[uú]mero\s+de\s+Contribuinte|Contribuinte)\s*[:#]*\s*(\d{9})\b/gi;
+var PERSONNUMMER_SE_RE = /\b(\d{6,8}[-+]\d{4})\b/g;
+var CPR_DK_RE = /\b(\d{6}-\d{4})\b/g;
+var HETU_FI_RE = /\b(\d{6}[+\-A]\d{3}[0-9A-FHJ-NPR-Y])\b/g;
 var SVNR_AT_RE = /\b(\d{10})\b/g;
 function validSvnrAt(s) {
   if (s.length !== 10 || !/^\d+$/.test(s)) return false;
@@ -367,17 +382,22 @@ var LOCALE_DETECTORS = {
     "company_name_tr",
     "mersis_no",
     "postal_code_tr",
-    "province_tr"
+    "province_tr",
+    "sgk_no"
   ]),
   us: /* @__PURE__ */ new Set(["ssn", "tax_id_us", "national_id_us", "phone_intl", "company_name_intl"]),
   eu: /* @__PURE__ */ new Set(["phone_intl", "iban_intl", "company_name_intl"]),
-  de: /* @__PURE__ */ new Set(["tax_id_de", "social_id_de"]),
-  fr: /* @__PURE__ */ new Set(["siret_fr", "company_id_fr", "social_id_fr"]),
+  de: /* @__PURE__ */ new Set(["tax_id_de", "social_id_de", "social_id_at"]),
+  fr: /* @__PURE__ */ new Set(["siret_fr", "company_id_fr", "social_id_fr", "national_id_be"]),
   it: /* @__PURE__ */ new Set(["national_id_it", "tax_id_it"]),
-  nl: /* @__PURE__ */ new Set(["national_id_nl", "company_id_nl"]),
+  nl: /* @__PURE__ */ new Set(["national_id_nl", "company_id_nl", "national_id_be"]),
   es: /* @__PURE__ */ new Set(["national_id_es", "tax_id_es"]),
   uk: /* @__PURE__ */ new Set(["social_id_uk", "tax_id_uk"]),
-  pl: /* @__PURE__ */ new Set(["national_id_pl"]),
+  pl: /* @__PURE__ */ new Set(["national_id_pl", "tax_id_pl"]),
+  pt: /* @__PURE__ */ new Set(["tax_id_pt"]),
+  sv: /* @__PURE__ */ new Set(["national_id_se"]),
+  da: /* @__PURE__ */ new Set(["national_id_dk"]),
+  fi: /* @__PURE__ */ new Set(["national_id_fi"]),
   at: /* @__PURE__ */ new Set(["social_id_at"]),
   be: /* @__PURE__ */ new Set(["national_id_be"])
 };
@@ -490,6 +510,14 @@ function detectPii(text, locale = "und") {
       findings.push({ type: "mersis_no", value: m[1], start: m.index, end: m.index + m[1].length });
     }
   }
+  if (active.has("sgk_no")) {
+    SGK_RE.lastIndex = 0;
+    let m;
+    while ((m = SGK_RE.exec(t)) !== null) {
+      const start = m.index + m[0].lastIndexOf(m[1]);
+      findings.push({ type: "sgk_no", value: m[1], start, end: start + m[1].length });
+    }
+  }
   if (active.has("postal_code_tr")) {
     POSTAL_CODE_TR_RE.lastIndex = 0;
     let m;
@@ -638,6 +666,45 @@ function detectPii(text, locale = "und") {
       if (validPeselPl(m[1])) findings.push({ type: "national_id_pl", value: m[1], start: m.index, end: m.index + m[1].length });
     }
   }
+  if (active.has("tax_id_pl")) {
+    NIP_PL_RE.lastIndex = 0;
+    let m;
+    while ((m = NIP_PL_RE.exec(t)) !== null) {
+      const start = m.index + m[0].lastIndexOf(m[1]);
+      findings.push({ type: "tax_id_pl", value: m[1], start, end: start + m[1].length });
+    }
+  }
+  if (active.has("tax_id_pt")) {
+    NIF_PT_RE.lastIndex = 0;
+    let m;
+    while ((m = NIF_PT_RE.exec(t)) !== null) {
+      if (validNifPt(m[1])) {
+        const start = m.index + m[0].lastIndexOf(m[1]);
+        findings.push({ type: "tax_id_pt", value: m[1], start, end: start + m[1].length });
+      }
+    }
+  }
+  if (active.has("national_id_se")) {
+    PERSONNUMMER_SE_RE.lastIndex = 0;
+    let m;
+    while ((m = PERSONNUMMER_SE_RE.exec(t)) !== null) {
+      findings.push({ type: "national_id_se", value: m[1], start: m.index, end: m.index + m[1].length });
+    }
+  }
+  if (active.has("national_id_dk")) {
+    CPR_DK_RE.lastIndex = 0;
+    let m;
+    while ((m = CPR_DK_RE.exec(t)) !== null) {
+      findings.push({ type: "national_id_dk", value: m[1], start: m.index, end: m.index + m[1].length });
+    }
+  }
+  if (active.has("national_id_fi")) {
+    HETU_FI_RE.lastIndex = 0;
+    let m;
+    while ((m = HETU_FI_RE.exec(t)) !== null) {
+      findings.push({ type: "national_id_fi", value: m[1], start: m.index, end: m.index + m[1].length });
+    }
+  }
   if (active.has("social_id_at")) {
     SVNR_AT_RE.lastIndex = 0;
     let m;
@@ -786,7 +853,7 @@ function applyMask(text, findings, strategy = "redact") {
 }
 
 // src/index.ts
-var version = "0.6.0";
+var version = "0.7.0";
 function computeQualityScore(completeness, avgLength, garbageRatio) {
   const lengthScore = Math.min(avgLength / 500, 1);
   const noiseScore = Math.max(0, 1 - garbageRatio * 10);
@@ -842,6 +909,16 @@ async function* auditStream(texts, options = {}) {
     yield audit(text, options);
   }
 }
+function redactForLlm(text, options = {}) {
+  const { strategy, ...auditOptions } = options;
+  const result = audit(text, auditOptions);
+  return mask(text, result.pii, { strategy });
+}
+function estimateTokens(text) {
+  if (!text || !text.trim()) return 0;
+  const words = text.trim().split(/\s+/).length;
+  return Math.max(1, Math.round(words * 4 / 3));
+}
 var HIGH_RISK_TYPES = /* @__PURE__ */ new Set([
   "national_id_tr",
   "ssn",
@@ -903,9 +980,11 @@ function complianceReport(result) {
   auditStream,
   complianceReport,
   detectPii,
+  estimateTokens,
   mask,
   noiseMetrics,
   noiseRatio,
   qualityMetrics,
+  redactForLlm,
   version
 });

package/dist/index.d.cts

@@ -54,7 +54,7 @@ declare function applyMask(text: string, findings: PiiFinding[], strategy?: Mask
  * // "Contact: [REDACTED_EMAIL]"
  */
 
-declare const version = "0.6.0";
+declare const version = "0.7.0";
 type QualityGrade = "A" | "B" | "C" | "D";
 interface PiiSummaryEntry {
     type: string;
@@ -130,6 +130,30 @@ declare function mask(text: string, findings: PiiFinding[], options?: MaskOption
  * }
  */
 declare function auditStream(texts: AsyncIterable<string>, options?: AuditOptions): AsyncGenerator<AuditResult>;
+/**
+ * Audit *text* and return a PII-free version ready for LLM processing.
+ *
+ * One-shot convenience wrapper around audit() + mask(). Equivalent to:
+ *   const result = audit(text, { locale })
+ *   return mask(text, result.pii, { strategy })
+ *
+ * @example
+ * const clean = redactForLlm("TCKN: 12345678950, email: ali@example.com", { locale: "tr" })
+ * // "TCKN: [REDACTED_NATIONAL_ID_TR], email: [REDACTED_EMAIL]"
+ */
+declare function redactForLlm(text: string, options?: AuditOptions & MaskOptions): string;
+/**
+ * Estimate the token count of *text* using a word-based heuristic.
+ *
+ * Uses the standard approximation: 1 token ≈ 0.75 words (words × 4/3).
+ * No external dependencies — accuracy within ~15% of real tokenizers for
+ * English and most European languages. Treat as a planning estimate.
+ *
+ * @example
+ * estimateTokens("The quick brown fox")   // → 7
+ * estimateTokens("")                       // → 0
+ */
+declare function estimateTokens(text: string): number;
 type RiskLevel = "none" | "low" | "medium" | "high";
 interface ComplianceReport {
     has_pii: boolean;
@@ -145,4 +169,4 @@ interface ComplianceReport {
  */
 declare function complianceReport(result: AuditResult): ComplianceReport;
 
-export { type AuditOptions, type AuditResult, type BatchAuditResult, type ComplianceReport, type MaskOptions, type MaskStrategy, type NoiseMetrics, type PiiFinding, type PiiSummaryEntry, type QualityGrade, type QualityMetrics, type RiskLevel, applyMask, audit, auditBatch, auditStream, complianceReport, detectPii, mask, noiseMetrics, noiseRatio, qualityMetrics, version };
+export { type AuditOptions, type AuditResult, type BatchAuditResult, type ComplianceReport, type MaskOptions, type MaskStrategy, type NoiseMetrics, type PiiFinding, type PiiSummaryEntry, type QualityGrade, type QualityMetrics, type RiskLevel, applyMask, audit, auditBatch, auditStream, complianceReport, detectPii, estimateTokens, mask, noiseMetrics, noiseRatio, qualityMetrics, redactForLlm, version };

package/dist/index.d.ts

@@ -54,7 +54,7 @@ declare function applyMask(text: string, findings: PiiFinding[], strategy?: Mask
  * // "Contact: [REDACTED_EMAIL]"
  */
 
-declare const version = "0.6.0";
+declare const version = "0.7.0";
 type QualityGrade = "A" | "B" | "C" | "D";
 interface PiiSummaryEntry {
     type: string;
@@ -130,6 +130,30 @@ declare function mask(text: string, findings: PiiFinding[], options?: MaskOption
  * }
  */
 declare function auditStream(texts: AsyncIterable<string>, options?: AuditOptions): AsyncGenerator<AuditResult>;
+/**
+ * Audit *text* and return a PII-free version ready for LLM processing.
+ *
+ * One-shot convenience wrapper around audit() + mask(). Equivalent to:
+ *   const result = audit(text, { locale })
+ *   return mask(text, result.pii, { strategy })
+ *
+ * @example
+ * const clean = redactForLlm("TCKN: 12345678950, email: ali@example.com", { locale: "tr" })
+ * // "TCKN: [REDACTED_NATIONAL_ID_TR], email: [REDACTED_EMAIL]"
+ */
+declare function redactForLlm(text: string, options?: AuditOptions & MaskOptions): string;
+/**
+ * Estimate the token count of *text* using a word-based heuristic.
+ *
+ * Uses the standard approximation: 1 token ≈ 0.75 words (words × 4/3).
+ * No external dependencies — accuracy within ~15% of real tokenizers for
+ * English and most European languages. Treat as a planning estimate.
+ *
+ * @example
+ * estimateTokens("The quick brown fox")   // → 7
+ * estimateTokens("")                       // → 0
+ */
+declare function estimateTokens(text: string): number;
 type RiskLevel = "none" | "low" | "medium" | "high";
 interface ComplianceReport {
     has_pii: boolean;
@@ -145,4 +169,4 @@ interface ComplianceReport {
  */
 declare function complianceReport(result: AuditResult): ComplianceReport;
 
-export { type AuditOptions, type AuditResult, type BatchAuditResult, type ComplianceReport, type MaskOptions, type MaskStrategy, type NoiseMetrics, type PiiFinding, type PiiSummaryEntry, type QualityGrade, type QualityMetrics, type RiskLevel, applyMask, audit, auditBatch, auditStream, complianceReport, detectPii, mask, noiseMetrics, noiseRatio, qualityMetrics, version };
+export { type AuditOptions, type AuditResult, type BatchAuditResult, type ComplianceReport, type MaskOptions, type MaskStrategy, type NoiseMetrics, type PiiFinding, type PiiSummaryEntry, type QualityGrade, type QualityMetrics, type RiskLevel, applyMask, audit, auditBatch, auditStream, complianceReport, detectPii, estimateTokens, mask, noiseMetrics, noiseRatio, qualityMetrics, redactForLlm, version };

package/dist/index.js

@@ -21,6 +21,7 @@ var COMPANY_NAME_TR_RE = new RegExp(
   "gu"
 );
 var MERSIS_RE = /\b([1-9]\d{15})\b/g;
+var SGK_RE = /(?:SGK\s*(?:Sicil\s*No(?:su)?|No(?:su)?|Numara(?:s[ıi])?)?|Sigortal[ıi]\s*(?:Sicil\s*)?(?:No|Numara(?:s[ıi])?)|SSK\s*(?:No|Numara(?:s[ıi])?|Sicil))\s*[:#]*\s*(\d{10,11})\b/giu;
 var POSTAL_CODE_TR_RE = /\b((?:0[1-9]|[1-7]\d|80|81)\d{3})\b/g;
 var _TR_PROVINCES_SORTED = [
   "Afyonkarahisar",
@@ -300,12 +301,24 @@ function validEinUs(s) {
   return !_EIN_INVALID_PREFIXES.has(s.slice(0, 2));
 }
 var PESEL_PL_RE = /\b(\d{11})\b/g;
+var NIP_PL_RE = /(?:NIP|Numer\s+NIP|Numer\s+Identyfikacji\s+Podatkowej)\s*[:#]*\s*(\d{10})\b/gi;
 function validPeselPl(s) {
   if (s.length !== 11 || !/^\d+$/.test(s)) return false;
   const weights = [1, 3, 7, 9, 1, 3, 7, 9, 1, 3];
   const total = weights.reduce((sum, w, i) => sum + w * parseInt(s[i]), 0);
   return (10 - total % 10) % 10 === parseInt(s[10]);
 }
+function validNifPt(s) {
+  if (s.length !== 9 || !/^\d+$/.test(s) || s[0] === "0") return false;
+  let total = 0;
+  for (let i = 0; i < 8; i++) total += (9 - i) * parseInt(s[i]);
+  const check = (11 - total % 11) % 11;
+  return (check >= 10 ? 0 : check) === parseInt(s[8]);
+}
+var NIF_PT_RE = /(?:NIF|N[uú]mero\s+de\s+Contribuinte|Contribuinte)\s*[:#]*\s*(\d{9})\b/gi;
+var PERSONNUMMER_SE_RE = /\b(\d{6,8}[-+]\d{4})\b/g;
+var CPR_DK_RE = /\b(\d{6}-\d{4})\b/g;
+var HETU_FI_RE = /\b(\d{6}[+\-A]\d{3}[0-9A-FHJ-NPR-Y])\b/g;
 var SVNR_AT_RE = /\b(\d{10})\b/g;
 function validSvnrAt(s) {
   if (s.length !== 10 || !/^\d+$/.test(s)) return false;
@@ -331,17 +344,22 @@ var LOCALE_DETECTORS = {
     "company_name_tr",
     "mersis_no",
     "postal_code_tr",
-    "province_tr"
+    "province_tr",
+    "sgk_no"
   ]),
   us: /* @__PURE__ */ new Set(["ssn", "tax_id_us", "national_id_us", "phone_intl", "company_name_intl"]),
   eu: /* @__PURE__ */ new Set(["phone_intl", "iban_intl", "company_name_intl"]),
-  de: /* @__PURE__ */ new Set(["tax_id_de", "social_id_de"]),
-  fr: /* @__PURE__ */ new Set(["siret_fr", "company_id_fr", "social_id_fr"]),
+  de: /* @__PURE__ */ new Set(["tax_id_de", "social_id_de", "social_id_at"]),
+  fr: /* @__PURE__ */ new Set(["siret_fr", "company_id_fr", "social_id_fr", "national_id_be"]),
   it: /* @__PURE__ */ new Set(["national_id_it", "tax_id_it"]),
-  nl: /* @__PURE__ */ new Set(["national_id_nl", "company_id_nl"]),
+  nl: /* @__PURE__ */ new Set(["national_id_nl", "company_id_nl", "national_id_be"]),
   es: /* @__PURE__ */ new Set(["national_id_es", "tax_id_es"]),
   uk: /* @__PURE__ */ new Set(["social_id_uk", "tax_id_uk"]),
-  pl: /* @__PURE__ */ new Set(["national_id_pl"]),
+  pl: /* @__PURE__ */ new Set(["national_id_pl", "tax_id_pl"]),
+  pt: /* @__PURE__ */ new Set(["tax_id_pt"]),
+  sv: /* @__PURE__ */ new Set(["national_id_se"]),
+  da: /* @__PURE__ */ new Set(["national_id_dk"]),
+  fi: /* @__PURE__ */ new Set(["national_id_fi"]),
   at: /* @__PURE__ */ new Set(["social_id_at"]),
   be: /* @__PURE__ */ new Set(["national_id_be"])
 };
@@ -454,6 +472,14 @@ function detectPii(text, locale = "und") {
       findings.push({ type: "mersis_no", value: m[1], start: m.index, end: m.index + m[1].length });
     }
   }
+  if (active.has("sgk_no")) {
+    SGK_RE.lastIndex = 0;
+    let m;
+    while ((m = SGK_RE.exec(t)) !== null) {
+      const start = m.index + m[0].lastIndexOf(m[1]);
+      findings.push({ type: "sgk_no", value: m[1], start, end: start + m[1].length });
+    }
+  }
   if (active.has("postal_code_tr")) {
     POSTAL_CODE_TR_RE.lastIndex = 0;
     let m;
@@ -602,6 +628,45 @@ function detectPii(text, locale = "und") {
       if (validPeselPl(m[1])) findings.push({ type: "national_id_pl", value: m[1], start: m.index, end: m.index + m[1].length });
     }
   }
+  if (active.has("tax_id_pl")) {
+    NIP_PL_RE.lastIndex = 0;
+    let m;
+    while ((m = NIP_PL_RE.exec(t)) !== null) {
+      const start = m.index + m[0].lastIndexOf(m[1]);
+      findings.push({ type: "tax_id_pl", value: m[1], start, end: start + m[1].length });
+    }
+  }
+  if (active.has("tax_id_pt")) {
+    NIF_PT_RE.lastIndex = 0;
+    let m;
+    while ((m = NIF_PT_RE.exec(t)) !== null) {
+      if (validNifPt(m[1])) {
+        const start = m.index + m[0].lastIndexOf(m[1]);
+        findings.push({ type: "tax_id_pt", value: m[1], start, end: start + m[1].length });
+      }
+    }
+  }
+  if (active.has("national_id_se")) {
+    PERSONNUMMER_SE_RE.lastIndex = 0;
+    let m;
+    while ((m = PERSONNUMMER_SE_RE.exec(t)) !== null) {
+      findings.push({ type: "national_id_se", value: m[1], start: m.index, end: m.index + m[1].length });
+    }
+  }
+  if (active.has("national_id_dk")) {
+    CPR_DK_RE.lastIndex = 0;
+    let m;
+    while ((m = CPR_DK_RE.exec(t)) !== null) {
+      findings.push({ type: "national_id_dk", value: m[1], start: m.index, end: m.index + m[1].length });
+    }
+  }
+  if (active.has("national_id_fi")) {
+    HETU_FI_RE.lastIndex = 0;
+    let m;
+    while ((m = HETU_FI_RE.exec(t)) !== null) {
+      findings.push({ type: "national_id_fi", value: m[1], start: m.index, end: m.index + m[1].length });
+    }
+  }
   if (active.has("social_id_at")) {
     SVNR_AT_RE.lastIndex = 0;
     let m;
@@ -750,7 +815,7 @@ function applyMask(text, findings, strategy = "redact") {
 }
 
 // src/index.ts
-var version = "0.6.0";
+var version = "0.7.0";
 function computeQualityScore(completeness, avgLength, garbageRatio) {
   const lengthScore = Math.min(avgLength / 500, 1);
   const noiseScore = Math.max(0, 1 - garbageRatio * 10);
@@ -806,6 +871,16 @@ async function* auditStream(texts, options = {}) {
     yield audit(text, options);
   }
 }
+function redactForLlm(text, options = {}) {
+  const { strategy, ...auditOptions } = options;
+  const result = audit(text, auditOptions);
+  return mask(text, result.pii, { strategy });
+}
+function estimateTokens(text) {
+  if (!text || !text.trim()) return 0;
+  const words = text.trim().split(/\s+/).length;
+  return Math.max(1, Math.round(words * 4 / 3));
+}
 var HIGH_RISK_TYPES = /* @__PURE__ */ new Set([
   "national_id_tr",
   "ssn",
@@ -866,9 +941,11 @@ export {
   auditStream,
   complianceReport,
   detectPii,
+  estimateTokens,
   mask,
   noiseMetrics,
   noiseRatio,
   qualityMetrics,
+  redactForLlm,
   version
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flexorch/audit",
-  "version": "0.6.0",
+  "version": "0.8.1",
   "description": "Zero-dependency PII + quality + noise audit for LLM datasets (TR/EU/US)",
   "keywords": [
     "pii",