@flexorch/audit 0.2.0 → 0.3.0

package/README.md

@@ -35,6 +35,8 @@ const clean = mask(text, result.pii, { strategy: "redact" })
 npm install @flexorch/audit
 ```
 
+![demo](assets/demo.svg)
+
 ## Locale support
 
 | `locale` | Active detectors |

package/dist/index.cjs

@@ -22,6 +22,7 @@ var index_exports = {};
 __export(index_exports, {
   applyMask: () => applyMask,
   audit: () => audit,
+  auditBatch: () => auditBatch,
   detectPii: () => detectPii,
   mask: () => mask,
   noiseMetrics: () => noiseMetrics,
@@ -36,8 +37,14 @@ var PHONE_INTL_RE = /\+\d{1,3}[\s\-.]?\(?\d{1,4}\)?[\s\-.]?\d{3,4}[\s\-.]?\d{4}\
 var IBAN_RE = /\b[A-Z]{2}\d{2}[0-9A-Z]{11,30}\b/g;
 var CC_RE = /\b\d{4}[ \-]\d{4}[ \-]\d{4}[ \-]\d{4}\b/g;
 var IPV4_RE = /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g;
+var _H = "[0-9a-fA-F]{1,4}";
+var IPV6_RE = new RegExp(
+  `(?<![:\\.\\w])(?:(?:${_H}:){7}${_H}|(?:${_H}:){1,7}:|::(?:(?:${_H}:){0,6}${_H})?|(?:${_H}:){1,6}:${_H}|(?:${_H}:){1,5}(?::${_H}){1,2}|(?:${_H}:){1,4}(?::${_H}){1,3}|(?:${_H}:){1,3}(?::${_H}){1,4}|(?:${_H}:){1,2}(?::${_H}){1,5}|${_H}:(?::${_H}){1,6})(?![:\\.\\w])`,
+  "gi"
+);
 var PHONE_TR_RE = /\b(?:\+90|0)?\s*5\d{2}\s*\d{3}\s*\d{2}\s*\d{2}\b/g;
 var TCKN_RE = /\b([1-9]\d{10})\b/g;
+var VKN_RE = /\b([1-9]\d{9})\b/g;
 var NAME_PREFIX_TR = "(?:Ad[\u0131i]\\s*(?:Soyad[\u0131i])?|Soyad[\u0131i]|\u0130sim|M\xFC\u015Fteri\\s+Ad[\u0131i]|Yetkili(?:\\s+Ki\u015Fi)?|\xC7al\u0131\u015Fan\\s+Ad[\u0131i]|Personel\\s+Ad[\u0131i]|Ki\u015Fi\\s+Ad[\u0131i]|Sat\u0131c\u0131\\s+Ad[\u0131i]|Al\u0131c\u0131\\s+Ad[\u0131i]|\u0130lgili\\s+Ki\u015Fi|Hesap\\s+Sahibi)";
 var NAME_PREFIX_EN = "(?:Full\\s+Name|Customer\\s+Name|Employee\\s+Name|Contact\\s+Name|Authorized\\s+(?:By|Person)|Account\\s+Holder|(?<!\\bUser\\s)Name)";
 var NAME_VALUE = "([A-Z\xC7\u011E\u0130\xD6\u015E\xDC][a-z\xE7\u011F\u0131\u015F\xF6\u015F\xFC]+(?:\\s+[A-Z\xC7\u011E\u0130\xD6\u015E\xDC][a-z\xE7\u011F\u0131\u015F\xF6\u015F\xFC]+){0,2})";
@@ -54,6 +61,20 @@ function validTckn(s) {
   if ((sumOdd * 7 - sumEven) % 10 !== d[9]) return false;
   return d.slice(0, 10).reduce((a, b) => a + b, 0) % 10 === d[10];
 }
+function validVkn(s) {
+  if (s.length !== 10 || !/^\d+$/.test(s) || s[0] === "0") return false;
+  const d = s.split("").map(Number);
+  let total = 0;
+  for (let i = 0; i < 9; i++) {
+    const x = (d[i] + (9 - i)) % 10;
+    if (x !== 0) {
+      let y = x * Math.pow(2, 9 - i) % 9;
+      if (y === 0) y = 9;
+      total += y;
+    }
+  }
+  return (10 - total % 10) % 10 === d[9];
+}
 function luhn(number) {
   const digits = number.replace(/\D/g, "");
   if (digits.length < 13 || digits.length > 19) return false;
@@ -68,12 +89,26 @@ function luhn(number) {
   }
   return total % 10 === 0;
 }
+function validIban(s) {
+  const rearranged = s.slice(4) + s.slice(0, 4);
+  const numeric = rearranged.toUpperCase().split("").map((c) => {
+    const code = c.charCodeAt(0);
+    return code >= 65 && code <= 90 ? String(code - 55) : c;
+  }).join("");
+  let remainder = 0;
+  for (let i = 0; i < numeric.length; i += 9) {
+    const chunk = Number(String(remainder) + numeric.slice(i, i + 9));
+    if (!Number.isFinite(chunk)) return false;
+    remainder = chunk % 97;
+  }
+  return remainder === 1;
+}
 var LOCALE_DETECTORS = {
-  tr: /* @__PURE__ */ new Set(["national_id_tr", "phone_tr", "name"]),
+  tr: /* @__PURE__ */ new Set(["national_id_tr", "tax_id_tr", "phone_tr", "name"]),
   us: /* @__PURE__ */ new Set(["ssn", "phone"]),
   eu: /* @__PURE__ */ new Set(["phone"])
 };
-var UNIVERSAL = /* @__PURE__ */ new Set(["email", "iban", "credit_card", "ip"]);
+var UNIVERSAL = /* @__PURE__ */ new Set(["email", "iban", "credit_card", "ip", "ip_v6"]);
 function activeDetectors(locale) {
   if (locale === "all") {
     const active2 = new Set(UNIVERSAL);
@@ -111,7 +146,15 @@ function detectPii(text, locale = "tr") {
       }
     }
   }
-  if (active.has("iban")) findings.push(...findAll(IBAN_RE, t, "iban"));
+  if (active.has("iban")) {
+    IBAN_RE.lastIndex = 0;
+    let m;
+    while ((m = IBAN_RE.exec(t)) !== null) {
+      if (validIban(m[0])) {
+        findings.push({ type: "iban", value: m[0], start: m.index, end: m.index + m[0].length });
+      }
+    }
+  }
   if (active.has("credit_card")) {
     CC_RE.lastIndex = 0;
     let m;
@@ -122,6 +165,7 @@ function detectPii(text, locale = "tr") {
     }
   }
   if (active.has("ip")) findings.push(...findAll(IPV4_RE, t, "ip"));
+  if (active.has("ip_v6")) findings.push(...findAll(IPV6_RE, t, "ip_v6"));
   if (active.has("phone_tr")) findings.push(...findAll(PHONE_TR_RE, t, "phone_tr"));
   if (active.has("national_id_tr")) {
     TCKN_RE.lastIndex = 0;
@@ -132,6 +176,15 @@ function detectPii(text, locale = "tr") {
       }
     }
   }
+  if (active.has("tax_id_tr")) {
+    VKN_RE.lastIndex = 0;
+    let m;
+    while ((m = VKN_RE.exec(t)) !== null) {
+      if (validVkn(m[1])) {
+        findings.push({ type: "tax_id_tr", value: m[1], start: m.index, end: m.index + m[0].length });
+      }
+    }
+  }
   if (active.has("name")) {
     NAME_RE.lastIndex = 0;
     let m;
@@ -221,7 +274,7 @@ function applyMask(text, findings, strategy = "redact") {
 }
 
 // src/index.ts
-var version = "0.2.0";
+var version = "0.3.0";
 function computeQualityScore(completeness, avgLength, garbageRatio) {
   const lengthScore = Math.min(avgLength / 500, 1);
   const noiseScore = Math.max(0, 1 - garbageRatio * 10);
@@ -249,6 +302,25 @@ function audit(text, options = {}) {
   const pii_summary = Array.from(counts.entries()).sort(([a], [b]) => a.localeCompare(b)).map(([type, count]) => ({ type, count }));
   return { quality_grade, quality_score, pii_summary, pii, quality, noise };
 }
+function auditBatch(texts, options = {}) {
+  if (texts.length === 0) {
+    return { results: [], duplicate_ratio: 0, pii_summary: [], avg_quality_score: 0 };
+  }
+  const results = texts.map((t) => audit(t, options));
+  const seen = /* @__PURE__ */ new Set();
+  let dupCount = 0;
+  for (const t of texts) {
+    if (seen.has(t)) dupCount++;
+    else seen.add(t);
+  }
+  const duplicate_ratio = Math.round(dupCount / texts.length * 1e4) / 1e4;
+  const allPii = results.flatMap((r) => r.pii);
+  const counts = /* @__PURE__ */ new Map();
+  for (const f of allPii) counts.set(f.type, (counts.get(f.type) ?? 0) + 1);
+  const pii_summary = Array.from(counts.entries()).sort(([a], [b]) => a.localeCompare(b)).map(([type, count]) => ({ type, count }));
+  const avg_quality_score = Math.round(results.reduce((sum, r) => sum + r.quality_score, 0) / results.length * 1e4) / 1e4;
+  return { results, duplicate_ratio, pii_summary, avg_quality_score };
+}
 function mask(text, findings, options = {}) {
   return applyMask(text, findings, options.strategy ?? "redact");
 }
@@ -256,6 +328,7 @@ function mask(text, findings, options = {}) {
 0 && (module.exports = {
   applyMask,
   audit,
+  auditBatch,
   detectPii,
   mask,
   noiseMetrics,

package/dist/index.d.cts

@@ -45,7 +45,7 @@ declare function applyMask(text: string, findings: PiiFinding[], strategy?: Mask
  * // "Contact: [REDACTED_EMAIL]"
  */
 
-declare const version = "0.2.0";
+declare const version = "0.3.0";
 type QualityGrade = "A" | "B" | "C" | "D";
 interface PiiSummaryEntry {
     type: string;
@@ -54,12 +54,12 @@ interface PiiSummaryEntry {
 interface AuditOptions {
     /**
      * Active locale-specific detectors.
-     * - "tr"  — Turkish: TCKN, phone_tr, name  (default)
+     * - "tr"  — Turkish: TCKN, VKN, phone_tr, name  (default)
      * - "us"  — US: SSN, E.164 phone
      * - "eu"  — EU: E.164 phone
      * - "all" — All detectors
      *
-     * Universal detectors (email, iban, credit_card, ip) are always active.
+     * Universal detectors (email, iban, credit_card, ip, ip_v6) are always active.
      */
     locale?: string;
 }
@@ -83,9 +83,23 @@ interface MaskOptions {
  * Audit *text* for LLM dataset readiness.
  */
 declare function audit(text: string, options?: AuditOptions): AuditResult;
+interface BatchAuditResult {
+    /** One AuditResult per input text, in order. */
+    results: AuditResult[];
+    /** Fraction of texts that are exact duplicates (0.0–1.0). */
+    duplicate_ratio: number;
+    /** PII counts aggregated across all texts. */
+    pii_summary: PiiSummaryEntry[];
+    /** Mean quality_score across all texts. */
+    avg_quality_score: number;
+}
+/**
+ * Audit a list of texts and aggregate metrics — including duplicate_ratio.
+ */
+declare function auditBatch(texts: string[], options?: AuditOptions): BatchAuditResult;
 /**
  * Apply masking to PII findings in *text*.
  */
 declare function mask(text: string, findings: PiiFinding[], options?: MaskOptions): string;
 
-export { type AuditOptions, type AuditResult, type MaskOptions, type MaskStrategy, type NoiseMetrics, type PiiFinding, type PiiSummaryEntry, type QualityGrade, type QualityMetrics, applyMask, audit, detectPii, mask, noiseMetrics, qualityMetrics, version };
+export { type AuditOptions, type AuditResult, type BatchAuditResult, type MaskOptions, type MaskStrategy, type NoiseMetrics, type PiiFinding, type PiiSummaryEntry, type QualityGrade, type QualityMetrics, applyMask, audit, auditBatch, detectPii, mask, noiseMetrics, qualityMetrics, version };

package/dist/index.d.ts

@@ -45,7 +45,7 @@ declare function applyMask(text: string, findings: PiiFinding[], strategy?: Mask
  * // "Contact: [REDACTED_EMAIL]"
  */
 
-declare const version = "0.2.0";
+declare const version = "0.3.0";
 type QualityGrade = "A" | "B" | "C" | "D";
 interface PiiSummaryEntry {
     type: string;
@@ -54,12 +54,12 @@ interface PiiSummaryEntry {
 interface AuditOptions {
     /**
      * Active locale-specific detectors.
-     * - "tr"  — Turkish: TCKN, phone_tr, name  (default)
+     * - "tr"  — Turkish: TCKN, VKN, phone_tr, name  (default)
      * - "us"  — US: SSN, E.164 phone
      * - "eu"  — EU: E.164 phone
      * - "all" — All detectors
      *
-     * Universal detectors (email, iban, credit_card, ip) are always active.
+     * Universal detectors (email, iban, credit_card, ip, ip_v6) are always active.
      */
     locale?: string;
 }
@@ -83,9 +83,23 @@ interface MaskOptions {
  * Audit *text* for LLM dataset readiness.
  */
 declare function audit(text: string, options?: AuditOptions): AuditResult;
+interface BatchAuditResult {
+    /** One AuditResult per input text, in order. */
+    results: AuditResult[];
+    /** Fraction of texts that are exact duplicates (0.0–1.0). */
+    duplicate_ratio: number;
+    /** PII counts aggregated across all texts. */
+    pii_summary: PiiSummaryEntry[];
+    /** Mean quality_score across all texts. */
+    avg_quality_score: number;
+}
+/**
+ * Audit a list of texts and aggregate metrics — including duplicate_ratio.
+ */
+declare function auditBatch(texts: string[], options?: AuditOptions): BatchAuditResult;
 /**
  * Apply masking to PII findings in *text*.
  */
 declare function mask(text: string, findings: PiiFinding[], options?: MaskOptions): string;
 
-export { type AuditOptions, type AuditResult, type MaskOptions, type MaskStrategy, type NoiseMetrics, type PiiFinding, type PiiSummaryEntry, type QualityGrade, type QualityMetrics, applyMask, audit, detectPii, mask, noiseMetrics, qualityMetrics, version };
+export { type AuditOptions, type AuditResult, type BatchAuditResult, type MaskOptions, type MaskStrategy, type NoiseMetrics, type PiiFinding, type PiiSummaryEntry, type QualityGrade, type QualityMetrics, applyMask, audit, auditBatch, detectPii, mask, noiseMetrics, qualityMetrics, version };

package/dist/index.js

@@ -4,8 +4,14 @@ var PHONE_INTL_RE = /\+\d{1,3}[\s\-.]?\(?\d{1,4}\)?[\s\-.]?\d{3,4}[\s\-.]?\d{4}\
 var IBAN_RE = /\b[A-Z]{2}\d{2}[0-9A-Z]{11,30}\b/g;
 var CC_RE = /\b\d{4}[ \-]\d{4}[ \-]\d{4}[ \-]\d{4}\b/g;
 var IPV4_RE = /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g;
+var _H = "[0-9a-fA-F]{1,4}";
+var IPV6_RE = new RegExp(
+  `(?<![:\\.\\w])(?:(?:${_H}:){7}${_H}|(?:${_H}:){1,7}:|::(?:(?:${_H}:){0,6}${_H})?|(?:${_H}:){1,6}:${_H}|(?:${_H}:){1,5}(?::${_H}){1,2}|(?:${_H}:){1,4}(?::${_H}){1,3}|(?:${_H}:){1,3}(?::${_H}){1,4}|(?:${_H}:){1,2}(?::${_H}){1,5}|${_H}:(?::${_H}){1,6})(?![:\\.\\w])`,
+  "gi"
+);
 var PHONE_TR_RE = /\b(?:\+90|0)?\s*5\d{2}\s*\d{3}\s*\d{2}\s*\d{2}\b/g;
 var TCKN_RE = /\b([1-9]\d{10})\b/g;
+var VKN_RE = /\b([1-9]\d{9})\b/g;
 var NAME_PREFIX_TR = "(?:Ad[\u0131i]\\s*(?:Soyad[\u0131i])?|Soyad[\u0131i]|\u0130sim|M\xFC\u015Fteri\\s+Ad[\u0131i]|Yetkili(?:\\s+Ki\u015Fi)?|\xC7al\u0131\u015Fan\\s+Ad[\u0131i]|Personel\\s+Ad[\u0131i]|Ki\u015Fi\\s+Ad[\u0131i]|Sat\u0131c\u0131\\s+Ad[\u0131i]|Al\u0131c\u0131\\s+Ad[\u0131i]|\u0130lgili\\s+Ki\u015Fi|Hesap\\s+Sahibi)";
 var NAME_PREFIX_EN = "(?:Full\\s+Name|Customer\\s+Name|Employee\\s+Name|Contact\\s+Name|Authorized\\s+(?:By|Person)|Account\\s+Holder|(?<!\\bUser\\s)Name)";
 var NAME_VALUE = "([A-Z\xC7\u011E\u0130\xD6\u015E\xDC][a-z\xE7\u011F\u0131\u015F\xF6\u015F\xFC]+(?:\\s+[A-Z\xC7\u011E\u0130\xD6\u015E\xDC][a-z\xE7\u011F\u0131\u015F\xF6\u015F\xFC]+){0,2})";
@@ -22,6 +28,20 @@ function validTckn(s) {
   if ((sumOdd * 7 - sumEven) % 10 !== d[9]) return false;
   return d.slice(0, 10).reduce((a, b) => a + b, 0) % 10 === d[10];
 }
+function validVkn(s) {
+  if (s.length !== 10 || !/^\d+$/.test(s) || s[0] === "0") return false;
+  const d = s.split("").map(Number);
+  let total = 0;
+  for (let i = 0; i < 9; i++) {
+    const x = (d[i] + (9 - i)) % 10;
+    if (x !== 0) {
+      let y = x * Math.pow(2, 9 - i) % 9;
+      if (y === 0) y = 9;
+      total += y;
+    }
+  }
+  return (10 - total % 10) % 10 === d[9];
+}
 function luhn(number) {
   const digits = number.replace(/\D/g, "");
   if (digits.length < 13 || digits.length > 19) return false;
@@ -36,12 +56,26 @@ function luhn(number) {
   }
   return total % 10 === 0;
 }
+function validIban(s) {
+  const rearranged = s.slice(4) + s.slice(0, 4);
+  const numeric = rearranged.toUpperCase().split("").map((c) => {
+    const code = c.charCodeAt(0);
+    return code >= 65 && code <= 90 ? String(code - 55) : c;
+  }).join("");
+  let remainder = 0;
+  for (let i = 0; i < numeric.length; i += 9) {
+    const chunk = Number(String(remainder) + numeric.slice(i, i + 9));
+    if (!Number.isFinite(chunk)) return false;
+    remainder = chunk % 97;
+  }
+  return remainder === 1;
+}
 var LOCALE_DETECTORS = {
-  tr: /* @__PURE__ */ new Set(["national_id_tr", "phone_tr", "name"]),
+  tr: /* @__PURE__ */ new Set(["national_id_tr", "tax_id_tr", "phone_tr", "name"]),
   us: /* @__PURE__ */ new Set(["ssn", "phone"]),
   eu: /* @__PURE__ */ new Set(["phone"])
 };
-var UNIVERSAL = /* @__PURE__ */ new Set(["email", "iban", "credit_card", "ip"]);
+var UNIVERSAL = /* @__PURE__ */ new Set(["email", "iban", "credit_card", "ip", "ip_v6"]);
 function activeDetectors(locale) {
   if (locale === "all") {
     const active2 = new Set(UNIVERSAL);
@@ -79,7 +113,15 @@ function detectPii(text, locale = "tr") {
       }
     }
   }
-  if (active.has("iban")) findings.push(...findAll(IBAN_RE, t, "iban"));
+  if (active.has("iban")) {
+    IBAN_RE.lastIndex = 0;
+    let m;
+    while ((m = IBAN_RE.exec(t)) !== null) {
+      if (validIban(m[0])) {
+        findings.push({ type: "iban", value: m[0], start: m.index, end: m.index + m[0].length });
+      }
+    }
+  }
   if (active.has("credit_card")) {
     CC_RE.lastIndex = 0;
     let m;
@@ -90,6 +132,7 @@ function detectPii(text, locale = "tr") {
     }
   }
   if (active.has("ip")) findings.push(...findAll(IPV4_RE, t, "ip"));
+  if (active.has("ip_v6")) findings.push(...findAll(IPV6_RE, t, "ip_v6"));
   if (active.has("phone_tr")) findings.push(...findAll(PHONE_TR_RE, t, "phone_tr"));
   if (active.has("national_id_tr")) {
     TCKN_RE.lastIndex = 0;
@@ -100,6 +143,15 @@ function detectPii(text, locale = "tr") {
       }
     }
   }
+  if (active.has("tax_id_tr")) {
+    VKN_RE.lastIndex = 0;
+    let m;
+    while ((m = VKN_RE.exec(t)) !== null) {
+      if (validVkn(m[1])) {
+        findings.push({ type: "tax_id_tr", value: m[1], start: m.index, end: m.index + m[0].length });
+      }
+    }
+  }
   if (active.has("name")) {
     NAME_RE.lastIndex = 0;
     let m;
@@ -189,7 +241,7 @@ function applyMask(text, findings, strategy = "redact") {
 }
 
 // src/index.ts
-var version = "0.2.0";
+var version = "0.3.0";
 function computeQualityScore(completeness, avgLength, garbageRatio) {
   const lengthScore = Math.min(avgLength / 500, 1);
   const noiseScore = Math.max(0, 1 - garbageRatio * 10);
@@ -217,12 +269,32 @@ function audit(text, options = {}) {
   const pii_summary = Array.from(counts.entries()).sort(([a], [b]) => a.localeCompare(b)).map(([type, count]) => ({ type, count }));
   return { quality_grade, quality_score, pii_summary, pii, quality, noise };
 }
+function auditBatch(texts, options = {}) {
+  if (texts.length === 0) {
+    return { results: [], duplicate_ratio: 0, pii_summary: [], avg_quality_score: 0 };
+  }
+  const results = texts.map((t) => audit(t, options));
+  const seen = /* @__PURE__ */ new Set();
+  let dupCount = 0;
+  for (const t of texts) {
+    if (seen.has(t)) dupCount++;
+    else seen.add(t);
+  }
+  const duplicate_ratio = Math.round(dupCount / texts.length * 1e4) / 1e4;
+  const allPii = results.flatMap((r) => r.pii);
+  const counts = /* @__PURE__ */ new Map();
+  for (const f of allPii) counts.set(f.type, (counts.get(f.type) ?? 0) + 1);
+  const pii_summary = Array.from(counts.entries()).sort(([a], [b]) => a.localeCompare(b)).map(([type, count]) => ({ type, count }));
+  const avg_quality_score = Math.round(results.reduce((sum, r) => sum + r.quality_score, 0) / results.length * 1e4) / 1e4;
+  return { results, duplicate_ratio, pii_summary, avg_quality_score };
+}
 function mask(text, findings, options = {}) {
   return applyMask(text, findings, options.strategy ?? "redact");
 }
 export {
   applyMask,
   audit,
+  auditBatch,
   detectPii,
   mask,
   noiseMetrics,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flexorch/audit",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Zero-dependency PII + quality + noise audit for LLM datasets (TR/EU/US)",
   "keywords": [
     "pii",
@@ -17,7 +17,7 @@
   "homepage": "https://github.com/flexorch/flexorch-audit-js",
   "repository": {
     "type": "git",
-    "url": "https://github.com/flexorch/flexorch-audit-js.git"
+    "url": "git+https://github.com/flexorch/flexorch-audit-js.git"
   },
   "bugs": {
     "url": "https://github.com/flexorch/flexorch-audit-js/issues"