@besales/anonymizer 0.1.0

package/dist/locales/ru/data/stopwords.csv

@@ -0,0 +1,236 @@
+value
+август
+австралия
+австрия
+агентство
+азербайджан
+азия
+академия
+акционер
+анадырь
+аналитик
+апрель
+апрельский
+аргентина
+армения
+архангельск
+астрахань
+банк
+барнаул
+беларусь
+белгород
+бельгия
+биробиджан
+благовещенск
+болгария
+больница
+бразилия
+брянск
+бульвар
+бухгалтер
+великобритания
+венгрия
+владивосток
+владимир
+волгоград
+воронеж
+воскресенье
+восток
+вторник
+генеральный
+германия
+город
+греция
+грузия
+группа
+дальний восток
+дания
+декабрь
+департамент
+деревня
+директор
+европа
+екатеринбург
+заказчик
+запад
+иваново
+ижевск
+инвестор
+индия
+инженер
+институт
+интернет
+иркутск
+испания
+италия
+июль
+июнь
+июньский
+кавказ
+казань
+казахстан
+калининград
+канада
+кемерово
+киргизия
+киров
+китай
+клиент
+клиника
+комитет
+компания
+консультант
+корпорация
+край
+краснодар
+красноярск
+красноярская
+курск
+ленинградская
+липецк
+магадан
+магнитогорск
+май
+майский
+маркетолог
+март
+мартовский
+махачкала
+мексика
+менеджер
+министерство
+молдова
+москва
+москвич
+московская
+мурманск
+набережная
+набережные челны
+нальчик
+нарьян-мар
+нидерланды
+нижний новгород
+нижний тагил
+новокузнецк
+новосибирск
+новосибирская
+норвегия
+ноябрь
+область
+общество
+округ
+октябрь
+омск
+организация
+оренбург
+отдел
+партнер
+пенза
+переулок
+пермь
+петербуржец
+петрозаводск
+петропавловск-камчатский
+платформа
+площадь
+поволжье
+подмосковье
+подрядчик
+покупатель
+польша
+понедельник
+португалия
+поставщик
+почта
+предприятие
+президент
+приложение
+продавец
+продукт
+проект
+проспект
+пятница
+разработчик
+район
+республика
+решение
+россия
+россиянин
+россиянка
+ростов-на-дону
+руководитель
+румыния
+рязань
+салехард
+самара
+санкт-петербург
+саратов
+свердловская
+севастополь
+север
+село
+сентябрь
+сербия
+сервис
+сибирь
+симферополь
+система
+служба
+сотрудник
+сочи
+специалист
+среда
+ставрополь
+суббота
+сша
+сыктывкар
+таджикистан
+тверь
+телефон
+технолог
+технология
+тольятти
+томск
+тула
+туркменистан
+турция
+тюменская
+тюмень
+узбекистан
+украина
+улан-удэ
+улица
+ульяновск
+университет
+управление
+урал
+уфа
+учредитель
+февраль
+февральский
+финляндия
+фирма
+франция
+хабаровск
+ханты-мансийск
+холдинг
+хорватия
+центр
+чебоксары
+челябинск
+четверг
+чехия
+чита
+швейцария
+швеция
+школа
+шоссе
+экономист
+юг
+южно-сахалинск
+юрист
+якутск
+январский
+январь
+япония
+ярославль

package/dist/locales/ru/index.d.ts

@@ -0,0 +1,3 @@
+import { LocaleConfig } from '../locale.interface';
+export declare const RU_LOCALE: LocaleConfig;
+//# sourceMappingURL=index.d.ts.map

package/dist/locales/ru/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/locales/ru/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAiFnD,eAAO,MAAM,SAAS,EAAE,YA2JvB,CAAC"}

package/dist/locales/ru/index.js

@@ -0,0 +1,227 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RU_LOCALE = void 0;
+const path = __importStar(require("path"));
+const detection_context_1 = require("../../types/detection-context");
+const entity_type_1 = require("../../types/entity-type");
+const PHONE_PATTERN = /(?:\+7|8)[\s-]?\(?[\d]{3}\)?[\s-]?[\d]{3}[\s-]?[\d]{2}[\s-]?[\d]{2}/g;
+const EMAIL_PATTERN = /[a-zA-Z0-9._%+-]{1,64}@[a-zA-Z0-9.-]{1,253}\.[a-zA-Z]{2,63}/g;
+const INN_PATTERN = /\b\d{10}(?:\d{2})?\b/g;
+const INN_CONTEXT = /[иИ][нН]{2}/i;
+const CARD_PATTERN = /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/g;
+const PASSPORT_PATTERN = /\b\d{2}\s?\d{2}\s+\d{6}\b/g;
+const PASSPORT_CONTEXT = /паспорт|серия|документ/i;
+const DOB_PATTERN = /\b\d{2}[./]\d{2}[./]\d{4}\b/g;
+const DOB_CONTEXT = /рожд|д\.?\s?р\.?|born/i;
+const AMOUNT_PATTERN = /\b\d[\d\s.,]{0,20}?\s*(?:руб|₽|\$|€|[KкКk]|млн|тыс|000)(?:\b|(?=\s|$|[^а-яёa-z]))/gi;
+const AMOUNT_CONTEXT = /сделк|бюджет|чек|оплат|стоимост|цен|сумм|платёж|платеж/i;
+const ACCOUNT_PATTERN = /\b\d{20}\b/g;
+const ACCOUNT_CONTEXT = /сч[её]т|р[/.]с|расч[её]тн/i;
+const CONTEXT_RADIUS_INN = 30;
+const CONTEXT_RADIUS_PASSPORT = 50;
+const CONTEXT_RADIUS_DOB = 50;
+const CONTEXT_RADIUS_AMOUNT = 80;
+const CONTEXT_RADIUS_ACCOUNT = 50;
+const CONFIDENCE_HIGH = 0.95;
+const CONFIDENCE_INN = 0.90;
+const CONFIDENCE_PASSPORT = 0.85;
+const CONFIDENCE_DOB = 0.85;
+const CONFIDENCE_AMOUNT = 0.80;
+const CONFIDENCE_ACCOUNT = 0.90;
+const MAX_EXAMPLE_TOKENS = 5;
+function buildRuInstruction(activeTokens) {
+    if (activeTokens.length === 0)
+        return '';
+    const examples = activeTokens.slice(0, MAX_EXAMPLE_TOKENS).join(', ');
+    return `---
+ВАЖНО: В тексте используются токены-заполнители для персональных данных.
+Ты ОБЯЗАН сохранять их ТОЧНО в оригинальном формате: [ТИП_НОМЕР]
+Примеры токенов в этом диалоге: ${examples}
+Правила:
+- НЕ заменяй токены на реальные данные или описания
+- НЕ изменяй регистр, скобки или формат токенов
+- Используй токены как есть в своих ответах
+- Если нужно упомянуть человека — используй его токен, например [PERSON_1]
+---`;
+}
+exports.RU_LOCALE = {
+    code: 'ru',
+    regexRules: [
+        {
+            type: entity_type_1.ENTITY_TYPE.PHONE,
+            pattern: PHONE_PATTERN,
+            contextPattern: null,
+            contextRadius: 0,
+            contextDirection: 'both',
+            confidence: CONFIDENCE_HIGH,
+        },
+        {
+            type: entity_type_1.ENTITY_TYPE.EMAIL,
+            pattern: EMAIL_PATTERN,
+            contextPattern: null,
+            contextRadius: 0,
+            contextDirection: 'both',
+            confidence: CONFIDENCE_HIGH,
+        },
+        {
+            type: entity_type_1.ENTITY_TYPE.INN,
+            pattern: INN_PATTERN,
+            contextPattern: INN_CONTEXT,
+            contextRadius: CONTEXT_RADIUS_INN,
+            contextDirection: 'before',
+            confidence: CONFIDENCE_INN,
+        },
+        {
+            type: entity_type_1.ENTITY_TYPE.CARD,
+            pattern: CARD_PATTERN,
+            contextPattern: null,
+            contextRadius: 0,
+            contextDirection: 'both',
+            confidence: CONFIDENCE_HIGH,
+        },
+        {
+            type: entity_type_1.ENTITY_TYPE.PASSPORT,
+            pattern: PASSPORT_PATTERN,
+            contextPattern: PASSPORT_CONTEXT,
+            contextRadius: CONTEXT_RADIUS_PASSPORT,
+            contextDirection: 'both',
+            confidence: CONFIDENCE_PASSPORT,
+        },
+        {
+            type: entity_type_1.ENTITY_TYPE.DATE_OF_BIRTH,
+            pattern: DOB_PATTERN,
+            contextPattern: DOB_CONTEXT,
+            contextRadius: CONTEXT_RADIUS_DOB,
+            contextDirection: 'both',
+            confidence: CONFIDENCE_DOB,
+        },
+        {
+            type: entity_type_1.ENTITY_TYPE.AMOUNT,
+            pattern: AMOUNT_PATTERN,
+            contextPattern: AMOUNT_CONTEXT,
+            contextRadius: CONTEXT_RADIUS_AMOUNT,
+            contextDirection: 'both',
+            confidence: CONFIDENCE_AMOUNT,
+        },
+        {
+            type: entity_type_1.ENTITY_TYPE.ACCOUNT,
+            pattern: ACCOUNT_PATTERN,
+            contextPattern: ACCOUNT_CONTEXT,
+            contextRadius: CONTEXT_RADIUS_ACCOUNT,
+            contextDirection: 'both',
+            confidence: CONFIDENCE_ACCOUNT,
+        },
+    ],
+    contextualRules: [
+        {
+            id: 'C1',
+            activeIn: [
+                detection_context_1.DETECTION_CONTEXT.SALES_CALL_TRANSCRIPT,
+                detection_context_1.DETECTION_CONTEXT.LEAK_ANALYSIS,
+                detection_context_1.DETECTION_CONTEXT.TRAINING_SCENARIO,
+            ],
+            entityType: entity_type_1.ENTITY_TYPE.AMOUNT,
+            triggerPattern: /сделк[аи]|бюджет|чек|оплат|стоимост/gi,
+            capturePattern: /\d[\d\s,.]*\s*(?:руб|₽|тыс|млн|[KkКк])?/,
+            direction: 'after',
+            maxDistance: 50,
+            confidence: 0.70,
+        },
+        {
+            id: 'C2',
+            activeIn: [
+                detection_context_1.DETECTION_CONTEXT.SALES_CALL_TRANSCRIPT,
+                detection_context_1.DETECTION_CONTEXT.OUTREACH_MESSAGE,
+                detection_context_1.DETECTION_CONTEXT.LEAK_ANALYSIS,
+                detection_context_1.DETECTION_CONTEXT.TRAINING_SCENARIO,
+            ],
+            entityType: entity_type_1.ENTITY_TYPE.COMPANY,
+            triggerPattern: /компани[яию]|организаци[яию]|ООО|ИП|АО|ЗАО/gi,
+            capturePattern: /[А-ЯЁ"][а-яёА-ЯЁ\s\-"]+/,
+            direction: 'after',
+            maxDistance: 30,
+            confidence: 0.70,
+        },
+        {
+            id: 'C3',
+            activeIn: [
+                detection_context_1.DETECTION_CONTEXT.OUTREACH_MESSAGE,
+                detection_context_1.DETECTION_CONTEXT.TRAINING_SCENARIO,
+            ],
+            entityType: entity_type_1.ENTITY_TYPE.PERSON,
+            triggerPattern: /уважаем\S*|здравствуйте|добрый день/gi,
+            capturePattern: /[А-ЯЁ][а-яё]+\s+[А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+)?/,
+            direction: 'after',
+            maxDistance: 50,
+            confidence: 0.75,
+        },
+        {
+            id: 'C4',
+            activeIn: [
+                detection_context_1.DETECTION_CONTEXT.SALES_CALL_TRANSCRIPT,
+                detection_context_1.DETECTION_CONTEXT.CHAT_MESSAGE,
+                detection_context_1.DETECTION_CONTEXT.TRAINING_SCENARIO,
+            ],
+            entityType: entity_type_1.ENTITY_TYPE.PERSON,
+            triggerPattern: /клиент|заказчик|покупатель/gi,
+            capturePattern: /[А-ЯЁ][а-яё]+(?:\s+[А-ЯЁ][а-яё]+)?/,
+            direction: 'after',
+            maxDistance: 30,
+            confidence: 0.65,
+        },
+        {
+            id: 'C6',
+            activeIn: [
+                detection_context_1.DETECTION_CONTEXT.LEAK_ANALYSIS,
+            ],
+            entityType: entity_type_1.ENTITY_TYPE.AMOUNT,
+            triggerPattern: /выручк|доход|убыт|потер/gi,
+            capturePattern: /\d[\d\s,.]*\s*(?:руб|₽|тыс|млн|[KkКк])?/,
+            direction: 'after',
+            maxDistance: 80,
+            confidence: 0.70,
+        },
+    ],
+    dictionary: {
+        namesPath: path.join(__dirname, 'data', 'names.csv'),
+        stopwordsPath: path.join(__dirname, 'data', 'stopwords.csv'),
+        tokenSplitPattern: /[^а-яёА-ЯЁa-zA-Z-]+/,
+    },
+    systemPrompt: {
+        buildInstruction: buildRuInstruction,
+    },
+};
+//# sourceMappingURL=index.js.map

package/dist/locales/ru/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/locales/ru/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAC7B,qEAAkE;AAClE,yDAAsD;AAMtD,MAAM,aAAa,GACjB,sEAAsE,CAAC;AAGzE,MAAM,aAAa,GACjB,8DAA8D,CAAC;AAGjE,MAAM,WAAW,GAAG,uBAAuB,CAAC;AAC5C,MAAM,WAAW,GAAG,cAAc,CAAC;AAGnC,MAAM,YAAY,GAAG,6CAA6C,CAAC;AAGnE,MAAM,gBAAgB,GAAG,4BAA4B,CAAC;AACtD,MAAM,gBAAgB,GAAG,yBAAyB,CAAC;AAGnD,MAAM,WAAW,GAAG,8BAA8B,CAAC;AACnD,MAAM,WAAW,GAAG,wBAAwB,CAAC;AAG7C,MAAM,cAAc,GAClB,qFAAqF,CAAC;AACxF,MAAM,cAAc,GAClB,yDAAyD,CAAC;AAG5D,MAAM,eAAe,GAAG,aAAa,CAAC;AACtC,MAAM,eAAe,GAAG,4BAA4B,CAAC;AAErD,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAC/B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAYhC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,SAAS,kBAAkB,CAAC,YAAsB;IAChD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEzC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEtE,OAAO;;;kCAGyB,QAAQ;;;;;;IAMtC,CAAC;AACL,CAAC;AAIY,QAAA,SAAS,GAAiB;IACrC,IAAI,EAAE,IAAI;IAEV,UAAU,EAAE;QACV;YACE,IAAI,EAAE,yBAAW,CAAC,KAAK;YACvB,OAAO,EAAE,aAAa;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,MAAM;YACxB,UAAU,EAAE,eAAe;SAC5B;QACD;YACE,IAAI,EAAE,yBAAW,CAAC,KAAK;YACvB,OAAO,EAAE,aAAa;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,MAAM;YACxB,UAAU,EAAE,eAAe;SAC5B;QACD;YACE,IAAI,EAAE,yBAAW,CAAC,GAAG;YACrB,OAAO,EAAE,WAAW;YACpB,cAAc,EAAE,WAAW;YAC3B,aAAa,EAAE,kBAAkB;YACjC,gBAAgB,EAAE,QAAQ;YAC1B,UAAU,EAAE,cAAc;SAC3B;QACD;YACE,IAAI,EAAE,yBAAW,CAAC,IAAI;YACtB,OAAO,EAAE,YAAY;YACrB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,MAAM;YACxB,UAAU,EAAE,eAAe;SAC5B;QACD;YACE,IAAI,EAAE,yBAAW,CAAC,QAAQ;YAC1B,OAAO,EAAE,gBAAgB;YACzB,cAAc,EAAE,gBAAgB;YAChC,aAAa,EAAE,uBAAuB;YACtC,gBAAgB,EAAE,MAAM;YACxB,UAAU,EAAE,mBAAmB;SAChC;QACD;YACE,IAAI,EAAE,yBAAW,CAAC,aAAa;YAC/B,OAAO,EAAE,WAAW;YACpB,cAAc,EAAE,WAAW;YAC3B,aAAa,EAAE,kBAAkB;YACjC,gBAAgB,EAAE,MAAM;YACxB,UAAU,EAAE,cAAc;SAC3B;QACD;YACE,IAAI,EAAE,yBAAW,CAAC,MAAM;YACxB,OAAO,EAAE,cAAc;YACvB,cAAc,EAAE,cAAc;YAC9B,aAAa,EAAE,qBAAqB;YACpC,gBAAgB,EAAE,MAAM;YACxB,UAAU,EAAE,iBAAiB;SAC9B;QACD;YACE,IAAI,EAAE,yBAAW,CAAC,OAAO;YACzB,OAAO,EAAE,eAAe;YACxB,cAAc,EAAE,eAAe;YAC/B,aAAa,EAAE,sBAAsB;YACrC,gBAAgB,EAAE,MAAM;YACxB,UAAU,EAAE,kBAAkB;SAC/B;KACF;IAED,eAAe,EAAE;QAEf;YACE,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE;gBACR,qCAAiB,CAAC,qBAAqB;gBACvC,qCAAiB,CAAC,aAAa;gBAC/B,qCAAiB,CAAC,iBAAiB;aACpC;YACD,UAAU,EAAE,yBAAW,CAAC,MAAM;YAC9B,cAAc,EAAE,uCAAuC;YACvD,cAAc,EAAE,yCAAyC;YACzD,SAAS,EAAE,OAAO;YAClB,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,IAAI;SACjB;QAED;YACE,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE;gBACR,qCAAiB,CAAC,qBAAqB;gBACvC,qCAAiB,CAAC,gBAAgB;gBAClC,qCAAiB,CAAC,aAAa;gBAC/B,qCAAiB,CAAC,iBAAiB;aACpC;YACD,UAAU,EAAE,yBAAW,CAAC,OAAO;YAC/B,cAAc,EAAE,8CAA8C;YAC9D,cAAc,EAAE,yBAAyB;YACzC,SAAS,EAAE,OAAO;YAClB,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,IAAI;SACjB;QAED;YACE,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE;gBACR,qCAAiB,CAAC,gBAAgB;gBAClC,qCAAiB,CAAC,iBAAiB;aACpC;YACD,UAAU,EAAE,yBAAW,CAAC,MAAM;YAC9B,cAAc,EAAE,uCAAuC;YACvD,cAAc,EAAE,oDAAoD;YACpE,SAAS,EAAE,OAAO;YAClB,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,IAAI;SACjB;QAED;YACE,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE;gBACR,qCAAiB,CAAC,qBAAqB;gBACvC,qCAAiB,CAAC,YAAY;gBAC9B,qCAAiB,CAAC,iBAAiB;aACpC;YACD,UAAU,EAAE,yBAAW,CAAC,MAAM;YAC9B,cAAc,EAAE,8BAA8B;YAC9C,cAAc,EAAE,oCAAoC;YACpD,SAAS,EAAE,OAAO;YAClB,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,IAAI;SACjB;QAED;YACE,EAAE,EAAE,IAAI;YACR,QAAQ,EAAE;gBACR,qCAAiB,CAAC,aAAa;aAChC;YACD,UAAU,EAAE,yBAAW,CAAC,MAAM;YAC9B,cAAc,EAAE,2BAA2B;YAC3C,cAAc,EAAE,yCAAyC;YACzD,SAAS,EAAE,OAAO;YAClB,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,IAAI;SACjB;KACF;IAED,UAAU,EAAE;QACV,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC;QACpD,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,eAAe,CAAC;QAC5D,iBAAiB,EAAE,qBAAqB;KACzC;IAED,YAAY,EAAE;QACZ,gBAAgB,EAAE,kBAAkB;KACrC;CACF,CAAC"}

package/dist/tokenizer/tokenizer.d.ts

@@ -0,0 +1,13 @@
+import { DetectedEntity } from '../types/detected-entity.interface';
+import { MappingData } from '../types/mapping-data.interface';
+export interface TokenizeResult {
+    tokenized: string;
+    updatedMapping: MappingData;
+}
+export declare function normalizeValue(value: string): string;
+export declare function tokenize(text: string, entities: DetectedEntity[], existingMapping: MappingData): TokenizeResult;
+export declare function detokenize(text: string, mappingData: MappingData): {
+    restored: string;
+    unresolvedTokens: string[];
+};
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/tokenizer/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/tokenizer/tokenizer.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,WAAW,CAAC;CAC7B;AAMD,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAMD,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,cAAc,EAAE,EAC1B,eAAe,EAAE,WAAW,GAC3B,cAAc,CA8ChB;AAKD,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,WAAW,GACvB;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,gBAAgB,EAAE,MAAM,EAAE,CAAA;CAAE,CAgBlD"}

package/dist/tokenizer/tokenizer.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeValue = normalizeValue;
+exports.tokenize = tokenize;
+exports.detokenize = detokenize;
+const constants_1 = require("../constants");
+function normalizeValue(value) {
+    return value.toLowerCase().trim().replace(/\s+/g, ' ');
+}
+function tokenize(text, entities, existingMapping) {
+    const mapping = JSON.parse(JSON.stringify(existingMapping));
+    if (entities.length === 0) {
+        return { tokenized: text, updatedMapping: mapping };
+    }
+    const sorted = [...entities].sort((a, b) => b.position.start - a.position.start);
+    let result = text;
+    for (const entity of sorted) {
+        const normalized = normalizeValue(entity.value);
+        let tokenKey = mapping.reverseIndex[normalized];
+        if (!tokenKey) {
+            const counter = mapping.nextCounters[entity.type] ?? 1;
+            if (counter > constants_1.MAX_TOKENS_PER_TYPE) {
+                throw new Error(`Превышен лимит токенов для типа ${entity.type}: максимум ${constants_1.MAX_TOKENS_PER_TYPE}`);
+            }
+            tokenKey = (0, constants_1.MAPPING_KEY_TEMPLATE)(entity.type, counter);
+            mapping.tokens[tokenKey] = entity.value;
+            mapping.reverseIndex[normalized] = tokenKey;
+            mapping.nextCounters[entity.type] = counter + 1;
+        }
+        const tokenText = `[${tokenKey}]`;
+        result =
+            result.slice(0, entity.position.start) +
+                tokenText +
+                result.slice(entity.position.end);
+    }
+    return { tokenized: result, updatedMapping: mapping };
+}
+function detokenize(text, mappingData) {
+    const unresolvedTokens = [];
+    const restored = text.replace(constants_1.TOKEN_PATTERN, (match, key) => {
+        const original = mappingData.tokens[key];
+        if (original !== undefined) {
+            return original;
+        }
+        unresolvedTokens.push(match);
+        return match;
+    });
+    return { restored, unresolvedTokens };
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/tokenizer/tokenizer.ts"],"names":[],"mappings":";;AAiBA,wCAEC;AAMD,4BAkDC;AAKD,gCAmBC;AAnGD,4CAIsB;AAatB,SAAgB,cAAc,CAAC,KAAa;IAC1C,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AACzD,CAAC;AAMD,SAAgB,QAAQ,CACtB,IAAY,EACZ,QAA0B,EAC1B,eAA4B;IAI5B,MAAM,OAAO,GAAgB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;IAEzE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;IACtD,CAAC;IAGD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAC9C,CAAC;IAEF,IAAI,MAAM,GAAG,IAAI,CAAC;IAElB,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAEhD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEvD,IAAI,OAAO,GAAG,+BAAmB,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CACb,mCAAmC,MAAM,CAAC,IAAI,cAAc,+BAAmB,EAAE,CAClF,CAAC;YACJ,CAAC;YAED,QAAQ,GAAG,IAAA,gCAAoB,EAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;YACxC,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC;YAC5C,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC;QAClD,CAAC;QAID,MAAM,SAAS,GAAG,IAAI,QAAQ,GAAG,CAAC;QAElC,MAAM;YACJ,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACtC,SAAS;gBACT,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;AACxD,CAAC;AAKD,SAAgB,UAAU,CACxB,IAAY,EACZ,WAAwB;IAExB,MAAM,gBAAgB,GAAa,EAAE,CAAC;IAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAC3B,yBAAa,EACb,CAAC,KAAa,EAAE,GAAW,EAAE,EAAE;QAC7B,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC,CACF,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;AACxC,CAAC"}

package/dist/traversers/completions.traverser.d.ts

@@ -0,0 +1,7 @@
+import { CompletionsMessage } from '../types/llm-message.interface';
+import { TraverserPort, ExtractedString } from './traverser.interface';
+export declare class CompletionsTraverser implements TraverserPort<CompletionsMessage[]> {
+    extractStrings(messages: CompletionsMessage[]): ExtractedString[];
+    reassemble(messages: CompletionsMessage[], processed: Map<string, string>): CompletionsMessage[];
+}
+//# sourceMappingURL=completions.traverser.d.ts.map

package/dist/traversers/completions.traverser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"completions.traverser.d.ts","sourceRoot":"","sources":["../../src/traversers/completions.traverser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAOvE,qBAAa,oBACX,YAAW,aAAa,CAAC,kBAAkB,EAAE,CAAC;IAE9C,cAAc,CAAC,QAAQ,EAAE,kBAAkB,EAAE,GAAG,eAAe,EAAE;IA4CjE,UAAU,CACR,QAAQ,EAAE,kBAAkB,EAAE,EAC9B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B,kBAAkB,EAAE;CAyCxB"}

package/dist/traversers/completions.traverser.js

@@ -0,0 +1,111 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CompletionsTraverser = void 0;
+class CompletionsTraverser {
+    extractStrings(messages) {
+        const result = [];
+        messages.forEach((msg, i) => {
+            if (typeof msg.content === 'string' && msg.content.length > 0) {
+                result.push({
+                    path: `messages[${i}].content`,
+                    value: msg.content,
+                });
+            }
+            if (Array.isArray(msg.content)) {
+                msg.content.forEach((part, j) => {
+                    if (part.type === 'text' && part.text && part.text.length > 0) {
+                        result.push({
+                            path: `messages[${i}].content[${j}].text`,
+                            value: part.text,
+                        });
+                    }
+                });
+            }
+            if (msg.tool_calls) {
+                msg.tool_calls.forEach((tc, k) => {
+                    if (tc.function?.arguments) {
+                        const basePath = `messages[${i}].tool_calls[${k}].function.arguments`;
+                        try {
+                            const parsed = JSON.parse(tc.function.arguments);
+                            extractFromValue(parsed, basePath, result);
+                        }
+                        catch {
+                            result.push({ path: basePath, value: tc.function.arguments });
+                        }
+                    }
+                });
+            }
+        });
+        return result;
+    }
+    reassemble(messages, processed) {
+        const cloned = structuredClone(messages);
+        cloned.forEach((msg, i) => {
+            const contentPath = `messages[${i}].content`;
+            if (typeof msg.content === 'string' && processed.has(contentPath)) {
+                msg.content = processed.get(contentPath);
+            }
+            if (Array.isArray(msg.content)) {
+                msg.content.forEach((part, j) => {
+                    const partPath = `messages[${i}].content[${j}].text`;
+                    if (part.type === 'text' && processed.has(partPath)) {
+                        part.text = processed.get(partPath);
+                    }
+                });
+            }
+            if (msg.tool_calls) {
+                msg.tool_calls.forEach((tc, k) => {
+                    const basePath = `messages[${i}].tool_calls[${k}].function.arguments`;
+                    if (tc.function?.arguments) {
+                        try {
+                            const parsed = JSON.parse(tc.function.arguments);
+                            const rebuilt = reassembleValue(parsed, basePath, processed);
+                            tc.function.arguments = JSON.stringify(rebuilt);
+                        }
+                        catch {
+                            if (processed.has(basePath)) {
+                                tc.function.arguments = processed.get(basePath);
+                            }
+                        }
+                    }
+                });
+            }
+        });
+        return cloned;
+    }
+}
+exports.CompletionsTraverser = CompletionsTraverser;
+function extractFromValue(value, path, result) {
+    if (typeof value === 'string' && value.length > 0) {
+        result.push({ path, value });
+        return;
+    }
+    if (Array.isArray(value)) {
+        value.forEach((item, i) => {
+            extractFromValue(item, `${path}[${i}]`, result);
+        });
+        return;
+    }
+    if (value !== null && typeof value === 'object') {
+        for (const [key, val] of Object.entries(value)) {
+            extractFromValue(val, `${path}.${key}`, result);
+        }
+    }
+}
+function reassembleValue(value, path, processed) {
+    if (typeof value === 'string') {
+        return processed.has(path) ? processed.get(path) : value;
+    }
+    if (Array.isArray(value)) {
+        return value.map((item, i) => reassembleValue(item, `${path}[${i}]`, processed));
+    }
+    if (value !== null && typeof value === 'object') {
+        const result = {};
+        for (const [key, val] of Object.entries(value)) {
+            result[key] = reassembleValue(val, `${path}.${key}`, processed);
+        }
+        return result;
+    }
+    return value;
+}
+//# sourceMappingURL=completions.traverser.js.map