@application-kit/kafka 0.2.0 → 0.4.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @application-kit/kafka
 
+## 0.4.0
+
+### Minor Changes
+
+- bedf9ac: `eventIdOf` и `eventVersionOfHeaders` читают заголовок Debezium в его настоящем формате `{"schema":…,"payload":…}`: `KafkaParser` превращает такой заголовок в объект, и идентификатор события был `null`, а версия контракта всегда `1`. Новый экспорт `debeziumHeaderText` достаёт текст такого заголовка.
+
+  После обновления проверка версии контракта начинает работать: сообщение чужой версии уходит в DLQ, а сообщение v2 в топике `.v2` принимается. Журнал аудита начинает получать `event_id`, записи без него остаются как есть.
+
+## 0.3.0
+
+### Minor Changes
+
+- e9411eb: Спеки-гейты продукта переехали в пакеты: спека теперь — вызов помощника с параметрами продукта, а нарушение приходит ошибкой со списком.
+
+  `ui/testing`: `assertTokensDeclared`. `kafka/testing` (новый вход): `assertConsumerTopicsConsistent`. `authorization/testing` (новый вход): `assertAbilityRegistryConsistent`. `nestjs-vite/static-assets-gate` (новый вход, отдельно от `/testing`, чтобы спеки с фейком рендерера не грузили `vite`): `assertStaticAssetsLayout`. `lint-config` разрешает импорт нового входа `nestjs-vite/static-assets-gate`.
+
 ## 0.2.0
 
 ### Minor Changes

package/README.md

@@ -101,3 +101,20 @@ export class UserConsumer extends ConsumerAbstract {
 Правила версии живут и отдельным входом `@application-kit/kafka/event-contract` — без `kafkajs`,
 `@nestjs/microservices` и побочных эффектов. Его берёт производитель события (web-процесс, outbox),
 которому транспорт не нужен; корень пакета экспортирует то же самое.
+
+## Гейт реестра топиков
+
+Имя очереди живёт в манифесте модуля, статике `queueName` события, подписке консьюмера литералом и ключе
+`kafka.eventQueues` values, и типом эти носители не связаны. `assertConsumerTopicsConsistent` из входа
+`@application-kit/kafka/testing` читает их как текст и падает со списком расхождений:
+
+```ts
+await assertConsumerTopicsConsistent({
+  chartValuesFile: 'ops/.helm/values-product.yaml',
+  consumerSources: '*/user-interface/consumers/*.ts',
+  eventSources: '**/domain/events/*.event.ts',
+  manifests: CONSUMER_MANIFESTS,
+  modulesDirectory: 'src/modules',
+  rootDirectory: process.cwd(),
+})
+```

package/dist/cjs/event-contract/debezium-header-text.d.ts

@@ -0,0 +1 @@
+export declare const debeziumHeaderText: (raw: unknown) => null | string;

package/dist/cjs/event-contract/debezium-header-text.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.debeziumHeaderText = void 0;
+const debeziumHeaderText = (raw) => {
+    const value = typeof raw === 'object' && raw !== null && 'payload' in raw ? raw.payload : raw;
+    const text = Buffer.isBuffer(value) ? value.toString('utf8') : value;
+    return typeof text === 'string' && text !== '' ? text : null;
+};
+exports.debeziumHeaderText = debeziumHeaderText;

package/dist/cjs/event-contract/event-contract-version.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.versionedQueueName = exports.eventVersionOfHeaders = exports.eventVersionOfTopic = exports.EVENT_VERSION_HEADER = exports.DEFAULT_EVENT_VERSION = void 0;
+const debezium_header_text_js_1 = require("./debezium-header-text.js");
 exports.DEFAULT_EVENT_VERSION = '1';
 exports.EVENT_VERSION_HEADER = 'x-event-version';
 const VERSION_SEGMENT = /^v\d+$/u;
@@ -10,11 +11,7 @@ const eventVersionOfTopic = (topic) => {
     return segment === undefined ? exports.DEFAULT_EVENT_VERSION : segment.slice(1);
 };
 exports.eventVersionOfTopic = eventVersionOfTopic;
-const eventVersionOfHeaders = (headers) => {
-    const raw = headers?.[exports.EVENT_VERSION_HEADER];
-    const value = Buffer.isBuffer(raw) ? raw.toString('utf8') : raw;
-    return typeof value === 'string' && value !== '' ? value : exports.DEFAULT_EVENT_VERSION;
-};
+const eventVersionOfHeaders = (headers) => (0, debezium_header_text_js_1.debeziumHeaderText)(headers?.[exports.EVENT_VERSION_HEADER]) ?? exports.DEFAULT_EVENT_VERSION;
 exports.eventVersionOfHeaders = eventVersionOfHeaders;
 const versionedQueueName = (queueName, version) => `${queueName}${versionSegment(version)}`;
 exports.versionedQueueName = versionedQueueName;

package/dist/cjs/event-contract/index.d.ts

@@ -1 +1,2 @@
+export * from './debezium-header-text.js';
 export * from './event-contract-version.js';

package/dist/cjs/event-contract/index.js

@@ -14,4 +14,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./debezium-header-text.js"), exports);
 __exportStar(require("./event-contract-version.js"), exports);

package/dist/cjs/libs/kafka-message-context.js

@@ -1,11 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.eventIdOf = void 0;
-const eventIdOf = (context) => {
-    const raw = context.getMessage().headers?.id;
-    if (Buffer.isBuffer(raw)) {
-        return raw.toString('utf8');
-    }
-    return typeof raw === 'string' && raw !== '' ? raw : null;
-};
+const debezium_header_text_js_1 = require("../event-contract/debezium-header-text.js");
+const eventIdOf = (context) => (0, debezium_header_text_js_1.debeziumHeaderText)(context.getMessage().headers?.id);
 exports.eventIdOf = eventIdOf;

package/dist/cjs/testing/assert-consumer-topics-consistent.d.ts

@@ -0,0 +1,13 @@
+export interface ConsumerTopicsManifestInterface {
+    readonly name: string;
+    readonly queues: readonly string[];
+}
+export interface ConsumerTopicsOptionsInterface {
+    readonly chartValuesFile: string;
+    readonly consumerSources: string;
+    readonly eventSources: string;
+    readonly manifests: readonly ConsumerTopicsManifestInterface[];
+    readonly modulesDirectory: string;
+    readonly rootDirectory: string;
+}
+export declare const assertConsumerTopicsConsistent: (options: ConsumerTopicsOptionsInterface) => Promise<void>;

package/dist/cjs/testing/assert-consumer-topics-consistent.js

@@ -0,0 +1,95 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assertConsumerTopicsConsistent = void 0;
+const promises_1 = require("node:fs/promises");
+const node_path_1 = __importDefault(require("node:path"));
+const EVENT_QUEUE = /static readonly queueName\b[^\n]*'(?<queue>[^\n']+)'/u;
+const SUBSCRIBED_QUEUE = /eventTopics\([^\n]*'(?<queue>[^\n']+)'/gu;
+const CHART_QUEUES_BLOCK = /^ {2}eventQueues:\n(?<block>(?:(?: {4}.*)?\n)*)/mu;
+const CHART_QUEUE = /^ {4}(?<queue>[\w\-.]+):/gmu;
+const readSources = async (directory, pattern) => {
+    const files = await Array.fromAsync((0, promises_1.glob)(pattern, { cwd: directory }));
+    const sources = files.filter((file) => !file.endsWith('.spec.ts'));
+    return Promise.all(sources.map(async (file) => ({
+        file,
+        text: await (0, promises_1.readFile)(node_path_1.default.join(directory, file), 'utf8'),
+    })));
+};
+const missingFrom = (actual, expected) => {
+    const rest = [...actual];
+    return expected.filter((queue) => {
+        const index = rest.indexOf(queue);
+        if (index === -1) {
+            return true;
+        }
+        rest.splice(index, 1);
+        return false;
+    });
+};
+const chartViolations = (values, declared) => {
+    const block = CHART_QUEUES_BLOCK.exec(values)?.groups?.block ?? '';
+    const chartQueues = [...block.matchAll(CHART_QUEUE)].map((match) => match.groups?.queue ?? '');
+    return [
+        ...missingFrom(chartQueues, declared).map((queue) => `очередь ${queue} из манифеста без ключа kafka.eventQueues: автосоздание топиков выключено, консьюмер упадёт`),
+        ...missingFrom(declared, chartQueues).map((queue) => `ключ kafka.eventQueues.${queue} без очереди в манифесте: топик без производителя`),
+    ];
+};
+const queueOf = ({ text }) => EVENT_QUEUE.exec(text)?.groups?.queue;
+const ownerOf = ({ file }) => file.split('/', 1).join('');
+const producedQueues = (events) => {
+    const produced = new Map();
+    for (const event of events) {
+        const queue = queueOf(event);
+        if (queue !== undefined) {
+            produced.set(ownerOf(event), [...produced.get(ownerOf(event)) ?? [], queue]);
+        }
+    }
+    return produced;
+};
+const ownerViolations = (events, manifests) => {
+    const produced = producedQueues(events);
+    const declared = new Map(manifests.map(({ name, queues }) => [name, queues]));
+    const owners = [...new Set([...produced.keys(), ...declared.keys()])];
+    return [
+        ...events.filter((event) => queueOf(event) === undefined).map(({ file }) => `событие без queueName (${file})`),
+        ...owners.flatMap((owner) => {
+            const ownQueues = produced.get(owner) ?? [];
+            const declaredQueues = declared.get(owner) ?? [];
+            return [
+                ...missingFrom(declaredQueues, ownQueues).map((queue) => `модуль ${owner}: очередь ${queue} есть у события, но не в манифесте модуля`),
+                ...missingFrom(ownQueues, declaredQueues).map((queue) => `модуль ${owner}: очередь ${queue} в манифесте без события модуля`),
+            ];
+        }),
+    ];
+};
+const subscriptionViolations = (consumers, declared) => consumers.flatMap(({ file, text }) => [...text.matchAll(SUBSCRIBED_QUEUE)]
+    .map((match) => match.groups?.queue ?? '')
+    .filter((queue) => !declared.includes(queue))
+    .map((queue) => `подписка на очередь ${queue} вне реестра: топик никто не производит (${file})`));
+const assertConsumerTopicsConsistent = async (options) => {
+    const { chartValuesFile, consumerSources, eventSources, manifests, modulesDirectory, rootDirectory } = options;
+    const directory = node_path_1.default.join(rootDirectory, modulesDirectory);
+    const [values, events, consumers] = await Promise.all([
+        (0, promises_1.readFile)(node_path_1.default.join(rootDirectory, chartValuesFile), 'utf8'),
+        readSources(directory, eventSources),
+        readSources(directory, consumerSources),
+    ]);
+    const declared = manifests.flatMap(({ queues }) => queues);
+    const emptyPatterns = [
+        ...events.length === 0 ? [`шаблон событий ${eventSources} не нашёл файлов`] : [],
+        ...consumers.length === 0 ? [`шаблон консьюмеров ${consumerSources} не нашёл файлов`] : [],
+    ];
+    const violations = [
+        ...emptyPatterns,
+        ...chartViolations(values, declared),
+        ...ownerViolations(events, manifests),
+        ...subscriptionViolations(consumers, declared),
+    ];
+    if (violations.length > 0) {
+        throw new Error(`Гейт реестра топиков Kafka:\n${violations.join('\n')}`);
+    }
+};
+exports.assertConsumerTopicsConsistent = assertConsumerTopicsConsistent;

package/dist/cjs/testing/index.d.ts

@@ -0,0 +1 @@
+export * from './assert-consumer-topics-consistent.js';

package/dist/cjs/testing/index.js

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./assert-consumer-topics-consistent.js"), exports);

package/dist/esm/event-contract/debezium-header-text.d.ts

@@ -0,0 +1 @@
+export declare const debeziumHeaderText: (raw: unknown) => null | string;

package/dist/esm/event-contract/debezium-header-text.js

@@ -0,0 +1,5 @@
+export const debeziumHeaderText = (raw) => {
+    const value = typeof raw === 'object' && raw !== null && 'payload' in raw ? raw.payload : raw;
+    const text = Buffer.isBuffer(value) ? value.toString('utf8') : value;
+    return typeof text === 'string' && text !== '' ? text : null;
+};

package/dist/esm/event-contract/event-contract-version.js

@@ -1,3 +1,4 @@
+import { debeziumHeaderText } from './debezium-header-text.js';
 export const DEFAULT_EVENT_VERSION = '1';
 export const EVENT_VERSION_HEADER = 'x-event-version';
 const VERSION_SEGMENT = /^v\d+$/u;
@@ -6,9 +7,5 @@ export const eventVersionOfTopic = (topic) => {
     const segment = topic.split('.').find((part) => VERSION_SEGMENT.test(part));
     return segment === undefined ? DEFAULT_EVENT_VERSION : segment.slice(1);
 };
-export const eventVersionOfHeaders = (headers) => {
-    const raw = headers?.[EVENT_VERSION_HEADER];
-    const value = Buffer.isBuffer(raw) ? raw.toString('utf8') : raw;
-    return typeof value === 'string' && value !== '' ? value : DEFAULT_EVENT_VERSION;
-};
+export const eventVersionOfHeaders = (headers) => debeziumHeaderText(headers?.[EVENT_VERSION_HEADER]) ?? DEFAULT_EVENT_VERSION;
 export const versionedQueueName = (queueName, version) => `${queueName}${versionSegment(version)}`;

package/dist/esm/event-contract/index.d.ts

@@ -1 +1,2 @@
+export * from './debezium-header-text.js';
 export * from './event-contract-version.js';

package/dist/esm/event-contract/index.js

@@ -1 +1,2 @@
+export * from './debezium-header-text.js';
 export * from './event-contract-version.js';

package/dist/esm/libs/kafka-message-context.js

@@ -1,7 +1,2 @@
-export const eventIdOf = (context) => {
-    const raw = context.getMessage().headers?.id;
-    if (Buffer.isBuffer(raw)) {
-        return raw.toString('utf8');
-    }
-    return typeof raw === 'string' && raw !== '' ? raw : null;
-};
+import { debeziumHeaderText } from '../event-contract/debezium-header-text.js';
+export const eventIdOf = (context) => debeziumHeaderText(context.getMessage().headers?.id);

package/dist/esm/testing/assert-consumer-topics-consistent.d.ts

@@ -0,0 +1,13 @@
+export interface ConsumerTopicsManifestInterface {
+    readonly name: string;
+    readonly queues: readonly string[];
+}
+export interface ConsumerTopicsOptionsInterface {
+    readonly chartValuesFile: string;
+    readonly consumerSources: string;
+    readonly eventSources: string;
+    readonly manifests: readonly ConsumerTopicsManifestInterface[];
+    readonly modulesDirectory: string;
+    readonly rootDirectory: string;
+}
+export declare const assertConsumerTopicsConsistent: (options: ConsumerTopicsOptionsInterface) => Promise<void>;

package/dist/esm/testing/assert-consumer-topics-consistent.js

@@ -0,0 +1,88 @@
+import { glob, readFile } from 'node:fs/promises';
+import path from 'node:path';
+const EVENT_QUEUE = /static readonly queueName\b[^\n]*'(?<queue>[^\n']+)'/u;
+const SUBSCRIBED_QUEUE = /eventTopics\([^\n]*'(?<queue>[^\n']+)'/gu;
+const CHART_QUEUES_BLOCK = /^ {2}eventQueues:\n(?<block>(?:(?: {4}.*)?\n)*)/mu;
+const CHART_QUEUE = /^ {4}(?<queue>[\w\-.]+):/gmu;
+const readSources = async (directory, pattern) => {
+    const files = await Array.fromAsync(glob(pattern, { cwd: directory }));
+    const sources = files.filter((file) => !file.endsWith('.spec.ts'));
+    return Promise.all(sources.map(async (file) => ({
+        file,
+        text: await readFile(path.join(directory, file), 'utf8'),
+    })));
+};
+const missingFrom = (actual, expected) => {
+    const rest = [...actual];
+    return expected.filter((queue) => {
+        const index = rest.indexOf(queue);
+        if (index === -1) {
+            return true;
+        }
+        rest.splice(index, 1);
+        return false;
+    });
+};
+const chartViolations = (values, declared) => {
+    const block = CHART_QUEUES_BLOCK.exec(values)?.groups?.block ?? '';
+    const chartQueues = [...block.matchAll(CHART_QUEUE)].map((match) => match.groups?.queue ?? '');
+    return [
+        ...missingFrom(chartQueues, declared).map((queue) => `очередь ${queue} из манифеста без ключа kafka.eventQueues: автосоздание топиков выключено, консьюмер упадёт`),
+        ...missingFrom(declared, chartQueues).map((queue) => `ключ kafka.eventQueues.${queue} без очереди в манифесте: топик без производителя`),
+    ];
+};
+const queueOf = ({ text }) => EVENT_QUEUE.exec(text)?.groups?.queue;
+const ownerOf = ({ file }) => file.split('/', 1).join('');
+const producedQueues = (events) => {
+    const produced = new Map();
+    for (const event of events) {
+        const queue = queueOf(event);
+        if (queue !== undefined) {
+            produced.set(ownerOf(event), [...produced.get(ownerOf(event)) ?? [], queue]);
+        }
+    }
+    return produced;
+};
+const ownerViolations = (events, manifests) => {
+    const produced = producedQueues(events);
+    const declared = new Map(manifests.map(({ name, queues }) => [name, queues]));
+    const owners = [...new Set([...produced.keys(), ...declared.keys()])];
+    return [
+        ...events.filter((event) => queueOf(event) === undefined).map(({ file }) => `событие без queueName (${file})`),
+        ...owners.flatMap((owner) => {
+            const ownQueues = produced.get(owner) ?? [];
+            const declaredQueues = declared.get(owner) ?? [];
+            return [
+                ...missingFrom(declaredQueues, ownQueues).map((queue) => `модуль ${owner}: очередь ${queue} есть у события, но не в манифесте модуля`),
+                ...missingFrom(ownQueues, declaredQueues).map((queue) => `модуль ${owner}: очередь ${queue} в манифесте без события модуля`),
+            ];
+        }),
+    ];
+};
+const subscriptionViolations = (consumers, declared) => consumers.flatMap(({ file, text }) => [...text.matchAll(SUBSCRIBED_QUEUE)]
+    .map((match) => match.groups?.queue ?? '')
+    .filter((queue) => !declared.includes(queue))
+    .map((queue) => `подписка на очередь ${queue} вне реестра: топик никто не производит (${file})`));
+export const assertConsumerTopicsConsistent = async (options) => {
+    const { chartValuesFile, consumerSources, eventSources, manifests, modulesDirectory, rootDirectory } = options;
+    const directory = path.join(rootDirectory, modulesDirectory);
+    const [values, events, consumers] = await Promise.all([
+        readFile(path.join(rootDirectory, chartValuesFile), 'utf8'),
+        readSources(directory, eventSources),
+        readSources(directory, consumerSources),
+    ]);
+    const declared = manifests.flatMap(({ queues }) => queues);
+    const emptyPatterns = [
+        ...events.length === 0 ? [`шаблон событий ${eventSources} не нашёл файлов`] : [],
+        ...consumers.length === 0 ? [`шаблон консьюмеров ${consumerSources} не нашёл файлов`] : [],
+    ];
+    const violations = [
+        ...emptyPatterns,
+        ...chartViolations(values, declared),
+        ...ownerViolations(events, manifests),
+        ...subscriptionViolations(consumers, declared),
+    ];
+    if (violations.length > 0) {
+        throw new Error(`Гейт реестра топиков Kafka:\n${violations.join('\n')}`);
+    }
+};

package/dist/esm/testing/index.d.ts

@@ -0,0 +1 @@
+export * from './assert-consumer-topics-consistent.js';

package/dist/esm/testing/index.js

@@ -0,0 +1 @@
+export * from './assert-consumer-topics-consistent.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@application-kit/kafka",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Kafka-транспорт для NestJS: консьюмер-процесс с heartbeat, auto-commit, retry/DLQ и контракт версии события",
   "keywords": [
     "consumer",
@@ -42,7 +42,17 @@
         "default": "./dist/cjs/event-contract/index.js"
       }
     },
-    "./package.json": "./package.json"
+    "./package.json": "./package.json",
+    "./testing": {
+      "import": {
+        "types": "./dist/esm/testing/index.d.ts",
+        "default": "./dist/esm/testing/index.js"
+      },
+      "require": {
+        "types": "./dist/cjs/testing/index.d.ts",
+        "default": "./dist/cjs/testing/index.js"
+      }
+    }
   },
   "main": "./dist/cjs/index.js",
   "module": "./dist/esm/index.js",
@@ -51,7 +61,7 @@
     "dist"
   ],
   "scripts": {
-    "build": "rm -rf ./dist && node ../../scripts/barrels.mjs create ./src && tsc --project tsconfig.build.json && tsc --project tsconfig.build.cjs.json && node ../../scripts/barrels.mjs clean ./src && node ../../scripts/mark-cjs.mjs ./dist/cjs",
+    "build": "rm -rf ./dist && node ../../scripts/barrels.mjs create ./src --entry-only=testing && tsc --project tsconfig.build.json && tsc --project tsconfig.build.cjs.json && node ../../scripts/barrels.mjs clean ./src && node ../../scripts/mark-cjs.mjs ./dist/cjs",
     "lint:tsc": "tsc --noEmit --pretty --project .",
     "test:mutation": "stryker run",
     "test:unit": "vitest run"