@eventcatalog/core 3.25.5 → 3.26.0

package/eventcatalog/src/enterprise/fields/field-indexer.ts

@@ -0,0 +1,131 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { FieldsDatabase } from './fields-db';
+import { extractSchemaFieldsDeep } from './field-extractor';
+
+function detectFormat(fileName: string): string {
+  const ext = path.extname(fileName).toLowerCase();
+  if (ext === '.proto') return 'proto';
+  if (ext === '.avro' || ext === '.avsc') return 'avro';
+  return 'json-schema';
+}
+
+interface Warning {
+  messageId: string;
+  version: string;
+  error: string;
+}
+
+/**
+ * Build the fields SQLite index using the EventCatalog SDK.
+ * This runs BEFORE Astro starts (no Astro dependencies).
+ *
+ * @param catalogDir - Path to the EventCatalog content directory
+ * @param outputDir - Path to write the .eventcatalog/fields.db (defaults to catalogDir)
+ */
+export async function buildFieldsIndex(catalogDir: string, outputDir?: string): Promise<{ dbPath: string; warnings: Warning[] }> {
+  // Dynamic import to avoid bundling the SDK into the Astro build
+  const sdkModule = await import('@eventcatalog/sdk');
+  const sdk = sdkModule.default(catalogDir);
+
+  const dbDir = path.join(outputDir || catalogDir, '.eventcatalog');
+  const dbPath = path.join(dbDir, 'fields.db');
+
+  if (!fs.existsSync(dbDir)) {
+    fs.mkdirSync(dbDir, { recursive: true });
+  }
+
+  const db = new FieldsDatabase(dbPath, { recreate: true });
+  const warnings: Warning[] = [];
+
+  try {
+    // Fetch all messages (latest version only) using the SDK
+    const [events, commands, queries] = await Promise.all([
+      sdk.getEvents({ latestOnly: true }),
+      sdk.getCommands({ latestOnly: true }),
+      sdk.getQueries({ latestOnly: true }),
+    ]);
+
+    const collections = [
+      { entries: events, type: 'event' as const },
+      { entries: commands, type: 'command' as const },
+      { entries: queries, type: 'query' as const },
+    ];
+
+    for (const { entries, type } of collections) {
+      for (const entry of entries) {
+        const msgId = entry.id;
+        const msgVersion = entry.version;
+
+        // Get schema content via SDK
+        let schemaData: { schema: string; fileName: string } | undefined;
+        try {
+          schemaData = await sdk.getSchemaForMessage(msgId, msgVersion);
+        } catch {
+          // No schema for this message
+          continue;
+        }
+
+        if (!schemaData) continue;
+
+        const { schema: content, fileName } = schemaData;
+        const format = detectFormat(fileName);
+
+        try {
+          const fields = extractSchemaFieldsDeep(content, format);
+
+          for (const field of fields) {
+            db.insertField({
+              path: field.path,
+              type: field.type,
+              description: field.description,
+              required: field.required,
+              schemaFormat: format,
+              messageId: msgId,
+              messageVersion: msgVersion,
+              messageType: type,
+              messageName: entry.name || msgId,
+              messageSummary: entry.summary || '',
+              messageOwners: entry.owners || [],
+            });
+          }
+
+          // Get producers and consumers via SDK
+          const { producers, consumers } = await sdk.getProducersAndConsumersForMessage(msgId, msgVersion);
+
+          for (const producer of producers) {
+            db.insertProducer(
+              msgId,
+              msgVersion,
+              producer.id,
+              producer.version,
+              producer.name || producer.id,
+              producer.summary || '',
+              producer.owners || []
+            );
+          }
+          for (const consumer of consumers) {
+            db.insertConsumer(
+              msgId,
+              msgVersion,
+              consumer.id,
+              consumer.version,
+              consumer.name || consumer.id,
+              consumer.summary || '',
+              consumer.owners || []
+            );
+          }
+        } catch (err: any) {
+          warnings.push({ messageId: msgId, version: msgVersion, error: err.message });
+        }
+      }
+    }
+
+    db.rebuildFts();
+    db.close();
+    return { dbPath, warnings };
+  } catch (err) {
+    db.close();
+    throw err;
+  }
+}

package/eventcatalog/src/enterprise/fields/fields-db.test.ts

@@ -0,0 +1,186 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { FieldsDatabase } from './fields-db';
+import path from 'node:path';
+import fs from 'node:fs';
+import os from 'node:os';
+
+describe('FieldsDatabase', () => {
+  let db: FieldsDatabase;
+  let tmpDir: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fields-db-'));
+    db = new FieldsDatabase(path.join(tmpDir, 'fields.db'), { recreate: true });
+  });
+
+  afterEach(() => {
+    db.close();
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('creates the schema tables on initialization', () => {
+    const tables = db.db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all();
+    const tableNames = tables.map((t: any) => t.name);
+    expect(tableNames).toContain('fields');
+    expect(tableNames).toContain('message_producers');
+    expect(tableNames).toContain('message_consumers');
+  });
+
+  it('inserts fields and retrieves them with pagination', () => {
+    db.insertField({
+      path: 'orderId',
+      type: 'string',
+      description: 'The order ID',
+      required: true,
+      schemaFormat: 'json-schema',
+      messageId: 'OrderCreated',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.insertField({
+      path: 'amount',
+      type: 'number',
+      description: '',
+      required: false,
+      schemaFormat: 'json-schema',
+      messageId: 'OrderCreated',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.rebuildFts();
+    const result = db.queryFields({});
+    expect(result.fields).toHaveLength(2);
+    expect(result.total).toBe(2);
+  });
+
+  it('filters fields by full-text search matching field path', () => {
+    db.insertField({
+      path: 'orderId',
+      type: 'string',
+      description: '',
+      required: true,
+      schemaFormat: 'json-schema',
+      messageId: 'OrderCreated',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.insertField({
+      path: 'amount',
+      type: 'number',
+      description: '',
+      required: false,
+      schemaFormat: 'json-schema',
+      messageId: 'OrderCreated',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.rebuildFts();
+    const result = db.queryFields({ q: 'orderId' });
+    expect(result.fields).toHaveLength(1);
+    expect(result.fields[0].path).toBe('orderId');
+  });
+
+  it('filters fields that appear in multiple messages when shared flag is set', () => {
+    db.insertField({
+      path: 'customerId',
+      type: 'string',
+      description: '',
+      required: true,
+      schemaFormat: 'json-schema',
+      messageId: 'OrderCreated',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.insertField({
+      path: 'customerId',
+      type: 'string',
+      description: '',
+      required: true,
+      schemaFormat: 'json-schema',
+      messageId: 'PaymentProcessed',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.insertField({
+      path: 'amount',
+      type: 'number',
+      description: '',
+      required: false,
+      schemaFormat: 'json-schema',
+      messageId: 'PaymentProcessed',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.rebuildFts();
+    const result = db.queryFields({ shared: true });
+    expect(result.fields.every((f) => f.path === 'customerId')).toBe(true);
+  });
+
+  it('returns facet counts reflecting current filters', () => {
+    db.insertField({
+      path: 'orderId',
+      type: 'string',
+      description: '',
+      required: true,
+      schemaFormat: 'json-schema',
+      messageId: 'OrderCreated',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.insertField({
+      path: 'status',
+      type: 'string',
+      description: '',
+      required: true,
+      schemaFormat: 'avro',
+      messageId: 'OrderUpdated',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.rebuildFts();
+    const result = db.queryFields({});
+    expect(result.facets.formats).toContainEqual({ value: 'json-schema', count: 1 });
+    expect(result.facets.formats).toContainEqual({ value: 'avro', count: 1 });
+  });
+
+  it('paginates results using keyset cursor', () => {
+    for (let i = 0; i < 5; i++) {
+      db.insertField({
+        path: `field${i}`,
+        type: 'string',
+        description: '',
+        required: false,
+        schemaFormat: 'json-schema',
+        messageId: 'TestMsg',
+        messageVersion: '1.0.0',
+        messageType: 'event',
+      });
+    }
+    db.rebuildFts();
+    const page1 = db.queryFields({ pageSize: 2 });
+    expect(page1.fields).toHaveLength(2);
+    expect(page1.cursor).toBeDefined();
+    const page2 = db.queryFields({ pageSize: 2, cursor: page1.cursor });
+    expect(page2.fields).toHaveLength(2);
+    expect(page2.fields[0].path).not.toBe(page1.fields[0].path);
+  });
+
+  it('joins producer and consumer data for each field', () => {
+    db.insertField({
+      path: 'orderId',
+      type: 'string',
+      description: '',
+      required: true,
+      schemaFormat: 'json-schema',
+      messageId: 'OrderCreated',
+      messageVersion: '1.0.0',
+      messageType: 'event',
+    });
+    db.insertProducer('OrderCreated', '1.0.0', 'OrderService', '1.0.0');
+    db.insertConsumer('OrderCreated', '1.0.0', 'InventoryService', '2.0.0');
+    db.rebuildFts();
+    const result = db.queryFields({});
+    expect(result.fields[0].producers).toEqual([{ id: 'OrderService', version: '1.0.0' }]);
+    expect(result.fields[0].consumers).toEqual([{ id: 'InventoryService', version: '2.0.0' }]);
+  });
+});