@dudousxd/nestjs-catalog 0.1.0

package/dist/catalog.events.d.ts

@@ -0,0 +1,179 @@
+/**
+ * What this package publishes on `aviary:catalog:*`.
+ *
+ * Diagnostics rather than a logger, and certainly rather than a direct
+ * Telescope dependency: the channel is the neutral seam the ecosystem is built
+ * on, so a host that wants these in Telescope installs
+ * `@dudousxd/nestjs-catalog-telescope`, one that wants them in its own tracing
+ * subscribes directly, and one that wants neither pays nothing — `emit` builds
+ * no envelope unless the channel has subscribers.
+ *
+ * These events are also the lineage feed. Since the catalog deliberately has no
+ * dataset history to walk (see the Iceberg decision), the record of who loaded
+ * what and when only exists because it was emitted as it happened.
+ */
+export declare const CATALOG_LIB = "catalog";
+/** Every event name this package emits. Exported so a watcher can claim them. */
+export declare const CATALOG_EVENTS: readonly ["schema.changed", "snapshot.written", "snapshot.committed", "snapshot.dropped", "type.curated", "connector.run.started", "connector.run.finished", "transform.changed", "workflow.changed"];
+export type CatalogEvent = (typeof CATALOG_EVENTS)[number];
+/**
+ * Where each event sits in the life of one load.
+ *
+ * This exists because a timestamp is not enough to order a trace. A fast load
+ * emits its whole story inside a single tick of whatever clock the recorder
+ * writes — the bundled MySQL store keeps whole seconds — so sorting by time
+ * alone leaves the events in insertion order, and insertion order is arbitrary.
+ * The observed result is a trace that reads `finished → written → committed →
+ * started`: causally impossible, rendered with total confidence. Sorting by
+ * time *and then* by this rank puts a same-tick story back in the only order it
+ * could have happened in, and leaves genuinely separated events alone.
+ *
+ * It is a rank, not a schedule. Nothing here says an event must occur, or must
+ * occur once — a load writes many batches, and a retried run emits a second
+ * `connector.run.started` — it only says which came first when the clock cannot.
+ */
+export declare const CATALOG_EVENT_PHASE: Record<CatalogEvent, number>;
+/**
+ * Where an event nobody here declared is placed: in the middle, with the work.
+ *
+ * Not at either end deliberately. An unknown event sorted last would be read as
+ * the thing that ended the trace, and one sorted first as the thing that caused
+ * it — both are claims this library has no basis to make about a name it does
+ * not recognise.
+ */
+export declare const CATALOG_EVENT_PHASE_FALLBACK = 4;
+export declare function catalogEventPhase(event: string): number;
+export interface CatalogEventPayloads {
+    /** DDL was applied to an object type's physical table. Always additive. */
+    'schema.changed': {
+        typeName: string;
+        table: string;
+        addedColumns: string[];
+        created: boolean;
+    };
+    /** A batch landed. Fires per batch, so a large load emits many. */
+    'snapshot.written': {
+        typeName: string;
+        snapshotId: string;
+        principalId: string;
+        rows: number;
+    };
+    /** A load became the one readers get. */
+    'snapshot.committed': {
+        typeName: string;
+        snapshotId: string;
+        principalId: string;
+        rowCount: number;
+    };
+    'snapshot.dropped': {
+        typeName: string;
+        snapshotId: string;
+    };
+    /**
+     * Someone changed a label, description, unit or visibility.
+     *
+     * Presentation-only, and emitted anyway: "who renamed this column and when"
+     * is a governance question, and the answer is otherwise nowhere.
+     */
+    'type.curated': {
+        typeName: string;
+        property?: string;
+        changed: string[];
+    };
+    /** A connector began pulling. */
+    'connector.run.started': {
+        connectorId: string;
+        connectorName: string;
+        typeName: string;
+        snapshotId: string;
+        principalId: string;
+    };
+    /**
+     * It finished, either way.
+     *
+     * One event for both outcomes rather than two: a watcher that cares about
+     * failures has to subscribe to successes anyway to notice the ones that never
+     * finished.
+     */
+    'connector.run.finished': {
+        connectorId: string;
+        connectorName: string;
+        typeName: string;
+        snapshotId: string;
+        principalId: string;
+        status: 'succeeded' | 'failed';
+        fetched: number;
+        written: number;
+        error?: string;
+        transformVersion?: number;
+    };
+    /**
+     * Someone changed code that shapes stored data.
+     *
+     * The one event on this channel that is not about data movement, and the most
+     * important for governance: a load that produced surprising numbers is
+     * investigated afterwards, and "who changed the transform, and when" is the
+     * first question.
+     */
+    'transform.changed': {
+        transformId: string;
+        name: string;
+        language: string;
+        version: number;
+        changedBy: string;
+    };
+    /**
+     * Someone changed the wiring rather than the code.
+     *
+     * Emitted for the same governance reason as `transform.changed`, and it is
+     * genuinely a second question: a load can change behaviour because a
+     * transform was edited *or* because a node was rewired between two transforms
+     * that both stayed exactly as they were. The graph hash rides along because a
+     * version number only identifies a graph within one catalog database, and
+     * these events are read across environments.
+     */
+    'workflow.changed': {
+        workflowId: string;
+        name: string;
+        version: number;
+        graphHash: string;
+        /** The type its sink writes, so a lineage view can place the edit. */
+        targetType: string;
+        nodeCount: number;
+        changedBy: string;
+    };
+}
+/**
+ * Teach the diagnostics channel what this library puts on it.
+ *
+ * Without this, `CatalogEventPayloads` only helps callers who go through
+ * `emitCatalog`. Anyone subscribing the way the ecosystem actually subscribes —
+ * generic `emit`/`trace`/watcher code that names the lib as a string — gets
+ * `unknown` and narrows by hand, which is where a payload field gets misspelled
+ * and nothing says so until the panel is blank in production.
+ *
+ * Pointed at the interface rather than restating its members. The sibling
+ * libraries hand-list every event here, which is a second copy of a map that
+ * already exists, and a second copy is the thing that stops matching. There is
+ * nothing to keep in step: this *is* the map.
+ */
+declare module '@dudousxd/nestjs-diagnostics' {
+    interface ChannelRegistry {
+        catalog: CatalogEventPayloads;
+    }
+}
+/**
+ * The channel an event is published on.
+ *
+ * Exported so a recorder or a watcher can subscribe without rebuilding the
+ * `aviary:<lib>:<event>` convention by hand — the one place that string is
+ * assembled is the one place it can drift.
+ */
+export declare function channelNameFor(event: CatalogEvent | string): string;
+/**
+ * Typed wrapper over `emit`.
+ *
+ * Never throws — observability that can break a load is worse than no
+ * observability — which `emit` already guarantees; this exists for the types.
+ */
+export declare function emitCatalog<E extends CatalogEvent>(event: E, payload: CatalogEventPayloads[E]): void;

package/dist/catalog.events.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CATALOG_EVENT_PHASE_FALLBACK = exports.CATALOG_EVENT_PHASE = exports.CATALOG_EVENTS = exports.CATALOG_LIB = void 0;
+exports.catalogEventPhase = catalogEventPhase;
+exports.channelNameFor = channelNameFor;
+exports.emitCatalog = emitCatalog;
+const nestjs_diagnostics_1 = require("@dudousxd/nestjs-diagnostics");
+/**
+ * What this package publishes on `aviary:catalog:*`.
+ *
+ * Diagnostics rather than a logger, and certainly rather than a direct
+ * Telescope dependency: the channel is the neutral seam the ecosystem is built
+ * on, so a host that wants these in Telescope installs
+ * `@dudousxd/nestjs-catalog-telescope`, one that wants them in its own tracing
+ * subscribes directly, and one that wants neither pays nothing — `emit` builds
+ * no envelope unless the channel has subscribers.
+ *
+ * These events are also the lineage feed. Since the catalog deliberately has no
+ * dataset history to walk (see the Iceberg decision), the record of who loaded
+ * what and when only exists because it was emitted as it happened.
+ */
+exports.CATALOG_LIB = 'catalog';
+/** Every event name this package emits. Exported so a watcher can claim them. */
+exports.CATALOG_EVENTS = [
+    'schema.changed',
+    'snapshot.written',
+    'snapshot.committed',
+    'snapshot.dropped',
+    'type.curated',
+    'connector.run.started',
+    'connector.run.finished',
+    'transform.changed',
+    'workflow.changed',
+];
+/**
+ * Where each event sits in the life of one load.
+ *
+ * This exists because a timestamp is not enough to order a trace. A fast load
+ * emits its whole story inside a single tick of whatever clock the recorder
+ * writes — the bundled MySQL store keeps whole seconds — so sorting by time
+ * alone leaves the events in insertion order, and insertion order is arbitrary.
+ * The observed result is a trace that reads `finished → written → committed →
+ * started`: causally impossible, rendered with total confidence. Sorting by
+ * time *and then* by this rank puts a same-tick story back in the only order it
+ * could have happened in, and leaves genuinely separated events alone.
+ *
+ * It is a rank, not a schedule. Nothing here says an event must occur, or must
+ * occur once — a load writes many batches, and a retried run emits a second
+ * `connector.run.started` — it only says which came first when the clock cannot.
+ */
+exports.CATALOG_EVENT_PHASE = {
+    'connector.run.started': 0,
+    'transform.changed': 1,
+    // The same rank as the transform edit above, which is allowed and meant:
+    // these ranks order a same-tick trace, and editing a graph and editing the
+    // code inside it are the same kind of thing at the same point in the story —
+    // an authoring event that precedes whatever run is being investigated. Giving
+    // one an earlier rank than the other would assert a causal order between two
+    // edits that have none.
+    'workflow.changed': 1,
+    // Curation carries no snapshot id, so it never lands inside a trace and this
+    // rank is never consulted. It is written out rather than left to the fallback
+    // because the type demands every event have one, and a `Record` that has to
+    // be complete is exactly the mechanism that will fail the build the day a new
+    // event is added and nobody thinks about where it belongs.
+    'type.curated': 2,
+    'schema.changed': 3,
+    'snapshot.written': 4,
+    'snapshot.committed': 5,
+    'snapshot.dropped': 6,
+    'connector.run.finished': 7,
+};
+/**
+ * Where an event nobody here declared is placed: in the middle, with the work.
+ *
+ * Not at either end deliberately. An unknown event sorted last would be read as
+ * the thing that ended the trace, and one sorted first as the thing that caused
+ * it — both are claims this library has no basis to make about a name it does
+ * not recognise.
+ */
+exports.CATALOG_EVENT_PHASE_FALLBACK = 4;
+function catalogEventPhase(event) {
+    const phase = Reflect.get(exports.CATALOG_EVENT_PHASE, event);
+    return typeof phase === 'number' ? phase : exports.CATALOG_EVENT_PHASE_FALLBACK;
+}
+/**
+ * The channel an event is published on.
+ *
+ * Exported so a recorder or a watcher can subscribe without rebuilding the
+ * `aviary:<lib>:<event>` convention by hand — the one place that string is
+ * assembled is the one place it can drift.
+ */
+function channelNameFor(event) {
+    return (0, nestjs_diagnostics_1.channelName)(exports.CATALOG_LIB, event);
+}
+/**
+ * Typed wrapper over `emit`.
+ *
+ * Never throws — observability that can break a load is worse than no
+ * observability — which `emit` already guarantees; this exists for the types.
+ */
+function emitCatalog(event, payload) {
+    // The type arguments are named rather than inferred. `emit` types its payload
+    // as `PayloadOf<TLib, TEvent>`, a conditional that stays unresolved while
+    // `TEvent` is still an open generic — so inference leaves the compiler unable
+    // to see that the registry entry and `CatalogEventPayloads` are now the same
+    // type. Passing the whole event union makes the conditional distribute into a
+    // union of every payload, which the one being emitted is a member of.
+    (0, nestjs_diagnostics_1.emit)(exports.CATALOG_LIB, event, payload);
+}

package/dist/catalog.module.d.ts

@@ -0,0 +1,5 @@
+import { type DynamicModule } from '@nestjs/common';
+import { type CatalogModuleOptions } from './catalog.options';
+export declare class CatalogModule {
+    static forRoot(options?: CatalogModuleOptions): DynamicModule;
+}

package/dist/catalog.module.js

@@ -0,0 +1,71 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var CatalogModule_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CatalogModule = void 0;
+const node_path_1 = require("node:path");
+const common_1 = require("@nestjs/common");
+const catalog_controller_1 = require("./catalog.controller");
+const catalog_options_1 = require("./catalog.options");
+const catalog_overlay_store_1 = require("./catalog.overlay-store");
+const catalog_overlay_store_token_1 = require("./catalog.overlay-store.token");
+const catalog_registry_1 = require("./catalog.registry");
+const catalog_registry_base_1 = require("./catalog.registry.base");
+const catalog_service_1 = require("./catalog.service");
+const catalog_store_1 = require("./catalog.store");
+const mikro_orm_read_store_1 = require("./stores/mikro-orm-read.store");
+let CatalogModule = CatalogModule_1 = class CatalogModule {
+    static forRoot(options = {}) {
+        const path = options.path ?? 'api/catalog';
+        const mountController = options.controller !== false;
+        const controllers = mountController
+            ? [(0, catalog_controller_1.createCatalogController)(path, options.guards ?? [], options.decorators ?? [])]
+            : [];
+        const overlayStore = options.overlayStore ??
+            new catalog_overlay_store_1.FileCatalogOverlayStore(options.overlayPath ?? (0, node_path_1.join)(process.cwd(), '.catalog', 'overlay.json'));
+        return {
+            module: CatalogModule_1,
+            imports: options.imports ?? [],
+            controllers,
+            providers: [
+                { provide: catalog_options_1.CATALOG_OPTIONS, useValue: options },
+                { provide: catalog_overlay_store_token_1.CATALOG_OVERLAY_STORE, useValue: overlayStore },
+                // The MikroORM-backed defaults are registered only when nothing
+                // overrides them. Registering them unconditionally would be worse than
+                // wasteful: a provider declared in this module shadows the same token
+                // exported by an imported one, so the default would quietly win over
+                // the host's override and the failure would look like the override
+                // never took effect.
+                //
+                // Derived is the right default — the model comes from the ORM of the
+                // application that owns the tables, and the rows come from those same
+                // tables, so nothing is stale and no infrastructure is needed. A
+                // warehouse overrides both: its model arrived over the wire and there
+                // are no entity classes to reflect over.
+                ...(options.registry
+                    ? [options.registry]
+                    : [
+                        catalog_registry_1.MikroOrmCatalogRegistry,
+                        {
+                            provide: catalog_registry_base_1.CatalogRegistry,
+                            useExisting: catalog_registry_1.MikroOrmCatalogRegistry,
+                        },
+                    ]),
+                ...(options.store
+                    ? [options.store]
+                    : [mikro_orm_read_store_1.MikroOrmReadStore, { provide: catalog_store_1.CATALOG_STORE, useExisting: mikro_orm_read_store_1.MikroOrmReadStore }]),
+                catalog_service_1.CatalogService,
+            ],
+            exports: [catalog_registry_base_1.CatalogRegistry, catalog_service_1.CatalogService, catalog_store_1.CATALOG_STORE],
+        };
+    }
+};
+exports.CatalogModule = CatalogModule;
+exports.CatalogModule = CatalogModule = CatalogModule_1 = __decorate([
+    (0, common_1.Module)({})
+], CatalogModule);

package/dist/catalog.options.d.ts

@@ -0,0 +1,79 @@
+import type { DynamicModule, ForwardReference, Provider, Type } from '@nestjs/common';
+import type { CatalogOverlayStore } from './catalog.overlay-store';
+export declare const CATALOG_OPTIONS: unique symbol;
+export interface CatalogModuleOptions {
+    /** Only expose these entity class names. Empty/undefined means all of them. */
+    include?: string[];
+    /** Never expose these, whatever else says. */
+    exclude?: string[];
+    /** Group for types that declare none. */
+    defaultGroup?: string;
+    /** Where tier-0 edits are persisted. Defaults to a JSON file. */
+    overlayPath?: string;
+    overlayStore?: CatalogOverlayStore;
+    /**
+     * Route prefix for the introspection + object-read API.
+     * Defaults to `api/catalog`.
+     */
+    path?: string;
+    /**
+     * Guards applied to every catalog route. The library ships none: an
+     * introspection endpoint that lists every table in the system is exactly the
+     * kind of thing that should never be open by default, and only the host app
+     * knows what its auth looks like.
+     */
+    guards?: Type<unknown>[];
+    /**
+     * Extra class decorators applied to the generated controller — the escape
+     * hatch for host-specific metadata (roles, rate limits, audit tags) that a
+     * generic library has no business knowing the shape of.
+     */
+    decorators?: ClassDecorator[];
+    /**
+     * Modules this dynamic module should import.
+     *
+     * Needed because Nest instantiates a guard in the injector of the module that
+     * *declares* the controller — which, for a library that generates its own
+     * controller, is the library's module and not yours. Whatever your guards
+     * inject has to be resolvable from here.
+     */
+    imports?: Array<Type<unknown> | DynamicModule | Promise<DynamicModule> | ForwardReference>;
+    /**
+     * Where the objects live. A provider for `CATALOG_STORE`.
+     *
+     * Defaults to reading the host application's own tables through MikroORM,
+     * which is always current and needs no infrastructure. Supply a warehouse
+     * adapter instead when the catalog should hold its own copy and keep history
+     * — the trade is a load to schedule and a staleness window to explain.
+     */
+    store?: Provider;
+    /**
+     * Where the type definitions come from. A provider for `CatalogRegistry`.
+     *
+     * Defaults to deriving them from the host's MikroORM metadata. A warehouse
+     * that receives its model from other applications binds a stored
+     * implementation here instead.
+     */
+    registry?: Provider;
+    /**
+     * Mount the built-in controller. Default true.
+     *
+     * Set false to keep the registry and reads but publish your own routes:
+     * inject `CatalogService` by class and shape the HTTP surface however your
+     * app already shapes it. Everything the built-in controller does is on that
+     * one service.
+     */
+    controller?: boolean;
+    /** Hard cap on generic object reads, so the explorer cannot pull a table. */
+    maxPageSize?: number;
+    /** Hard cap on rows an ad-hoc query may return. Default 1000. */
+    maxQueryRows?: number;
+    /**
+     * How long a query may run. Default 15s.
+     *
+     * A console where anyone can write a cartesian join needs this more than it
+     * needs a bigger row cap — a query that returns nothing for four minutes is
+     * holding a connection the whole time.
+     */
+    queryTimeoutMs?: number;
+}

package/dist/catalog.options.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CATALOG_OPTIONS = void 0;
+exports.CATALOG_OPTIONS = Symbol('CATALOG_OPTIONS');

package/dist/catalog.overlay-store.d.ts

@@ -0,0 +1,25 @@
+import type { CatalogOverlay } from './catalog.types';
+/**
+ * Where tier-0 edits live.
+ *
+ * Deliberately not the application database. The overlay is control-plane
+ * state, and mixing it into the operational schema is the first step toward a
+ * service that owns a database connection it should not have.
+ */
+export interface CatalogOverlayStore {
+    load(): Promise<CatalogOverlay>;
+    save(overlay: CatalogOverlay): Promise<void>;
+}
+/** The default: a JSON file. Good enough until there is a control plane. */
+export declare class FileCatalogOverlayStore implements CatalogOverlayStore {
+    private readonly path;
+    constructor(path: string);
+    load(): Promise<CatalogOverlay>;
+    save(overlay: CatalogOverlay): Promise<void>;
+}
+/** For tests, and for deployments that want the catalog strictly read-only. */
+export declare class InMemoryCatalogOverlayStore implements CatalogOverlayStore {
+    private overlay;
+    load(): Promise<CatalogOverlay>;
+    save(overlay: CatalogOverlay): Promise<void>;
+}

package/dist/catalog.overlay-store.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InMemoryCatalogOverlayStore = exports.FileCatalogOverlayStore = void 0;
+const promises_1 = require("node:fs/promises");
+const node_path_1 = require("node:path");
+/** The default: a JSON file. Good enough until there is a control plane. */
+class FileCatalogOverlayStore {
+    path;
+    constructor(path) {
+        this.path = path;
+    }
+    async load() {
+        try {
+            const raw = await (0, promises_1.readFile)(this.path, 'utf8');
+            const parsed = JSON.parse(raw);
+            if (parsed &&
+                typeof parsed === 'object' &&
+                'types' in parsed &&
+                typeof parsed.types === 'object') {
+                return parsed;
+            }
+            return { types: {} };
+        }
+        catch {
+            return { types: {} };
+        }
+    }
+    async save(overlay) {
+        await (0, promises_1.mkdir)((0, node_path_1.dirname)(this.path), { recursive: true });
+        await (0, promises_1.writeFile)(this.path, `${JSON.stringify(overlay, null, 2)}\n`, 'utf8');
+    }
+}
+exports.FileCatalogOverlayStore = FileCatalogOverlayStore;
+/** For tests, and for deployments that want the catalog strictly read-only. */
+class InMemoryCatalogOverlayStore {
+    overlay = { types: {} };
+    async load() {
+        return this.overlay;
+    }
+    async save(overlay) {
+        this.overlay = overlay;
+    }
+}
+exports.InMemoryCatalogOverlayStore = InMemoryCatalogOverlayStore;

package/dist/catalog.overlay-store.token.d.ts

@@ -0,0 +1 @@
+export declare const CATALOG_OVERLAY_STORE: unique symbol;

package/dist/catalog.overlay-store.token.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CATALOG_OVERLAY_STORE = void 0;
+exports.CATALOG_OVERLAY_STORE = Symbol('CATALOG_OVERLAY_STORE');