@dudousxd/nestjs-catalog 0.6.0 → 0.7.0

package/dist/catalog.decorators.d.ts

@@ -31,6 +31,23 @@ export declare function CatalogType(options?: CatalogTypeOptions): ClassDecorato
  * Enriches one property. Everything it sets is tier 0 — the overlay can
  * override any of it at runtime without a migration, which is precisely why
  * these live in metadata rather than in the column definition.
+ *
+ * **This is also how a relation is enriched, and there is deliberately no
+ * `@CatalogRelation`.** The metadata is keyed by property name, and a
+ * `@ManyToOne` is a property; the registry looks the options up before it
+ * decides whether the field is a scalar or a link, so
+ * `@CatalogProperty({ displayName: 'Home base' })` on `Mvr.base` labels the link
+ * exactly as it labels a column. A second decorator would be a synonym for this
+ * one.
+ *
+ * A decorator that declared a relation *outright* — target, kind, join column —
+ * was considered and rejected twice over. Everything an ORM models is already
+ * derived, and a hand-written line that could restate it is a line that can
+ * disagree with the schema, which is the one thing this model does not allow.
+ * And the links an ORM genuinely cannot see are, in practice, the ones that
+ * cross applications: neither side's ORM holds both ends, so no decorator in
+ * either codebase can assert them. That is a curation act in the console, and it
+ * needs a route that does not exist yet.
  */
 export declare function CatalogProperty(options?: CatalogPropertyOptions): PropertyDecorator;
 export declare function readTypeOptions(target: unknown): CatalogTypeOptions;

package/dist/catalog.decorators.js

@@ -27,6 +27,23 @@ function CatalogType(options = {}) {
  * Enriches one property. Everything it sets is tier 0 — the overlay can
  * override any of it at runtime without a migration, which is precisely why
  * these live in metadata rather than in the column definition.
+ *
+ * **This is also how a relation is enriched, and there is deliberately no
+ * `@CatalogRelation`.** The metadata is keyed by property name, and a
+ * `@ManyToOne` is a property; the registry looks the options up before it
+ * decides whether the field is a scalar or a link, so
+ * `@CatalogProperty({ displayName: 'Home base' })` on `Mvr.base` labels the link
+ * exactly as it labels a column. A second decorator would be a synonym for this
+ * one.
+ *
+ * A decorator that declared a relation *outright* — target, kind, join column —
+ * was considered and rejected twice over. Everything an ORM models is already
+ * derived, and a hand-written line that could restate it is a line that can
+ * disagree with the schema, which is the one thing this model does not allow.
+ * And the links an ORM genuinely cannot see are, in practice, the ones that
+ * cross applications: neither side's ORM holds both ends, so no decorator in
+ * either codebase can assert them. That is a curation act in the console, and it
+ * needs a route that does not exist yet.
  */
 function CatalogProperty(options = {}) {
     return (target, propertyKey) => {

package/dist/catalog.environment.d.ts

@@ -291,6 +291,45 @@ export interface PromotableObjectType {
         unit?: string;
         classification?: string;
     }>;
+    /**
+     * The links the type declares, in the shape they are stored in.
+     *
+     * Absent from this interface until it was noticed that a promoted type arrived
+     * complete in every visible way — right properties, right table, right owner —
+     * and sat in the target's graph as an island. Nothing errored, nothing was
+     * reported, and the plan the operator approved said "nothing to promote" for a
+     * release whose only content was a link.
+     *
+     * Spelled out here rather than imported from the store package, which is the
+     * same reason `properties` is: the dependency runs the other way, and this
+     * side is the one both a MikroORM environment and anything else that ever
+     * holds a catalog have to satisfy. `kind` is a bare `string` for the reason the
+     * stored row gives — what comes back out of a JSON column is whatever some
+     * earlier version of some publisher put in, and it is narrowed where it is
+     * read, not asserted here.
+     *
+     * Optional, and read as `[]` wherever it is used. An environment whose rows
+     * predate the relations column holds `NULL` there, and requiring the field
+     * would force every hand-built promotable set to name something it has nothing
+     * to say about. "Absent" and "empty" are deliberately the same statement here,
+     * unlike on the publish wire: this shape is built by `readPromotable` in this
+     * process rather than arriving from a client of unknown vintage, and the plan
+     * an operator approves has to describe exactly what the apply will do. See
+     * `promoteType`.
+     */
+    relations?: Array<{
+        name: string;
+        displayName: string;
+        description?: string;
+        kind: string;
+        targetType: string;
+        localKey?: string;
+        nullable: boolean;
+        hidden: boolean;
+        position: number;
+        owner: boolean;
+        inverseName?: string;
+    }>;
 }
 export interface PromotableTransform {
     id: string;

package/dist/catalog.environment.js

@@ -732,5 +732,55 @@ function diffObjectType(existing, incoming) {
             to: gone,
         });
     }
+    diffs.push(...diffRelations(existing, incoming));
+    return diffs;
+}
+/**
+ * The links, compared the same way the properties above are — and named
+ * differently on purpose.
+ *
+ * **Why this exists at all.** Without it a promotion whose only difference is a
+ * link reports `unchanged`, so the plan says there is nothing to promote and the
+ * apply, driven by that plan, does nothing. The change is real, the operator is
+ * shown an empty diff, and the link stays behind in dev. A plan is what somebody
+ * approves; a change invisible in the plan is a change nobody approved.
+ *
+ * **`relations.removed`, not `relations.absentFromSource`.** The properties
+ * above borrow the softer word because nothing acts on them: `ensureType` is
+ * additive, so a column that vanished from the source keeps its data in the
+ * target. A link that vanished really is deleted there — `promoteType` assigns
+ * the source's links rather than merging them, because a link holds no data and
+ * keeping one the source deliberately dropped means the target asserts a join
+ * the schema no longer has. The field name has to say which of those two a
+ * reviewer is looking at.
+ *
+ * **`to` repeats the removed names rather than being empty.** The fingerprint an
+ * approval is compared against hashes each field's `to` value; an empty one
+ * would make "drops the link to Base" and "drops the link to Depot" hash
+ * identically, so a plan approved for one would be applicable as the other.
+ */
+function diffRelations(existing, incoming) {
+    const diffs = [];
+    // `?? []` on both sides, so an environment predating the relations column
+    // compares as holding none rather than as differing from everything.
+    const before = new Map((existing?.relations ?? []).map((relation) => [relation.name, relation]));
+    const after = new Map((incoming.relations ?? []).map((relation) => [relation.name, relation]));
+    const added = [...after.keys()].filter((name) => !before.has(name));
+    const changed = [...after.entries()]
+        .filter(([name, relation]) => {
+        const previous = before.get(name);
+        return previous !== undefined && stable(previous) !== stable(relation);
+    })
+        .map(([name]) => name);
+    const removed = [...before.keys()].filter((name) => !after.has(name));
+    if (added.length) {
+        diffs.push({ field: 'relations.added', from: [], to: added });
+    }
+    if (changed.length) {
+        diffs.push({ field: 'relations.changed', from: changed, to: changed });
+    }
+    if (removed.length) {
+        diffs.push({ field: 'relations.removed', from: removed, to: removed });
+    }
     return diffs;
 }

package/dist/catalog.pipeline.d.ts

@@ -84,6 +84,25 @@ export interface CatalogConnector {
      * view atomically. `incremental` reads only what changed since the last run
      * and carries the rest forward, which is cheaper but needs the source to
      * offer a watermark and the type to have a primary key to merge on.
+     *
+     * **`incremental` is blind to deletes, and that is structural rather than a
+     * gap in any particular fetcher.** A run asks its source for what changed
+     * since a watermark; a row physically removed from the source never changes
+     * again, so it is never returned again, so the carry-forward copies it into
+     * every subsequent snapshot indefinitely. Nothing goes wrong at any single
+     * step — the catalog simply never finds out, and every count and dashboard
+     * built on the type is quietly wrong from then on.
+     *
+     * Because that failure is silent, the pipeline **refuses an incremental load
+     * of a type for which no reconciliation strategy has been declared**: a full
+     * read on an interval, a source that soft-deletes where the watermark can see
+     * it, or an explicit "stale rows are acceptable here, because …". The
+     * declaration is per object type and lives in the host's
+     * `CATALOG_LOAD_EXPECTATIONS` (see `load-expectations.ts` in
+     * `@dudousxd/nestjs-catalog-pipeline`), because it is a statement about the
+     * data rather than about the connector reading it — the same type loaded by a
+     * workflow sink or by an application POSTing to the publish API has exactly
+     * the same problem.
      */
     mode?: 'full' | 'incremental';
     /**

package/dist/catalog.registry.js

@@ -27,6 +27,50 @@ const RELATION_KINDS = ['1:1', '1:m', 'm:1', 'm:n'];
 function isRelationKind(kind) {
     return RELATION_KINDS.includes(kind);
 }
+/**
+ * Which end of the link holds the key.
+ *
+ * Not simply `prop.owner`, and the difference matters. MikroORM sets that flag
+ * while resolving the *pair*, so it is dependable for `1:1` and `m:n` — where
+ * either side could plausibly own the key and only the mapping says which — and
+ * beside the point for the two kinds that have no choice: a `m:1` is the many
+ * end and therefore always holds the column, a `1:m` is the one end and
+ * therefore never does. Deriving those two from the kind rather than from a flag
+ * also means metadata assembled by hand (an `EntitySchema`, or a test) answers
+ * correctly without having to know the flag exists.
+ *
+ * `mappedBy` is checked first because it is unambiguous wherever it appears: the
+ * ORM only ever writes it on the inverse side.
+ */
+function isOwningSide(prop, kind) {
+    if (prop.mappedBy)
+        return false;
+    if (kind === '1:m')
+        return false;
+    if (kind === 'm:1')
+        return true;
+    return Boolean(prop.owner);
+}
+/**
+ * A key both ends of one link agree on, so the graph can draw it once.
+ *
+ * The owning end names the link — `Mvr.base` — and the inverse end, which knows
+ * the owner's property through `mappedBy`, arrives at the same string. That is
+ * the whole trick, and it is why `inverseName` is carried on the def at all.
+ *
+ * The fallback covers metadata that names neither end of the pair: both rows
+ * then reduce to the unordered pair plus the property name, which collapses the
+ * symmetric case (two `m:n` sides spelled alike) and leaves genuinely different
+ * names as two links. Guessing harder than that would mean pairing links by
+ * shape, and drawing one line where the schema has two is the worse error.
+ */
+function linkKey(holder, relation) {
+    if (relation.owner)
+        return `${holder}.${relation.name}`;
+    if (relation.inverseName)
+        return `${relation.targetType}.${relation.inverseName}`;
+    return `${[holder, relation.targetType].sort().join('::')}::${relation.name}`;
+}
 /** Classify one type name. Returns "unknown" when nothing matches. */
 function classify(raw) {
     const t = raw.toLowerCase();
@@ -156,7 +200,6 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
     }
     getGraph() {
         const snapshot = this.getSnapshot();
-        const known = new Set(snapshot.types.map((t) => t.name));
         const nodes = snapshot.types.map((t) => ({
             id: t.name,
             label: t.displayName,
@@ -165,29 +208,7 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
             propertyCount: t.properties.length,
             relationCount: t.relations.length,
         }));
-        // Both ends of a relation are declared, so every link shows up twice. Keep
-        // one edge per unordered pair per name so the graph does not double up.
-        const seen = new Set();
-        const edges = [];
-        for (const type of snapshot.types) {
-            for (const relation of type.relations) {
-                if (!known.has(relation.targetType))
-                    continue;
-                const pair = [type.name, relation.targetType].sort().join('::');
-                const key = `${pair}::${relation.name}`;
-                if (seen.has(key))
-                    continue;
-                seen.add(key);
-                edges.push({
-                    id: `${type.name}.${relation.name}`,
-                    source: type.name,
-                    target: relation.targetType,
-                    label: relation.displayName,
-                    kind: relation.kind,
-                });
-            }
-        }
-        return { nodes, edges };
+        return { nodes, edges: buildEdges(snapshot.types) };
     }
     /** Tier-0 edit on a type. Never touches the database. */
     async patchType(typeName, patch) {
@@ -245,13 +266,21 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
         // here yields an empty array and a catalog that silently contains nothing.
         const all = this.orm.getMetadata().getAll();
         const types = [];
+        // Two passes, because a relation cannot be described without knowing the
+        // whole catalog: whether its target is published is a fact about the
+        // catalog, not about the entity being read, and a single pass would answer
+        // it differently depending on discovery order.
         this.entityClasses.clear();
+        const included = [];
         for (const meta of all.values()) {
             if (!this.shouldInclude(meta))
                 continue;
             this.entityClasses.set(meta.className, meta.class);
-            types.push(this.buildType(meta));
+            included.push(meta);
         }
+        const published = new Set(included.map((meta) => meta.className));
+        for (const meta of included)
+            types.push(this.buildType(meta, published));
         types.sort((a, b) => a.group.localeCompare(b.group) || a.displayName.localeCompare(b.displayName));
         this.version += 1;
         this.snapshot = {
@@ -279,7 +308,7 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
             return false;
         return true;
     }
-    buildType(meta) {
+    buildType(meta, published) {
         const entityClass = meta.class;
         const declared = (0, catalog_decorators_1.readTypeOptions)(entityClass);
         const declaredProps = (0, catalog_decorators_1.readPropertyOptions)(entityClass);
@@ -298,16 +327,28 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
             const fromOverlay = overlayProps[prop.name];
             const { displayName, description, hidden, order } = resolveFieldPresentation(prop.name, index, fromDecorator, fromOverlay);
             if (isRelationKind(prop.kind)) {
+                const targetType = prop.targetMeta?.className ?? String(prop.type);
                 relations.push({
                     name: prop.name,
                     displayName,
                     description,
                     kind: prop.kind,
-                    targetType: prop.targetMeta?.className ?? String(prop.type),
+                    targetType,
                     localKey: prop.fieldNames?.[0],
                     nullable: Boolean(prop.nullable),
                     hidden,
                     order,
+                    owner: isOwningSide(prop, prop.kind),
+                    // Either name identifies the same thing — the property at the other
+                    // end — and only one of them is ever set, on the side the ORM decided
+                    // is inverse. Read as one field because callers pairing the two ends
+                    // do not care which of the two spellings carried it.
+                    inverseName: prop.mappedBy || prop.inversedBy || undefined,
+                    targetPublished: published.has(targetType),
+                    // A relation is enriched on the same terms as a scalar: somebody said
+                    // something about it. Structure is not enrichment — every relation
+                    // here was derived, so its existence proves nothing about curation.
+                    enriched: Boolean(fromDecorator || fromOverlay),
                 });
                 return;
             }
@@ -341,7 +382,12 @@ let MikroOrmCatalogRegistry = MikroOrmCatalogRegistry_1 = class MikroOrmCatalogR
             primaryKey: meta.primaryKeys ?? [],
             enriched: Object.keys(declared).length > 0 ||
                 Object.keys(overlay).length > 0 ||
-                properties.some((p) => p.enriched),
+                properties.some((p) => p.enriched) ||
+                // Relations count too. A type whose only human input is "this link is
+                // called Home base" has been worked on, and leaving it out of the tally
+                // put it back on the "nobody has named this" list the curator uses to
+                // decide what to do next.
+                relations.some((r) => r.enriched),
             properties,
             relations,
         };
@@ -354,6 +400,49 @@ exports.MikroOrmCatalogRegistry = MikroOrmCatalogRegistry = MikroOrmCatalogRegis
     __param(2, (0, common_1.Inject)(catalog_overlay_store_token_1.CATALOG_OVERLAY_STORE)),
     __metadata("design:paramtypes", [core_1.MikroORM, Object, Object])
 ], MikroOrmCatalogRegistry);
+/**
+ * The links, as lines to draw.
+ *
+ * Two rules, and both exist because the naive version of this drew a picture
+ * that was wrong in a way nobody would notice:
+ *
+ * 1. **One edge per link.** A link declared at both ends produces two rows, and
+ *    keying the de-duplication on the property name only caught the case where
+ *    both ends happened to be spelled alike — so `Mvr.base` plus `Base.mvrs`,
+ *    the ordinary shape, drew two lines between the same pair of nodes. See
+ *    {@link linkKey}.
+ * 2. **Drawn from the end that holds the key**, so the arrow points the way a
+ *    join is written. Both ends are collected before either is chosen, because
+ *    otherwise the direction depends on which type was discovered first.
+ *
+ * Hidden relations are deliberately still drawn. Hiding is a statement about a
+ * table cell; a graph that quietly dropped edges would be a picture nobody could
+ * read as complete, which is the only thing a graph is for.
+ */
+function buildEdges(types) {
+    const byLink = new Map();
+    for (const type of types) {
+        for (const relation of type.relations) {
+            if (!relation.targetPublished)
+                continue;
+            const key = linkKey(type.name, relation);
+            const seen = byLink.get(key);
+            // Replace only when this row is the owning one and the row already held
+            // is not: anything else keeps the first, so the edge order stays the type
+            // order rather than shuffling with every rebuild.
+            if (seen && (seen.relation.owner || !relation.owner))
+                continue;
+            byLink.set(key, { holder: type.name, relation });
+        }
+    }
+    return [...byLink.values()].map(({ holder, relation }) => ({
+        id: `${holder}.${relation.name}`,
+        source: holder,
+        target: relation.targetType,
+        label: relation.displayName,
+        kind: relation.kind,
+    }));
+}
 /**
  * How one field is presented, resolved across the tiers.
  *

package/dist/catalog.types.d.ts

@@ -54,7 +54,24 @@ export interface CatalogPropertyDef {
     /** True when the value came from a hand-written decorator rather than the ORM. */
     enriched: boolean;
 }
-/** A link between two object types. Derived entirely from the ORM. */
+/**
+ * A link between two object types — the thing that makes this an ontology
+ * rather than a list of tables.
+ *
+ * **Structure derived, semantics declared**, the same split as everywhere else.
+ * A `@ManyToOne` already names its target, its kind and its join column, so none
+ * of that is ever written by hand — a decorator that could restate it is a
+ * decorator that can disagree with the schema. What a human adds is what they
+ * add to a scalar, a label and a meaning, through `@CatalogProperty` or the
+ * overlay; both key on the property name and so reach a relation without having
+ * to know it is one.
+ *
+ * **One row per declaration, not per link.** `@ManyToOne(() => Base)` on `Mvr`
+ * with the matching `@OneToMany` on `Base` is two rows describing one link.
+ * Collapsing them here would mean `Base` could not carry its own label for the
+ * end it declares, and a type could not say what it points at without consulting
+ * every other type. The graph collapses them instead — see {@link CatalogGraph}.
+ */
 export interface CatalogRelationDef {
     name: string;
     displayName: string;
@@ -67,6 +84,39 @@ export interface CatalogRelationDef {
     nullable: boolean;
     hidden: boolean;
     order: number;
+    /**
+     * True when this side physically holds the key.
+     *
+     * The two ends of a link are not interchangeable. The owning end is where the
+     * foreign key actually is, so it is the end a join is written from, the end
+     * whose column can be indexed, and the end whose removal breaks the link. A
+     * `1:m` is never the owner — the key lives on the many side.
+     */
+    owner: boolean;
+    /**
+     * The property on {@link targetType} that is the other end of this same link,
+     * when the ORM knows it (MikroORM's `mappedBy` / `inversedBy`).
+     *
+     * This is what lets two rows be recognised as one link. Pairing them by name
+     * instead only works for the accident of both ends being spelled the same:
+     * `Mvr.base` and `Base.mvrs` are one link and would otherwise draw two edges,
+     * which is exactly the picture a graph is supposed to prevent.
+     */
+    inverseName?: string;
+    /**
+     * Whether {@link targetType} is a type this catalog actually holds.
+     *
+     * False when the target was excluded by configuration, or belongs to an
+     * application that has not published it. The relation is still reported: that
+     * an MVR points at something called `Base` is true, and when the other end is
+     * missing that is the most useful single fact about it. Dropping it silently
+     * would leave a type looking unlinked when it is really linked to something
+     * out of reach — but drawing it as a navigable edge promises a node that
+     * cannot be opened, so the graph omits it and the type page keeps it, marked.
+     */
+    targetPublished: boolean;
+    /** True when a human has labelled or described this link. */
+    enriched: boolean;
 }
 /** One node of the ontology. */
 export interface CatalogObjectTypeDef {
@@ -97,6 +147,40 @@ export interface CatalogObjectTypeDef {
      * came from a regex and are probably wrong.
      */
     enriched: boolean;
+    /**
+     * When readers started seeing the data this type currently serves.
+     *
+     * `committedAt` of the newest committed snapshot, not `createdAt`: a load that
+     * was written and never committed is not what anybody is reading, and dating
+     * the type by it would report freshness nobody has.
+     *
+     * **Absent means no committed snapshot ever**, which is a different statement
+     * from "committed a year ago" — a type published by a schema and never loaded
+     * looked exactly like a type loaded daily until this field existed, and both
+     * looked exactly like a type whose publisher was deleted six months ago.
+     *
+     * Carried on the type rather than fetched per type on demand. The value of
+     * this signal is that it arrives without anybody going to look for it, and a
+     * field costing one request per row on a screen listing every type is a field
+     * that screen will not use.
+     */
+    lastCommittedAt?: string;
+    /**
+     * Rows in that snapshot.
+     *
+     * Here for a failure the timestamp cannot show: a connector that starts
+     * returning 12 rows where it returned 40,000 produces data that is wrong and
+     * *fresh*, so every staleness signal reports it as healthy. The count next to
+     * the date is what makes that visible, and it is the same read.
+     */
+    rowCount?: number;
+    /**
+     * Which application committed it.
+     *
+     * Because "stale since March" is never the last question — "so who was
+     * loading this?" is — and the answer is already in the row being read.
+     */
+    lastPrincipalId?: string;
     properties: CatalogPropertyDef[];
     relations: CatalogRelationDef[];
 }
@@ -108,12 +192,27 @@ export interface CatalogSnapshot {
     stats: {
         types: number;
         properties: number;
+        /**
+         * Declared relations, summed over the types — **not** distinct links. A link
+         * declared at both ends counts twice, because that is what this number is
+         * derived from and quietly halving it would make it disagree with the rows
+         * on the type pages that produce it. `getGraph().edges.length` is the count
+         * of links.
+         */
         relations: number;
         enrichedTypes: number;
     };
     types: CatalogObjectTypeDef[];
 }
-/** Nodes and edges, for drawing the ontology. */
+/**
+ * Nodes and edges, for drawing the ontology.
+ *
+ * One edge per **link**, not per declaration: a link declared at both ends is
+ * one line on the picture, drawn from the end that holds the key so the arrow
+ * points the way a join is written. And every edge lands on a node that is
+ * present — a target this catalog does not hold produces no edge, because an
+ * edge to nowhere is a node the reader will try to click.
+ */
 export interface CatalogGraph {
     nodes: Array<{
         id: string;
@@ -143,6 +242,12 @@ export interface CatalogOverlay {
         icon?: string;
         group?: string;
         titleProperty?: string;
+        /**
+         * Keyed by property name — and a relation is a property to whoever is
+         * looking, so a link's label and description are curated through this map
+         * too, under the relation's own name. That is why curating a link needs no
+         * new route and no new patch shape: `patchProperty` already accepts one.
+         */
         properties?: Record<string, {
             displayName?: string;
             description?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dudousxd/nestjs-catalog",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "A metadata registry for NestJS: object types, properties and relations, derived from your ORM and enriched with decorators.",
   "license": "MIT",
   "author": "Davide Carvalho",