@segmentstream-cloud/ontology 0.12.0 → 0.13.0

package/dist/warehouse-backed.js

@@ -1,14 +1,30 @@
-import { SNAKE_CASE_NAME } from "./names.js";
+import { isSystemProperty, SNAKE_CASE_NAME } from "./names.js";
 const columnName = /^[A-Za-z_][A-Za-z0-9_]*$/;
 const types = new Set(["text", "integer", "number", "boolean", "date", "timestamp", "select"]);
+/**
+ * Types a join key may have. Joins deduplicate their target with `PARTITION BY` the key, which
+ * BigQuery refuses for FLOAT64; booleans, JSON and files identify nothing.
+ */
+const joinTypes = new Set(["text", "email", "url", "select", "integer", "date", "timestamp"]);
+/** Types `m.countDistinct` counts: the join key types, whose values compare exactly. */
+const distinctTypes = joinTypes;
 /** Cross-field validation runs after relationships have resolved their target entities. */
 export function checkWarehouseSource(entities, report) {
     for (const entity of entities) {
         const { source, measures = {} } = entity.entity.definition;
         const path = entity.name;
         if (source === undefined) {
-            if (Object.keys(measures).length) {
-                report("invalid-measure", path, "Measures belong to warehouse-backed entities.");
+            if (entity.builtIn && Object.keys(measures).length) {
+                report("invalid-measure", path, "User cannot declare measures.");
+            }
+            else {
+                checkMeasures({
+                    entity,
+                    measures,
+                    properties: new Map(entity.properties.map((property) => [property.name, property.definition])),
+                    hasCurrency: false,
+                    report,
+                });
             }
             if (entity.properties.some(({ definition }) => definition.column !== undefined || definition.scope !== undefined)) {
                 report("invalid-column", path, "Column mappings and scopes belong to warehouse-backed entities.");
@@ -39,11 +55,9 @@ export function checkWarehouseSource(entities, report) {
                 else if (!columnName.test(property.column)) {
                     report("invalid-column", field, "Invalid warehouse column.");
                 }
-                if (!types.has(property.type) ||
-                    property.unique ||
-                    property.managed ||
-                    "default" in property) {
-                    report("invalid-property", field, "Warehouse-backed properties must use a supported scalar type without unique, managed or default.");
+                // `unique` is allowed: an unenforced assertion that makes the property a join target.
+                if (!types.has(property.type) || property.managed || "default" in property) {
+                    report("invalid-property", field, "Warehouse-backed properties must use a supported scalar type without managed or default.");
                 }
                 let remainingScopes = 10_000;
                 const visit = (key, ancestors) => {
@@ -71,127 +85,214 @@ export function checkWarehouseSource(entities, report) {
             if (source?.currency !== undefined && !hasCurrency) {
                 report("invalid-source", path, "Currency must name a text property.");
             }
-            for (const [name, measure] of Object.entries(measures)) {
-                const field = `${path}.measures.${name}`;
-                if (!SNAKE_CASE_NAME.test(name) ||
-                    properties.has(name) ||
-                    entity.relationships.some((r) => r.name === name) ||
-                    !measure.description?.trim()) {
-                    report("invalid-measure", field, "Measures require unique snake_case names and a description.");
-                }
-                if (measure.unit !== undefined &&
-                    !["number", "currency", "ratio", "percent"].includes(measure.unit)) {
-                    report("invalid-measure", field, "Invalid measure unit.");
-                }
-                if (measure.format !== undefined &&
-                    !["decimal", "currency", "percent"].includes(measure.format)) {
-                    report("invalid-measure", field, "Invalid measure format.");
+            checkMeasures({ entity, measures, properties, hasCurrency, report });
+        }
+        for (const resolved of entity.relationships) {
+            checkRelationship({ entity, resolved, entities, report });
+        }
+    }
+}
+/** The aggregates a measure expression may contain. */
+const AGGREGATES = new Set(["sum", "countDistinct"]);
+const OPERATORS = new Set([...AGGREGATES, "add", "subtract", "multiply", "divide"]);
+/**
+ * The most dimensions and measures the platform accepts in one aggregate model. It also refuses a
+ * model definition over 60 KiB, which the SDK cannot measure exactly.
+ */
+const MAXIMUM_MODEL_MEMBERS = 100;
+/** Stored property types that are not dimensions: their values cannot be grouped by. */
+const UNGROUPABLE_TYPES = new Set(["json", "blob"]);
+/**
+ * The measures of a warehouse-backed or stored entity. A measure's columns name properties of the
+ * entity. `hasCurrency` says whether the entity maps a text `source.currency`, which stored entities
+ * never do, so their measures cannot be currency amounts.
+ */
+function checkMeasures(input) {
+    const { entity, measures, properties, hasCurrency, report } = input;
+    // A stored entity's rows carry its system fields, whose names a measure cannot take.
+    const stored = entity.entity.definition.source === undefined;
+    const measureCount = Object.keys(measures).length;
+    // A stored entity's dimensions are its id and its groupable properties.
+    const dimensionCount = stored
+        ? 1 + [...properties.values()].filter(({ type }) => !UNGROUPABLE_TYPES.has(type)).length
+        : properties.size;
+    if (measureCount > 0 && dimensionCount + measureCount > MAXIMUM_MODEL_MEMBERS) {
+        report("invalid-measure", `${entity.name}.measures`, `${entity.name} has ${dimensionCount + measureCount} dimensions and measures; an entity with ` +
+            `measures may have at most ${MAXIMUM_MODEL_MEMBERS}${stored ? ", counting its id" : ""}, ` +
+            "and its definition at most 60 KiB.");
+    }
+    for (const [name, measure] of Object.entries(measures)) {
+        const field = `${entity.name}.measures.${name}`;
+        if (!SNAKE_CASE_NAME.test(name) ||
+            properties.has(name) ||
+            (stored && isSystemProperty(name)) ||
+            entity.relationships.some((r) => r.name === name) ||
+            !measure.description?.trim()) {
+            report("invalid-measure", field, "Measures require unique snake_case names and a description.");
+        }
+        if (measure.unit !== undefined &&
+            !["number", "currency", "ratio", "percent"].includes(measure.unit)) {
+            report("invalid-measure", field, "Invalid measure unit.");
+        }
+        if (measure.format !== undefined &&
+            !["decimal", "currency", "percent"].includes(measure.format)) {
+            report("invalid-measure", field, "Invalid measure format.");
+        }
+        let remaining = 10_000;
+        /** `aggregate` is the aggregate the expression is an argument of, if any. */
+        const visit = (expression, references, depth, aggregate) => {
+            if (--remaining < 0 ||
+                depth > 30 ||
+                references.length > 20 ||
+                !expression ||
+                typeof expression !== "object") {
+                report("invalid-measure", field, "Invalid or overly complex expression.");
+                return false;
+            }
+            switch (expression.op) {
+                case "count":
+                    if (aggregate !== undefined) {
+                        report("invalid-measure", field, "Nested aggregates are unsupported.");
+                    }
+                    return true;
+                case "literal":
+                    if (typeof expression.value !== "string" ||
+                        expression.value.length >= 100 ||
+                        !/^[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?$/.test(expression.value) ||
+                        !Number.isFinite(Number(expression.value))) {
+                        report("invalid-measure", field, "Literals must be finite numbers.");
+                    }
+                    return false;
+                case "column": {
+                    const type = properties.get(expression.name)?.type ?? "";
+                    if (aggregate === "countDistinct") {
+                        if (!distinctTypes.has(type)) {
+                            report("invalid-measure", field, "Distinct counts require a text, email, url, select, integer, date or timestamp " +
+                                "property.");
+                        }
+                    }
+                    else if (aggregate === undefined || !["integer", "number"].includes(type)) {
+                        report("invalid-measure", field, "Sums require a numeric property.");
+                    }
+                    return false;
                 }
-                let remaining = 10_000;
-                const visit = (expression, references, depth, inside = false) => {
-                    if (--remaining < 0 ||
-                        depth > 30 ||
-                        references.length > 20 ||
-                        !expression ||
-                        typeof expression !== "object") {
-                        report("invalid-measure", field, "Invalid or overly complex expression.");
+                case "measure": {
+                    const target = measures[expression.name];
+                    if (!target) {
+                        report("unknown-measure", field, "Unknown measure reference.");
                         return false;
                     }
-                    switch (expression.op) {
-                        case "count":
-                            if (inside) {
-                                report("invalid-measure", field, "Nested aggregates are unsupported.");
-                            }
-                            return true;
-                        case "literal":
-                            if (typeof expression.value !== "string" ||
-                                expression.value.length >= 100 ||
-                                !/^[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?$/.test(expression.value) ||
-                                !Number.isFinite(Number(expression.value))) {
-                                report("invalid-measure", field, "Literals must be finite numbers.");
-                            }
-                            return false;
-                        case "column":
-                            if (!inside ||
-                                !["integer", "number"].includes(properties.get(expression.name)?.type ?? "")) {
-                                report("invalid-measure", field, "Sums require a numeric property.");
-                            }
-                            return false;
-                        case "measure": {
-                            const target = measures[expression.name];
-                            if (!target) {
-                                report("unknown-measure", field, "Unknown measure reference.");
-                                return false;
-                            }
-                            if (references.includes(expression.name)) {
-                                report("circular-measure", field, "Circular measure reference.");
-                                return false;
-                            }
-                            if (inside) {
-                                report("invalid-measure", field, "Measure references cannot be aggregated again.");
-                            }
-                            if (target.unit === "currency" && !hasCurrency) {
-                                report("invalid-measure", field, "Currency measures require source.currency.");
-                            }
-                            return visit(target, [...references, expression.name], depth + 1);
-                        }
-                        default: {
-                            if (!["sum", "add", "subtract", "multiply", "divide"].includes(expression.op) ||
-                                !Array.isArray(expression.args) ||
-                                expression.args.length !== (expression.op === "sum" ? 1 : 2)) {
-                                report("invalid-measure", field, "Invalid expression operator or arity.");
-                                return false;
-                            }
-                            if (inside && expression.op === "sum") {
-                                report("invalid-measure", field, "Nested aggregates are unsupported.");
-                            }
-                            const aggregated = expression.args
-                                .map((arg) => visit(arg, references, depth + 1, inside || expression.op === "sum"))
-                                .some(Boolean);
-                            return expression.op === "sum" || aggregated;
-                        }
+                    if (references.includes(expression.name)) {
+                        report("circular-measure", field, "Circular measure reference.");
+                        return false;
                     }
-                };
-                if (!visit(measure, [name], 0)) {
-                    report("invalid-measure", field, "A measure must contain an aggregate.");
+                    if (aggregate !== undefined) {
+                        report("invalid-measure", field, "Measure references cannot be aggregated again.");
+                    }
+                    if (target.unit === "currency" && !hasCurrency) {
+                        report("invalid-measure", field, "Currency measures require source.currency.");
+                    }
+                    return visit(target, [...references, expression.name], depth + 1);
                 }
-                if (measure.unit === "currency" && !hasCurrency) {
-                    report("invalid-measure", field, "Currency measures require source.currency.");
+                default: {
+                    const aggregates = AGGREGATES.has(expression.op);
+                    if (!OPERATORS.has(expression.op) ||
+                        !Array.isArray(expression.args) ||
+                        expression.args.length !== (aggregates ? 1 : 2)) {
+                        report("invalid-measure", field, "Invalid expression operator or arity.");
+                        return false;
+                    }
+                    if (aggregates && aggregate !== undefined) {
+                        report("invalid-measure", field, "Nested aggregates are unsupported.");
+                    }
+                    if (expression.op === "countDistinct" && expression.args[0]?.op !== "column") {
+                        report("invalid-measure", field, "Distinct counts take one property.");
+                    }
+                    const aggregated = expression.args
+                        .map((arg) => visit(arg, references, depth + 1, aggregates ? expression.op : aggregate))
+                        .some(Boolean);
+                    return aggregates || aggregated;
                 }
             }
+        };
+        if (!visit(measure, [name], 0)) {
+            report("invalid-measure", field, "A measure must contain an aggregate.");
         }
-        for (const { name, relationship, target } of entity.relationships) {
-            const landing = entities.find((candidate) => candidate.name === target);
-            const field = `${path}.relationships.${name}`;
-            if (landing?.entity.definition.source !== undefined) {
-                report("invalid-join", field, "Relationships cannot target warehouse-backed entities in this release.");
-            }
-            if (source === undefined) {
-                if (relationship.cardinality === "one" && relationship.on !== undefined) {
-                    report("invalid-join", field, "Stored relationships cannot declare on.");
-                }
-                continue;
-            }
-            if (relationship.cardinality !== "one") {
-                report("invalid-join", field, "Warehouse-backed entities support rel.one only.");
-                continue;
-            }
-            const pairs = Object.entries(relationship.on ?? {});
-            if (pairs.length !== 1) {
-                report(pairs.length ? "invalid-join" : "missing-join", field, "Declare one property pair in on.");
-                continue;
-            }
-            const [own, other] = pairs[0];
-            const ownType = entity.properties.find((p) => p.name === own)?.definition.type;
-            const targetProperty = landing?.properties.find((p) => p.name === other)?.definition;
-            const targetType = other === "id" ? "text" : targetProperty?.type;
-            const text = (type) => ["email", "url", "select"].includes(type ?? "") ? "text" : type;
-            if (!ownType ||
-                !targetType ||
-                text(ownType) !== text(targetType) ||
-                (other !== "id" && !targetProperty?.unique)) {
-                report("invalid-join", field, "Join properties must have compatible types and target id or a unique property.");
-            }
+        if (measure.unit === "currency" && !hasCurrency) {
+            report("invalid-measure", field, "Currency measures require source.currency.");
         }
     }
 }
+/**
+ * One case per pairing. Stored → stored relationships never declare `on`. A relationship with a
+ * warehouse-backed end is value-matched: a `rel.one` with one `on` pair and no delete rule or
+ * optionality, because no key column or foreign key backs it.
+ */
+function checkRelationship(input) {
+    const { entity, resolved, entities, report } = input;
+    const { name, relationship, target } = resolved;
+    const landing = entities.find((candidate) => candidate.name === target);
+    const field = `${entity.name}.relationships.${name}`;
+    const storedOwner = entity.entity.definition.source === undefined;
+    const targetSource = landing?.entity.definition.source;
+    if (storedOwner && targetSource === undefined) {
+        if (relationship.cardinality === "one" && relationship.on !== undefined) {
+            report("invalid-join", field, "Stored relationships cannot declare on.");
+        }
+        return;
+    }
+    if (relationship.cardinality !== "one") {
+        const message = storedOwner
+            ? "Relationships to warehouse-backed entities support rel.one only."
+            : "Warehouse-backed entities support rel.one only.";
+        report("invalid-join", field, message);
+        return;
+    }
+    const pairs = Object.entries(relationship.on ?? {});
+    const [own, other] = pairs.length === 1 ? pairs[0] : [undefined, undefined];
+    if (relationship.onDelete !== undefined || relationship.optional) {
+        const message = storedOwner
+            ? "A relationship to a warehouse-backed entity has no delete rule; its optionality follows " +
+                `${own ?? "the property named in on"}.`
+            : "Relationships of warehouse-backed entities have no delete rule or optionality: their " +
+                "joins never drop a row.";
+        report("invalid-join", field, message);
+    }
+    if (own === undefined || other === undefined) {
+        report(pairs.length ? "invalid-join" : "missing-join", field, "Declare one property pair in on.");
+        return;
+    }
+    const ownProperty = entity.properties.find((property) => property.name === own)?.definition;
+    if (storedOwner && ownProperty === undefined) {
+        report("invalid-join", field, `The own side of on must be a declared property of ${entity.name}: it holds the reference.`);
+        return;
+    }
+    const targetProperty = landing?.properties.find((property) => property.name === other);
+    const text = (type) => ["email", "url", "select"].includes(type ?? "") ? "text" : type;
+    let targetType;
+    let keyed;
+    if (targetSource === undefined) {
+        targetType = other === "id" ? "text" : targetProperty?.definition.type;
+        keyed = other === "id" || targetProperty?.definition.unique === true;
+    }
+    else {
+        const { key } = targetSource;
+        const singleKey = Array.isArray(key) && key.length === 1 ? key[0] : undefined;
+        targetType = targetProperty?.definition.type;
+        keyed =
+            targetProperty !== undefined &&
+                (targetProperty.definition.unique === true ||
+                    (singleKey !== undefined && targetProperty.definition.column === singleKey));
+    }
+    if ((ownProperty !== undefined && !joinTypes.has(ownProperty.type)) ||
+        (targetType !== undefined && !joinTypes.has(targetType))) {
+        report("invalid-join", field, "Join properties must be text, integer, date or timestamp.");
+        return;
+    }
+    if (!ownProperty || !targetType || text(ownProperty.type) !== text(targetType) || !keyed) {
+        const accepted = targetSource === undefined
+            ? "id or a unique property"
+            : "the property that maps its single source key column, or a unique property";
+        report("invalid-join", field, `Join properties must have compatible types and target ${accepted}.`);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@segmentstream-cloud/ontology",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Define a SegmentStream workspace app's ontology: entities, relationships, roles and actions, written with Effect.",
   "license": "Apache-2.0",
   "author": "SegmentStream",