@segmentstream-cloud/ontology 0.16.0 → 0.17.0

package/dist/service/schemas.js

@@ -1,10 +1,12 @@
 import { BlobValue } from "../blob.js";
 import { Schema, SchemaTransformation } from "effect";
 import { wireStruct } from "../wire.js";
+import { AGGREGATE_FILTER_OPERATORS, ENTITY_AGGREGATE_FILTER_OPERATORS, filterValuesIssue, MAXIMUM_FILTER_VALUES, } from "../aggregate-filters.js";
 import { isValueMatched, OntologyManifestWire } from "../manifest.js";
 import { entityQuerySchemas, queryLinks } from "../query.js";
 import { ValidationIssue as RuntimeValidationIssue } from "../runtime/errors.js";
 import { Edit, HistoryEntry as RuntimeHistoryEntry } from "../runtime/history.js";
+export { AGGREGATE_FILTER_OPERATORS, AGGREGATE_PERIOD_UNITS, AGGREGATE_PERIODS, ENTITY_AGGREGATE_FILTER_OPERATORS, } from "../aggregate-filters.js";
 /** `lead` → `Lead`, `pilot_request` → `PilotRequest`: component names in the OpenAPI document. */
 function pascalCase(name) {
     return name
@@ -312,54 +314,29 @@ export const AGGREGATE_LIMITS = {
     measures: 30,
     dimensions: 20,
     filters: 50,
-    values: 100,
+    values: MAXIMUM_FILTER_VALUES,
     orderBy: 50,
     rows: 1000,
 };
-export const AGGREGATE_FILTER_OPERATORS = [
-    "equals",
-    "in",
-    "isNull",
-    "isNotNull",
-    "between",
-];
-/** The operators of the per-entity shorthand, which bounds dates with `dateRange` instead. */
-export const ENTITY_AGGREGATE_FILTER_OPERATORS = ["equals", "in", "isNull", "isNotNull"];
-export const AGGREGATE_GRANULARITIES = ["day", "week", "month"];
+/**
+ * The granularities a date or timestamp dimension buckets by; `hour` applies to timestamps only.
+ */
+export const AGGREGATE_GRANULARITIES = ["hour", "day", "week", "month", "quarter", "year"];
+/** The longest `timezone` a request names. */
+export const MAXIMUM_TIMEZONE_LENGTH = 64;
 const AggregateValue = Schema.Union([Schema.String, Schema.Number, Schema.Boolean]);
-/** Which values each operator takes, with SegmentStream's own messages (`planner.ts`). */
-function filterValuesIssue(filter) {
-    const { operator, value, values } = filter;
-    switch (operator) {
-        case "equals":
-            return value === undefined || values !== undefined
-                ? "Equality filters require one value."
-                : undefined;
-        case "in":
-            return values === undefined ||
-                values.length === 0 ||
-                values.length > AGGREGATE_LIMITS.values ||
-                value !== undefined
-                ? "Membership filters require between 1 and 100 values."
-                : undefined;
-        case "between":
-            return value !== undefined || values?.length !== 2
-                ? "between needs two values in order."
-                : undefined;
-        default:
-            return value !== undefined || values !== undefined
-                ? "Null filters do not accept values."
-                : undefined;
-    }
-}
-function aggregateFilter(operators) {
+/**
+ * A filter of either request. Which values each operator takes is checked with SegmentStream's own
+ * messages (`filterValuesIssue`); whether they suit the dimension's type is SegmentStream's to check.
+ */
+function aggregateFilter(operators, descriptions) {
     const filter = closedStruct({
         dimension: Schema.String.annotate({
             description: "A property, or a path of to-one relationships ending in one.",
         }),
         operator: Schema.Literals(operators),
-        value: Schema.optionalKey(AggregateValue).annotate({ description: "The value of equals." }),
-        values: Schema.optionalKey(Schema.Array(AggregateValue).check(Schema.isMaxLength(AGGREGATE_LIMITS.values))).annotate({ description: "The values of in, or the two inclusive bounds of between." }),
+        value: Schema.optionalKey(AggregateValue).annotate({ description: descriptions.value }),
+        values: Schema.optionalKey(Schema.Array(AggregateValue).check(Schema.isMaxLength(AGGREGATE_LIMITS.values))).annotate({ description: descriptions.values }),
     }, "Unknown key; use dimension, operator, value and values");
     return filter.check(Schema.makeFilter((candidate) => filterValuesIssue(candidate) ?? true));
 }
@@ -374,6 +351,11 @@ const aggregateFields = {
     granularityOf: Schema.optionalKey(Schema.String).annotate({
         description: "The requested date or timestamp dimension that granularity buckets.",
     }),
+    timezone: Schema.optionalKey(Schema.String.check(Schema.isMaxLength(MAXIMUM_TIMEZONE_LENGTH))).annotate({
+        description: "An IANA timezone, such as Europe/Moscow, or UTC. Buckets of timestamps, date bounds on " +
+            "timestamps and inPeriod's today follow it instead of each entity's timezone; measure " +
+            "conditions never do.",
+    }),
     orderBy: Schema.optionalKey(Schema.Array(AggregateOrder).check(Schema.isMaxLength(AGGREGATE_LIMITS.orderBy))),
     limit: Schema.optionalKey(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: AGGREGATE_LIMITS.rows }))).annotate({ description: "At most 1000 rows; 100 by default." }),
 };
@@ -389,9 +371,15 @@ export const AggregateRequest = closedStruct({
         description: "<entity>.<property>, or a route from a measure's entity: " +
             "<entity>.<relationship>[.<relationship>].<property>.",
     }),
-    filters: Schema.optionalKey(Schema.Array(aggregateFilter(AGGREGATE_FILTER_OPERATORS)).check(Schema.isMaxLength(AGGREGATE_LIMITS.filters))).annotate({ description: "All must hold." }),
+    filters: Schema.optionalKey(Schema.Array(aggregateFilter(AGGREGATE_FILTER_OPERATORS, {
+        value: "The value of equals and notEquals; the bound of gt, gte, lt and lte; a non-empty " +
+            "string for contains, startsWith and endsWith; the period of inPeriod, such as " +
+            "last 7 days.",
+        values: "The values of in and notIn, or the two inclusive bounds of between.",
+    })).check(Schema.isMaxLength(AGGREGATE_LIMITS.filters))).annotate({ description: "All must hold." }),
     ...aggregateFields,
-}, "Unknown key; use measures, dimensions, filters, granularity, granularityOf, orderBy and limit").annotate({ identifier: "AggregateRequest" });
+}, "Unknown key; use measures, dimensions, filters, granularity, granularityOf, timezone, orderBy " +
+    "and limit").annotate({ identifier: "AggregateRequest" });
 /**
  * `POST /api/ontology/<entity>/aggregate`: SegmentStream's per-entity request, unchanged. Names are
  * relative to the entity (`revenue`, `product.brand`), and `dateRange` bounds a partitioned entity.
@@ -409,10 +397,14 @@ export const EntityAggregateRequest = closedStruct({
         from: Schema.String.annotate({ format: "date" }),
         to: Schema.String.annotate({ format: "date" }),
     }, "Unknown key; use from and to")).annotate({ description: "Inclusive days of the entity's partition property." }),
-    filters: Schema.optionalKey(Schema.Array(aggregateFilter(ENTITY_AGGREGATE_FILTER_OPERATORS)).check(Schema.isMaxLength(AGGREGATE_LIMITS.filters))).annotate({ description: "All must hold." }),
+    filters: Schema.optionalKey(Schema.Array(aggregateFilter(ENTITY_AGGREGATE_FILTER_OPERATORS, {
+        value: "The value of equals and notEquals; the bound of gt, gte, lt and lte; a non-empty " +
+            "string for contains, startsWith and endsWith.",
+        values: "The values of in and notIn.",
+    })).check(Schema.isMaxLength(AGGREGATE_LIMITS.filters))).annotate({ description: "All must hold." }),
     ...aggregateFields,
 }, "Unknown key; use dimensions, measures, dateRange, filters, granularity, granularityOf, " +
-    "orderBy and limit").annotate({ identifier: "EntityAggregateRequest" });
+    "timezone, orderBy and limit").annotate({ identifier: "EntityAggregateRequest" });
 /** SegmentStream's answer to an aggregate, which the service forwards unchanged. */
 export const AggregateResult = Schema.Struct({
     fields: Schema.Array(Schema.Struct({
@@ -429,6 +421,10 @@ export const AggregateResult = Schema.Struct({
         filters: Schema.optionalKey(Schema.Array(Schema.String)).annotate({
             description: "The filters the measure applied.",
         }),
+        balanceOver: Schema.optionalKey(Schema.String).annotate({
+            description: "The date or timestamp dimension a balance measure was grouped by or filtered to one " +
+                "day of.",
+        }),
     })),
     rows: Schema.Array(Schema.Record(Schema.String, Schema.NullOr(Schema.Union([Schema.String, Schema.Boolean])))),
     truncated: Schema.Boolean,
@@ -440,6 +436,13 @@ export const AggregateResult = Schema.Struct({
             description: "For example, a filter that did not apply to some measure's entity.",
         }),
         sources: Schema.optionalKey(Schema.Array(Schema.Struct({ entity: Schema.String, asOf: Schema.String }))).annotate({ description: "How fresh the replica of each stored entity read is." }),
+        periods: Schema.optionalKey(Schema.Array(Schema.Struct({
+            dimension: Schema.String,
+            value: Schema.String,
+            from: Schema.String.annotate({ format: "date" }),
+            to: Schema.String.annotate({ format: "date" }),
+            timezone: Schema.String,
+        }))).annotate({ description: "The inclusive dates each inPeriod filter used, in request order." }),
     }),
     deploymentId: Schema.NullOr(Schema.String),
     definitionDigest: Schema.String,

package/dist/warehouse-backed.js

@@ -1,4 +1,7 @@
-import { isSystemProperty, SNAKE_CASE_NAME } from "./names.js";
+import { dimensionTypeOf, filterTypeIssue, filterValuesIssue, MEASURE_CONDITION_OPERATORS, } from "./aggregate-filters.js";
+import { measuresManifest } from "./manifest.js";
+import { balanceOf, conditionProperty, conditionsOf, MEASURE_AGGREGATES, } from "./measure.js";
+import { humanize, isSystemProperty, SNAKE_CASE_NAME } from "./names.js";
 const columnName = /^[A-Za-z_][A-Za-z0-9_]*$/;
 const types = new Set(["text", "integer", "number", "boolean", "date", "timestamp", "select"]);
 /**
@@ -92,23 +95,135 @@ export function checkWarehouseSource(entities, report) {
         }
     }
 }
-/** The aggregates a measure expression may contain. */
-const AGGREGATES = new Set(["sum", "countDistinct"]);
+/**
+ * The aggregates of one property, which take one argument. `countDistinct` counts what compares
+ * exactly; `sum`, `avg`, `min` and `max` take numbers.
+ */
+const AGGREGATES = new Set(["sum", "countDistinct", "avg", "min", "max"]);
 const OPERATORS = new Set([...AGGREGATES, "add", "subtract", "multiply", "divide"]);
+/** The most conditions one aggregate takes. */
+const MAXIMUM_CONDITIONS = 10;
+const CONDITION_OPERATORS = new Set(MEASURE_CONDITION_OPERATORS);
+const CONDITION_PROPERTIES = "Measure conditions name properties of the entity.";
 /**
  * The most dimensions and measures the platform accepts in one aggregate model. It also refuses a
- * model definition over 60 KiB, which the SDK cannot measure exactly.
+ * model definition over {@link MAXIMUM_MODEL_BYTES}.
  */
 const MAXIMUM_MODEL_MEMBERS = 100;
 /** Stored property types that are not dimensions: their values cannot be grouped by. */
 const UNGROUPABLE_TYPES = new Set(["json", "blob"]);
+/** The most JSON the platform reads as one entity's model: its dimensions and measures. */
+const MAXIMUM_MODEL_BYTES = 60 * 1024;
+/**
+ * An entity's model as the platform decodes it (`warehouse-backed-entities.ts`): a warehouse-backed
+ * entity's properties as dimensions, or a stored entity's id and groupable properties over its
+ * replication view, and its measures as the manifest writes them.
+ */
+function modelDefinition(entity) {
+    const { definition } = entity.entity;
+    const { source } = definition;
+    const label = (name, property) => property.label ?? humanize(name);
+    const identity = {
+        name: entity.name,
+        label: definition.label,
+        description: definition.description,
+    };
+    const measures = measuresManifest(entity);
+    if (source === undefined) {
+        const entityLabel = typeof definition.label === "string" ? definition.label.toLowerCase() : "";
+        const id = {
+            name: "id",
+            column: "id",
+            type: "string",
+            label: "ID",
+            description: `The ${entityLabel}'s id.`,
+            scope: [],
+        };
+        return {
+            ...identity,
+            source: `ontology/${entity.name}`,
+            dimensions: [
+                id,
+                ...entity.properties.flatMap(({ name, definition: property }) => {
+                    const type = dimensionTypeOf(property.type);
+                    return type === undefined
+                        ? []
+                        : [
+                            {
+                                name,
+                                column: name,
+                                type,
+                                label: label(name, property),
+                                description: property.description ?? "",
+                                scope: [],
+                            },
+                        ];
+                }),
+            ],
+            measures,
+        };
+    }
+    return {
+        ...identity,
+        source: source.asset,
+        ...(source.currency ? { currencyDimension: source.currency } : {}),
+        dimensions: entity.properties.map(({ name, definition: property }) => ({
+            name,
+            label: label(name, property),
+            description: property.description ?? "",
+            type: dimensionTypeOf(property.type),
+            column: property.column,
+            scope: property.scope ?? [],
+        })),
+        measures,
+    };
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/** Whether an expression is an aggregate: `count`, or an aggregate of one property. */
+function isAggregate(expression) {
+    return MEASURE_AGGREGATES.includes(expression.op);
+}
+/**
+ * Why a measure condition is refused, or `undefined`. It names a property of the entity, never a
+ * path, with an operator and values a request filter on that property would take, and never a
+ * period relative to today.
+ */
+function conditionIssue(condition, properties) {
+    if (!isRecord(condition)) {
+        return CONDITION_PROPERTIES;
+    }
+    const property = conditionProperty(condition);
+    const definition = typeof property === "string" ? properties.get(property) : undefined;
+    const type = definition === undefined ? undefined : dimensionTypeOf(definition.type);
+    const { operator, value, values } = condition;
+    if (type === undefined) {
+        return CONDITION_PROPERTIES;
+    }
+    if (operator === "inPeriod") {
+        return "Measure conditions cannot be relative to today.";
+    }
+    if (typeof operator !== "string" || !CONDITION_OPERATORS.has(operator)) {
+        return `Measure conditions use ${MEASURE_CONDITION_OPERATORS.join(", ")}.`;
+    }
+    // Values that are not a list are as wrong as an empty list, which no operator takes.
+    const listed = values === undefined || Array.isArray(values) ? values : [];
+    const filter = { operator, value, values: listed };
+    return filterValuesIssue(filter) ?? filterTypeIssue(filter, type);
+}
 /**
  * The measures of a warehouse-backed or stored entity. A measure's columns name properties of the
  * entity. `hasCurrency` says whether the entity maps a text `source.currency`, which stored entities
  * never do, so their measures cannot be currency amounts.
  */
 function checkMeasures(input) {
-    const { entity, measures, properties, hasCurrency, report } = input;
+    const { entity, measures, properties, hasCurrency } = input;
+    let refusals = 0;
+    const report = (code, path, message) => {
+        refusals++;
+        input.report(code, path, message);
+    };
     // A stored entity's rows carry its system fields, whose names a measure cannot take.
     const stored = entity.entity.definition.source === undefined;
     const measureCount = Object.keys(measures).length;
@@ -139,6 +254,35 @@ function checkMeasures(input) {
             report("invalid-measure", field, "Invalid measure format.");
         }
         let remaining = 10_000;
+        /** The balance properties of the measure's aggregates and of the measures it references. */
+        const balances = new Set();
+        /** An aggregate's balance property, which must be a date or timestamp property. */
+        const checkBalance = (balance) => {
+            const type = typeof balance === "string" ? properties.get(balance)?.type : undefined;
+            if (type !== "date" && type !== "timestamp") {
+                report("invalid-measure", field, "balanceOver names a date or timestamp property.");
+            }
+            if (typeof balance === "string") {
+                balances.add(balance);
+            }
+        };
+        /** An aggregate's conditions, from its `where` option or the `filtered` expression around it. */
+        const checkConditions = (where) => {
+            if (!Array.isArray(where) || where.length === 0) {
+                report("invalid-measure", field, "Measure conditions take at least one entry.");
+                return;
+            }
+            if (where.length > MAXIMUM_CONDITIONS) {
+                report("invalid-measure", field, "Measure conditions take at most 10 entries.");
+                return;
+            }
+            for (const condition of where) {
+                const issue = conditionIssue(condition, properties);
+                if (issue !== undefined) {
+                    report("invalid-measure", field, issue);
+                }
+            }
+        };
         /** `aggregate` is the aggregate the expression is an argument of, if any. */
         const visit = (expression, references, depth, aggregate) => {
             if (--remaining < 0 ||
@@ -149,6 +293,24 @@ function checkMeasures(input) {
                 report("invalid-measure", field, "Invalid or overly complex expression.");
                 return false;
             }
+            const conditions = conditionsOf(expression);
+            const balance = balanceOf(expression);
+            if (conditions !== undefined && expression.op !== "filtered") {
+                if (isAggregate(expression)) {
+                    checkConditions(conditions);
+                }
+                else {
+                    report("invalid-measure", field, "Only count, sum, countDistinct, avg, min and max take where.");
+                }
+            }
+            if (balance !== undefined) {
+                if (isAggregate(expression)) {
+                    checkBalance(balance);
+                }
+                else {
+                    report("invalid-measure", field, "Only count, sum, countDistinct, avg, min and max take balanceOver.");
+                }
+            }
             switch (expression.op) {
                 case "count":
                     if (aggregate !== undefined) {
@@ -171,9 +333,12 @@ function checkMeasures(input) {
                                 "property.");
                         }
                     }
-                    else if (aggregate === undefined || !["integer", "number"].includes(type)) {
+                    else if (aggregate === undefined) {
                         report("invalid-measure", field, "Sums require a numeric property.");
                     }
+                    else if (!["integer", "number"].includes(type)) {
+                        report("invalid-measure", field, "avg, min, max and sum require a numeric property.");
+                    }
                     return false;
                 }
                 case "measure": {
@@ -194,6 +359,22 @@ function checkMeasures(input) {
                     }
                     return visit(target, [...references, expression.name], depth + 1);
                 }
+                case "filtered": {
+                    // The wrapper counts as the aggregate it wraps, which must not declare conditions too.
+                    const args = Array.isArray(expression.args)
+                        ? expression.args
+                        : [];
+                    const [inner] = args;
+                    if (args.length !== 1 ||
+                        !isRecord(inner) ||
+                        !isAggregate(inner) ||
+                        conditionsOf(inner) !== undefined) {
+                        report("invalid-measure", field, "Invalid expression operator or arity.");
+                        return false;
+                    }
+                    checkConditions(expression.where);
+                    return visit(inner, references, depth + 1, aggregate);
+                }
                 default: {
                     const aggregates = AGGREGATES.has(expression.op);
                     if (!OPERATORS.has(expression.op) ||
@@ -221,6 +402,18 @@ function checkMeasures(input) {
         if (measure.unit === "currency" && !hasCurrency) {
             report("invalid-measure", field, "Currency measures require source.currency.");
         }
+        if (balances.size > 1) {
+            report("invalid-measure", field, "A measure reads balances over two properties.");
+        }
+    }
+    // The platform builds a model of every warehouse-backed entity, and of each stored entity with
+    // measures. Measured only once the measures are valid, which writing them requires.
+    if (refusals === 0 && (!stored || measureCount > 0)) {
+        const bytes = new TextEncoder().encode(JSON.stringify(modelDefinition(entity))).byteLength;
+        if (bytes > MAXIMUM_MODEL_BYTES) {
+            input.report("invalid-measure", entity.name, `${entity.name}'s model, its dimensions and measures as SegmentStream reads them, takes ` +
+                `${(bytes / 1024).toFixed(1)} KiB; the limit is ${MAXIMUM_MODEL_BYTES / 1024} KiB.`);
+        }
     }
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@segmentstream-cloud/ontology",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Define a SegmentStream workspace app's ontology: entities, relationships, roles and actions, written with Effect.",
   "license": "Apache-2.0",
   "author": "SegmentStream",