@segmentstream-cloud/ontology 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -1
- package/dist/aggregate-filters.d.ts +48 -0
- package/dist/aggregate-filters.js +252 -0
- package/dist/analyze.js +5 -4
- package/dist/authoring-records.d.ts +5 -0
- package/dist/authoring-records.js +5 -1
- package/dist/authoring.js +32 -3
- package/dist/blob.d.ts +8 -1
- package/dist/blob.js +34 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/manifest.d.ts +109 -60
- package/dist/manifest.js +43 -27
- package/dist/measure.d.ts +86 -7
- package/dist/measure.js +92 -15
- package/dist/migrate/model.d.ts +12 -8
- package/dist/property.d.ts +4 -3
- package/dist/property.js +6 -1
- package/dist/render.d.ts +6 -3
- package/dist/render.js +158 -2
- package/dist/service/mcp/server.js +2 -2
- package/dist/service/mcp/tools.js +46 -11
- package/dist/service/schemas.d.ts +40 -22
- package/dist/service/schemas.js +46 -43
- package/dist/storage/attachments.js +2 -2
- package/dist/storage/sessions.js +7 -5
- package/dist/warehouse-backed.js +199 -6
- package/package.json +1 -1
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
|
+
import { AGGREGATE_FILTER_OPERATORS, ENTITY_AGGREGATE_FILTER_OPERATORS, type AggregateValue } from "../aggregate-filters.js";
|
|
2
3
|
import type { ApiPlan } from "./plan.js";
|
|
4
|
+
export { AGGREGATE_FILTER_OPERATORS, AGGREGATE_PERIOD_UNITS, AGGREGATE_PERIODS, ENTITY_AGGREGATE_FILTER_OPERATORS, } from "../aggregate-filters.js";
|
|
3
5
|
/** A record id in a path: a UUID. */
|
|
4
6
|
export declare const IdParam: Schema.String;
|
|
5
7
|
/** The schemas of one entity's records and queries. */
|
|
@@ -149,6 +151,17 @@ export declare const ActionResponse: Schema.Struct<{
|
|
|
149
151
|
}>;
|
|
150
152
|
export declare const Catalog: Schema.decodeTo<Schema.Struct<{
|
|
151
153
|
readonly entities: Schema.$Array<Schema.encodeKeys<Schema.Struct<{
|
|
154
|
+
readonly measures: Schema.optionalKey<Schema.$Array<Schema.encodeKeys<Schema.Struct<{
|
|
155
|
+
readonly name: Schema.String;
|
|
156
|
+
readonly label: Schema.String;
|
|
157
|
+
readonly description: Schema.String;
|
|
158
|
+
readonly unit: Schema.Literals<readonly ["number", "currency", "ratio", "percent"]>;
|
|
159
|
+
readonly format: Schema.Literals<readonly ["decimal", "currency", "percent"]>;
|
|
160
|
+
readonly expression: Schema.Codec<Schema.Json, Schema.Json, never, never>;
|
|
161
|
+
readonly balanceOver: Schema.optionalKey<Schema.String>;
|
|
162
|
+
}>, {
|
|
163
|
+
readonly balanceOver: "balance_over";
|
|
164
|
+
}>>>;
|
|
152
165
|
readonly properties: Schema.$Array<Schema.encodeKeys<Schema.Struct<{
|
|
153
166
|
readonly column: Schema.optionalKey<Schema.String>;
|
|
154
167
|
readonly scope: Schema.optionalKey<Schema.$Array<Schema.String>>;
|
|
@@ -201,14 +214,6 @@ export declare const Catalog: Schema.decodeTo<Schema.Struct<{
|
|
|
201
214
|
readonly key: Schema.$Array<Schema.String>;
|
|
202
215
|
readonly currency: Schema.optionalKey<Schema.String>;
|
|
203
216
|
}>>;
|
|
204
|
-
readonly measures: Schema.optionalKey<Schema.$Array<Schema.Struct<{
|
|
205
|
-
readonly name: Schema.String;
|
|
206
|
-
readonly label: Schema.String;
|
|
207
|
-
readonly description: Schema.String;
|
|
208
|
-
readonly unit: Schema.Literals<readonly ["number", "currency", "ratio", "percent"]>;
|
|
209
|
-
readonly format: Schema.Literals<readonly ["decimal", "currency", "percent"]>;
|
|
210
|
-
readonly expression: Schema.Codec<Schema.Json, Schema.Json, never, never>;
|
|
211
|
-
}>>>;
|
|
212
217
|
readonly id: Schema.String;
|
|
213
218
|
readonly name: Schema.String;
|
|
214
219
|
readonly label: Schema.String;
|
|
@@ -264,6 +269,17 @@ export declare const Catalog: Schema.decodeTo<Schema.Struct<{
|
|
|
264
269
|
}>>>;
|
|
265
270
|
}>, Schema.Struct<{
|
|
266
271
|
readonly entities: Schema.toEncoded<Schema.$Array<Schema.encodeKeys<Schema.Struct<{
|
|
272
|
+
readonly measures: Schema.optionalKey<Schema.$Array<Schema.encodeKeys<Schema.Struct<{
|
|
273
|
+
readonly name: Schema.String;
|
|
274
|
+
readonly label: Schema.String;
|
|
275
|
+
readonly description: Schema.String;
|
|
276
|
+
readonly unit: Schema.Literals<readonly ["number", "currency", "ratio", "percent"]>;
|
|
277
|
+
readonly format: Schema.Literals<readonly ["decimal", "currency", "percent"]>;
|
|
278
|
+
readonly expression: Schema.Codec<Schema.Json, Schema.Json, never, never>;
|
|
279
|
+
readonly balanceOver: Schema.optionalKey<Schema.String>;
|
|
280
|
+
}>, {
|
|
281
|
+
readonly balanceOver: "balance_over";
|
|
282
|
+
}>>>;
|
|
267
283
|
readonly properties: Schema.$Array<Schema.encodeKeys<Schema.Struct<{
|
|
268
284
|
readonly column: Schema.optionalKey<Schema.String>;
|
|
269
285
|
readonly scope: Schema.optionalKey<Schema.$Array<Schema.String>>;
|
|
@@ -316,14 +332,6 @@ export declare const Catalog: Schema.decodeTo<Schema.Struct<{
|
|
|
316
332
|
readonly key: Schema.$Array<Schema.String>;
|
|
317
333
|
readonly currency: Schema.optionalKey<Schema.String>;
|
|
318
334
|
}>>;
|
|
319
|
-
readonly measures: Schema.optionalKey<Schema.$Array<Schema.Struct<{
|
|
320
|
-
readonly name: Schema.String;
|
|
321
|
-
readonly label: Schema.String;
|
|
322
|
-
readonly description: Schema.String;
|
|
323
|
-
readonly unit: Schema.Literals<readonly ["number", "currency", "ratio", "percent"]>;
|
|
324
|
-
readonly format: Schema.Literals<readonly ["decimal", "currency", "percent"]>;
|
|
325
|
-
readonly expression: Schema.Codec<Schema.Json, Schema.Json, never, never>;
|
|
326
|
-
}>>>;
|
|
327
335
|
readonly id: Schema.String;
|
|
328
336
|
readonly name: Schema.String;
|
|
329
337
|
readonly label: Schema.String;
|
|
@@ -430,11 +438,12 @@ export declare const AGGREGATE_LIMITS: {
|
|
|
430
438
|
readonly orderBy: 50;
|
|
431
439
|
readonly rows: 1000;
|
|
432
440
|
};
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
export declare const AGGREGATE_GRANULARITIES: readonly ["day", "week", "month"];
|
|
437
|
-
|
|
441
|
+
/**
|
|
442
|
+
* The granularities a date or timestamp dimension buckets by; `hour` applies to timestamps only.
|
|
443
|
+
*/
|
|
444
|
+
export declare const AGGREGATE_GRANULARITIES: readonly ["hour", "day", "week", "month", "quarter", "year"];
|
|
445
|
+
/** The longest `timezone` a request names. */
|
|
446
|
+
export declare const MAXIMUM_TIMEZONE_LENGTH = 64;
|
|
438
447
|
interface AggregateFilter<Operator extends string> {
|
|
439
448
|
readonly dimension: string;
|
|
440
449
|
readonly operator: Operator;
|
|
@@ -458,6 +467,7 @@ export interface AggregateRequest {
|
|
|
458
467
|
readonly filters?: ReadonlyArray<AggregateFilter<(typeof AGGREGATE_FILTER_OPERATORS)[number]>>;
|
|
459
468
|
readonly granularity?: (typeof AGGREGATE_GRANULARITIES)[number];
|
|
460
469
|
readonly granularityOf?: string;
|
|
470
|
+
readonly timezone?: string;
|
|
461
471
|
readonly orderBy?: ReadonlyArray<AggregateOrdering>;
|
|
462
472
|
readonly limit?: number;
|
|
463
473
|
}
|
|
@@ -476,6 +486,7 @@ export interface EntityAggregateRequest {
|
|
|
476
486
|
readonly filters?: ReadonlyArray<AggregateFilter<(typeof ENTITY_AGGREGATE_FILTER_OPERATORS)[number]>>;
|
|
477
487
|
readonly granularity?: (typeof AGGREGATE_GRANULARITIES)[number];
|
|
478
488
|
readonly granularityOf?: string;
|
|
489
|
+
readonly timezone?: string;
|
|
479
490
|
readonly orderBy?: ReadonlyArray<AggregateOrdering>;
|
|
480
491
|
readonly limit?: number;
|
|
481
492
|
}
|
|
@@ -491,6 +502,7 @@ export declare const AggregateResult: Schema.Struct<{
|
|
|
491
502
|
readonly implicit: Schema.optionalKey<Schema.Boolean>;
|
|
492
503
|
readonly additive: Schema.optionalKey<Schema.Boolean>;
|
|
493
504
|
readonly filters: Schema.optionalKey<Schema.$Array<Schema.String>>;
|
|
505
|
+
readonly balanceOver: Schema.optionalKey<Schema.String>;
|
|
494
506
|
}>>;
|
|
495
507
|
readonly rows: Schema.$Array<Schema.$Record<Schema.String, Schema.NullOr<Schema.Union<readonly [Schema.String, Schema.Boolean]>>>>;
|
|
496
508
|
readonly truncated: Schema.Boolean;
|
|
@@ -503,6 +515,13 @@ export declare const AggregateResult: Schema.Struct<{
|
|
|
503
515
|
readonly entity: Schema.String;
|
|
504
516
|
readonly asOf: Schema.String;
|
|
505
517
|
}>>>;
|
|
518
|
+
readonly periods: Schema.optionalKey<Schema.$Array<Schema.Struct<{
|
|
519
|
+
readonly dimension: Schema.String;
|
|
520
|
+
readonly value: Schema.String;
|
|
521
|
+
readonly from: Schema.String;
|
|
522
|
+
readonly to: Schema.String;
|
|
523
|
+
readonly timezone: Schema.String;
|
|
524
|
+
}>>>;
|
|
506
525
|
}>;
|
|
507
526
|
readonly deploymentId: Schema.NullOr<Schema.String>;
|
|
508
527
|
readonly definitionDigest: Schema.String;
|
|
@@ -549,4 +568,3 @@ export declare const ErrorSchemas: {
|
|
|
549
568
|
connectionResponseInvalid: Schema.Top;
|
|
550
569
|
connectionUnavailable: Schema.Top;
|
|
551
570
|
};
|
|
552
|
-
export {};
|
package/dist/service/schemas.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { BlobValue } from "../blob.js";
|
|
2
2
|
import { Schema, SchemaTransformation } from "effect";
|
|
3
3
|
import { wireStruct } from "../wire.js";
|
|
4
|
+
import { AGGREGATE_FILTER_OPERATORS, ENTITY_AGGREGATE_FILTER_OPERATORS, filterValuesIssue, MAXIMUM_FILTER_VALUES, } from "../aggregate-filters.js";
|
|
4
5
|
import { isValueMatched, OntologyManifestWire } from "../manifest.js";
|
|
5
6
|
import { entityQuerySchemas, queryLinks } from "../query.js";
|
|
6
7
|
import { ValidationIssue as RuntimeValidationIssue } from "../runtime/errors.js";
|
|
7
8
|
import { Edit, HistoryEntry as RuntimeHistoryEntry } from "../runtime/history.js";
|
|
9
|
+
export { AGGREGATE_FILTER_OPERATORS, AGGREGATE_PERIOD_UNITS, AGGREGATE_PERIODS, ENTITY_AGGREGATE_FILTER_OPERATORS, } from "../aggregate-filters.js";
|
|
8
10
|
/** `lead` → `Lead`, `pilot_request` → `PilotRequest`: component names in the OpenAPI document. */
|
|
9
11
|
function pascalCase(name) {
|
|
10
12
|
return name
|
|
@@ -312,54 +314,29 @@ export const AGGREGATE_LIMITS = {
|
|
|
312
314
|
measures: 30,
|
|
313
315
|
dimensions: 20,
|
|
314
316
|
filters: 50,
|
|
315
|
-
values:
|
|
317
|
+
values: MAXIMUM_FILTER_VALUES,
|
|
316
318
|
orderBy: 50,
|
|
317
319
|
rows: 1000,
|
|
318
320
|
};
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
];
|
|
326
|
-
/** The operators of the per-entity shorthand, which bounds dates with `dateRange` instead. */
|
|
327
|
-
export const ENTITY_AGGREGATE_FILTER_OPERATORS = ["equals", "in", "isNull", "isNotNull"];
|
|
328
|
-
export const AGGREGATE_GRANULARITIES = ["day", "week", "month"];
|
|
321
|
+
/**
|
|
322
|
+
* The granularities a date or timestamp dimension buckets by; `hour` applies to timestamps only.
|
|
323
|
+
*/
|
|
324
|
+
export const AGGREGATE_GRANULARITIES = ["hour", "day", "week", "month", "quarter", "year"];
|
|
325
|
+
/** The longest `timezone` a request names. */
|
|
326
|
+
export const MAXIMUM_TIMEZONE_LENGTH = 64;
|
|
329
327
|
const AggregateValue = Schema.Union([Schema.String, Schema.Number, Schema.Boolean]);
|
|
330
|
-
/**
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
return value === undefined || values !== undefined
|
|
336
|
-
? "Equality filters require one value."
|
|
337
|
-
: undefined;
|
|
338
|
-
case "in":
|
|
339
|
-
return values === undefined ||
|
|
340
|
-
values.length === 0 ||
|
|
341
|
-
values.length > AGGREGATE_LIMITS.values ||
|
|
342
|
-
value !== undefined
|
|
343
|
-
? "Membership filters require between 1 and 100 values."
|
|
344
|
-
: undefined;
|
|
345
|
-
case "between":
|
|
346
|
-
return value !== undefined || values?.length !== 2
|
|
347
|
-
? "between needs two values in order."
|
|
348
|
-
: undefined;
|
|
349
|
-
default:
|
|
350
|
-
return value !== undefined || values !== undefined
|
|
351
|
-
? "Null filters do not accept values."
|
|
352
|
-
: undefined;
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
function aggregateFilter(operators) {
|
|
328
|
+
/**
|
|
329
|
+
* A filter of either request. Which values each operator takes is checked with SegmentStream's own
|
|
330
|
+
* messages (`filterValuesIssue`); whether they suit the dimension's type is SegmentStream's to check.
|
|
331
|
+
*/
|
|
332
|
+
function aggregateFilter(operators, descriptions) {
|
|
356
333
|
const filter = closedStruct({
|
|
357
334
|
dimension: Schema.String.annotate({
|
|
358
335
|
description: "A property, or a path of to-one relationships ending in one.",
|
|
359
336
|
}),
|
|
360
337
|
operator: Schema.Literals(operators),
|
|
361
|
-
value: Schema.optionalKey(AggregateValue).annotate({ description:
|
|
362
|
-
values: Schema.optionalKey(Schema.Array(AggregateValue).check(Schema.isMaxLength(AGGREGATE_LIMITS.values))).annotate({ description:
|
|
338
|
+
value: Schema.optionalKey(AggregateValue).annotate({ description: descriptions.value }),
|
|
339
|
+
values: Schema.optionalKey(Schema.Array(AggregateValue).check(Schema.isMaxLength(AGGREGATE_LIMITS.values))).annotate({ description: descriptions.values }),
|
|
363
340
|
}, "Unknown key; use dimension, operator, value and values");
|
|
364
341
|
return filter.check(Schema.makeFilter((candidate) => filterValuesIssue(candidate) ?? true));
|
|
365
342
|
}
|
|
@@ -374,6 +351,11 @@ const aggregateFields = {
|
|
|
374
351
|
granularityOf: Schema.optionalKey(Schema.String).annotate({
|
|
375
352
|
description: "The requested date or timestamp dimension that granularity buckets.",
|
|
376
353
|
}),
|
|
354
|
+
timezone: Schema.optionalKey(Schema.String.check(Schema.isMaxLength(MAXIMUM_TIMEZONE_LENGTH))).annotate({
|
|
355
|
+
description: "An IANA timezone, such as Europe/Moscow, or UTC. Buckets of timestamps, date bounds on " +
|
|
356
|
+
"timestamps and inPeriod's today follow it instead of each entity's timezone; measure " +
|
|
357
|
+
"conditions never do.",
|
|
358
|
+
}),
|
|
377
359
|
orderBy: Schema.optionalKey(Schema.Array(AggregateOrder).check(Schema.isMaxLength(AGGREGATE_LIMITS.orderBy))),
|
|
378
360
|
limit: Schema.optionalKey(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: AGGREGATE_LIMITS.rows }))).annotate({ description: "At most 1000 rows; 100 by default." }),
|
|
379
361
|
};
|
|
@@ -389,9 +371,15 @@ export const AggregateRequest = closedStruct({
|
|
|
389
371
|
description: "<entity>.<property>, or a route from a measure's entity: " +
|
|
390
372
|
"<entity>.<relationship>[.<relationship>].<property>.",
|
|
391
373
|
}),
|
|
392
|
-
filters: Schema.optionalKey(Schema.Array(aggregateFilter(AGGREGATE_FILTER_OPERATORS
|
|
374
|
+
filters: Schema.optionalKey(Schema.Array(aggregateFilter(AGGREGATE_FILTER_OPERATORS, {
|
|
375
|
+
value: "The value of equals and notEquals; the bound of gt, gte, lt and lte; a non-empty " +
|
|
376
|
+
"string for contains, startsWith and endsWith; the period of inPeriod, such as " +
|
|
377
|
+
"last 7 days.",
|
|
378
|
+
values: "The values of in and notIn, or the two inclusive bounds of between.",
|
|
379
|
+
})).check(Schema.isMaxLength(AGGREGATE_LIMITS.filters))).annotate({ description: "All must hold." }),
|
|
393
380
|
...aggregateFields,
|
|
394
|
-
}, "Unknown key; use measures, dimensions, filters, granularity, granularityOf, orderBy
|
|
381
|
+
}, "Unknown key; use measures, dimensions, filters, granularity, granularityOf, timezone, orderBy " +
|
|
382
|
+
"and limit").annotate({ identifier: "AggregateRequest" });
|
|
395
383
|
/**
|
|
396
384
|
* `POST /api/ontology/<entity>/aggregate`: SegmentStream's per-entity request, unchanged. Names are
|
|
397
385
|
* relative to the entity (`revenue`, `product.brand`), and `dateRange` bounds a partitioned entity.
|
|
@@ -409,10 +397,14 @@ export const EntityAggregateRequest = closedStruct({
|
|
|
409
397
|
from: Schema.String.annotate({ format: "date" }),
|
|
410
398
|
to: Schema.String.annotate({ format: "date" }),
|
|
411
399
|
}, "Unknown key; use from and to")).annotate({ description: "Inclusive days of the entity's partition property." }),
|
|
412
|
-
filters: Schema.optionalKey(Schema.Array(aggregateFilter(ENTITY_AGGREGATE_FILTER_OPERATORS
|
|
400
|
+
filters: Schema.optionalKey(Schema.Array(aggregateFilter(ENTITY_AGGREGATE_FILTER_OPERATORS, {
|
|
401
|
+
value: "The value of equals and notEquals; the bound of gt, gte, lt and lte; a non-empty " +
|
|
402
|
+
"string for contains, startsWith and endsWith.",
|
|
403
|
+
values: "The values of in and notIn.",
|
|
404
|
+
})).check(Schema.isMaxLength(AGGREGATE_LIMITS.filters))).annotate({ description: "All must hold." }),
|
|
413
405
|
...aggregateFields,
|
|
414
406
|
}, "Unknown key; use dimensions, measures, dateRange, filters, granularity, granularityOf, " +
|
|
415
|
-
"orderBy and limit").annotate({ identifier: "EntityAggregateRequest" });
|
|
407
|
+
"timezone, orderBy and limit").annotate({ identifier: "EntityAggregateRequest" });
|
|
416
408
|
/** SegmentStream's answer to an aggregate, which the service forwards unchanged. */
|
|
417
409
|
export const AggregateResult = Schema.Struct({
|
|
418
410
|
fields: Schema.Array(Schema.Struct({
|
|
@@ -429,6 +421,10 @@ export const AggregateResult = Schema.Struct({
|
|
|
429
421
|
filters: Schema.optionalKey(Schema.Array(Schema.String)).annotate({
|
|
430
422
|
description: "The filters the measure applied.",
|
|
431
423
|
}),
|
|
424
|
+
balanceOver: Schema.optionalKey(Schema.String).annotate({
|
|
425
|
+
description: "The date or timestamp dimension a balance measure was grouped by or filtered to one " +
|
|
426
|
+
"day of.",
|
|
427
|
+
}),
|
|
432
428
|
})),
|
|
433
429
|
rows: Schema.Array(Schema.Record(Schema.String, Schema.NullOr(Schema.Union([Schema.String, Schema.Boolean])))),
|
|
434
430
|
truncated: Schema.Boolean,
|
|
@@ -440,6 +436,13 @@ export const AggregateResult = Schema.Struct({
|
|
|
440
436
|
description: "For example, a filter that did not apply to some measure's entity.",
|
|
441
437
|
}),
|
|
442
438
|
sources: Schema.optionalKey(Schema.Array(Schema.Struct({ entity: Schema.String, asOf: Schema.String }))).annotate({ description: "How fresh the replica of each stored entity read is." }),
|
|
439
|
+
periods: Schema.optionalKey(Schema.Array(Schema.Struct({
|
|
440
|
+
dimension: Schema.String,
|
|
441
|
+
value: Schema.String,
|
|
442
|
+
from: Schema.String.annotate({ format: "date" }),
|
|
443
|
+
to: Schema.String.annotate({ format: "date" }),
|
|
444
|
+
timezone: Schema.String,
|
|
445
|
+
}))).annotate({ description: "The inclusive dates each inPeriod filter used, in request order." }),
|
|
443
446
|
}),
|
|
444
447
|
deploymentId: Schema.NullOr(Schema.String),
|
|
445
448
|
definitionDigest: Schema.String,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { Effect } from "effect";
|
|
3
|
-
import { BLOB_UPLOAD_ANNOTATION, MAX_BLOB_BYTES } from "../blob.js";
|
|
3
|
+
import { acceptsBlobContentType, BLOB_UPLOAD_ANNOTATION, MAX_BLOB_BYTES } from "../blob.js";
|
|
4
4
|
import { ActionRejected, Conflict, NotFound } from "../errors.js";
|
|
5
5
|
import { canonicalJson } from "../manifest.js";
|
|
6
6
|
import { actionInputJsonSchema } from "../runtime/generated-actions.js";
|
|
@@ -357,7 +357,7 @@ export function attachmentRuntime(ontology, sql, provider) {
|
|
|
357
357
|
if (row.state !== "ready" || row.expired) {
|
|
358
358
|
return yield* unavailable("The attachment expired or was discarded.");
|
|
359
359
|
}
|
|
360
|
-
if (!property.accept
|
|
360
|
+
if (!acceptsBlobContentType(property.accept, row.content_type) ||
|
|
361
361
|
Number(row.size) > (property.maxBytes ?? MAX_BLOB_BYTES)) {
|
|
362
362
|
return yield* new ActionRejected({
|
|
363
363
|
message: `${field.entity}.${field.property} accepts ${(property.accept ?? []).join(", ")} ` +
|
package/dist/storage/sessions.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { Effect, Schema } from "effect";
|
|
3
|
-
import { uploadInput, BLOB_UPLOAD_ANNOTATION, MAX_BLOB_BYTES } from "../blob.js";
|
|
3
|
+
import { acceptsBlobContentType, uploadInput, BLOB_UPLOAD_ANNOTATION, MAX_BLOB_BYTES, } from "../blob.js";
|
|
4
4
|
import { wireStruct } from "../wire.js";
|
|
5
5
|
import { ActionRejected, Conflict, NotFound } from "../errors.js";
|
|
6
6
|
import { authorizeBlobOperation } from "./authorization.js";
|
|
@@ -50,7 +50,7 @@ function allowsUpload(schema, entity, property) {
|
|
|
50
50
|
}
|
|
51
51
|
function sniff(bytes, contentType) {
|
|
52
52
|
const starts = (magic) => magic.every((byte, index) => bytes[index] === byte);
|
|
53
|
-
switch (contentType) {
|
|
53
|
+
switch (contentType.toLowerCase()) {
|
|
54
54
|
case "application/pdf":
|
|
55
55
|
return starts([37, 80, 68, 70, 45]);
|
|
56
56
|
case "image/png":
|
|
@@ -67,7 +67,8 @@ function sniff(bytes, contentType) {
|
|
|
67
67
|
return false;
|
|
68
68
|
}
|
|
69
69
|
default:
|
|
70
|
-
|
|
70
|
+
// Other MIME types describe opaque bytes; storage does not parse their formats.
|
|
71
|
+
return true;
|
|
71
72
|
}
|
|
72
73
|
}
|
|
73
74
|
/** Persisted phases surround every provider call; no external I/O holds a database transaction. */
|
|
@@ -108,7 +109,7 @@ export function blobRuntime(ontology, sql, provider) {
|
|
|
108
109
|
entity.source ||
|
|
109
110
|
!property ||
|
|
110
111
|
!found ||
|
|
111
|
-
!property.accept
|
|
112
|
+
!acceptsBlobContentType(property.accept, input.contentType) ||
|
|
112
113
|
input.size > (property.maxBytes ?? MAX_BLOB_BYTES) ||
|
|
113
114
|
Boolean(input.recordId) === Boolean(input.createIntent) ||
|
|
114
115
|
/[\x00-\x1f\x7f/\\]/.test(input.filename)) {
|
|
@@ -334,7 +335,8 @@ export function blobRuntime(ontology, sql, provider) {
|
|
|
334
335
|
key: blob.key,
|
|
335
336
|
filename: blob.filename,
|
|
336
337
|
contentType: blob.content_type,
|
|
337
|
-
inline: input.inline === true &&
|
|
338
|
+
inline: input.inline === true &&
|
|
339
|
+
["image/jpeg", "image/png"].includes(blob.content_type.toLowerCase()),
|
|
338
340
|
});
|
|
339
341
|
});
|
|
340
342
|
return { begin, status, part, complete, cancel, download };
|
package/dist/warehouse-backed.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { dimensionTypeOf, filterTypeIssue, filterValuesIssue, MEASURE_CONDITION_OPERATORS, } from "./aggregate-filters.js";
|
|
2
|
+
import { measuresManifest } from "./manifest.js";
|
|
3
|
+
import { balanceOf, conditionProperty, conditionsOf, MEASURE_AGGREGATES, } from "./measure.js";
|
|
4
|
+
import { humanize, isSystemProperty, SNAKE_CASE_NAME } from "./names.js";
|
|
2
5
|
const columnName = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
3
6
|
const types = new Set(["text", "integer", "number", "boolean", "date", "timestamp", "select"]);
|
|
4
7
|
/**
|
|
@@ -92,23 +95,135 @@ export function checkWarehouseSource(entities, report) {
|
|
|
92
95
|
}
|
|
93
96
|
}
|
|
94
97
|
}
|
|
95
|
-
/**
|
|
96
|
-
|
|
98
|
+
/**
|
|
99
|
+
* The aggregates of one property, which take one argument. `countDistinct` counts what compares
|
|
100
|
+
* exactly; `sum`, `avg`, `min` and `max` take numbers.
|
|
101
|
+
*/
|
|
102
|
+
const AGGREGATES = new Set(["sum", "countDistinct", "avg", "min", "max"]);
|
|
97
103
|
const OPERATORS = new Set([...AGGREGATES, "add", "subtract", "multiply", "divide"]);
|
|
104
|
+
/** The most conditions one aggregate takes. */
|
|
105
|
+
const MAXIMUM_CONDITIONS = 10;
|
|
106
|
+
const CONDITION_OPERATORS = new Set(MEASURE_CONDITION_OPERATORS);
|
|
107
|
+
const CONDITION_PROPERTIES = "Measure conditions name properties of the entity.";
|
|
98
108
|
/**
|
|
99
109
|
* The most dimensions and measures the platform accepts in one aggregate model. It also refuses a
|
|
100
|
-
* model definition over
|
|
110
|
+
* model definition over {@link MAXIMUM_MODEL_BYTES}.
|
|
101
111
|
*/
|
|
102
112
|
const MAXIMUM_MODEL_MEMBERS = 100;
|
|
103
113
|
/** Stored property types that are not dimensions: their values cannot be grouped by. */
|
|
104
114
|
const UNGROUPABLE_TYPES = new Set(["json", "blob"]);
|
|
115
|
+
/** The most JSON the platform reads as one entity's model: its dimensions and measures. */
|
|
116
|
+
const MAXIMUM_MODEL_BYTES = 60 * 1024;
|
|
117
|
+
/**
|
|
118
|
+
* An entity's model as the platform decodes it (`warehouse-backed-entities.ts`): a warehouse-backed
|
|
119
|
+
* entity's properties as dimensions, or a stored entity's id and groupable properties over its
|
|
120
|
+
* replication view, and its measures as the manifest writes them.
|
|
121
|
+
*/
|
|
122
|
+
function modelDefinition(entity) {
|
|
123
|
+
const { definition } = entity.entity;
|
|
124
|
+
const { source } = definition;
|
|
125
|
+
const label = (name, property) => property.label ?? humanize(name);
|
|
126
|
+
const identity = {
|
|
127
|
+
name: entity.name,
|
|
128
|
+
label: definition.label,
|
|
129
|
+
description: definition.description,
|
|
130
|
+
};
|
|
131
|
+
const measures = measuresManifest(entity);
|
|
132
|
+
if (source === undefined) {
|
|
133
|
+
const entityLabel = typeof definition.label === "string" ? definition.label.toLowerCase() : "";
|
|
134
|
+
const id = {
|
|
135
|
+
name: "id",
|
|
136
|
+
column: "id",
|
|
137
|
+
type: "string",
|
|
138
|
+
label: "ID",
|
|
139
|
+
description: `The ${entityLabel}'s id.`,
|
|
140
|
+
scope: [],
|
|
141
|
+
};
|
|
142
|
+
return {
|
|
143
|
+
...identity,
|
|
144
|
+
source: `ontology/${entity.name}`,
|
|
145
|
+
dimensions: [
|
|
146
|
+
id,
|
|
147
|
+
...entity.properties.flatMap(({ name, definition: property }) => {
|
|
148
|
+
const type = dimensionTypeOf(property.type);
|
|
149
|
+
return type === undefined
|
|
150
|
+
? []
|
|
151
|
+
: [
|
|
152
|
+
{
|
|
153
|
+
name,
|
|
154
|
+
column: name,
|
|
155
|
+
type,
|
|
156
|
+
label: label(name, property),
|
|
157
|
+
description: property.description ?? "",
|
|
158
|
+
scope: [],
|
|
159
|
+
},
|
|
160
|
+
];
|
|
161
|
+
}),
|
|
162
|
+
],
|
|
163
|
+
measures,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
return {
|
|
167
|
+
...identity,
|
|
168
|
+
source: source.asset,
|
|
169
|
+
...(source.currency ? { currencyDimension: source.currency } : {}),
|
|
170
|
+
dimensions: entity.properties.map(({ name, definition: property }) => ({
|
|
171
|
+
name,
|
|
172
|
+
label: label(name, property),
|
|
173
|
+
description: property.description ?? "",
|
|
174
|
+
type: dimensionTypeOf(property.type),
|
|
175
|
+
column: property.column,
|
|
176
|
+
scope: property.scope ?? [],
|
|
177
|
+
})),
|
|
178
|
+
measures,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
function isRecord(value) {
|
|
182
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
183
|
+
}
|
|
184
|
+
/** Whether an expression is an aggregate: `count`, or an aggregate of one property. */
|
|
185
|
+
function isAggregate(expression) {
|
|
186
|
+
return MEASURE_AGGREGATES.includes(expression.op);
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Why a measure condition is refused, or `undefined`. It names a property of the entity, never a
|
|
190
|
+
* path, with an operator and values a request filter on that property would take, and never a
|
|
191
|
+
* period relative to today.
|
|
192
|
+
*/
|
|
193
|
+
function conditionIssue(condition, properties) {
|
|
194
|
+
if (!isRecord(condition)) {
|
|
195
|
+
return CONDITION_PROPERTIES;
|
|
196
|
+
}
|
|
197
|
+
const property = conditionProperty(condition);
|
|
198
|
+
const definition = typeof property === "string" ? properties.get(property) : undefined;
|
|
199
|
+
const type = definition === undefined ? undefined : dimensionTypeOf(definition.type);
|
|
200
|
+
const { operator, value, values } = condition;
|
|
201
|
+
if (type === undefined) {
|
|
202
|
+
return CONDITION_PROPERTIES;
|
|
203
|
+
}
|
|
204
|
+
if (operator === "inPeriod") {
|
|
205
|
+
return "Measure conditions cannot be relative to today.";
|
|
206
|
+
}
|
|
207
|
+
if (typeof operator !== "string" || !CONDITION_OPERATORS.has(operator)) {
|
|
208
|
+
return `Measure conditions use ${MEASURE_CONDITION_OPERATORS.join(", ")}.`;
|
|
209
|
+
}
|
|
210
|
+
// Values that are not a list are as wrong as an empty list, which no operator takes.
|
|
211
|
+
const listed = values === undefined || Array.isArray(values) ? values : [];
|
|
212
|
+
const filter = { operator, value, values: listed };
|
|
213
|
+
return filterValuesIssue(filter) ?? filterTypeIssue(filter, type);
|
|
214
|
+
}
|
|
105
215
|
/**
|
|
106
216
|
* The measures of a warehouse-backed or stored entity. A measure's columns name properties of the
|
|
107
217
|
* entity. `hasCurrency` says whether the entity maps a text `source.currency`, which stored entities
|
|
108
218
|
* never do, so their measures cannot be currency amounts.
|
|
109
219
|
*/
|
|
110
220
|
function checkMeasures(input) {
|
|
111
|
-
const { entity, measures, properties, hasCurrency
|
|
221
|
+
const { entity, measures, properties, hasCurrency } = input;
|
|
222
|
+
let refusals = 0;
|
|
223
|
+
const report = (code, path, message) => {
|
|
224
|
+
refusals++;
|
|
225
|
+
input.report(code, path, message);
|
|
226
|
+
};
|
|
112
227
|
// A stored entity's rows carry its system fields, whose names a measure cannot take.
|
|
113
228
|
const stored = entity.entity.definition.source === undefined;
|
|
114
229
|
const measureCount = Object.keys(measures).length;
|
|
@@ -139,6 +254,35 @@ function checkMeasures(input) {
|
|
|
139
254
|
report("invalid-measure", field, "Invalid measure format.");
|
|
140
255
|
}
|
|
141
256
|
let remaining = 10_000;
|
|
257
|
+
/** The balance properties of the measure's aggregates and of the measures it references. */
|
|
258
|
+
const balances = new Set();
|
|
259
|
+
/** An aggregate's balance property, which must be a date or timestamp property. */
|
|
260
|
+
const checkBalance = (balance) => {
|
|
261
|
+
const type = typeof balance === "string" ? properties.get(balance)?.type : undefined;
|
|
262
|
+
if (type !== "date" && type !== "timestamp") {
|
|
263
|
+
report("invalid-measure", field, "balanceOver names a date or timestamp property.");
|
|
264
|
+
}
|
|
265
|
+
if (typeof balance === "string") {
|
|
266
|
+
balances.add(balance);
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
/** An aggregate's conditions, from its `where` option or the `filtered` expression around it. */
|
|
270
|
+
const checkConditions = (where) => {
|
|
271
|
+
if (!Array.isArray(where) || where.length === 0) {
|
|
272
|
+
report("invalid-measure", field, "Measure conditions take at least one entry.");
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
if (where.length > MAXIMUM_CONDITIONS) {
|
|
276
|
+
report("invalid-measure", field, "Measure conditions take at most 10 entries.");
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
for (const condition of where) {
|
|
280
|
+
const issue = conditionIssue(condition, properties);
|
|
281
|
+
if (issue !== undefined) {
|
|
282
|
+
report("invalid-measure", field, issue);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
};
|
|
142
286
|
/** `aggregate` is the aggregate the expression is an argument of, if any. */
|
|
143
287
|
const visit = (expression, references, depth, aggregate) => {
|
|
144
288
|
if (--remaining < 0 ||
|
|
@@ -149,6 +293,24 @@ function checkMeasures(input) {
|
|
|
149
293
|
report("invalid-measure", field, "Invalid or overly complex expression.");
|
|
150
294
|
return false;
|
|
151
295
|
}
|
|
296
|
+
const conditions = conditionsOf(expression);
|
|
297
|
+
const balance = balanceOf(expression);
|
|
298
|
+
if (conditions !== undefined && expression.op !== "filtered") {
|
|
299
|
+
if (isAggregate(expression)) {
|
|
300
|
+
checkConditions(conditions);
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
report("invalid-measure", field, "Only count, sum, countDistinct, avg, min and max take where.");
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (balance !== undefined) {
|
|
307
|
+
if (isAggregate(expression)) {
|
|
308
|
+
checkBalance(balance);
|
|
309
|
+
}
|
|
310
|
+
else {
|
|
311
|
+
report("invalid-measure", field, "Only count, sum, countDistinct, avg, min and max take balanceOver.");
|
|
312
|
+
}
|
|
313
|
+
}
|
|
152
314
|
switch (expression.op) {
|
|
153
315
|
case "count":
|
|
154
316
|
if (aggregate !== undefined) {
|
|
@@ -171,9 +333,12 @@ function checkMeasures(input) {
|
|
|
171
333
|
"property.");
|
|
172
334
|
}
|
|
173
335
|
}
|
|
174
|
-
else if (aggregate === undefined
|
|
336
|
+
else if (aggregate === undefined) {
|
|
175
337
|
report("invalid-measure", field, "Sums require a numeric property.");
|
|
176
338
|
}
|
|
339
|
+
else if (!["integer", "number"].includes(type)) {
|
|
340
|
+
report("invalid-measure", field, "avg, min, max and sum require a numeric property.");
|
|
341
|
+
}
|
|
177
342
|
return false;
|
|
178
343
|
}
|
|
179
344
|
case "measure": {
|
|
@@ -194,6 +359,22 @@ function checkMeasures(input) {
|
|
|
194
359
|
}
|
|
195
360
|
return visit(target, [...references, expression.name], depth + 1);
|
|
196
361
|
}
|
|
362
|
+
case "filtered": {
|
|
363
|
+
// The wrapper counts as the aggregate it wraps, which must not declare conditions too.
|
|
364
|
+
const args = Array.isArray(expression.args)
|
|
365
|
+
? expression.args
|
|
366
|
+
: [];
|
|
367
|
+
const [inner] = args;
|
|
368
|
+
if (args.length !== 1 ||
|
|
369
|
+
!isRecord(inner) ||
|
|
370
|
+
!isAggregate(inner) ||
|
|
371
|
+
conditionsOf(inner) !== undefined) {
|
|
372
|
+
report("invalid-measure", field, "Invalid expression operator or arity.");
|
|
373
|
+
return false;
|
|
374
|
+
}
|
|
375
|
+
checkConditions(expression.where);
|
|
376
|
+
return visit(inner, references, depth + 1, aggregate);
|
|
377
|
+
}
|
|
197
378
|
default: {
|
|
198
379
|
const aggregates = AGGREGATES.has(expression.op);
|
|
199
380
|
if (!OPERATORS.has(expression.op) ||
|
|
@@ -221,6 +402,18 @@ function checkMeasures(input) {
|
|
|
221
402
|
if (measure.unit === "currency" && !hasCurrency) {
|
|
222
403
|
report("invalid-measure", field, "Currency measures require source.currency.");
|
|
223
404
|
}
|
|
405
|
+
if (balances.size > 1) {
|
|
406
|
+
report("invalid-measure", field, "A measure reads balances over two properties.");
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
// The platform builds a model of every warehouse-backed entity, and of each stored entity with
|
|
410
|
+
// measures. Measured only once the measures are valid, which writing them requires.
|
|
411
|
+
if (refusals === 0 && (!stored || measureCount > 0)) {
|
|
412
|
+
const bytes = new TextEncoder().encode(JSON.stringify(modelDefinition(entity))).byteLength;
|
|
413
|
+
if (bytes > MAXIMUM_MODEL_BYTES) {
|
|
414
|
+
input.report("invalid-measure", entity.name, `${entity.name}'s model, its dimensions and measures as SegmentStream reads them, takes ` +
|
|
415
|
+
`${(bytes / 1024).toFixed(1)} KiB; the limit is ${MAXIMUM_MODEL_BYTES / 1024} KiB.`);
|
|
416
|
+
}
|
|
224
417
|
}
|
|
225
418
|
}
|
|
226
419
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@segmentstream-cloud/ontology",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.0",
|
|
4
4
|
"description": "Define a SegmentStream workspace app's ontology: entities, relationships, roles and actions, written with Effect.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "SegmentStream",
|