@segmentstream-cloud/ontology 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -16
- package/dist/aggregate-routes.d.ts +95 -0
- package/dist/aggregate-routes.js +332 -0
- package/dist/analyze.js +83 -16
- package/dist/entity.d.ts +31 -15
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/manifest.d.ts +57 -23
- package/dist/manifest.js +77 -30
- package/dist/measure.d.ts +11 -2
- package/dist/measure.js +14 -1
- package/dist/migrate/plan.d.ts +1 -1
- package/dist/migrate/plan.js +38 -13
- package/dist/migrate/storage.d.ts +3 -2
- package/dist/migrate/storage.js +15 -5
- package/dist/names.d.ts +4 -2
- package/dist/names.js +4 -1
- package/dist/query.d.ts +5 -2
- package/dist/query.js +6 -2
- package/dist/relationship.d.ts +36 -5
- package/dist/relationship.js +5 -3
- package/dist/render.d.ts +47 -3
- package/dist/render.js +117 -11
- package/dist/replication/model.d.ts +1 -1
- package/dist/replication/model.js +11 -1
- package/dist/service/aggregate-platform.d.ts +60 -0
- package/dist/service/aggregate-platform.js +189 -0
- package/dist/service/aggregate.d.ts +42 -0
- package/dist/service/aggregate.js +145 -0
- package/dist/service/api.d.ts +5 -3
- package/dist/service/api.js +52 -6
- package/dist/service/catalog.d.ts +10 -1
- package/dist/service/catalog.js +12 -3
- package/dist/service/config.d.ts +13 -1
- package/dist/service/config.js +11 -1
- package/dist/service/errors.d.ts +32 -0
- package/dist/service/errors.js +102 -0
- package/dist/service/handlers.d.ts +2 -1
- package/dist/service/handlers.js +66 -28
- package/dist/service/mcp/server.d.ts +3 -0
- package/dist/service/mcp/server.js +60 -8
- package/dist/service/mcp/tools.d.ts +13 -2
- package/dist/service/mcp/tools.js +165 -6
- package/dist/service/plan.js +7 -1
- package/dist/service/schemas.d.ts +99 -1
- package/dist/service/schemas.js +150 -2
- package/dist/service/service.js +8 -2
- package/dist/user.d.ts +1 -1
- package/dist/warehouse-backed.js +222 -121
- package/package.json +1 -1
package/dist/render.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import type { ActionManifest, OntologyManifest } from "./manifest.js";
|
|
1
|
+
import type { ActionManifest, EntityManifest, OntologyManifest } from "./manifest.js";
|
|
2
2
|
export interface RenderOptions {
|
|
3
3
|
/**
|
|
4
|
-
* The permissions of the reader. Only the actions these permissions may run are listed
|
|
5
|
-
*
|
|
4
|
+
* The permissions of the reader. Only the actions these permissions may run are listed, and only
|
|
5
|
+
* the measures and routes of entities they may read. Without it, every action is listed with the
|
|
6
|
+
* permission it requires and the roles that grant it, and every measure and route.
|
|
6
7
|
*/
|
|
7
8
|
readonly as?: ReadonlyArray<string>;
|
|
8
9
|
}
|
|
@@ -16,3 +17,46 @@ export declare function actionPermission(entity: string, action: ActionManifest)
|
|
|
16
17
|
* the console, the CLI and the service describe the same ontology.
|
|
17
18
|
*/
|
|
18
19
|
export declare function renderCatalog(manifest: OntologyManifest, options?: RenderOptions): string;
|
|
20
|
+
/**
|
|
21
|
+
* The catalog as `describe` answers a reader: {@link renderCatalog} with the reader's permissions,
|
|
22
|
+
* or everything without them.
|
|
23
|
+
*/
|
|
24
|
+
export declare function describeCatalog(manifest: OntologyManifest, permissions?: ReadonlyArray<string>): string;
|
|
25
|
+
type MeasureManifest = NonNullable<EntityManifest["measures"]>[number];
|
|
26
|
+
/** A measure as aggregates name it: `<entity>.<measure>`. */
|
|
27
|
+
export interface NamedMeasure {
|
|
28
|
+
readonly name: string;
|
|
29
|
+
readonly entity: EntityManifest;
|
|
30
|
+
readonly measure: MeasureManifest;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* An entity a measure entity reaches, and the shortest routes there the reader may follow:
|
|
34
|
+
* `product.region`. `ambiguous` when the resolver finds several shortest routes, readable or not,
|
|
35
|
+
* so a request must write the route.
|
|
36
|
+
*/
|
|
37
|
+
export interface Reach {
|
|
38
|
+
readonly target: string;
|
|
39
|
+
readonly routes: ReadonlyArray<string>;
|
|
40
|
+
readonly ambiguous: boolean;
|
|
41
|
+
}
|
|
42
|
+
/** The rule every surface repeats (docs/ontology-aggregates-spec.md, "Three ways to ask"). */
|
|
43
|
+
export declare const RECORDS_OR_AGGREGATES = "look at or change records through find and actions; count, sum or compare through aggregate";
|
|
44
|
+
/**
|
|
45
|
+
* Every measure of the entities the reader may read, warehouse-backed and stored alike, in
|
|
46
|
+
* declaration order. Without permissions, every measure.
|
|
47
|
+
*/
|
|
48
|
+
export declare function measuresFor(manifest: OntologyManifest, permissions?: ReadonlyArray<string>): ReadonlyArray<NamedMeasure>;
|
|
49
|
+
/**
|
|
50
|
+
* The entities one entity reaches, with the shortest routes to each: exactly the routes aggregate
|
|
51
|
+
* resolution weighs ({@link shortestRoutesFrom}). Only then is the reader applied: a target they may
|
|
52
|
+
* not read is left out, and so is any route through an entity they may not read, which the service
|
|
53
|
+
* would refuse. A target keeps `ambiguous` from before that filter, so a route is never shown as the
|
|
54
|
+
* only one when the resolver would ask for it to be written.
|
|
55
|
+
*/
|
|
56
|
+
export declare function reachesOf(manifest: OntologyManifest, from: string, permissions?: ReadonlyArray<string>): ReadonlyArray<Reach>;
|
|
57
|
+
/**
|
|
58
|
+
* The aggregate part of the catalog's prose: every measure the reader may read, model-wide, and
|
|
59
|
+
* per measure entity the entities it reaches. Empty when the reader may read no measure.
|
|
60
|
+
*/
|
|
61
|
+
export declare function renderAggregates(manifest: OntologyManifest, permissions?: ReadonlyArray<string>): string;
|
|
62
|
+
export {};
|
package/dist/render.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { shortestRoutesFrom } from "./aggregate-routes.js";
|
|
1
2
|
import { isAllowed, permissionOf, READ_VERB, rolesGranting } from "./permissions.js";
|
|
2
3
|
import { REFERENCE_ANNOTATION } from "./property.js";
|
|
3
4
|
/** What an action of the manifest requires; version 1 manifests leave it implicit. */
|
|
@@ -24,11 +25,14 @@ function withDescription(head, description) {
|
|
|
24
25
|
function formatValue(value) {
|
|
25
26
|
return typeof value === "string" ? value : JSON.stringify(value);
|
|
26
27
|
}
|
|
27
|
-
|
|
28
|
+
/** A warehouse-backed property's `unique` is an assertion the platform does not enforce. */
|
|
29
|
+
function propertyLines(input) {
|
|
30
|
+
const { property, warehouseBacked } = input;
|
|
31
|
+
const unique = warehouseBacked ? "unique, not enforced" : "unique";
|
|
28
32
|
const traits = [
|
|
29
33
|
property.type,
|
|
30
34
|
...(property.optional ? ["optional"] : []),
|
|
31
|
-
...(property.unique ? [
|
|
35
|
+
...(property.unique ? [unique] : []),
|
|
32
36
|
...(property.default !== undefined ? [`default ${formatValue(property.default)}`] : []),
|
|
33
37
|
...(property.min !== undefined ? [`min ${property.min}`] : []),
|
|
34
38
|
...(property.max !== undefined ? [`max ${property.max}`] : []),
|
|
@@ -43,18 +47,40 @@ function propertyLines(property) {
|
|
|
43
47
|
...options.map((option) => `${INDENT}${option}`),
|
|
44
48
|
];
|
|
45
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* The inverse of a relationship as its target reads it. A value-matched inverse, one with `on`, has
|
|
52
|
+
* no route: a warehouse-backed source is read through aggregates, a stored one by its `on` property.
|
|
53
|
+
*/
|
|
54
|
+
function inverseLine(source, relationship) {
|
|
55
|
+
const { on, phrase, inverse } = relationship;
|
|
56
|
+
if (on === undefined) {
|
|
57
|
+
return `${phrase[1]} many ${source.plural} (${inverse})`;
|
|
58
|
+
}
|
|
59
|
+
if (source.source !== undefined) {
|
|
60
|
+
return `${phrase[1]} ${source.plural} (${inverse}), through aggregates`;
|
|
61
|
+
}
|
|
62
|
+
return `${phrase[1]} ${source.plural} (${inverse}), matched on ${on.property}`;
|
|
63
|
+
}
|
|
46
64
|
function relationshipLines(entity, entities, all) {
|
|
47
65
|
const declared = entity.relationships.map((relationship) => {
|
|
66
|
+
const { on, phrase, name } = relationship;
|
|
48
67
|
const target = entities.get(relationship.to);
|
|
68
|
+
const label = target?.label ?? relationship.to;
|
|
49
69
|
const optional = relationship.optional === true ? ", optional" : "";
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
70
|
+
let head = `${phrase[0]} one ${label} (${name})${optional}`;
|
|
71
|
+
if (relationship.cardinality === "many") {
|
|
72
|
+
head = `${phrase[0]} many ${target?.plural ?? relationship.to} (${name})`;
|
|
73
|
+
}
|
|
74
|
+
else if (on !== undefined) {
|
|
75
|
+
// Value-matched: nothing promises a match, and the key is the declared property pair.
|
|
76
|
+
const key = `${on.property} = ${label}.${on.targetProperty}`;
|
|
77
|
+
head = `${phrase[0]} at most one ${label} (${name}), matched on ${key}`;
|
|
78
|
+
}
|
|
53
79
|
return withDescription(head, relationship.description);
|
|
54
80
|
});
|
|
55
81
|
const inverses = all.flatMap((source) => source.relationships
|
|
56
82
|
.filter((relationship) => relationship.to === entity.name)
|
|
57
|
-
.map((relationship) =>
|
|
83
|
+
.map((relationship) => inverseLine(source, relationship)));
|
|
58
84
|
return [...declared, ...inverses];
|
|
59
85
|
}
|
|
60
86
|
function inputType(schema) {
|
|
@@ -143,10 +169,8 @@ function renderEntity(entity, manifest, entities, options) {
|
|
|
143
169
|
...(entity.source
|
|
144
170
|
? [`${INDENT}Source: ${entity.source.asset}; key: ${entity.source.key.join(", ")}.`]
|
|
145
171
|
: []),
|
|
146
|
-
...section("Properties", entity.properties.flatMap(propertyLines)),
|
|
147
|
-
...(entity.
|
|
148
|
-
? section("Measures", (entity.measures ?? []).map((measure) => `${measure.name}: ${measure.description} (${measure.unit}).`))
|
|
149
|
-
: []),
|
|
172
|
+
...section("Properties", entity.properties.flatMap((property) => propertyLines({ property, warehouseBacked: entity.source !== undefined }))),
|
|
173
|
+
...section("Measures", (entity.measures ?? []).map((measure) => `${measure.name}: ${measure.description} (${measure.unit}).`)),
|
|
150
174
|
...section("Relationships", relationshipLines(entity, entities, manifest.entities)),
|
|
151
175
|
...section(reader === undefined ? "Actions" : "Actions you may run", actions.flatMap((action) => actionLines(action, reader === undefined
|
|
152
176
|
? requirement("Requires", actionPermission(entity.name, action), manifest.roles)
|
|
@@ -180,5 +204,87 @@ export function renderCatalog(manifest, options = {}) {
|
|
|
180
204
|
options.as === undefined ||
|
|
181
205
|
isAllowed(entity.permissions?.read ?? `${entity.name}:read`, { permissions: options.as }))
|
|
182
206
|
.map((entity) => renderEntity(entity, manifest, entities, options));
|
|
183
|
-
|
|
207
|
+
const aggregates = renderAggregates(manifest, options.as);
|
|
208
|
+
return `${[...sections, renderRoles(manifest.roles), ...(aggregates === "" ? [] : [aggregates])].join("\n\n")}\n`;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* The catalog as `describe` answers a reader: {@link renderCatalog} with the reader's permissions,
|
|
212
|
+
* or everything without them.
|
|
213
|
+
*/
|
|
214
|
+
export function describeCatalog(manifest, permissions) {
|
|
215
|
+
return renderCatalog(manifest, permissions === undefined ? {} : { as: permissions });
|
|
216
|
+
}
|
|
217
|
+
/** The rule every surface repeats (docs/ontology-aggregates-spec.md, "Three ways to ask"). */
|
|
218
|
+
export const RECORDS_OR_AGGREGATES = "look at or change records through find and actions; count, sum or compare through aggregate";
|
|
219
|
+
/** Whether the reader may read the entity; without a reader, everything is described. */
|
|
220
|
+
function readable(entity, permissions) {
|
|
221
|
+
return permissions === undefined || isAllowed(permissionOf(entity, READ_VERB), { permissions });
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Every measure of the entities the reader may read, warehouse-backed and stored alike, in
|
|
225
|
+
* declaration order. Without permissions, every measure.
|
|
226
|
+
*/
|
|
227
|
+
export function measuresFor(manifest, permissions) {
|
|
228
|
+
return manifest.entities
|
|
229
|
+
.filter((entity) => readable(entity.name, permissions))
|
|
230
|
+
.flatMap((entity) => (entity.measures ?? []).map((measure) => ({
|
|
231
|
+
name: `${entity.name}.${measure.name}`,
|
|
232
|
+
entity,
|
|
233
|
+
measure,
|
|
234
|
+
})));
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* The entities one entity reaches, with the shortest routes to each: exactly the routes aggregate
|
|
238
|
+
* resolution weighs ({@link shortestRoutesFrom}). Only then is the reader applied: a target they may
|
|
239
|
+
* not read is left out, and so is any route through an entity they may not read, which the service
|
|
240
|
+
* would refuse. A target keeps `ambiguous` from before that filter, so a route is never shown as the
|
|
241
|
+
* only one when the resolver would ask for it to be written.
|
|
242
|
+
*/
|
|
243
|
+
export function reachesOf(manifest, from, permissions) {
|
|
244
|
+
return [...shortestRoutesFrom(manifest, from)].flatMap(([target, routes]) => {
|
|
245
|
+
const followed = routes.filter((route) => route.entities.every((entity) => readable(entity, permissions)));
|
|
246
|
+
return followed.length === 0
|
|
247
|
+
? []
|
|
248
|
+
: [
|
|
249
|
+
{
|
|
250
|
+
target,
|
|
251
|
+
routes: followed.map((route) => route.relationships.join(".")),
|
|
252
|
+
ambiguous: routes.length > 1,
|
|
253
|
+
},
|
|
254
|
+
];
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
/** `sales reaches product (product), region (product.region or listing.region; write the route).` */
|
|
258
|
+
function reachLine(entity, reaches) {
|
|
259
|
+
const targets = reaches.map(({ target, routes, ambiguous }) => `${target} (${routes.join(" or ")}${ambiguous ? "; write the route" : ""})`);
|
|
260
|
+
return `${entity} reaches ${targets.join(", ")}.`;
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* The aggregate part of the catalog's prose: every measure the reader may read, model-wide, and
|
|
264
|
+
* per measure entity the entities it reaches. Empty when the reader may read no measure.
|
|
265
|
+
*/
|
|
266
|
+
export function renderAggregates(manifest, permissions) {
|
|
267
|
+
const measures = measuresFor(manifest, permissions);
|
|
268
|
+
if (measures.length === 0) {
|
|
269
|
+
return "";
|
|
270
|
+
}
|
|
271
|
+
const owners = [...new Set(measures.map(({ entity }) => entity.name))];
|
|
272
|
+
const reachLines = owners.flatMap((owner) => {
|
|
273
|
+
const reaches = reachesOf(manifest, owner, permissions);
|
|
274
|
+
return reaches.length === 0 ? [] : [reachLine(owner, reaches)];
|
|
275
|
+
});
|
|
276
|
+
const measureLines = measures.map(({ name, measure }) => `${name} (${measure.unit}): ${sentence(measure.description)}`);
|
|
277
|
+
return [
|
|
278
|
+
`Aggregates: ${RECORDS_OR_AGGREGATES} (POST /api/ontology/aggregate, or the aggregate tool).`,
|
|
279
|
+
`${INDENT}Measures, as <entity>.<measure>:`,
|
|
280
|
+
...measureLines.map((line) => `${INDENT}${INDENT}${line}`),
|
|
281
|
+
...(reachLines.length === 0
|
|
282
|
+
? []
|
|
283
|
+
: [
|
|
284
|
+
`${INDENT}Group and filter by <entity>.<property>. Where a target says write the route, ` +
|
|
285
|
+
"name it from the measure's entity: <entity>.<relationship>.<property>. Through " +
|
|
286
|
+
"to-one relationships:",
|
|
287
|
+
...reachLines.map((line) => `${INDENT}${INDENT}${line}`),
|
|
288
|
+
]),
|
|
289
|
+
].join("\n");
|
|
184
290
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type OntologyManifest } from "../manifest.js";
|
|
2
2
|
/** The resolved projection for one immutable generation. `id` is always included. */
|
|
3
3
|
export interface ReplicationSelection {
|
|
4
4
|
readonly entityId: string;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { isValueMatched } from "../manifest.js";
|
|
2
3
|
import { queryFields } from "../query.js";
|
|
3
4
|
export function resolveReplicationSelection(manifest, exclusions = []) {
|
|
4
5
|
const entities = manifest.entities.filter((entity) => !entity.builtIn && !entity.source);
|
|
@@ -18,13 +19,22 @@ export function resolveReplicationSelection(manifest, exclusions = []) {
|
|
|
18
19
|
if (item.relationships?.some((relationship) => !relationships.has(relationship))) {
|
|
19
20
|
throw new Error(`Unknown replication relationship on ${entity.name}.`);
|
|
20
21
|
}
|
|
22
|
+
// A value-matched relationship has no `<name>_id`: its reference is the property `on` names.
|
|
23
|
+
const matched = entity.relationships.find((relationship) => item.relationships?.includes(relationship.name) && isValueMatched(relationship));
|
|
24
|
+
if (matched !== undefined) {
|
|
25
|
+
throw new Error(`${entity.name}.${matched.name} is matched by value; exclude its property ` +
|
|
26
|
+
`${matched.on?.property ?? ""} instead.`);
|
|
27
|
+
}
|
|
21
28
|
}
|
|
22
29
|
const included = entities.filter((entity) => !exclusions.some((item) => item.entityId === entity.id && item.disabled));
|
|
23
30
|
const selection = included.map((entity) => {
|
|
24
31
|
const excluded = exclusions.find((item) => item.entityId === entity.id);
|
|
32
|
+
const keyed = entity.relationships.filter((relationship) => relationship.cardinality === "one" && !isValueMatched(relationship));
|
|
25
33
|
const excludedFields = new Set([
|
|
26
34
|
...(excluded?.fields ?? []),
|
|
27
|
-
...
|
|
35
|
+
...keyed
|
|
36
|
+
.filter((relationship) => excluded?.relationships?.includes(relationship.name))
|
|
37
|
+
.map((relationship) => `${relationship.name}_id`),
|
|
28
38
|
]);
|
|
29
39
|
return {
|
|
30
40
|
entityId: entity.id,
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { Effect, Redacted } from "effect";
|
|
2
|
+
import { AggregateUpstreamUnavailable } from "./errors.js";
|
|
3
|
+
/**
|
|
4
|
+
* How long one aggregate may take end to end, the response included. BigQuery answers most within
|
|
5
|
+
* seconds; the margin covers a cold warehouse and the hops to and from SegmentStream.
|
|
6
|
+
*/
|
|
7
|
+
export declare const AGGREGATE_TIMEOUT_MS = 60000;
|
|
8
|
+
/**
|
|
9
|
+
* The hash of the manifest the service resolved and checked the request against. SegmentStream
|
|
10
|
+
* answers `409 ontology_version_mismatch` when its deployed manifest differs, so a request checked
|
|
11
|
+
* against one ontology never runs against another.
|
|
12
|
+
*/
|
|
13
|
+
export declare const MANIFEST_HASH_HEADER = "x-ontology-manifest-hash";
|
|
14
|
+
/**
|
|
15
|
+
* The entities the service checked `<entity>:read` on, sorted and comma-separated. SegmentStream
|
|
16
|
+
* answers `409 aggregate_scope_mismatch` when it would read any other, so the service's resolution
|
|
17
|
+
* and SegmentStream's can never drift apart unnoticed.
|
|
18
|
+
*/
|
|
19
|
+
export declare const TOUCHED_HEADER = "x-ontology-touched";
|
|
20
|
+
/** SegmentStream would read beyond what the service checked: this deployment's SDK is behind. */
|
|
21
|
+
export declare const SCOPE_MISMATCH = "aggregate_scope_mismatch";
|
|
22
|
+
/** SegmentStream's answer: its status, its JSON body as sent, and that body parsed. */
|
|
23
|
+
export interface PlatformAnswer {
|
|
24
|
+
readonly status: number;
|
|
25
|
+
readonly text: string;
|
|
26
|
+
readonly body: Readonly<Record<string, unknown>>;
|
|
27
|
+
/** SegmentStream's `Retry-After`, on a 429 or a 503 that sent one. */
|
|
28
|
+
readonly retryAfter?: string;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Sends a checked aggregate to SegmentStream: model-level, or the per-entity shorthand of `entity`.
|
|
32
|
+
* `touched` is every entity the service required `<entity>:read` on.
|
|
33
|
+
*/
|
|
34
|
+
export type AggregateTransport = (input: {
|
|
35
|
+
readonly entity?: string;
|
|
36
|
+
readonly body: unknown;
|
|
37
|
+
readonly requestId: string;
|
|
38
|
+
readonly touched: ReadonlyArray<string>;
|
|
39
|
+
}) => Effect.Effect<PlatformAnswer, AggregateUpstreamUnavailable>;
|
|
40
|
+
/**
|
|
41
|
+
* Where a deployment sends aggregates: its platform, its workspace and its app credential, with the
|
|
42
|
+
* hash of the manifest it serves.
|
|
43
|
+
*/
|
|
44
|
+
export interface AggregateEnvironment {
|
|
45
|
+
readonly segmentstreamUrl: string;
|
|
46
|
+
readonly workspaceId: string;
|
|
47
|
+
readonly appCredential?: Redacted.Redacted<string>;
|
|
48
|
+
readonly manifestHash: string;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* SegmentStream's `POST /api/v1/workspaces/{id}/aggregate`, and its per-entity
|
|
52
|
+
* `…/entities/{entity}/aggregate`, called with the workspace app credential. `undefined` when the deployment has none, or when SegmentStream's origin is plain
|
|
53
|
+
* http to another machine, which is logged once. Every request names the manifest it was checked
|
|
54
|
+
* against, and a redirect is refused, so the credential never leaves for another origin.
|
|
55
|
+
*
|
|
56
|
+
* Every answer but a 2xx is logged with both request ids, the status and the start of the body; a
|
|
57
|
+
* failed call with its message, stack and causes. The credential is redacted from all of them. A
|
|
58
|
+
* caller who goes away interrupts the call, which is not SegmentStream's failure.
|
|
59
|
+
*/
|
|
60
|
+
export declare function platformAggregates(environment: AggregateEnvironment, fetchImpl?: typeof fetch): AggregateTransport | undefined;
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { Data, Effect, Redacted } from "effect";
|
|
2
|
+
import { AggregateUpstreamUnavailable } from "./errors.js";
|
|
3
|
+
/**
|
|
4
|
+
* How long one aggregate may take end to end, the response included. BigQuery answers most within
|
|
5
|
+
* seconds; the margin covers a cold warehouse and the hops to and from SegmentStream.
|
|
6
|
+
*/
|
|
7
|
+
export const AGGREGATE_TIMEOUT_MS = 60_000;
|
|
8
|
+
/** The header that carries the service's request id to SegmentStream, as connection calls do. */
|
|
9
|
+
const REQUEST_ID_HEADER = "x-segmentstream-request-id";
|
|
10
|
+
/**
|
|
11
|
+
* The hash of the manifest the service resolved and checked the request against. SegmentStream
|
|
12
|
+
* answers `409 ontology_version_mismatch` when its deployed manifest differs, so a request checked
|
|
13
|
+
* against one ontology never runs against another.
|
|
14
|
+
*/
|
|
15
|
+
export const MANIFEST_HASH_HEADER = "x-ontology-manifest-hash";
|
|
16
|
+
/**
|
|
17
|
+
* The entities the service checked `<entity>:read` on, sorted and comma-separated. SegmentStream
|
|
18
|
+
* answers `409 aggregate_scope_mismatch` when it would read any other, so the service's resolution
|
|
19
|
+
* and SegmentStream's can never drift apart unnoticed.
|
|
20
|
+
*/
|
|
21
|
+
export const TOUCHED_HEADER = "x-ontology-touched";
|
|
22
|
+
/**
|
|
23
|
+
* Statuses that refuse the deployment rather than the caller: its credential, its admission or the
|
|
24
|
+
* route itself. Passing them on would make the caller sign in again or look for a missing route.
|
|
25
|
+
*/
|
|
26
|
+
const DEPLOYMENT_FAULTS = {
|
|
27
|
+
401: "credential_rejected",
|
|
28
|
+
403: "credential_rejected",
|
|
29
|
+
404: "upstream_misconfigured",
|
|
30
|
+
};
|
|
31
|
+
/** SegmentStream would read beyond what the service checked: this deployment's SDK is behind. */
|
|
32
|
+
export const SCOPE_MISMATCH = "aggregate_scope_mismatch";
|
|
33
|
+
/** Statuses whose `Retry-After` the caller receives as SegmentStream sent it. */
|
|
34
|
+
const RETRY_AFTER_STATUSES = new Set([429, 503]);
|
|
35
|
+
/** A `Retry-After` value: seconds or an HTTP date. Anything else is not passed on. */
|
|
36
|
+
const RETRY_AFTER_VALUE = /^[0-9A-Za-z ,:+-]{1,64}$/;
|
|
37
|
+
/** The most of SegmentStream's body an operator log keeps. */
|
|
38
|
+
const SNIPPET_LENGTH = 500;
|
|
39
|
+
/** How many causes deep an operator log follows a failed call. */
|
|
40
|
+
const CAUSE_DEPTH = 3;
|
|
41
|
+
/** Hosts a deployment may reach over plain http: the machine itself, in development. */
|
|
42
|
+
const LOOPBACK_HOSTS = new Set(["localhost", "[::1]"]);
|
|
43
|
+
/** The call ended without an answer; `aborted` when the caller went away first. */
|
|
44
|
+
class CallFailed extends Data.TaggedError("CallFailed") {
|
|
45
|
+
}
|
|
46
|
+
function isRecord(value) {
|
|
47
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
48
|
+
}
|
|
49
|
+
/** The credential travels only to SegmentStream over TLS, or over plain http to this machine. */
|
|
50
|
+
function securePlatform(origin) {
|
|
51
|
+
const { protocol, hostname } = new URL(origin);
|
|
52
|
+
if (protocol === "https:") {
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
return (protocol === "http:" && (LOOPBACK_HOSTS.has(hostname) || /^127(?:\.\d{1,3}){3}$/.test(hostname)));
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* A failed call for the operator: name, message and stack, then each cause, with every occurrence
|
|
59
|
+
* of the credential replaced.
|
|
60
|
+
*/
|
|
61
|
+
function describeError(error, redact, depth = 0) {
|
|
62
|
+
if (!(error instanceof Error)) {
|
|
63
|
+
return redact(String(error));
|
|
64
|
+
}
|
|
65
|
+
const code = error.code;
|
|
66
|
+
const head = error.stack ?? `${error.name}: ${error.message}`;
|
|
67
|
+
const own = typeof code === "string" ? `${head} [${code}]` : head;
|
|
68
|
+
const cause = error.cause !== undefined && depth < CAUSE_DEPTH
|
|
69
|
+
? `\ncaused by: ${describeError(error.cause, redact, depth + 1)}`
|
|
70
|
+
: "";
|
|
71
|
+
return redact(`${own}${cause}`);
|
|
72
|
+
}
|
|
73
|
+
function parseObject(text) {
|
|
74
|
+
try {
|
|
75
|
+
const parsed = JSON.parse(text);
|
|
76
|
+
return isRecord(parsed) ? { body: parsed } : { error: new TypeError("Not a JSON object.") };
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
return { error };
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* SegmentStream's `POST /api/v1/workspaces/{id}/aggregate`, and its per-entity
|
|
84
|
+
* `…/entities/{entity}/aggregate`, called with the workspace app credential. `undefined` when the deployment has none, or when SegmentStream's origin is plain
|
|
85
|
+
* http to another machine, which is logged once. Every request names the manifest it was checked
|
|
86
|
+
* against, and a redirect is refused, so the credential never leaves for another origin.
|
|
87
|
+
*
|
|
88
|
+
* Every answer but a 2xx is logged with both request ids, the status and the start of the body; a
|
|
89
|
+
* failed call with its message, stack and causes. The credential is redacted from all of them. A
|
|
90
|
+
* caller who goes away interrupts the call, which is not SegmentStream's failure.
|
|
91
|
+
*/
|
|
92
|
+
export function platformAggregates(environment, fetchImpl = fetch) {
|
|
93
|
+
const { appCredential, segmentstreamUrl } = environment;
|
|
94
|
+
if (appCredential === undefined) {
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
if (!securePlatform(segmentstreamUrl)) {
|
|
98
|
+
Effect.runSync(Effect.logError("ontology.aggregate.insecure_platform").pipe(Effect.annotateLogs({
|
|
99
|
+
diagnostic: "SEGMENTSTREAM_URL is not https; aggregates are unavailable.",
|
|
100
|
+
platform: segmentstreamUrl,
|
|
101
|
+
})));
|
|
102
|
+
return undefined;
|
|
103
|
+
}
|
|
104
|
+
const credential = Redacted.value(appCredential);
|
|
105
|
+
const redact = (text) => text.split(credential).join("[redacted]");
|
|
106
|
+
const workspace = `${segmentstreamUrl}/api/v1/workspaces/` + encodeURIComponent(environment.workspaceId);
|
|
107
|
+
const urlOf = (entity) => entity === undefined
|
|
108
|
+
? `${workspace}/aggregate`
|
|
109
|
+
: `${workspace}/entities/${encodeURIComponent(entity)}/aggregate`;
|
|
110
|
+
const exchange = (input) => Effect.tryPromise({
|
|
111
|
+
try: async (signal) => {
|
|
112
|
+
try {
|
|
113
|
+
const response = await fetchImpl(urlOf(input.entity), {
|
|
114
|
+
method: "POST",
|
|
115
|
+
headers: {
|
|
116
|
+
authorization: `Bearer ${credential}`,
|
|
117
|
+
"content-type": "application/json",
|
|
118
|
+
accept: "application/json",
|
|
119
|
+
[REQUEST_ID_HEADER]: input.requestId,
|
|
120
|
+
[MANIFEST_HASH_HEADER]: environment.manifestHash,
|
|
121
|
+
[TOUCHED_HEADER]: input.touched.join(","),
|
|
122
|
+
},
|
|
123
|
+
body: JSON.stringify(input.body),
|
|
124
|
+
redirect: "error",
|
|
125
|
+
signal: AbortSignal.any([signal, AbortSignal.timeout(AGGREGATE_TIMEOUT_MS)]),
|
|
126
|
+
});
|
|
127
|
+
const text = await response.text();
|
|
128
|
+
return {
|
|
129
|
+
status: response.status,
|
|
130
|
+
text,
|
|
131
|
+
retryAfter: response.headers.get("retry-after"),
|
|
132
|
+
platformRequestId: response.headers.get("x-request-id"),
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
throw new CallFailed({ error, aborted: signal.aborted });
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
catch: (error) => error instanceof CallFailed ? error : new CallFailed({ error, aborted: false }),
|
|
140
|
+
});
|
|
141
|
+
/** No answer: a timeout or an unreachable SegmentStream. */
|
|
142
|
+
const unanswered = (error, requestId) => {
|
|
143
|
+
const reason = error instanceof DOMException && error.name === "TimeoutError" ? "timeout" : "unreachable";
|
|
144
|
+
return Effect.logWarning("ontology.aggregate.upstream_failed").pipe(Effect.annotateLogs({ requestId, diagnostic: reason, error: describeError(error, redact) }), Effect.andThen(Effect.fail(new AggregateUpstreamUnavailable({ reason }))));
|
|
145
|
+
};
|
|
146
|
+
/** An answer: passed on, or a 502 when it faults the deployment or is not a JSON object. */
|
|
147
|
+
const judge = (answer, requestId) => Effect.gen(function* () {
|
|
148
|
+
const parsed = parseObject(answer.text);
|
|
149
|
+
const body = "body" in parsed ? parsed.body : undefined;
|
|
150
|
+
const fault = DEPLOYMENT_FAULTS[answer.status] ?? (body === undefined ? "invalid_response" : undefined);
|
|
151
|
+
const platformRequestId = typeof body?.requestId === "string" ? body.requestId : answer.platformRequestId;
|
|
152
|
+
if (fault !== undefined || answer.status < 200 || answer.status >= 300) {
|
|
153
|
+
const code = typeof body?.error === "string" ? body.error : undefined;
|
|
154
|
+
const level = fault !== undefined || code === SCOPE_MISMATCH
|
|
155
|
+
? Effect.logError
|
|
156
|
+
: answer.status >= 500
|
|
157
|
+
? Effect.logWarning
|
|
158
|
+
: Effect.logInfo;
|
|
159
|
+
yield* level(fault === undefined
|
|
160
|
+
? "ontology.aggregate.platform_refused"
|
|
161
|
+
: "ontology.aggregate.upstream_failed").pipe(Effect.annotateLogs({
|
|
162
|
+
requestId,
|
|
163
|
+
platformRequestId: platformRequestId ?? "none",
|
|
164
|
+
status: answer.status,
|
|
165
|
+
diagnostic: fault ?? code ?? "refused",
|
|
166
|
+
body: redact(answer.text.slice(0, SNIPPET_LENGTH)),
|
|
167
|
+
...("error" in parsed ? { error: describeError(parsed.error, redact) } : {}),
|
|
168
|
+
}));
|
|
169
|
+
}
|
|
170
|
+
if (fault !== undefined || body === undefined) {
|
|
171
|
+
return yield* new AggregateUpstreamUnavailable({
|
|
172
|
+
reason: fault ?? "invalid_response",
|
|
173
|
+
status: answer.status,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
const retryAfter = RETRY_AFTER_STATUSES.has(answer.status) &&
|
|
177
|
+
answer.retryAfter !== null &&
|
|
178
|
+
RETRY_AFTER_VALUE.test(answer.retryAfter)
|
|
179
|
+
? answer.retryAfter
|
|
180
|
+
: undefined;
|
|
181
|
+
return {
|
|
182
|
+
status: answer.status,
|
|
183
|
+
text: answer.text,
|
|
184
|
+
body,
|
|
185
|
+
...(retryAfter === undefined ? {} : { retryAfter }),
|
|
186
|
+
};
|
|
187
|
+
});
|
|
188
|
+
return (input) => exchange(input).pipe(Effect.catchTag("CallFailed", (failed) => failed.aborted ? Effect.interrupt : unanswered(failed.error, input.requestId)), Effect.flatMap((answer) => judge(answer, input.requestId)));
|
|
189
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import type { OntologyManifest } from "../manifest.js";
|
|
3
|
+
import type { AggregateTransport, PlatformAnswer } from "./aggregate-platform.js";
|
|
4
|
+
import { CurrentPrincipal } from "./authentication.js";
|
|
5
|
+
import { AggregateLimited, AggregateUnavailable, AggregateUpstreamUnavailable, InvalidAggregateQuery, InvalidRequestBody, ReadDenied, RequestId } from "./errors.js";
|
|
6
|
+
import { type RateLimit, type RateLimiter } from "./rate-limit.js";
|
|
7
|
+
/**
|
|
8
|
+
* Aggregates one principal may run per minute, through both routes and the MCP tool together. Each
|
|
9
|
+
* runs a warehouse query with the deployment's credential, so one caller must not spend without
|
|
10
|
+
* bound.
|
|
11
|
+
*/
|
|
12
|
+
export declare const AGGREGATE_LIMIT: RateLimit;
|
|
13
|
+
/** Every way the facade refuses or fails an aggregate before SegmentStream answers it. */
|
|
14
|
+
export type AggregateFailure = AggregateLimited | InvalidRequestBody | InvalidAggregateQuery | ReadDenied | AggregateUnavailable | AggregateUpstreamUnavailable;
|
|
15
|
+
/**
|
|
16
|
+
* Runs an aggregate as the caller, whatever the transport: the HTTP route passes the parsed body,
|
|
17
|
+
* the MCP tool its arguments. Its answer is SegmentStream's, error statuses included.
|
|
18
|
+
*/
|
|
19
|
+
export type Aggregate = (body: unknown) => Effect.Effect<PlatformAnswer, AggregateFailure, CurrentPrincipal | RequestId>;
|
|
20
|
+
/** The service's aggregates: model-level, and the per-entity shorthand of each entity. */
|
|
21
|
+
export interface Aggregates {
|
|
22
|
+
/** `POST /api/ontology/aggregate` and the MCP `aggregate` tool. */
|
|
23
|
+
readonly model: Aggregate;
|
|
24
|
+
/** `POST /api/ontology/<entity>/aggregate`, the shorthand `segmentstream.ontology.<entity>`. */
|
|
25
|
+
readonly entity: (entity: string) => Aggregate;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The ontology service's aggregate facade (docs/ontology-aggregates-spec.md, "Surfaces"). Both
|
|
29
|
+
* routes and the MCP tool run as the caller:
|
|
30
|
+
* 0. the call counts against the caller's {@link AGGREGATE_LIMIT}, whichever way it came;
|
|
31
|
+
* 1. the request's shape and bounds are checked, and its names resolved against the service's own
|
|
32
|
+
* manifest into every entity it reads;
|
|
33
|
+
* 2. the caller needs `<entity>:read` on all of them;
|
|
34
|
+
* 3. the body goes to SegmentStream unchanged with the workspace app credential, naming the entities
|
|
35
|
+
* checked, or the answer is `aggregate_unavailable` where the deployment has no credential.
|
|
36
|
+
*/
|
|
37
|
+
export declare function makeAggregates(input: {
|
|
38
|
+
readonly manifest: OntologyManifest;
|
|
39
|
+
readonly transport: AggregateTransport | undefined;
|
|
40
|
+
/** One bucket per principal; {@link AGGREGATE_LIMIT} by default. */
|
|
41
|
+
readonly limiter?: RateLimiter;
|
|
42
|
+
}): Aggregates;
|