@segmentstream-cloud/ontology 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +27 -2
  2. package/dist/aggregate-routes.d.ts +46 -9
  3. package/dist/aggregate-routes.js +186 -20
  4. package/dist/analyze.d.ts +12 -1
  5. package/dist/analyze.js +3 -1
  6. package/dist/diagnostics.d.ts +3 -3
  7. package/dist/diagnostics.js +2 -0
  8. package/dist/index.d.ts +3 -2
  9. package/dist/index.js +2 -1
  10. package/dist/manifest.d.ts +113 -2
  11. package/dist/manifest.js +33 -2
  12. package/dist/measure.d.ts +57 -6
  13. package/dist/measure.js +7 -2
  14. package/dist/migrate/apply.js +2 -1
  15. package/dist/migrate/model.d.ts +22 -0
  16. package/dist/migrate/plan.js +9 -1
  17. package/dist/migrate/storage.d.ts +3 -0
  18. package/dist/migrate/storage.js +10 -0
  19. package/dist/model-measures.d.ts +22 -0
  20. package/dist/model-measures.js +543 -0
  21. package/dist/ontology.d.ts +5 -4
  22. package/dist/permissions.d.ts +3 -0
  23. package/dist/permissions.js +8 -1
  24. package/dist/render.d.ts +15 -3
  25. package/dist/render.js +56 -3
  26. package/dist/runtime/history.d.ts +22 -0
  27. package/dist/runtime/history.js +2 -0
  28. package/dist/runtime/pipeline.js +6 -2
  29. package/dist/runtime/principals.d.ts +2 -0
  30. package/dist/runtime/principals.js +55 -0
  31. package/dist/runtime/reads.js +9 -1
  32. package/dist/runtime/service-binding.js +2 -1
  33. package/dist/runtime/workload.d.ts +23 -0
  34. package/dist/runtime/workload.js +22 -0
  35. package/dist/runtime/writes.js +7 -1
  36. package/dist/runtime.d.ts +3 -0
  37. package/dist/service/authentication.d.ts +11 -0
  38. package/dist/service/authentication.js +114 -5
  39. package/dist/service/catalog.d.ts +2 -1
  40. package/dist/service/catalog.js +7 -3
  41. package/dist/service/config.d.ts +4 -0
  42. package/dist/service/config.js +11 -0
  43. package/dist/service/handlers.js +2 -1
  44. package/dist/service/mcp/server.js +10 -6
  45. package/dist/service/mcp/tools.js +25 -10
  46. package/dist/service/operations.d.ts +25 -5
  47. package/dist/service/runtime-contract.d.ts +1 -0
  48. package/dist/service/schemas.d.ts +143 -0
  49. package/dist/service/schemas.js +11 -2
  50. package/dist/storage/attachments.js +4 -4
  51. package/dist/storage/authorization.d.ts +1 -1
  52. package/dist/storage/authorization.js +9 -4
  53. package/dist/storage/sessions.js +3 -3
  54. package/dist/storage/writes.js +1 -1
  55. package/dist/warehouse-backed.js +4 -0
  56. package/package.json +1 -1
package/dist/render.js CHANGED
@@ -1,4 +1,4 @@
1
- import { shortestRoutesFrom } from "./aggregate-routes.js";
1
+ import { inputEntities, shortestRoutesFrom } from "./aggregate-routes.js";
2
2
  import { isAllowed, permissionOf, READ_VERB, rolesGranting } from "./permissions.js";
3
3
  import { REFERENCE_ANNOTATION } from "./property.js";
4
4
  /** What an action of the manifest requires; version 1 manifests leave it implicit. */
@@ -389,6 +389,51 @@ export function measuresFor(manifest, permissions) {
389
389
  measure,
390
390
  })));
391
391
  }
392
+ /**
393
+ * The measures of the model whose every input entity the reader may read, in declaration order.
394
+ * Reading a model measure is reading its inputs. Without permissions, every one.
395
+ */
396
+ export function modelMeasuresFor(manifest, permissions) {
397
+ return (manifest.measures ?? []).filter((measure) => inputEntities(measure).every((entity) => readable(entity, permissions)));
398
+ }
399
+ /** `a`, `a and b`, `a, b and c`. */
400
+ function listed(values) {
401
+ return values.length < 2
402
+ ? values.join("")
403
+ : `${values.slice(0, -1).join(", ")} and ${values[values.length - 1]}`;
404
+ }
405
+ /**
406
+ * How one alignment group reads: `sales.order_date aligns with plans.month`, and for an entity
407
+ * without a counterpart `listings has no counterpart for sales.order_date`. Members follow the
408
+ * measure's input entities, whatever order the document keeps them in.
409
+ */
410
+ function alignmentClauses(measure, group) {
411
+ const order = inputEntities(measure);
412
+ const members = Object.entries(group).sort(([left], [right]) => order.indexOf(left) - order.indexOf(right));
413
+ const present = members.flatMap(([entity, property]) => property === null ? [] : [`${entity}.${property}`]);
414
+ const absent = members.flatMap(([entity, property]) => (property === null ? [entity] : []));
415
+ const [first, ...rest] = present;
416
+ if (first === undefined) {
417
+ return [];
418
+ }
419
+ return [
420
+ ...(rest.length > 0 ? [`${first} aligns with ${listed(rest)}`] : []),
421
+ ...absent.map((entity) => `${entity} has no counterpart for ${first}`),
422
+ ];
423
+ }
424
+ /**
425
+ * A measure of the model as `describe`, the MCP tool and the instructions show it: its description,
426
+ * the entity measures it reads and how their properties align. `Ordered over planned units. Reads
427
+ * sales.ordered and plans.planned_units; sales.order_date aligns with plans.month.`
428
+ */
429
+ export function modelMeasureDescription(measure) {
430
+ const clauses = [
431
+ `Reads ${listed(measure.inputs)}`,
432
+ ...measure.align.flatMap((group) => alignmentClauses(measure, group)),
433
+ ];
434
+ const described = measure.description === "" ? [] : [sentence(measure.description)];
435
+ return [...described, `${clauses.join("; ")}.`].join(" ");
436
+ }
392
437
  /**
393
438
  * The entities one entity reaches, with the shortest routes to each: exactly the routes aggregate
394
439
  * resolution weighs ({@link shortestRoutesFrom}). Only then is the reader applied: a target they may
@@ -416,8 +461,9 @@ function reachLine(entity, reaches) {
416
461
  return `${entity} reaches ${targets.join(", ")}.`;
417
462
  }
418
463
  /**
419
- * The aggregate part of the catalog's prose: every measure the reader may read, model-wide, and
420
- * per measure entity the entities it reaches. Empty when the reader may read no measure.
464
+ * The aggregate part of the catalog's prose: every measure the reader may read, model-wide, the
465
+ * measures of the model whose every input they may read, and per measure entity the entities it
466
+ * reaches. Empty when the reader may read no measure.
421
467
  */
422
468
  export function renderAggregates(manifest, permissions) {
423
469
  const measures = measuresFor(manifest, permissions);
@@ -430,10 +476,17 @@ export function renderAggregates(manifest, permissions) {
430
476
  return reaches.length === 0 ? [] : [reachLine(owner, reaches)];
431
477
  });
432
478
  const measureLines = measures.map(({ name, entity, measure }) => `${name} (${measure.unit}): ${sentence(measureDescription(entity, measure))}`);
479
+ const modelLines = modelMeasuresFor(manifest, permissions).map((measure) => `${measure.name} (${measure.unit}): ${modelMeasureDescription(measure)}`);
433
480
  return [
434
481
  `Aggregates: ${RECORDS_OR_AGGREGATES} (POST /api/ontology/aggregate, or the aggregate tool).`,
435
482
  `${INDENT}Measures, as <entity>.<measure>:`,
436
483
  ...measureLines.map((line) => `${INDENT}${INDENT}${line}`),
484
+ ...(modelLines.length === 0
485
+ ? []
486
+ : [
487
+ `${INDENT}Measures of the model, by name:`,
488
+ ...modelLines.map((line) => `${INDENT}${INDENT}${line}`),
489
+ ]),
437
490
  ...(reachLines.length === 0
438
491
  ? []
439
492
  : [
@@ -1,4 +1,5 @@
1
1
  import { Schema } from "effect";
2
+ import { WorkloadExecution } from "./workload.js";
2
3
  import type { StoredRecord } from "../runtime.js";
3
4
  import type { EntityModel } from "./model.js";
4
5
  import type { Params } from "./statement.js";
@@ -31,6 +32,26 @@ export declare const Edit: Schema.Struct<{
31
32
  export type Edit = typeof Edit.Type;
32
33
  /** A row of `<entity>_history`. */
33
34
  export declare const HistoryEntry: Schema.Struct<{
35
+ readonly execution: Schema.optionalKey<Schema.Struct<{
36
+ readonly issuer: Schema.String;
37
+ readonly audience: Schema.String;
38
+ readonly organizationId: Schema.String;
39
+ readonly workspaceId: Schema.String;
40
+ readonly subject: Schema.String;
41
+ readonly deploymentId: Schema.String;
42
+ readonly ontologyDeploymentId: Schema.String;
43
+ readonly servingOntologyDeploymentId: Schema.String;
44
+ readonly ontologyOrigin: Schema.String;
45
+ readonly environment: Schema.Literal<"production">;
46
+ readonly jobName: Schema.String;
47
+ readonly runId: Schema.String;
48
+ readonly executionId: Schema.NullOr<Schema.String>;
49
+ readonly attemptId: Schema.String;
50
+ readonly generation: Schema.Number;
51
+ readonly tokenId: Schema.String;
52
+ readonly issuedAt: Schema.Number;
53
+ readonly expiresAt: Schema.Number;
54
+ }>>;
34
55
  readonly id: Schema.String;
35
56
  readonly recordId: Schema.String;
36
57
  /** The qualified action, for example `lead.changeStatus`. */
@@ -54,6 +75,7 @@ export interface HistoryWrite {
54
75
  readonly edit: Edit;
55
76
  }
56
77
  export interface HistoryScope {
78
+ readonly execution?: WorkloadExecution;
57
79
  readonly action: string;
58
80
  readonly actionId: string;
59
81
  readonly principalId: string | null;
@@ -1,5 +1,6 @@
1
1
  import { Schema } from "effect";
2
2
  import { canonicalJson } from "../manifest.js";
3
+ import { WorkloadExecution } from "./workload.js";
3
4
  import { quoteIdentifier } from "../migrate/storage.js";
4
5
  import { uuidv7 } from "./ids.js";
5
6
  export const EditKind = Schema.Literals(["create", "update", "delete", "link", "unlink"]);
@@ -21,6 +22,7 @@ export const Edit = Schema.Struct({
21
22
  });
22
23
  /** A row of `<entity>_history`. */
23
24
  export const HistoryEntry = Schema.Struct({
25
+ execution: Schema.optionalKey(WorkloadExecution),
24
26
  id: Schema.String,
25
27
  recordId: Schema.String,
26
28
  /** The qualified action, for example `lead.changeStatus`. */
@@ -141,7 +141,7 @@ export function runAction(ontology, request) {
141
141
  if (consumesUpload) {
142
142
  const saved = yield* sql
143
143
  .withTransaction(Effect.gen(function* () {
144
- yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions);
144
+ yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions, request.principal.execution);
145
145
  const [saved] = yield* sql.unsafe(`SELECT fingerprint, outcome FROM "_blob_action_results"
146
146
  WHERE principal_id = $1::uuid AND key = $2`, [request.principal.id, key]);
147
147
  if (saved && saved.fingerprint !== fingerprint) {
@@ -168,6 +168,7 @@ export function runAction(ontology, request) {
168
168
  action: qualified,
169
169
  actionId,
170
170
  principalId: request.principal.id,
171
+ ...(request.principal.execution ? { execution: request.principal.execution } : {}),
171
172
  ...(key ? { blobActionKey: key } : {}),
172
173
  ...(request.branchId ? { branchId: request.branchId } : {}),
173
174
  ...(request.options?.createIntent ? { createIntent: request.options.createIntent } : {}),
@@ -177,8 +178,11 @@ export function runAction(ontology, request) {
177
178
  const handler = Effect.suspend(() => handle({ ...context, prepared }));
178
179
  let replay;
179
180
  const transactionalHandler = Effect.gen(function* () {
181
+ if (request.principal.execution !== undefined) {
182
+ yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions, request.principal.execution);
183
+ }
180
184
  if (consumesUpload) {
181
- yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions);
185
+ yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions, request.principal.execution);
182
186
  yield* sql.unsafe(`INSERT INTO "_blob_action_results" (principal_id, key, fingerprint)
183
187
  VALUES ($1::uuid, $2, $3) ON CONFLICT DO NOTHING`, [request.principal.id, key, fingerprint]);
184
188
  const [saved] = yield* sql.unsafe(`SELECT fingerprint, outcome FROM "_blob_action_results" WHERE principal_id = $1::uuid AND key = $2
@@ -49,6 +49,8 @@ export declare const SIGN_IN_RECORD_INTERVAL_MS: number;
49
49
  * {@link SIGN_IN_RECORD_INTERVAL_MS}, so history keeps it. Inactive users are rejected.
50
50
  */
51
51
  export declare function resolvePrincipalByIdentity(ontology: Ontology, claims: IdentityClaims): Effect.Effect<Principal, PrincipalRejected, SqlClient.SqlClient>;
52
+ /** Resolve the workspace's stable keyless pipeline service, without customer roles or sign-in. */
53
+ export declare function resolvePipelineWorker(ontology: Ontology, execution: import("./workload.js").WorkloadExecution): Effect.Effect<Principal, PrincipalRejected, SqlClient.SqlClient>;
52
54
  /**
53
55
  * The principal of an API key, with the permissions its roles have in the ontology. Unknown and
54
56
  * revoked keys and inactive users are rejected.
@@ -207,6 +207,61 @@ export function resolvePrincipalByIdentity(ontology, claims) {
207
207
  return yield* activePrincipal(ontology, sql, user);
208
208
  }).pipe(Effect.catchIf(isSqlError, (error) => Effect.die(error)));
209
209
  }
210
+ /** Resolve the workspace's stable keyless pipeline service, without customer roles or sign-in. */
211
+ export function resolvePipelineWorker(ontology, execution) {
212
+ return Effect.gen(function* () {
213
+ const sql = yield* SqlClient.SqlClient;
214
+ const provider = "segmentstream-ontology-workload";
215
+ const subject = `pipeline-workers:${execution.workspaceId}`;
216
+ const runtime = platformRuntime(ontology, sql, "user.resolve_pipeline_worker");
217
+ const principal = (user) => user.kind !== "service" || user.active !== true
218
+ ? Effect.fail(new PrincipalRejected({
219
+ reason: "inactive_user",
220
+ message: "The pipeline worker principal is inactive or has an invalid kind.",
221
+ }))
222
+ : Effect.succeed({
223
+ id: user.id,
224
+ kind: "service",
225
+ roles: [],
226
+ permissions: ["*:*"],
227
+ execution,
228
+ });
229
+ const find = (forUpdate) => runtime.find({
230
+ entity: "user",
231
+ options: {
232
+ where: { identity_provider: provider, identity_subject: subject },
233
+ limit: 1,
234
+ forUpdate,
235
+ },
236
+ });
237
+ const [known] = yield* find(false);
238
+ // Returning requests only read the principal; writes revalidate it in their transaction.
239
+ if (known !== undefined) {
240
+ return yield* principal(known);
241
+ }
242
+ return yield* sql
243
+ .withTransaction(Effect.gen(function* () {
244
+ // The replication lock precedes every business lock, including principal initialization.
245
+ yield* beginReplicationWrite(sql);
246
+ yield* sql.unsafe("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [
247
+ provider + ":" + subject,
248
+ ]);
249
+ const [existing] = yield* find(true);
250
+ const user = existing ??
251
+ (yield* runtime.create({
252
+ entity: "user",
253
+ values: {
254
+ name: "Pipeline workers",
255
+ kind: "service",
256
+ identity_provider: provider,
257
+ identity_subject: subject,
258
+ },
259
+ }));
260
+ return yield* principal(user);
261
+ }))
262
+ .pipe(Effect.catchTag("Conflict", (error) => Effect.die(error)), Effect.catchTag("NotFound", (error) => Effect.die(error)));
263
+ }).pipe(Effect.catchIf(isSqlError, (error) => Effect.die(error)));
264
+ }
210
265
  /**
211
266
  * The principal of an API key, with the permissions its roles have in the ontology. Unknown and
212
267
  * revoked keys and inactive users are rejected.
@@ -238,17 +238,25 @@ export function readHistory(context, entity, input) {
238
238
  const at = params.add(cursor[0], "timestamptz");
239
239
  conditions.push(`("h"."at" < ${at} OR ("h"."at" = ${at} AND "h"."id" < ${params.add(cursor[1], "uuid")}))`);
240
240
  }
241
+ const presenceParams = new Params();
242
+ const [presence] = yield* context.run("SELECT to_regclass('_ontology_action_attribution') IS NOT NULL AS present", presenceParams);
243
+ const attribution = presence?.present === true;
241
244
  const rows = yield* context.run(`SELECT "h"."id"::text AS "id", "h"."record_id"::text AS "recordId", "h"."action" AS "action", ` +
242
245
  `"h"."action_id"::text AS "actionId", "h"."kind" AS "kind", ` +
243
246
  `"h"."principal_id"::text AS "principalId", ` +
244
247
  `to_char("h"."at" AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS "at", ` +
245
248
  `"h"."changes"::text AS "changes" ` +
249
+ (attribution ? `, "a"."execution" AS "execution" ` : "") +
246
250
  `FROM ${quoteIdentifier(entity.historyTable)} AS "h" ` +
251
+ (attribution
252
+ ? `LEFT JOIN "_ontology_action_attribution" AS "a" ON "a"."action_id" = "h"."action_id" `
253
+ : "") +
247
254
  `WHERE ${conditions.join(" AND ")} ` +
248
255
  `ORDER BY "h"."at" DESC, "h"."id" DESC ` +
249
256
  `LIMIT ${params.add(limit + 1, "bigint")}`, params);
250
257
  const items = rows.slice(0, limit).map((row) => Schema.decodeUnknownSync(HistoryEntry)({
251
- ...row,
258
+ ...Object.fromEntries(Object.entries(row).filter(([key, value]) => key !== "execution" || value != null)),
259
+ ...(row.execution == null ? {} : { execution: row.execution }),
252
260
  at: canonicalInstant(String(row.at)),
253
261
  changes: JSON.parse(String(row.changes)),
254
262
  }));
@@ -11,7 +11,7 @@ import { sqlAgentStore } from "./agents.js";
11
11
  import { generatedActions } from "./generated-actions.js";
12
12
  import { sqlOAuthStore } from "./oauth.js";
13
13
  import { runAction } from "./pipeline.js";
14
- import { checkSignIn, resolvePrincipalByApiKey, resolvePrincipalByIdentity } from "./principals.js";
14
+ import { checkSignIn, resolvePrincipalByApiKey, resolvePrincipalByIdentity, resolvePipelineWorker, } from "./principals.js";
15
15
  import { postgresTls } from "./postgres-tls.js";
16
16
  import { makeOntologyRuntime } from "./sql-runtime.js";
17
17
  /** Connections one service instance holds to the database's pooler. */
@@ -61,6 +61,7 @@ export function backendLayer(ontology, blobStorage) {
61
61
  })).pipe(Effect.catchTag("PrincipalRejected", (rejection) => rejection.reason === "not_invited"
62
62
  ? rejectedAs("sign_in")(rejection).pipe(Effect.andThen(Effect.fail(new NotInvited({ message: rejection.message }))))
63
63
  : rejectedAs("sign_in")(rejection))),
64
+ resolvePipelineWorker: (execution) => withSql(resolvePipelineWorker(ontology, execution)).pipe(Effect.catchTag("PrincipalRejected", rejectedAs("pipeline_worker"))),
64
65
  checkSignIn: (email) => withSql(checkSignIn(ontology, email)),
65
66
  resolveApiKey: (key) => withSql(resolvePrincipalByApiKey(ontology, Redacted.value(key))).pipe(Effect.catchTag("PrincipalRejected", rejectedAs("api_key"))),
66
67
  appliedHash: readCatalog({ sql }).pipe(Effect.map((manifest) => manifest?.hash), Effect.orDie),
@@ -0,0 +1,23 @@
1
+ import { Schema } from "effect";
2
+ /** Verified platform claims retained with each write; never supplied by request headers. */
3
+ export declare const WorkloadExecution: Schema.Struct<{
4
+ readonly issuer: Schema.String;
5
+ readonly audience: Schema.String;
6
+ readonly organizationId: Schema.String;
7
+ readonly workspaceId: Schema.String;
8
+ readonly subject: Schema.String;
9
+ readonly deploymentId: Schema.String;
10
+ readonly ontologyDeploymentId: Schema.String;
11
+ readonly servingOntologyDeploymentId: Schema.String;
12
+ readonly ontologyOrigin: Schema.String;
13
+ readonly environment: Schema.Literal<"production">;
14
+ readonly jobName: Schema.String;
15
+ readonly runId: Schema.String;
16
+ readonly executionId: Schema.NullOr<Schema.String>;
17
+ readonly attemptId: Schema.String;
18
+ readonly generation: Schema.Number;
19
+ readonly tokenId: Schema.String;
20
+ readonly issuedAt: Schema.Number;
21
+ readonly expiresAt: Schema.Number;
22
+ }>;
23
+ export type WorkloadExecution = typeof WorkloadExecution.Type;
@@ -0,0 +1,22 @@
1
+ import { Schema } from "effect";
2
+ /** Verified platform claims retained with each write; never supplied by request headers. */
3
+ export const WorkloadExecution = Schema.Struct({
4
+ issuer: Schema.String,
5
+ audience: Schema.String,
6
+ organizationId: Schema.String,
7
+ workspaceId: Schema.String,
8
+ subject: Schema.String,
9
+ deploymentId: Schema.String,
10
+ ontologyDeploymentId: Schema.String,
11
+ servingOntologyDeploymentId: Schema.String,
12
+ ontologyOrigin: Schema.String,
13
+ environment: Schema.Literal("production"),
14
+ jobName: Schema.String,
15
+ runId: Schema.String,
16
+ executionId: Schema.NullOr(Schema.String),
17
+ attemptId: Schema.String,
18
+ generation: Schema.Number,
19
+ tokenId: Schema.String,
20
+ issuedAt: Schema.Number,
21
+ expiresAt: Schema.Number,
22
+ });
@@ -1,7 +1,7 @@
1
1
  import { Effect } from "effect";
2
2
  import { resolveBlobWrites } from "../storage/writes.js";
3
3
  import { Conflict } from "../errors.js";
4
- import { quoteIdentifier } from "../migrate/storage.js";
4
+ import { ACTION_ATTRIBUTION_TABLE, quoteIdentifier } from "../migrate/storage.js";
5
5
  import { InvalidValues } from "./errors.js";
6
6
  import { uuidv7 } from "./ids.js";
7
7
  import { diffRecords, insertHistory } from "./history.js";
@@ -13,6 +13,12 @@ import { columnOf, Params } from "./statement.js";
13
13
  function recordEdit(context, entity, edit) {
14
14
  return Effect.gen(function* () {
15
15
  const params = new Params();
16
+ if (context.scope.execution !== undefined) {
17
+ const attribution = new Params();
18
+ yield* context.run(`INSERT INTO "${ACTION_ATTRIBUTION_TABLE}" (action_id, execution) VALUES (` +
19
+ `${attribution.add(context.scope.actionId, "uuid")}, ` +
20
+ `${attribution.add(JSON.stringify(context.scope.execution), "jsonb")}) ON CONFLICT DO NOTHING`, attribution);
21
+ }
16
22
  yield* context.run(insertHistory(context.scope, { entity, edit }, params), params);
17
23
  context.edits.push(edit);
18
24
  });
package/dist/runtime.d.ts CHANGED
@@ -1,8 +1,11 @@
1
1
  import { Context, Effect } from "effect";
2
2
  import type { Conflict, NotFound } from "./errors.js";
3
3
  import type { Where } from "./query.js";
4
+ import type { WorkloadExecution } from "./runtime/workload.js";
4
5
  /** Who runs an action: an app member or a service principal, both `user` records. */
5
6
  export interface Principal {
7
+ /** Present only for a verified managed pipeline worker. */
8
+ readonly execution?: WorkloadExecution;
6
9
  /** The id of the principal's `user` record. */
7
10
  readonly id: string;
8
11
  readonly kind: "member" | "service";
@@ -2,6 +2,7 @@ import { Context, Effect, Layer, Redacted } from "effect";
2
2
  import { HttpServerRequest, type HttpServerResponse } from "effect/unstable/http";
3
3
  import { HttpApiMiddleware, HttpApiSecurity } from "effect/unstable/httpapi";
4
4
  import { type JWTVerifyGetKey } from "jose";
5
+ import type { WorkloadExecution } from "../runtime/workload.js";
5
6
  import type { Principal } from "../runtime.js";
6
7
  import type { NeonAuthEnvironment } from "./config.js";
7
8
  import { RequestId } from "./errors.js";
@@ -32,6 +33,10 @@ declare const Authentication_base: HttpApiMiddleware.ServiceClass<Authentication
32
33
  export declare class Authentication extends Authentication_base {
33
34
  }
34
35
  export interface TokenVerifierConfig {
36
+ /** Production workload activation and the deployment actually serving the request. */
37
+ readonly workloadTarget?: {
38
+ readonly deploymentId: string;
39
+ };
35
40
  /** The control plane: the issuer of app tokens and the host of their JWKS. */
36
41
  readonly segmentstreamUrl: string;
37
42
  readonly applicationId: string;
@@ -39,6 +44,9 @@ export interface TokenVerifierConfig {
39
44
  readonly organizationId: string;
40
45
  }
41
46
  export type TokenVerification = {
47
+ readonly status: "workload";
48
+ readonly execution: WorkloadExecution;
49
+ } | {
42
50
  readonly status: "valid";
43
51
  readonly identity: SignInIdentity;
44
52
  } | {
@@ -56,6 +64,9 @@ export declare function neonAuthKeySet(jwksUrl: string): JWTVerifyGetKey;
56
64
  * `SEGMENTSTREAM_URL` for this application (`aud: app:<application id>`), organization and workspace.
57
65
  */
58
66
  export declare function makeTokenVerifier(config: TokenVerifierConfig, keySet: JWTVerifyGetKey): TokenVerifier;
67
+ export declare const WORKLOAD_TOKEN_TYPE = "segmentstream-ontology-workload+jwt";
68
+ /** A separate verifier: a workload token can never resolve as a human or act as an agent. */
69
+ export declare function makeWorkloadTokenVerifier(config: TokenVerifierConfig, keySet: JWTVerifyGetKey): TokenVerifier;
59
70
  /**
60
71
  * Neon Auth's issuer is the origin of its Auth URL: `https://ep-….neon.tech/ontology/auth` issues
61
72
  * tokens with `iss: https://ep-….neon.tech`.
@@ -1,7 +1,7 @@
1
1
  import { Cause, Context, Effect, Layer, Redacted } from "effect";
2
2
  import { HttpServerRequest } from "effect/unstable/http";
3
3
  import { HttpApiMiddleware, HttpApiSecurity, OpenApi } from "effect/unstable/httpapi";
4
- import { createRemoteJWKSet, decodeJwt, errors, jwtVerify, } from "jose";
4
+ import { createRemoteJWKSet, decodeJwt, decodeProtectedHeader, errors, jwtVerify, } from "jose";
5
5
  import { API_KEY_PREFIX } from "../runtime/principals.js";
6
6
  import { RequestId, crossOriginSession, forbidden, internalError, notInvited, toServerResponse, unauthorized, unavailable, } from "./errors.js";
7
7
  import { ACCESS_TOKEN_PREFIX, principalOfAccessToken } from "./oauth/protocol.js";
@@ -76,6 +76,7 @@ export function makeTokenVerifier(config, keySet) {
76
76
  try {
77
77
  const verified = await jwtVerify(token, keySet, {
78
78
  algorithms: ["ES256"],
79
+ typ: "JWT",
79
80
  issuer,
80
81
  audience,
81
82
  clockTolerance: CLOCK_TOLERANCE_SECONDS,
@@ -90,6 +91,96 @@ export function makeTokenVerifier(config, keySet) {
90
91
  return identity === undefined ? { status: "invalid" } : { status: "valid", identity };
91
92
  };
92
93
  }
94
+ export const WORKLOAD_TOKEN_TYPE = "segmentstream-ontology-workload+jwt";
95
+ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
96
+ /** A separate verifier: a workload token can never resolve as a human or act as an agent. */
97
+ export function makeWorkloadTokenVerifier(config, keySet) {
98
+ return async (token) => {
99
+ if (config.workloadTarget === undefined) {
100
+ return { status: "invalid" };
101
+ }
102
+ let payload;
103
+ try {
104
+ const verified = await jwtVerify(token, keySet, {
105
+ algorithms: ["ES256"],
106
+ typ: WORKLOAD_TOKEN_TYPE,
107
+ issuer: new URL(config.segmentstreamUrl).origin,
108
+ audience: `ontology:${config.applicationId}`,
109
+ clockTolerance: CLOCK_TOLERANCE_SECONDS,
110
+ requiredClaims: [
111
+ "sub",
112
+ "exp",
113
+ "iat",
114
+ "jti",
115
+ "org",
116
+ "wsp",
117
+ "deploymentId",
118
+ "ontologyDeploymentId",
119
+ "ontologyOrigin",
120
+ "environment",
121
+ "jobName",
122
+ "runId",
123
+ "executionId",
124
+ "attemptId",
125
+ "generation",
126
+ ],
127
+ });
128
+ payload = verified.payload;
129
+ }
130
+ catch (error) {
131
+ return keySetUnavailable(error) ? { status: "unavailable" } : { status: "invalid" };
132
+ }
133
+ const strings = ["jti", "jobName"];
134
+ const ids = ["deploymentId", "ontologyDeploymentId", "runId", "attemptId"];
135
+ if (payload.org !== config.organizationId ||
136
+ payload.wsp !== config.workspaceId ||
137
+ payload.sub !== `pipeline-workers:${config.workspaceId}` ||
138
+ payload.environment !== "production" ||
139
+ payload.aud !== `ontology:${config.applicationId}` ||
140
+ strings.some((name) => typeof payload[name] !== "string" || payload[name].trim() === "") ||
141
+ ids.some((name) => typeof payload[name] !== "string" || !UUID.test(payload[name])) ||
142
+ (payload.executionId !== null &&
143
+ (typeof payload.executionId !== "string" || !UUID.test(payload.executionId))) ||
144
+ !Number.isSafeInteger(payload.generation) ||
145
+ payload.generation < 1 ||
146
+ !Number.isSafeInteger(payload.iat) ||
147
+ !Number.isSafeInteger(payload.exp) ||
148
+ payload.iat > Math.floor(Date.now() / 1000) + CLOCK_TOLERANCE_SECONDS ||
149
+ payload.exp <= payload.iat ||
150
+ payload.exp - payload.iat > 300 ||
151
+ typeof payload.ontologyOrigin !== "string" ||
152
+ !URL.canParse(payload.ontologyOrigin)) {
153
+ return { status: "invalid" };
154
+ }
155
+ const origin = new URL(payload.ontologyOrigin);
156
+ if (origin.protocol !== "https:" || origin.origin !== payload.ontologyOrigin) {
157
+ return { status: "invalid" };
158
+ }
159
+ return {
160
+ status: "workload",
161
+ execution: {
162
+ issuer: payload.iss,
163
+ audience: payload.aud,
164
+ organizationId: config.organizationId,
165
+ workspaceId: config.workspaceId,
166
+ subject: payload.sub,
167
+ deploymentId: payload.deploymentId,
168
+ ontologyDeploymentId: payload.ontologyDeploymentId,
169
+ servingOntologyDeploymentId: config.workloadTarget.deploymentId,
170
+ ontologyOrigin: payload.ontologyOrigin,
171
+ environment: "production",
172
+ jobName: payload.jobName,
173
+ runId: payload.runId,
174
+ executionId: payload.executionId,
175
+ attemptId: payload.attemptId,
176
+ generation: payload.generation,
177
+ tokenId: payload.jti,
178
+ issuedAt: payload.iat,
179
+ expiresAt: payload.exp,
180
+ },
181
+ };
182
+ };
183
+ }
93
184
  /**
94
185
  * Neon Auth's issuer is the origin of its Auth URL: `https://ep-….neon.tech/ontology/auth` issues
95
186
  * tokens with `iss: https://ep-….neon.tech`.
@@ -166,11 +257,21 @@ export function verifyByIssuer(verifiers) {
166
257
  * once it has Neon Auth. Both are accepted while workspaces move to Neon Auth.
167
258
  */
168
259
  export function tokenVerifier(config) {
260
+ const platformKeys = remoteKeySet(config.segmentstreamUrl);
261
+ const human = makeTokenVerifier(config, platformKeys);
262
+ const workload = makeWorkloadTokenVerifier(config, platformKeys);
263
+ const platform = (token) => {
264
+ try {
265
+ return decodeProtectedHeader(token).typ === WORKLOAD_TOKEN_TYPE
266
+ ? workload(token)
267
+ : human(token);
268
+ }
269
+ catch {
270
+ return Promise.resolve({ status: "invalid" });
271
+ }
272
+ };
169
273
  const verifiers = new Map([
170
- [
171
- new URL(config.segmentstreamUrl).origin,
172
- makeTokenVerifier(config, remoteKeySet(config.segmentstreamUrl)),
173
- ],
274
+ [new URL(config.segmentstreamUrl).origin, platform],
174
275
  ]);
175
276
  if (config.neonAuth !== undefined) {
176
277
  verifiers.set(neonAuthIssuer(config.neonAuth.baseUrl), makeNeonAuthTokenVerifier(config.neonAuth, neonAuthKeySet(config.neonAuth.jwksUrl)));
@@ -337,6 +438,14 @@ export function authenticate(credential, verifyToken, options = {}) {
337
438
  }
338
439
  }
339
440
  const verification = yield* Effect.promise(() => verifyToken(token));
441
+ if (verification.status === "workload") {
442
+ const principal = backend.resolvePipelineWorker === undefined
443
+ ? undefined
444
+ : yield* backend.resolvePipelineWorker(verification.execution);
445
+ return principal === undefined
446
+ ? { status: "rejected" }
447
+ : { status: "authenticated", principal };
448
+ }
340
449
  if (verification.status !== "valid") {
341
450
  return verification.status === "invalid"
342
451
  ? { status: "rejected" }
@@ -1,7 +1,8 @@
1
1
  import { type OntologyManifest } from "../manifest.js";
2
2
  /**
3
3
  * The catalog for a reader: the manifest, with each entity's actions narrowed to the ones the given
4
- * permissions may run. Without permissions it is the manifest itself.
4
+ * permissions may run, and only the measures of the model whose every input entity they may read.
5
+ * Without permissions it is the manifest itself.
5
6
  */
6
7
  export declare function catalogFor(manifest: OntologyManifest, permissions?: ReadonlyArray<string>): OntologyManifest;
7
8
  export interface GraphRelationship {
@@ -1,16 +1,20 @@
1
1
  import { isValueMatched } from "../manifest.js";
2
2
  import { isAllowed } from "../permissions.js";
3
- import { actionPermission } from "../render.js";
3
+ import { actionPermission, modelMeasuresFor } from "../render.js";
4
4
  /**
5
5
  * The catalog for a reader: the manifest, with each entity's actions narrowed to the ones the given
6
- * permissions may run. Without permissions it is the manifest itself.
6
+ * permissions may run, and only the measures of the model whose every input entity they may read.
7
+ * Without permissions it is the manifest itself.
7
8
  */
8
9
  export function catalogFor(manifest, permissions) {
9
10
  if (permissions === undefined) {
10
11
  return manifest;
11
12
  }
13
+ const { measures: declared, ...rest } = manifest;
14
+ const measures = declared === undefined ? [] : modelMeasuresFor(manifest, permissions);
12
15
  return {
13
- ...manifest,
16
+ ...rest,
17
+ ...(measures.length === 0 ? {} : { measures }),
14
18
  entities: manifest.entities
15
19
  .filter((entity) => entity.source === undefined ||
16
20
  isAllowed(entity.permissions?.read ?? `${entity.name}:read`, { permissions }))
@@ -66,6 +66,10 @@ export interface OntologyEnvironment {
66
66
  * `segmentstreamUrl` for `workspaceId`; previews and local development have none.
67
67
  */
68
68
  readonly appCredential?: Redacted.Redacted<string>;
69
+ /** Production deployment identity; absent in local, preview and earlier deployments. */
70
+ readonly workloadTarget?: {
71
+ readonly deploymentId: string;
72
+ };
69
73
  /** Present once the workspace has Neon Auth: members then sign in with its tokens. */
70
74
  readonly neonAuth?: NeonAuthEnvironment;
71
75
  /**
@@ -134,6 +134,13 @@ export function configFromEnv(environment = process.env) {
134
134
  message: `The ontology service cannot start: ${list} ${verb} not set.`,
135
135
  });
136
136
  }
137
+ const deploymentId = read("SEGMENTSTREAM_ONTOLOGY_DEPLOYMENT_ID");
138
+ if (deploymentId !== "" &&
139
+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(deploymentId)) {
140
+ throw new OntologyServiceConfigError({
141
+ message: "SEGMENTSTREAM_ONTOLOGY_DEPLOYMENT_ID must be a UUID.",
142
+ });
143
+ }
137
144
  const neonAuth = neonAuthFromEnv(read);
138
145
  const runtimeIdentity = runtimeIdentityFromEnv(read);
139
146
  const localConnections = localConnectionsFromEnv(environment);
@@ -163,6 +170,10 @@ export function configFromEnv(environment = process.env) {
163
170
  ...(read(APP_CREDENTIAL_VARIABLE)
164
171
  ? { appCredential: Redacted.make(read(APP_CREDENTIAL_VARIABLE)) }
165
172
  : {}),
173
+ ...(environment.VERCEL_ENV === "production" &&
174
+ read("SEGMENTSTREAM_ONTOLOGY_DEPLOYMENT_ID") !== ""
175
+ ? { workloadTarget: { deploymentId: read("SEGMENTSTREAM_ONTOLOGY_DEPLOYMENT_ID") } }
176
+ : {}),
166
177
  ...(neonAuth === undefined ? {} : { neonAuth }),
167
178
  ...(runtimeIdentity === undefined ? {} : { runtimeIdentity }),
168
179
  ...(runtimeIdentity === undefined || read(AGENT_HOST_VARIABLE) === ""