@segmentstream-cloud/ontology 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -2
- package/dist/aggregate-routes.d.ts +46 -9
- package/dist/aggregate-routes.js +186 -20
- package/dist/analyze.d.ts +12 -1
- package/dist/analyze.js +3 -1
- package/dist/diagnostics.d.ts +3 -3
- package/dist/diagnostics.js +2 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/manifest.d.ts +113 -2
- package/dist/manifest.js +33 -2
- package/dist/measure.d.ts +57 -6
- package/dist/measure.js +7 -2
- package/dist/migrate/apply.js +2 -1
- package/dist/migrate/model.d.ts +22 -0
- package/dist/migrate/plan.js +9 -1
- package/dist/migrate/storage.d.ts +3 -0
- package/dist/migrate/storage.js +10 -0
- package/dist/model-measures.d.ts +22 -0
- package/dist/model-measures.js +543 -0
- package/dist/ontology.d.ts +5 -4
- package/dist/permissions.d.ts +3 -0
- package/dist/permissions.js +8 -1
- package/dist/render.d.ts +15 -3
- package/dist/render.js +56 -3
- package/dist/runtime/history.d.ts +22 -0
- package/dist/runtime/history.js +2 -0
- package/dist/runtime/pipeline.js +6 -2
- package/dist/runtime/principals.d.ts +2 -0
- package/dist/runtime/principals.js +55 -0
- package/dist/runtime/reads.js +9 -1
- package/dist/runtime/service-binding.js +2 -1
- package/dist/runtime/workload.d.ts +23 -0
- package/dist/runtime/workload.js +22 -0
- package/dist/runtime/writes.js +7 -1
- package/dist/runtime.d.ts +3 -0
- package/dist/service/authentication.d.ts +11 -0
- package/dist/service/authentication.js +114 -5
- package/dist/service/catalog.d.ts +2 -1
- package/dist/service/catalog.js +7 -3
- package/dist/service/config.d.ts +4 -0
- package/dist/service/config.js +11 -0
- package/dist/service/handlers.js +2 -1
- package/dist/service/mcp/server.js +10 -6
- package/dist/service/mcp/tools.js +25 -10
- package/dist/service/operations.d.ts +25 -5
- package/dist/service/runtime-contract.d.ts +1 -0
- package/dist/service/schemas.d.ts +143 -0
- package/dist/service/schemas.js +11 -2
- package/dist/storage/attachments.js +4 -4
- package/dist/storage/authorization.d.ts +1 -1
- package/dist/storage/authorization.js +9 -4
- package/dist/storage/sessions.js +3 -3
- package/dist/storage/writes.js +1 -1
- package/dist/warehouse-backed.js +4 -0
- package/package.json +1 -1
package/dist/render.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { shortestRoutesFrom } from "./aggregate-routes.js";
|
|
1
|
+
import { inputEntities, shortestRoutesFrom } from "./aggregate-routes.js";
|
|
2
2
|
import { isAllowed, permissionOf, READ_VERB, rolesGranting } from "./permissions.js";
|
|
3
3
|
import { REFERENCE_ANNOTATION } from "./property.js";
|
|
4
4
|
/** What an action of the manifest requires; version 1 manifests leave it implicit. */
|
|
@@ -389,6 +389,51 @@ export function measuresFor(manifest, permissions) {
|
|
|
389
389
|
measure,
|
|
390
390
|
})));
|
|
391
391
|
}
|
|
392
|
+
/**
|
|
393
|
+
* The measures of the model whose every input entity the reader may read, in declaration order.
|
|
394
|
+
* Reading a model measure is reading its inputs. Without permissions, every one.
|
|
395
|
+
*/
|
|
396
|
+
export function modelMeasuresFor(manifest, permissions) {
|
|
397
|
+
return (manifest.measures ?? []).filter((measure) => inputEntities(measure).every((entity) => readable(entity, permissions)));
|
|
398
|
+
}
|
|
399
|
+
/** `a`, `a and b`, `a, b and c`. */
|
|
400
|
+
function listed(values) {
|
|
401
|
+
return values.length < 2
|
|
402
|
+
? values.join("")
|
|
403
|
+
: `${values.slice(0, -1).join(", ")} and ${values[values.length - 1]}`;
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* How one alignment group reads: `sales.order_date aligns with plans.month`, and for an entity
|
|
407
|
+
* without a counterpart `listings has no counterpart for sales.order_date`. Members follow the
|
|
408
|
+
* measure's input entities, whatever order the document keeps them in.
|
|
409
|
+
*/
|
|
410
|
+
function alignmentClauses(measure, group) {
|
|
411
|
+
const order = inputEntities(measure);
|
|
412
|
+
const members = Object.entries(group).sort(([left], [right]) => order.indexOf(left) - order.indexOf(right));
|
|
413
|
+
const present = members.flatMap(([entity, property]) => property === null ? [] : [`${entity}.${property}`]);
|
|
414
|
+
const absent = members.flatMap(([entity, property]) => (property === null ? [entity] : []));
|
|
415
|
+
const [first, ...rest] = present;
|
|
416
|
+
if (first === undefined) {
|
|
417
|
+
return [];
|
|
418
|
+
}
|
|
419
|
+
return [
|
|
420
|
+
...(rest.length > 0 ? [`${first} aligns with ${listed(rest)}`] : []),
|
|
421
|
+
...absent.map((entity) => `${entity} has no counterpart for ${first}`),
|
|
422
|
+
];
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* A measure of the model as `describe`, the MCP tool and the instructions show it: its description,
|
|
426
|
+
* the entity measures it reads and how their properties align. `Ordered over planned units. Reads
|
|
427
|
+
* sales.ordered and plans.planned_units; sales.order_date aligns with plans.month.`
|
|
428
|
+
*/
|
|
429
|
+
export function modelMeasureDescription(measure) {
|
|
430
|
+
const clauses = [
|
|
431
|
+
`Reads ${listed(measure.inputs)}`,
|
|
432
|
+
...measure.align.flatMap((group) => alignmentClauses(measure, group)),
|
|
433
|
+
];
|
|
434
|
+
const described = measure.description === "" ? [] : [sentence(measure.description)];
|
|
435
|
+
return [...described, `${clauses.join("; ")}.`].join(" ");
|
|
436
|
+
}
|
|
392
437
|
/**
|
|
393
438
|
* The entities one entity reaches, with the shortest routes to each: exactly the routes aggregate
|
|
394
439
|
* resolution weighs ({@link shortestRoutesFrom}). Only then is the reader applied: a target they may
|
|
@@ -416,8 +461,9 @@ function reachLine(entity, reaches) {
|
|
|
416
461
|
return `${entity} reaches ${targets.join(", ")}.`;
|
|
417
462
|
}
|
|
418
463
|
/**
|
|
419
|
-
* The aggregate part of the catalog's prose: every measure the reader may read, model-wide,
|
|
420
|
-
*
|
|
464
|
+
* The aggregate part of the catalog's prose: every measure the reader may read, model-wide, the
|
|
465
|
+
* measures of the model whose every input they may read, and per measure entity the entities it
|
|
466
|
+
* reaches. Empty when the reader may read no measure.
|
|
421
467
|
*/
|
|
422
468
|
export function renderAggregates(manifest, permissions) {
|
|
423
469
|
const measures = measuresFor(manifest, permissions);
|
|
@@ -430,10 +476,17 @@ export function renderAggregates(manifest, permissions) {
|
|
|
430
476
|
return reaches.length === 0 ? [] : [reachLine(owner, reaches)];
|
|
431
477
|
});
|
|
432
478
|
const measureLines = measures.map(({ name, entity, measure }) => `${name} (${measure.unit}): ${sentence(measureDescription(entity, measure))}`);
|
|
479
|
+
const modelLines = modelMeasuresFor(manifest, permissions).map((measure) => `${measure.name} (${measure.unit}): ${modelMeasureDescription(measure)}`);
|
|
433
480
|
return [
|
|
434
481
|
`Aggregates: ${RECORDS_OR_AGGREGATES} (POST /api/ontology/aggregate, or the aggregate tool).`,
|
|
435
482
|
`${INDENT}Measures, as <entity>.<measure>:`,
|
|
436
483
|
...measureLines.map((line) => `${INDENT}${INDENT}${line}`),
|
|
484
|
+
...(modelLines.length === 0
|
|
485
|
+
? []
|
|
486
|
+
: [
|
|
487
|
+
`${INDENT}Measures of the model, by name:`,
|
|
488
|
+
...modelLines.map((line) => `${INDENT}${INDENT}${line}`),
|
|
489
|
+
]),
|
|
437
490
|
...(reachLines.length === 0
|
|
438
491
|
? []
|
|
439
492
|
: [
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
|
+
import { WorkloadExecution } from "./workload.js";
|
|
2
3
|
import type { StoredRecord } from "../runtime.js";
|
|
3
4
|
import type { EntityModel } from "./model.js";
|
|
4
5
|
import type { Params } from "./statement.js";
|
|
@@ -31,6 +32,26 @@ export declare const Edit: Schema.Struct<{
|
|
|
31
32
|
export type Edit = typeof Edit.Type;
|
|
32
33
|
/** A row of `<entity>_history`. */
|
|
33
34
|
export declare const HistoryEntry: Schema.Struct<{
|
|
35
|
+
readonly execution: Schema.optionalKey<Schema.Struct<{
|
|
36
|
+
readonly issuer: Schema.String;
|
|
37
|
+
readonly audience: Schema.String;
|
|
38
|
+
readonly organizationId: Schema.String;
|
|
39
|
+
readonly workspaceId: Schema.String;
|
|
40
|
+
readonly subject: Schema.String;
|
|
41
|
+
readonly deploymentId: Schema.String;
|
|
42
|
+
readonly ontologyDeploymentId: Schema.String;
|
|
43
|
+
readonly servingOntologyDeploymentId: Schema.String;
|
|
44
|
+
readonly ontologyOrigin: Schema.String;
|
|
45
|
+
readonly environment: Schema.Literal<"production">;
|
|
46
|
+
readonly jobName: Schema.String;
|
|
47
|
+
readonly runId: Schema.String;
|
|
48
|
+
readonly executionId: Schema.NullOr<Schema.String>;
|
|
49
|
+
readonly attemptId: Schema.String;
|
|
50
|
+
readonly generation: Schema.Number;
|
|
51
|
+
readonly tokenId: Schema.String;
|
|
52
|
+
readonly issuedAt: Schema.Number;
|
|
53
|
+
readonly expiresAt: Schema.Number;
|
|
54
|
+
}>>;
|
|
34
55
|
readonly id: Schema.String;
|
|
35
56
|
readonly recordId: Schema.String;
|
|
36
57
|
/** The qualified action, for example `lead.changeStatus`. */
|
|
@@ -54,6 +75,7 @@ export interface HistoryWrite {
|
|
|
54
75
|
readonly edit: Edit;
|
|
55
76
|
}
|
|
56
77
|
export interface HistoryScope {
|
|
78
|
+
readonly execution?: WorkloadExecution;
|
|
57
79
|
readonly action: string;
|
|
58
80
|
readonly actionId: string;
|
|
59
81
|
readonly principalId: string | null;
|
package/dist/runtime/history.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
2
|
import { canonicalJson } from "../manifest.js";
|
|
3
|
+
import { WorkloadExecution } from "./workload.js";
|
|
3
4
|
import { quoteIdentifier } from "../migrate/storage.js";
|
|
4
5
|
import { uuidv7 } from "./ids.js";
|
|
5
6
|
export const EditKind = Schema.Literals(["create", "update", "delete", "link", "unlink"]);
|
|
@@ -21,6 +22,7 @@ export const Edit = Schema.Struct({
|
|
|
21
22
|
});
|
|
22
23
|
/** A row of `<entity>_history`. */
|
|
23
24
|
export const HistoryEntry = Schema.Struct({
|
|
25
|
+
execution: Schema.optionalKey(WorkloadExecution),
|
|
24
26
|
id: Schema.String,
|
|
25
27
|
recordId: Schema.String,
|
|
26
28
|
/** The qualified action, for example `lead.changeStatus`. */
|
package/dist/runtime/pipeline.js
CHANGED
|
@@ -141,7 +141,7 @@ export function runAction(ontology, request) {
|
|
|
141
141
|
if (consumesUpload) {
|
|
142
142
|
const saved = yield* sql
|
|
143
143
|
.withTransaction(Effect.gen(function* () {
|
|
144
|
-
yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions);
|
|
144
|
+
yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions, request.principal.execution);
|
|
145
145
|
const [saved] = yield* sql.unsafe(`SELECT fingerprint, outcome FROM "_blob_action_results"
|
|
146
146
|
WHERE principal_id = $1::uuid AND key = $2`, [request.principal.id, key]);
|
|
147
147
|
if (saved && saved.fingerprint !== fingerprint) {
|
|
@@ -168,6 +168,7 @@ export function runAction(ontology, request) {
|
|
|
168
168
|
action: qualified,
|
|
169
169
|
actionId,
|
|
170
170
|
principalId: request.principal.id,
|
|
171
|
+
...(request.principal.execution ? { execution: request.principal.execution } : {}),
|
|
171
172
|
...(key ? { blobActionKey: key } : {}),
|
|
172
173
|
...(request.branchId ? { branchId: request.branchId } : {}),
|
|
173
174
|
...(request.options?.createIntent ? { createIntent: request.options.createIntent } : {}),
|
|
@@ -177,8 +178,11 @@ export function runAction(ontology, request) {
|
|
|
177
178
|
const handler = Effect.suspend(() => handle({ ...context, prepared }));
|
|
178
179
|
let replay;
|
|
179
180
|
const transactionalHandler = Effect.gen(function* () {
|
|
181
|
+
if (request.principal.execution !== undefined) {
|
|
182
|
+
yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions, request.principal.execution);
|
|
183
|
+
}
|
|
180
184
|
if (consumesUpload) {
|
|
181
|
-
yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions);
|
|
185
|
+
yield* authorizeBlobOperation(ontology, sql, request.principal.id, found.permissions, request.principal.execution);
|
|
182
186
|
yield* sql.unsafe(`INSERT INTO "_blob_action_results" (principal_id, key, fingerprint)
|
|
183
187
|
VALUES ($1::uuid, $2, $3) ON CONFLICT DO NOTHING`, [request.principal.id, key, fingerprint]);
|
|
184
188
|
const [saved] = yield* sql.unsafe(`SELECT fingerprint, outcome FROM "_blob_action_results" WHERE principal_id = $1::uuid AND key = $2
|
|
@@ -49,6 +49,8 @@ export declare const SIGN_IN_RECORD_INTERVAL_MS: number;
|
|
|
49
49
|
* {@link SIGN_IN_RECORD_INTERVAL_MS}, so history keeps it. Inactive users are rejected.
|
|
50
50
|
*/
|
|
51
51
|
export declare function resolvePrincipalByIdentity(ontology: Ontology, claims: IdentityClaims): Effect.Effect<Principal, PrincipalRejected, SqlClient.SqlClient>;
|
|
52
|
+
/** Resolve the workspace's stable keyless pipeline service, without customer roles or sign-in. */
|
|
53
|
+
export declare function resolvePipelineWorker(ontology: Ontology, execution: import("./workload.js").WorkloadExecution): Effect.Effect<Principal, PrincipalRejected, SqlClient.SqlClient>;
|
|
52
54
|
/**
|
|
53
55
|
* The principal of an API key, with the permissions its roles have in the ontology. Unknown and
|
|
54
56
|
* revoked keys and inactive users are rejected.
|
|
@@ -207,6 +207,61 @@ export function resolvePrincipalByIdentity(ontology, claims) {
|
|
|
207
207
|
return yield* activePrincipal(ontology, sql, user);
|
|
208
208
|
}).pipe(Effect.catchIf(isSqlError, (error) => Effect.die(error)));
|
|
209
209
|
}
|
|
210
|
+
/** Resolve the workspace's stable keyless pipeline service, without customer roles or sign-in. */
|
|
211
|
+
export function resolvePipelineWorker(ontology, execution) {
|
|
212
|
+
return Effect.gen(function* () {
|
|
213
|
+
const sql = yield* SqlClient.SqlClient;
|
|
214
|
+
const provider = "segmentstream-ontology-workload";
|
|
215
|
+
const subject = `pipeline-workers:${execution.workspaceId}`;
|
|
216
|
+
const runtime = platformRuntime(ontology, sql, "user.resolve_pipeline_worker");
|
|
217
|
+
const principal = (user) => user.kind !== "service" || user.active !== true
|
|
218
|
+
? Effect.fail(new PrincipalRejected({
|
|
219
|
+
reason: "inactive_user",
|
|
220
|
+
message: "The pipeline worker principal is inactive or has an invalid kind.",
|
|
221
|
+
}))
|
|
222
|
+
: Effect.succeed({
|
|
223
|
+
id: user.id,
|
|
224
|
+
kind: "service",
|
|
225
|
+
roles: [],
|
|
226
|
+
permissions: ["*:*"],
|
|
227
|
+
execution,
|
|
228
|
+
});
|
|
229
|
+
const find = (forUpdate) => runtime.find({
|
|
230
|
+
entity: "user",
|
|
231
|
+
options: {
|
|
232
|
+
where: { identity_provider: provider, identity_subject: subject },
|
|
233
|
+
limit: 1,
|
|
234
|
+
forUpdate,
|
|
235
|
+
},
|
|
236
|
+
});
|
|
237
|
+
const [known] = yield* find(false);
|
|
238
|
+
// Returning requests only read the principal; writes revalidate it in their transaction.
|
|
239
|
+
if (known !== undefined) {
|
|
240
|
+
return yield* principal(known);
|
|
241
|
+
}
|
|
242
|
+
return yield* sql
|
|
243
|
+
.withTransaction(Effect.gen(function* () {
|
|
244
|
+
// The replication lock precedes every business lock, including principal initialization.
|
|
245
|
+
yield* beginReplicationWrite(sql);
|
|
246
|
+
yield* sql.unsafe("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [
|
|
247
|
+
provider + ":" + subject,
|
|
248
|
+
]);
|
|
249
|
+
const [existing] = yield* find(true);
|
|
250
|
+
const user = existing ??
|
|
251
|
+
(yield* runtime.create({
|
|
252
|
+
entity: "user",
|
|
253
|
+
values: {
|
|
254
|
+
name: "Pipeline workers",
|
|
255
|
+
kind: "service",
|
|
256
|
+
identity_provider: provider,
|
|
257
|
+
identity_subject: subject,
|
|
258
|
+
},
|
|
259
|
+
}));
|
|
260
|
+
return yield* principal(user);
|
|
261
|
+
}))
|
|
262
|
+
.pipe(Effect.catchTag("Conflict", (error) => Effect.die(error)), Effect.catchTag("NotFound", (error) => Effect.die(error)));
|
|
263
|
+
}).pipe(Effect.catchIf(isSqlError, (error) => Effect.die(error)));
|
|
264
|
+
}
|
|
210
265
|
/**
|
|
211
266
|
* The principal of an API key, with the permissions its roles have in the ontology. Unknown and
|
|
212
267
|
* revoked keys and inactive users are rejected.
|
package/dist/runtime/reads.js
CHANGED
|
@@ -238,17 +238,25 @@ export function readHistory(context, entity, input) {
|
|
|
238
238
|
const at = params.add(cursor[0], "timestamptz");
|
|
239
239
|
conditions.push(`("h"."at" < ${at} OR ("h"."at" = ${at} AND "h"."id" < ${params.add(cursor[1], "uuid")}))`);
|
|
240
240
|
}
|
|
241
|
+
const presenceParams = new Params();
|
|
242
|
+
const [presence] = yield* context.run("SELECT to_regclass('_ontology_action_attribution') IS NOT NULL AS present", presenceParams);
|
|
243
|
+
const attribution = presence?.present === true;
|
|
241
244
|
const rows = yield* context.run(`SELECT "h"."id"::text AS "id", "h"."record_id"::text AS "recordId", "h"."action" AS "action", ` +
|
|
242
245
|
`"h"."action_id"::text AS "actionId", "h"."kind" AS "kind", ` +
|
|
243
246
|
`"h"."principal_id"::text AS "principalId", ` +
|
|
244
247
|
`to_char("h"."at" AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS "at", ` +
|
|
245
248
|
`"h"."changes"::text AS "changes" ` +
|
|
249
|
+
(attribution ? `, "a"."execution" AS "execution" ` : "") +
|
|
246
250
|
`FROM ${quoteIdentifier(entity.historyTable)} AS "h" ` +
|
|
251
|
+
(attribution
|
|
252
|
+
? `LEFT JOIN "_ontology_action_attribution" AS "a" ON "a"."action_id" = "h"."action_id" `
|
|
253
|
+
: "") +
|
|
247
254
|
`WHERE ${conditions.join(" AND ")} ` +
|
|
248
255
|
`ORDER BY "h"."at" DESC, "h"."id" DESC ` +
|
|
249
256
|
`LIMIT ${params.add(limit + 1, "bigint")}`, params);
|
|
250
257
|
const items = rows.slice(0, limit).map((row) => Schema.decodeUnknownSync(HistoryEntry)({
|
|
251
|
-
...row,
|
|
258
|
+
...Object.fromEntries(Object.entries(row).filter(([key, value]) => key !== "execution" || value != null)),
|
|
259
|
+
...(row.execution == null ? {} : { execution: row.execution }),
|
|
252
260
|
at: canonicalInstant(String(row.at)),
|
|
253
261
|
changes: JSON.parse(String(row.changes)),
|
|
254
262
|
}));
|
|
@@ -11,7 +11,7 @@ import { sqlAgentStore } from "./agents.js";
|
|
|
11
11
|
import { generatedActions } from "./generated-actions.js";
|
|
12
12
|
import { sqlOAuthStore } from "./oauth.js";
|
|
13
13
|
import { runAction } from "./pipeline.js";
|
|
14
|
-
import { checkSignIn, resolvePrincipalByApiKey, resolvePrincipalByIdentity } from "./principals.js";
|
|
14
|
+
import { checkSignIn, resolvePrincipalByApiKey, resolvePrincipalByIdentity, resolvePipelineWorker, } from "./principals.js";
|
|
15
15
|
import { postgresTls } from "./postgres-tls.js";
|
|
16
16
|
import { makeOntologyRuntime } from "./sql-runtime.js";
|
|
17
17
|
/** Connections one service instance holds to the database's pooler. */
|
|
@@ -61,6 +61,7 @@ export function backendLayer(ontology, blobStorage) {
|
|
|
61
61
|
})).pipe(Effect.catchTag("PrincipalRejected", (rejection) => rejection.reason === "not_invited"
|
|
62
62
|
? rejectedAs("sign_in")(rejection).pipe(Effect.andThen(Effect.fail(new NotInvited({ message: rejection.message }))))
|
|
63
63
|
: rejectedAs("sign_in")(rejection))),
|
|
64
|
+
resolvePipelineWorker: (execution) => withSql(resolvePipelineWorker(ontology, execution)).pipe(Effect.catchTag("PrincipalRejected", rejectedAs("pipeline_worker"))),
|
|
64
65
|
checkSignIn: (email) => withSql(checkSignIn(ontology, email)),
|
|
65
66
|
resolveApiKey: (key) => withSql(resolvePrincipalByApiKey(ontology, Redacted.value(key))).pipe(Effect.catchTag("PrincipalRejected", rejectedAs("api_key"))),
|
|
66
67
|
appliedHash: readCatalog({ sql }).pipe(Effect.map((manifest) => manifest?.hash), Effect.orDie),
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
/** Verified platform claims retained with each write; never supplied by request headers. */
|
|
3
|
+
export declare const WorkloadExecution: Schema.Struct<{
|
|
4
|
+
readonly issuer: Schema.String;
|
|
5
|
+
readonly audience: Schema.String;
|
|
6
|
+
readonly organizationId: Schema.String;
|
|
7
|
+
readonly workspaceId: Schema.String;
|
|
8
|
+
readonly subject: Schema.String;
|
|
9
|
+
readonly deploymentId: Schema.String;
|
|
10
|
+
readonly ontologyDeploymentId: Schema.String;
|
|
11
|
+
readonly servingOntologyDeploymentId: Schema.String;
|
|
12
|
+
readonly ontologyOrigin: Schema.String;
|
|
13
|
+
readonly environment: Schema.Literal<"production">;
|
|
14
|
+
readonly jobName: Schema.String;
|
|
15
|
+
readonly runId: Schema.String;
|
|
16
|
+
readonly executionId: Schema.NullOr<Schema.String>;
|
|
17
|
+
readonly attemptId: Schema.String;
|
|
18
|
+
readonly generation: Schema.Number;
|
|
19
|
+
readonly tokenId: Schema.String;
|
|
20
|
+
readonly issuedAt: Schema.Number;
|
|
21
|
+
readonly expiresAt: Schema.Number;
|
|
22
|
+
}>;
|
|
23
|
+
export type WorkloadExecution = typeof WorkloadExecution.Type;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
/** Verified platform claims retained with each write; never supplied by request headers. */
|
|
3
|
+
export const WorkloadExecution = Schema.Struct({
|
|
4
|
+
issuer: Schema.String,
|
|
5
|
+
audience: Schema.String,
|
|
6
|
+
organizationId: Schema.String,
|
|
7
|
+
workspaceId: Schema.String,
|
|
8
|
+
subject: Schema.String,
|
|
9
|
+
deploymentId: Schema.String,
|
|
10
|
+
ontologyDeploymentId: Schema.String,
|
|
11
|
+
servingOntologyDeploymentId: Schema.String,
|
|
12
|
+
ontologyOrigin: Schema.String,
|
|
13
|
+
environment: Schema.Literal("production"),
|
|
14
|
+
jobName: Schema.String,
|
|
15
|
+
runId: Schema.String,
|
|
16
|
+
executionId: Schema.NullOr(Schema.String),
|
|
17
|
+
attemptId: Schema.String,
|
|
18
|
+
generation: Schema.Number,
|
|
19
|
+
tokenId: Schema.String,
|
|
20
|
+
issuedAt: Schema.Number,
|
|
21
|
+
expiresAt: Schema.Number,
|
|
22
|
+
});
|
package/dist/runtime/writes.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Effect } from "effect";
|
|
2
2
|
import { resolveBlobWrites } from "../storage/writes.js";
|
|
3
3
|
import { Conflict } from "../errors.js";
|
|
4
|
-
import { quoteIdentifier } from "../migrate/storage.js";
|
|
4
|
+
import { ACTION_ATTRIBUTION_TABLE, quoteIdentifier } from "../migrate/storage.js";
|
|
5
5
|
import { InvalidValues } from "./errors.js";
|
|
6
6
|
import { uuidv7 } from "./ids.js";
|
|
7
7
|
import { diffRecords, insertHistory } from "./history.js";
|
|
@@ -13,6 +13,12 @@ import { columnOf, Params } from "./statement.js";
|
|
|
13
13
|
function recordEdit(context, entity, edit) {
|
|
14
14
|
return Effect.gen(function* () {
|
|
15
15
|
const params = new Params();
|
|
16
|
+
if (context.scope.execution !== undefined) {
|
|
17
|
+
const attribution = new Params();
|
|
18
|
+
yield* context.run(`INSERT INTO "${ACTION_ATTRIBUTION_TABLE}" (action_id, execution) VALUES (` +
|
|
19
|
+
`${attribution.add(context.scope.actionId, "uuid")}, ` +
|
|
20
|
+
`${attribution.add(JSON.stringify(context.scope.execution), "jsonb")}) ON CONFLICT DO NOTHING`, attribution);
|
|
21
|
+
}
|
|
16
22
|
yield* context.run(insertHistory(context.scope, { entity, edit }, params), params);
|
|
17
23
|
context.edits.push(edit);
|
|
18
24
|
});
|
package/dist/runtime.d.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { Context, Effect } from "effect";
|
|
2
2
|
import type { Conflict, NotFound } from "./errors.js";
|
|
3
3
|
import type { Where } from "./query.js";
|
|
4
|
+
import type { WorkloadExecution } from "./runtime/workload.js";
|
|
4
5
|
/** Who runs an action: an app member or a service principal, both `user` records. */
|
|
5
6
|
export interface Principal {
|
|
7
|
+
/** Present only for a verified managed pipeline worker. */
|
|
8
|
+
readonly execution?: WorkloadExecution;
|
|
6
9
|
/** The id of the principal's `user` record. */
|
|
7
10
|
readonly id: string;
|
|
8
11
|
readonly kind: "member" | "service";
|
|
@@ -2,6 +2,7 @@ import { Context, Effect, Layer, Redacted } from "effect";
|
|
|
2
2
|
import { HttpServerRequest, type HttpServerResponse } from "effect/unstable/http";
|
|
3
3
|
import { HttpApiMiddleware, HttpApiSecurity } from "effect/unstable/httpapi";
|
|
4
4
|
import { type JWTVerifyGetKey } from "jose";
|
|
5
|
+
import type { WorkloadExecution } from "../runtime/workload.js";
|
|
5
6
|
import type { Principal } from "../runtime.js";
|
|
6
7
|
import type { NeonAuthEnvironment } from "./config.js";
|
|
7
8
|
import { RequestId } from "./errors.js";
|
|
@@ -32,6 +33,10 @@ declare const Authentication_base: HttpApiMiddleware.ServiceClass<Authentication
|
|
|
32
33
|
export declare class Authentication extends Authentication_base {
|
|
33
34
|
}
|
|
34
35
|
export interface TokenVerifierConfig {
|
|
36
|
+
/** Production workload activation and the deployment actually serving the request. */
|
|
37
|
+
readonly workloadTarget?: {
|
|
38
|
+
readonly deploymentId: string;
|
|
39
|
+
};
|
|
35
40
|
/** The control plane: the issuer of app tokens and the host of their JWKS. */
|
|
36
41
|
readonly segmentstreamUrl: string;
|
|
37
42
|
readonly applicationId: string;
|
|
@@ -39,6 +44,9 @@ export interface TokenVerifierConfig {
|
|
|
39
44
|
readonly organizationId: string;
|
|
40
45
|
}
|
|
41
46
|
export type TokenVerification = {
|
|
47
|
+
readonly status: "workload";
|
|
48
|
+
readonly execution: WorkloadExecution;
|
|
49
|
+
} | {
|
|
42
50
|
readonly status: "valid";
|
|
43
51
|
readonly identity: SignInIdentity;
|
|
44
52
|
} | {
|
|
@@ -56,6 +64,9 @@ export declare function neonAuthKeySet(jwksUrl: string): JWTVerifyGetKey;
|
|
|
56
64
|
* `SEGMENTSTREAM_URL` for this application (`aud: app:<application id>`), organization and workspace.
|
|
57
65
|
*/
|
|
58
66
|
export declare function makeTokenVerifier(config: TokenVerifierConfig, keySet: JWTVerifyGetKey): TokenVerifier;
|
|
67
|
+
export declare const WORKLOAD_TOKEN_TYPE = "segmentstream-ontology-workload+jwt";
|
|
68
|
+
/** A separate verifier: a workload token can never resolve as a human or act as an agent. */
|
|
69
|
+
export declare function makeWorkloadTokenVerifier(config: TokenVerifierConfig, keySet: JWTVerifyGetKey): TokenVerifier;
|
|
59
70
|
/**
|
|
60
71
|
* Neon Auth's issuer is the origin of its Auth URL: `https://ep-….neon.tech/ontology/auth` issues
|
|
61
72
|
* tokens with `iss: https://ep-….neon.tech`.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Cause, Context, Effect, Layer, Redacted } from "effect";
|
|
2
2
|
import { HttpServerRequest } from "effect/unstable/http";
|
|
3
3
|
import { HttpApiMiddleware, HttpApiSecurity, OpenApi } from "effect/unstable/httpapi";
|
|
4
|
-
import { createRemoteJWKSet, decodeJwt, errors, jwtVerify, } from "jose";
|
|
4
|
+
import { createRemoteJWKSet, decodeJwt, decodeProtectedHeader, errors, jwtVerify, } from "jose";
|
|
5
5
|
import { API_KEY_PREFIX } from "../runtime/principals.js";
|
|
6
6
|
import { RequestId, crossOriginSession, forbidden, internalError, notInvited, toServerResponse, unauthorized, unavailable, } from "./errors.js";
|
|
7
7
|
import { ACCESS_TOKEN_PREFIX, principalOfAccessToken } from "./oauth/protocol.js";
|
|
@@ -76,6 +76,7 @@ export function makeTokenVerifier(config, keySet) {
|
|
|
76
76
|
try {
|
|
77
77
|
const verified = await jwtVerify(token, keySet, {
|
|
78
78
|
algorithms: ["ES256"],
|
|
79
|
+
typ: "JWT",
|
|
79
80
|
issuer,
|
|
80
81
|
audience,
|
|
81
82
|
clockTolerance: CLOCK_TOLERANCE_SECONDS,
|
|
@@ -90,6 +91,96 @@ export function makeTokenVerifier(config, keySet) {
|
|
|
90
91
|
return identity === undefined ? { status: "invalid" } : { status: "valid", identity };
|
|
91
92
|
};
|
|
92
93
|
}
|
|
94
|
+
export const WORKLOAD_TOKEN_TYPE = "segmentstream-ontology-workload+jwt";
|
|
95
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
96
|
+
/** A separate verifier: a workload token can never resolve as a human or act as an agent. */
|
|
97
|
+
export function makeWorkloadTokenVerifier(config, keySet) {
|
|
98
|
+
return async (token) => {
|
|
99
|
+
if (config.workloadTarget === undefined) {
|
|
100
|
+
return { status: "invalid" };
|
|
101
|
+
}
|
|
102
|
+
let payload;
|
|
103
|
+
try {
|
|
104
|
+
const verified = await jwtVerify(token, keySet, {
|
|
105
|
+
algorithms: ["ES256"],
|
|
106
|
+
typ: WORKLOAD_TOKEN_TYPE,
|
|
107
|
+
issuer: new URL(config.segmentstreamUrl).origin,
|
|
108
|
+
audience: `ontology:${config.applicationId}`,
|
|
109
|
+
clockTolerance: CLOCK_TOLERANCE_SECONDS,
|
|
110
|
+
requiredClaims: [
|
|
111
|
+
"sub",
|
|
112
|
+
"exp",
|
|
113
|
+
"iat",
|
|
114
|
+
"jti",
|
|
115
|
+
"org",
|
|
116
|
+
"wsp",
|
|
117
|
+
"deploymentId",
|
|
118
|
+
"ontologyDeploymentId",
|
|
119
|
+
"ontologyOrigin",
|
|
120
|
+
"environment",
|
|
121
|
+
"jobName",
|
|
122
|
+
"runId",
|
|
123
|
+
"executionId",
|
|
124
|
+
"attemptId",
|
|
125
|
+
"generation",
|
|
126
|
+
],
|
|
127
|
+
});
|
|
128
|
+
payload = verified.payload;
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
return keySetUnavailable(error) ? { status: "unavailable" } : { status: "invalid" };
|
|
132
|
+
}
|
|
133
|
+
const strings = ["jti", "jobName"];
|
|
134
|
+
const ids = ["deploymentId", "ontologyDeploymentId", "runId", "attemptId"];
|
|
135
|
+
if (payload.org !== config.organizationId ||
|
|
136
|
+
payload.wsp !== config.workspaceId ||
|
|
137
|
+
payload.sub !== `pipeline-workers:${config.workspaceId}` ||
|
|
138
|
+
payload.environment !== "production" ||
|
|
139
|
+
payload.aud !== `ontology:${config.applicationId}` ||
|
|
140
|
+
strings.some((name) => typeof payload[name] !== "string" || payload[name].trim() === "") ||
|
|
141
|
+
ids.some((name) => typeof payload[name] !== "string" || !UUID.test(payload[name])) ||
|
|
142
|
+
(payload.executionId !== null &&
|
|
143
|
+
(typeof payload.executionId !== "string" || !UUID.test(payload.executionId))) ||
|
|
144
|
+
!Number.isSafeInteger(payload.generation) ||
|
|
145
|
+
payload.generation < 1 ||
|
|
146
|
+
!Number.isSafeInteger(payload.iat) ||
|
|
147
|
+
!Number.isSafeInteger(payload.exp) ||
|
|
148
|
+
payload.iat > Math.floor(Date.now() / 1000) + CLOCK_TOLERANCE_SECONDS ||
|
|
149
|
+
payload.exp <= payload.iat ||
|
|
150
|
+
payload.exp - payload.iat > 300 ||
|
|
151
|
+
typeof payload.ontologyOrigin !== "string" ||
|
|
152
|
+
!URL.canParse(payload.ontologyOrigin)) {
|
|
153
|
+
return { status: "invalid" };
|
|
154
|
+
}
|
|
155
|
+
const origin = new URL(payload.ontologyOrigin);
|
|
156
|
+
if (origin.protocol !== "https:" || origin.origin !== payload.ontologyOrigin) {
|
|
157
|
+
return { status: "invalid" };
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
status: "workload",
|
|
161
|
+
execution: {
|
|
162
|
+
issuer: payload.iss,
|
|
163
|
+
audience: payload.aud,
|
|
164
|
+
organizationId: config.organizationId,
|
|
165
|
+
workspaceId: config.workspaceId,
|
|
166
|
+
subject: payload.sub,
|
|
167
|
+
deploymentId: payload.deploymentId,
|
|
168
|
+
ontologyDeploymentId: payload.ontologyDeploymentId,
|
|
169
|
+
servingOntologyDeploymentId: config.workloadTarget.deploymentId,
|
|
170
|
+
ontologyOrigin: payload.ontologyOrigin,
|
|
171
|
+
environment: "production",
|
|
172
|
+
jobName: payload.jobName,
|
|
173
|
+
runId: payload.runId,
|
|
174
|
+
executionId: payload.executionId,
|
|
175
|
+
attemptId: payload.attemptId,
|
|
176
|
+
generation: payload.generation,
|
|
177
|
+
tokenId: payload.jti,
|
|
178
|
+
issuedAt: payload.iat,
|
|
179
|
+
expiresAt: payload.exp,
|
|
180
|
+
},
|
|
181
|
+
};
|
|
182
|
+
};
|
|
183
|
+
}
|
|
93
184
|
/**
|
|
94
185
|
* Neon Auth's issuer is the origin of its Auth URL: `https://ep-….neon.tech/ontology/auth` issues
|
|
95
186
|
* tokens with `iss: https://ep-….neon.tech`.
|
|
@@ -166,11 +257,21 @@ export function verifyByIssuer(verifiers) {
|
|
|
166
257
|
* once it has Neon Auth. Both are accepted while workspaces move to Neon Auth.
|
|
167
258
|
*/
|
|
168
259
|
export function tokenVerifier(config) {
|
|
260
|
+
const platformKeys = remoteKeySet(config.segmentstreamUrl);
|
|
261
|
+
const human = makeTokenVerifier(config, platformKeys);
|
|
262
|
+
const workload = makeWorkloadTokenVerifier(config, platformKeys);
|
|
263
|
+
const platform = (token) => {
|
|
264
|
+
try {
|
|
265
|
+
return decodeProtectedHeader(token).typ === WORKLOAD_TOKEN_TYPE
|
|
266
|
+
? workload(token)
|
|
267
|
+
: human(token);
|
|
268
|
+
}
|
|
269
|
+
catch {
|
|
270
|
+
return Promise.resolve({ status: "invalid" });
|
|
271
|
+
}
|
|
272
|
+
};
|
|
169
273
|
const verifiers = new Map([
|
|
170
|
-
[
|
|
171
|
-
new URL(config.segmentstreamUrl).origin,
|
|
172
|
-
makeTokenVerifier(config, remoteKeySet(config.segmentstreamUrl)),
|
|
173
|
-
],
|
|
274
|
+
[new URL(config.segmentstreamUrl).origin, platform],
|
|
174
275
|
]);
|
|
175
276
|
if (config.neonAuth !== undefined) {
|
|
176
277
|
verifiers.set(neonAuthIssuer(config.neonAuth.baseUrl), makeNeonAuthTokenVerifier(config.neonAuth, neonAuthKeySet(config.neonAuth.jwksUrl)));
|
|
@@ -337,6 +438,14 @@ export function authenticate(credential, verifyToken, options = {}) {
|
|
|
337
438
|
}
|
|
338
439
|
}
|
|
339
440
|
const verification = yield* Effect.promise(() => verifyToken(token));
|
|
441
|
+
if (verification.status === "workload") {
|
|
442
|
+
const principal = backend.resolvePipelineWorker === undefined
|
|
443
|
+
? undefined
|
|
444
|
+
: yield* backend.resolvePipelineWorker(verification.execution);
|
|
445
|
+
return principal === undefined
|
|
446
|
+
? { status: "rejected" }
|
|
447
|
+
: { status: "authenticated", principal };
|
|
448
|
+
}
|
|
340
449
|
if (verification.status !== "valid") {
|
|
341
450
|
return verification.status === "invalid"
|
|
342
451
|
? { status: "rejected" }
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { type OntologyManifest } from "../manifest.js";
|
|
2
2
|
/**
|
|
3
3
|
* The catalog for a reader: the manifest, with each entity's actions narrowed to the ones the given
|
|
4
|
-
* permissions may run
|
|
4
|
+
* permissions may run, and only the measures of the model whose every input entity they may read.
|
|
5
|
+
* Without permissions it is the manifest itself.
|
|
5
6
|
*/
|
|
6
7
|
export declare function catalogFor(manifest: OntologyManifest, permissions?: ReadonlyArray<string>): OntologyManifest;
|
|
7
8
|
export interface GraphRelationship {
|
package/dist/service/catalog.js
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import { isValueMatched } from "../manifest.js";
|
|
2
2
|
import { isAllowed } from "../permissions.js";
|
|
3
|
-
import { actionPermission } from "../render.js";
|
|
3
|
+
import { actionPermission, modelMeasuresFor } from "../render.js";
|
|
4
4
|
/**
|
|
5
5
|
* The catalog for a reader: the manifest, with each entity's actions narrowed to the ones the given
|
|
6
|
-
* permissions may run
|
|
6
|
+
* permissions may run, and only the measures of the model whose every input entity they may read.
|
|
7
|
+
* Without permissions it is the manifest itself.
|
|
7
8
|
*/
|
|
8
9
|
export function catalogFor(manifest, permissions) {
|
|
9
10
|
if (permissions === undefined) {
|
|
10
11
|
return manifest;
|
|
11
12
|
}
|
|
13
|
+
const { measures: declared, ...rest } = manifest;
|
|
14
|
+
const measures = declared === undefined ? [] : modelMeasuresFor(manifest, permissions);
|
|
12
15
|
return {
|
|
13
|
-
...
|
|
16
|
+
...rest,
|
|
17
|
+
...(measures.length === 0 ? {} : { measures }),
|
|
14
18
|
entities: manifest.entities
|
|
15
19
|
.filter((entity) => entity.source === undefined ||
|
|
16
20
|
isAllowed(entity.permissions?.read ?? `${entity.name}:read`, { permissions }))
|
package/dist/service/config.d.ts
CHANGED
|
@@ -66,6 +66,10 @@ export interface OntologyEnvironment {
|
|
|
66
66
|
* `segmentstreamUrl` for `workspaceId`; previews and local development have none.
|
|
67
67
|
*/
|
|
68
68
|
readonly appCredential?: Redacted.Redacted<string>;
|
|
69
|
+
/** Production deployment identity; absent in local, preview and earlier deployments. */
|
|
70
|
+
readonly workloadTarget?: {
|
|
71
|
+
readonly deploymentId: string;
|
|
72
|
+
};
|
|
69
73
|
/** Present once the workspace has Neon Auth: members then sign in with its tokens. */
|
|
70
74
|
readonly neonAuth?: NeonAuthEnvironment;
|
|
71
75
|
/**
|
package/dist/service/config.js
CHANGED
|
@@ -134,6 +134,13 @@ export function configFromEnv(environment = process.env) {
|
|
|
134
134
|
message: `The ontology service cannot start: ${list} ${verb} not set.`,
|
|
135
135
|
});
|
|
136
136
|
}
|
|
137
|
+
const deploymentId = read("SEGMENTSTREAM_ONTOLOGY_DEPLOYMENT_ID");
|
|
138
|
+
if (deploymentId !== "" &&
|
|
139
|
+
!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(deploymentId)) {
|
|
140
|
+
throw new OntologyServiceConfigError({
|
|
141
|
+
message: "SEGMENTSTREAM_ONTOLOGY_DEPLOYMENT_ID must be a UUID.",
|
|
142
|
+
});
|
|
143
|
+
}
|
|
137
144
|
const neonAuth = neonAuthFromEnv(read);
|
|
138
145
|
const runtimeIdentity = runtimeIdentityFromEnv(read);
|
|
139
146
|
const localConnections = localConnectionsFromEnv(environment);
|
|
@@ -163,6 +170,10 @@ export function configFromEnv(environment = process.env) {
|
|
|
163
170
|
...(read(APP_CREDENTIAL_VARIABLE)
|
|
164
171
|
? { appCredential: Redacted.make(read(APP_CREDENTIAL_VARIABLE)) }
|
|
165
172
|
: {}),
|
|
173
|
+
...(environment.VERCEL_ENV === "production" &&
|
|
174
|
+
read("SEGMENTSTREAM_ONTOLOGY_DEPLOYMENT_ID") !== ""
|
|
175
|
+
? { workloadTarget: { deploymentId: read("SEGMENTSTREAM_ONTOLOGY_DEPLOYMENT_ID") } }
|
|
176
|
+
: {}),
|
|
166
177
|
...(neonAuth === undefined ? {} : { neonAuth }),
|
|
167
178
|
...(runtimeIdentity === undefined ? {} : { runtimeIdentity }),
|
|
168
179
|
...(runtimeIdentity === undefined || read(AGENT_HOST_VARIABLE) === ""
|