@segmentstream-cloud/ontology 0.11.0 → 0.13.0

package/dist/replication/model.js

@@ -0,0 +1,99 @@
+import { createHash } from "node:crypto";
+import { isValueMatched } from "../manifest.js";
+import { queryFields } from "../query.js";
+export function resolveReplicationSelection(manifest, exclusions = []) {
+    const entities = manifest.entities.filter((entity) => !entity.builtIn && !entity.source);
+    if (new Set(exclusions.map((item) => item.entityId)).size !== exclusions.length) {
+        throw new Error("Replication exclusions must name distinct entities.");
+    }
+    for (const item of exclusions) {
+        const entity = entities.find((candidate) => candidate.id === item.entityId);
+        if (!entity) {
+            throw new Error(`Unknown stored business entity in replication exclusions: ${item.entityId}.`);
+        }
+        const fields = new Set(queryFields(entity).map((field) => field.name));
+        const relationships = new Set(entity.relationships.map((relationship) => relationship.name));
+        if (item.fields?.some((field) => field === "id" || !fields.has(field))) {
+            throw new Error(`Unknown or required replication field on ${entity.name}.`);
+        }
+        if (item.relationships?.some((relationship) => !relationships.has(relationship))) {
+            throw new Error(`Unknown replication relationship on ${entity.name}.`);
+        }
+        // A value-matched relationship has no `<name>_id`: its reference is the property `on` names.
+        const matched = entity.relationships.find((relationship) => item.relationships?.includes(relationship.name) && isValueMatched(relationship));
+        if (matched !== undefined) {
+            throw new Error(`${entity.name}.${matched.name} is matched by value; exclude its property ` +
+                `${matched.on?.property ?? ""} instead.`);
+        }
+    }
+    const included = entities.filter((entity) => !exclusions.some((item) => item.entityId === entity.id && item.disabled));
+    const selection = included.map((entity) => {
+        const excluded = exclusions.find((item) => item.entityId === entity.id);
+        const keyed = entity.relationships.filter((relationship) => relationship.cardinality === "one" && !isValueMatched(relationship));
+        const excludedFields = new Set([
+            ...(excluded?.fields ?? []),
+            ...keyed
+                .filter((relationship) => excluded?.relationships?.includes(relationship.name))
+                .map((relationship) => `${relationship.name}_id`),
+        ]);
+        return {
+            entityId: entity.id,
+            fields: queryFields(entity)
+                .filter((field) => !excludedFields.has(field.name))
+                .map((field) => field.name),
+            relationships: entity.relationships
+                .filter((relationship) => relationship.cardinality === "many" &&
+                !excluded?.relationships?.includes(relationship.name) &&
+                included.some((target) => target.name === relationship.to))
+                .map((relationship) => relationship.name),
+        };
+    });
+    validateReplicationSelection(manifest, selection);
+    return selection;
+}
+export const REPLICATION_BATCH_SIZE = 100;
+export const REPLICATION_BATCH_BYTES = 1_000_000;
+// Leaves room for JSON escaping, the envelope and base64 inside a 1 MB publish request.
+export const REPLICATION_RECORD_BYTES = 200_000;
+export function validateReplicationSelection(manifest, selection) {
+    if (new Set(selection.map((s) => s.entityId)).size !== selection.length) {
+        throw new Error("Replicated entities must be distinct.");
+    }
+    const viewNames = new Set();
+    const addView = (name) => {
+        const key = name.toLowerCase();
+        if (viewNames.has(key)) {
+            throw new Error("Replicated entity and relationship view names collide.");
+        }
+        viewNames.add(key);
+    };
+    for (const item of selection) {
+        const entity = manifest.entities.find((e) => e.id === item.entityId);
+        if (!entity || entity.name === "user" || entity.source !== undefined) {
+            throw new Error(`Only stored business entities can be replicated: ${item.entityId}.`);
+        }
+        addView(entity.name);
+        const fields = new Set(queryFields(entity).map((f) => f.name));
+        if (item.fields.some((field) => !fields.has(field))) {
+            throw new Error(`Unknown replication field on ${entity.name}.`);
+        }
+        for (const name of item.relationships ?? []) {
+            const relationship = entity.relationships.find((r) => r.name === name);
+            const target = manifest.entities.find((e) => e.name === relationship?.to);
+            if (relationship?.cardinality !== "many" ||
+                !target ||
+                target.name === "user" ||
+                !selection.some((s) => s.entityId === target.id)) {
+                throw new Error(`Replication of ${entity.name}.${name} requires both business entities.`);
+            }
+            addView(`${entity.name}_${name}`);
+        }
+    }
+}
+export function replicationEventId(source, history, generation) {
+    return createHash("sha256").update(`${source}:${history}:${generation}`).digest("hex");
+}
+/** Includes base64 and JSON overhead, not just the original payload bytes. */
+export function publishMessage(event) {
+    return { data: Buffer.from(JSON.stringify(event)).toString("base64") };
+}

package/dist/replication/schema.d.ts

@@ -0,0 +1,7 @@
+import type { EntityManifest, OntologyManifest } from "../manifest.js";
+export declare const REPLICATION_SCHEMA_VERSION = 3;
+/** Installed only by the SDK migration runner, never at service startup. */
+export declare const REPLICATION_DDL: string[];
+export declare function snapshotExpression(entity: EntityManifest, alias: string): string;
+/** History is the durable source. Triggers also reject old writers while capture is enabled. */
+export declare function replicationEntityDdl(manifest: OntologyManifest): ReadonlyArray<string>;

package/dist/replication/schema.js

@@ -0,0 +1,123 @@
+import { createHash } from "node:crypto";
+import { queryFields } from "../query.js";
+import { columnName, historyTableName, quoteIdentifier as qi, quoteLiteral as ql, relationshipColumnName, SYSTEM_COLUMNS, tableName, joinTableName, } from "../migrate/storage.js";
+import { REPLICATION_RECORD_BYTES } from "./model.js";
+export const REPLICATION_SCHEMA_VERSION = 3;
+/** Installed only by the SDK migration runner, never at service startup. */
+export const REPLICATION_DDL = [
+    `CREATE TABLE IF NOT EXISTS _ontology_replication (
+    id boolean PRIMARY KEY DEFAULT true CHECK (id), version integer NOT NULL,
+    source_id uuid NOT NULL DEFAULT gen_random_uuid(), revision bigint NOT NULL DEFAULT 0,
+    selection jsonb NOT NULL DEFAULT '{}'::jsonb, enabled boolean NOT NULL DEFAULT false
+  )`,
+    `INSERT INTO _ontology_replication (id, version) VALUES (true, ${REPLICATION_SCHEMA_VERSION}) ON CONFLICT DO NOTHING`,
+    `ALTER TABLE _ontology_replication ADD COLUMN IF NOT EXISTS exclusions jsonb NOT NULL DEFAULT '[]'::jsonb`,
+    `ALTER TABLE _ontology_replication ADD COLUMN IF NOT EXISTS active_generation uuid`,
+    `CREATE TABLE IF NOT EXISTS _ontology_replication_targets (
+    generation uuid PRIMARY KEY, source_id uuid NOT NULL, from_revision bigint NOT NULL,
+    manifest jsonb NOT NULL, selection jsonb NOT NULL, created_at timestamptz NOT NULL DEFAULT now()
+  )`,
+    `CREATE TABLE IF NOT EXISTS _ontology_replication_baselines (
+    generation uuid NOT NULL REFERENCES _ontology_replication_targets(generation),
+    id uuid PRIMARY KEY DEFAULT gen_random_uuid(), event jsonb NOT NULL
+  )`,
+    `CREATE TABLE IF NOT EXISTS _ontology_replication_receipts (
+    generation uuid NOT NULL REFERENCES _ontology_replication_targets(generation),
+    event_id uuid NOT NULL, accepted_at timestamptz NOT NULL DEFAULT now(), verified_at timestamptz,
+    PRIMARY KEY (generation, event_id)
+  )`,
+    `CREATE OR REPLACE FUNCTION _ontology_replication_guard() RETURNS trigger LANGUAGE plpgsql AS $$
+  BEGIN
+    IF (SELECT enabled FROM _ontology_replication WHERE id) AND
+       current_setting('segmentstream.writer', true) IS DISTINCT FROM '1' THEN
+      RAISE EXCEPTION 'Replication requires a compatible ontology writer; pause replication before a data migration';
+    END IF;
+    RETURN NULL;
+  END $$`,
+];
+export function snapshotExpression(entity, alias) {
+    const values = queryFields(entity).flatMap((field) => {
+        const system = SYSTEM_COLUMNS[field.name];
+        const column = system ??
+            (entity.properties.some((p) => p.name === field.name)
+                ? columnName(field.name)
+                : relationshipColumnName(field.name.slice(0, -3)));
+        // JSON numbers cannot safely carry a PostgreSQL bigint across all transports.
+        const cast = field.storage === "integer" ? "::text" : "";
+        return [ql(field.name), `${alias}.${qi(column)}${cast}`];
+    });
+    const chunks = [];
+    for (let offset = 0; offset < values.length; offset += 80) {
+        chunks.push(`jsonb_build_object(${values.slice(offset, offset + 80).join(", ")})`);
+    }
+    return `(${chunks.join(" || ")})`;
+}
+/** History is the durable source. Triggers also reject old writers while capture is enabled. */
+export function replicationEntityDdl(manifest) {
+    return manifest.entities
+        .filter((e) => e.source === undefined && e.name !== "user")
+        .flatMap((entity) => {
+        const history = qi(historyTableName(entity.name));
+        const table = qi(tableName(entity.name));
+        // Keep names stable across renames and below PostgreSQL's 63-byte identifier limit.
+        const key = createHash("sha256").update(entity.id).digest("hex").slice(0, 40);
+        const fn = qi(`_replicate_${key}`);
+        const declared = entity.relationships
+            .filter((r) => r.cardinality === "many")
+            .map((r) => r.name);
+        return [
+            `ALTER TABLE ${history} ADD COLUMN IF NOT EXISTS replication jsonb`,
+            `CREATE INDEX IF NOT EXISTS ${qi(`_replication_${key}_idx`)} ON ${history}
+        (((replication->>'revision')::bigint)) WHERE replication IS NOT NULL`,
+            `CREATE OR REPLACE FUNCTION ${fn}() RETURNS trigger LANGUAGE plpgsql AS $$
+      DECLARE state _ontology_replication; policy jsonb; snapshot jsonb; relation text; other_id text;
+              record_key text; record_kind text := 'entity'; is_deleted boolean := NEW.kind = 'delete';
+      BEGIN
+        SELECT * INTO state FROM _ontology_replication WHERE id;
+        policy := state.selection->${ql(entity.id)};
+        IF NOT state.enabled OR policy IS NULL THEN RETURN NEW; END IF;
+        record_key := NEW.record_id::text;
+        IF NEW.kind IN ('link', 'unlink') THEN
+          SELECT key INTO relation FROM jsonb_object_keys(NEW.changes) AS key LIMIT 1;
+          IF NOT (${ql(JSON.stringify(declared))}::jsonb ? relation) OR
+             NOT (COALESCE(policy->'relationships', '[]'::jsonb) ? relation) THEN RETURN NEW; END IF;
+          other_id := COALESCE(NEW.changes->relation->>'after', NEW.changes->relation->>'before');
+          record_key := NEW.record_id::text || ':' || other_id;
+          record_kind := 'edge:' || relation;
+          is_deleted := NEW.kind = 'unlink';
+          snapshot := jsonb_build_object('source_id', NEW.record_id::text, 'target_id', other_id);
+        ELSIF is_deleted THEN snapshot := '{}'::jsonb;
+        ELSE
+          SELECT ${snapshotExpression(entity, "record")} INTO snapshot FROM ${table} AS record WHERE id = NEW.record_id;
+          SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) INTO snapshot
+            FROM jsonb_each(snapshot) WHERE key = 'id' OR (policy->'fields' ? key);
+          IF snapshot IS NULL OR snapshot = '{}'::jsonb THEN RAISE EXCEPTION 'Replication snapshot is missing'; END IF;
+        END IF;
+        IF octet_length(snapshot::text) > ${REPLICATION_RECORD_BYTES} THEN
+          RAISE EXCEPTION 'Projected ontology record exceeds the replication size limit';
+        END IF;
+        UPDATE _ontology_replication SET revision = revision + 1 WHERE id RETURNING * INTO state;
+        NEW.replication := jsonb_build_object('source_id', state.source_id::text,
+          'entity_id', ${ql(entity.id)}, 'record_key', record_key, 'record_kind', record_kind,
+          'revision', state.revision::text, 'deleted', is_deleted,
+          'schema_hash', ${ql(manifest.hash)}, 'payload', snapshot::text);
+        RETURN NEW;
+      END $$`,
+            `DROP TRIGGER IF EXISTS _ontology_capture ON ${history}`,
+            `CREATE TRIGGER _ontology_capture BEFORE INSERT ON ${history} FOR EACH ROW EXECUTE FUNCTION ${fn}()`,
+            `DROP TRIGGER IF EXISTS _ontology_writer_guard ON ${table}`,
+            `CREATE TRIGGER _ontology_writer_guard BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON ${table}
+        FOR EACH STATEMENT EXECUTE FUNCTION _ontology_replication_guard()`,
+            ...entity.relationships
+                .filter((r) => r.cardinality === "many")
+                .flatMap((r) => {
+                const join = qi(joinTableName(entity.name, r.name));
+                return [
+                    `DROP TRIGGER IF EXISTS _ontology_writer_guard ON ${join}`,
+                    `CREATE TRIGGER _ontology_writer_guard BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON ${join}
+              FOR EACH STATEMENT EXECUTE FUNCTION _ontology_replication_guard()`,
+                ];
+            }),
+        ];
+    });
+}

package/dist/replication/store.d.ts

@@ -0,0 +1,35 @@
+import { Effect } from "effect";
+import type { SqlClient } from "effect/unstable/sql";
+import { type ReplicationEvent, type ReplicationSelection, type ReplicationExclusion } from "./model.js";
+import { type ReplicationTarget } from "./baseline.js";
+export type { ReplicationTarget } from "./baseline.js";
+export declare const startReplication: (sql: SqlClient.SqlClient, input: {
+    readonly generation: string;
+    readonly exclusions?: ReadonlyArray<ReplicationExclusion>;
+    readonly selection?: ReadonlyArray<ReplicationSelection>;
+    readonly newSource?: boolean;
+}) => Effect.Effect<ReplicationTarget, import("../index.js").CatalogInvalid | import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const activeReplicationTarget: (sql: SqlClient.SqlClient) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const pauseReplication: (sql: SqlClient.SqlClient) => import("effect/unstable/sql/Statement").Statement<object>;
+export declare const replicationTarget: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;
+/** No timestamp/high-water cursor: every unacknowledged immutable event remains discoverable. */
+export declare function replicationEventsSql(target: ReplicationTarget): string;
+export declare const pendingReplicationEvents: (sql: SqlClient.SqlClient, target: ReplicationTarget) => Effect.Effect<{
+    readonly historyId: string;
+    readonly event: ReplicationEvent;
+}[], import("effect/unstable/sql/SqlError").SqlError, never>;
+/** Call only after Pub/Sub accepts the complete batch. Lost responses deliberately produce duplicates. */
+export declare const acknowledgeReplication: (sql: SqlClient.SqlClient, generation: string, ids: ReadonlyArray<string>) => import("effect/unstable/sql/Statement").Statement<object>;
+export declare const replayReplication: (sql: SqlClient.SqlClient, generation: string) => import("effect/unstable/sql/Statement").Statement<object>;
+export declare const replicationStatus: (sql: SqlClient.SqlClient, target: ReplicationTarget) => Effect.Effect<{
+    total: string;
+    accepted: string;
+    verified: string;
+}, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const unverifiedReplicationEvents: (sql: SqlClient.SqlClient, target: ReplicationTarget) => Effect.Effect<{
+    historyId: string;
+    eventId: string;
+}[], import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const verifyReplicationEvents: (sql: SqlClient.SqlClient, generation: string, ids: ReadonlyArray<string>) => import("effect/unstable/sql/Statement").Statement<object>;
+/** Retry accepted events that never appeared, including dead letters and transport expiry. */
+export declare const retryUnverifiedReplication: (sql: SqlClient.SqlClient, generation: string) => import("effect/unstable/sql/Statement").Statement<object>;

package/dist/replication/store.js

@@ -0,0 +1,88 @@
+import { Effect } from "effect";
+import { readCatalog } from "../migrate/apply.js";
+import { historyTableName, quoteIdentifier as qi } from "../migrate/storage.js";
+import { REPLICATION_BATCH_BYTES, REPLICATION_BATCH_SIZE, publishMessage, replicationEventId, resolveReplicationSelection, } from "./model.js";
+import { createReplicationBaseline } from "./baseline.js";
+export const startReplication = (sql, input) => sql.withTransaction(Effect.gen(function* () {
+    yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
+    yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
+    yield* sql.unsafe("SELECT id FROM _ontology_replication WHERE id FOR UPDATE");
+    const manifest = yield* readCatalog({ sql });
+    if (!manifest) {
+        return yield* Effect.die(new Error("The ontology has not been migrated."));
+    }
+    const selection = input.selection ??
+        (yield* Effect.sync(() => resolveReplicationSelection(manifest, input.exclusions)));
+    return yield* createReplicationBaseline(sql, { ...input, manifest, selection });
+}));
+export const activeReplicationTarget = (sql) => sql
+    .unsafe(`SELECT t.generation::text, t.source_id::text, t.from_revision::text, t.manifest, t.selection
+     FROM _ontology_replication_targets t JOIN _ontology_replication r ON r.active_generation = t.generation
+     WHERE r.id`)
+    .pipe(Effect.map((rows) => rows[0]));
+export const pauseReplication = (sql) => sql.unsafe("UPDATE _ontology_replication SET enabled = false WHERE id");
+export const replicationTarget = (sql, generation) => sql
+    .unsafe(`SELECT generation::text, source_id::text, from_revision::text, manifest, selection
+     FROM _ontology_replication_targets WHERE generation = $1::uuid`, [generation])
+    .pipe(Effect.map((rows) => rows[0]));
+/** No timestamp/high-water cursor: every unacknowledged immutable event remains discoverable. */
+export function replicationEventsSql(target) {
+    const histories = target.selection.map((item) => {
+        const entity = target.manifest.entities.find((e) => e.id === item.entityId);
+        return `SELECT id, replication AS event FROM ${qi(historyTableName(entity.name))}
+      WHERE replication IS NOT NULL AND replication->>'source_id' = $2
+      AND (replication->>'revision')::bigint > $3::bigint`;
+    });
+    const baseline = `SELECT id, event FROM _ontology_replication_baselines WHERE generation = $1::uuid
+    AND $2::text IS NOT NULL AND $3::bigint IS NOT NULL`;
+    return [baseline, ...histories].join(" UNION ALL ");
+}
+export const pendingReplicationEvents = (sql, target) => sql
+    .unsafe(`SELECT e.id::text, e.event FROM (${replicationEventsSql(target)}) e
+     WHERE NOT EXISTS (SELECT 1 FROM _ontology_replication_receipts r
+       WHERE r.generation = $1::uuid AND r.event_id = e.id)
+     ORDER BY (event->>'revision')::bigint, id LIMIT ${REPLICATION_BATCH_SIZE}`, [target.generation, target.source_id, target.from_revision])
+    .pipe(Effect.map((rows) => {
+    let size = 32;
+    const result = [];
+    for (const row of rows) {
+        const event = {
+            ...row.event,
+            generation: target.generation,
+            event_id: replicationEventId(target.source_id, row.id, target.generation),
+        };
+        const bytes = Buffer.byteLength(JSON.stringify(publishMessage(event))) + 1;
+        if (bytes + 32 > REPLICATION_BATCH_BYTES) {
+            throw new Error("A replication event exceeds the publish request limit.");
+        }
+        if (size + bytes > REPLICATION_BATCH_BYTES) {
+            break;
+        }
+        size += bytes;
+        result.push({ historyId: row.id, event });
+    }
+    return result;
+}));
+/** Call only after Pub/Sub accepts the complete batch. Lost responses deliberately produce duplicates. */
+export const acknowledgeReplication = (sql, generation, ids) => sql.unsafe(`INSERT INTO _ontology_replication_receipts (generation, event_id)
+     SELECT $1::uuid, value::uuid FROM jsonb_array_elements_text($2::jsonb) ON CONFLICT DO NOTHING`, [generation, JSON.stringify(ids)]);
+export const replayReplication = (sql, generation) => sql.unsafe("DELETE FROM _ontology_replication_receipts WHERE generation = $1::uuid", [
+    generation,
+]);
+export const replicationStatus = (sql, target) => sql
+    .unsafe(`SELECT count(*)::text AS total, count(r.event_id)::text AS accepted, count(r.verified_at)::text AS verified
+     FROM (${replicationEventsSql(target)}) e
+     LEFT JOIN _ontology_replication_receipts r ON r.generation = $1::uuid AND r.event_id = e.id`, [target.generation, target.source_id, target.from_revision])
+    .pipe(Effect.map((rows) => rows[0]));
+export const unverifiedReplicationEvents = (sql, target) => sql
+    .unsafe(`SELECT event_id::text AS id FROM _ontology_replication_receipts
+     WHERE generation = $1::uuid AND verified_at IS NULL ORDER BY accepted_at, event_id LIMIT 1000`, [target.generation])
+    .pipe(Effect.map((rows) => rows.map(({ id }) => ({
+    historyId: id,
+    eventId: replicationEventId(target.source_id, id, target.generation),
+}))));
+export const verifyReplicationEvents = (sql, generation, ids) => sql.unsafe(`UPDATE _ontology_replication_receipts SET verified_at = now()
+    WHERE generation = $1::uuid AND event_id IN (SELECT value::uuid FROM jsonb_array_elements_text($2::jsonb))`, [generation, JSON.stringify(ids)]);
+/** Retry accepted events that never appeared, including dead letters and transport expiry. */
+export const retryUnverifiedReplication = (sql, generation) => sql.unsafe(`DELETE FROM _ontology_replication_receipts WHERE generation = $1::uuid
+    AND verified_at IS NULL AND accepted_at < now() - interval '15 minutes'`, [generation]);

package/dist/replication/transaction.d.ts

@@ -0,0 +1,4 @@
+import { Effect } from "effect";
+import type { SqlClient } from "effect/unstable/sql";
+/** Must run inside the write transaction, before any business row locks. */
+export declare const beginReplicationWrite: (sql: SqlClient.SqlClient) => Effect.Effect<void, import("effect/unstable/sql/SqlError").SqlError, never>;

package/dist/replication/transaction.js

@@ -0,0 +1,11 @@
+import { Effect } from "effect";
+/** Must run inside the write transaction, before any business row locks. */
+export const beginReplicationWrite = (sql) => Effect.gen(function* () {
+    const [schema] = yield* sql.unsafe("SELECT to_regclass('_ontology_replication') IS NOT NULL AS present");
+    if (!schema?.present) {
+        return;
+    } // Platform principal operations also serve pre-upgrade databases.
+    yield* sql.unsafe("SELECT set_config('segmentstream.writer', '1', true)");
+    // Lock even while disabled: enabling and the baseline must exclude concurrent writes.
+    yield* sql.unsafe("SELECT id FROM _ontology_replication WHERE id FOR UPDATE");
+});

package/dist/runtime/pipeline.js

@@ -1,4 +1,5 @@
 import { createHash, randomUUID } from "node:crypto";
+import { beginReplicationWrite } from "../replication/transaction.js";
 import { Cause, Duration, Effect, Option, Schema } from "effect";
 import { SqlClient } from "effect/unstable/sql";
 import { isSqlError } from "effect/unstable/sql/SqlError";
@@ -204,7 +205,7 @@ export function runAction(ontology, request) {
             return result;
         });
         const result = yield* sql
-            .withTransaction(transactionalHandler)
+            .withTransaction(beginReplicationWrite(sql).pipe(Effect.andThen(transactionalHandler)))
             .pipe(Effect.catchIf(isSqlError, (error) => Effect.die(error)));
         if (replay) {
             return replay;

package/dist/runtime/principals.js

@@ -11,6 +11,7 @@ import { buildOntology } from "../ontology.js";
 import { permissionsOf } from "../permissions.js";
 import { EMAIL_PATTERN } from "../property.js";
 import { role } from "../role.js";
+import { beginReplicationWrite } from "../replication/transaction.js";
 import { User } from "../user.js";
 import { MemberNotFound, PrincipalRejected, UnknownRole } from "./errors.js";
 import { isRecordId } from "./records.js";
@@ -116,6 +117,7 @@ function signIn(ontology, sql, claims) {
     const email = claimedEmail !== undefined && EMAIL_PATTERN.test(claimedEmail) ? claimedEmail : undefined;
     return sql
         .withTransaction(Effect.gen(function* () {
+        yield* beginReplicationWrite(sql);
         const now = new Date();
         const signedIn = { last_sign_in_at: now.toISOString() };
         const [linked] = yield* runtime.find({
@@ -295,6 +297,7 @@ export function setUserRoles(ontology, input) {
         }
         const roles = [...new Set(input.roles)];
         yield* sql.withTransaction(Effect.gen(function* () {
+            yield* beginReplicationWrite(sql);
             yield* sql.unsafe(`DELETE FROM "${USER_ROLES_TABLE}" WHERE "user_id" = $1::uuid`, [
                 input.userId,
             ]);
@@ -313,6 +316,7 @@ export function createServiceUser(ontology, input) {
         const sql = yield* SqlClient.SqlClient;
         const runtime = platformRuntime(ontology, sql, "user.create_service");
         return yield* sql.withTransaction(Effect.gen(function* () {
+            yield* beginReplicationWrite(sql);
             const user = yield* runtime
                 .create({ entity: "user", values: { name: input.name, kind: "service" } })
                 .pipe(Effect.orDie);
@@ -576,6 +580,7 @@ export function updateMemberRoles(ontology, change) {
         yield* refuseMember(revoke === "all" ? [] : revoke);
         const runtime = platformRuntime(ontology, sql, "user.invite");
         const attempt = sql.withTransaction(Effect.gen(function* () {
+            yield* beginReplicationWrite(sql);
             const found = yield* lockUser(runtime, { email });
             if (found !== undefined && found.kind !== "member") {
                 return yield* new MemberNotFound({
@@ -636,6 +641,7 @@ export function setMemberActive(ontology, input) {
         const action = input.active ? "user.reactivate" : "user.deactivate";
         const runtime = platformRuntime(ontology, sql, action);
         return yield* sql.withTransaction(Effect.gen(function* () {
+            yield* beginReplicationWrite(sql);
             const user = yield* lockUser(runtime, input);
             if (user === undefined || user.kind !== "member") {
                 return yield* memberNotFound(input);

package/dist/runtime/service-binding.js

@@ -60,6 +60,14 @@ export function backendLayer(ontology, blobStorage) {
             checkSignIn: (email) => withSql(checkSignIn(ontology, email)),
             resolveApiKey: (key) => withSql(resolvePrincipalByApiKey(ontology, Redacted.value(key))).pipe(Effect.catchTag("PrincipalRejected", rejectedAs("api_key"))),
             appliedHash: readCatalog({ sql }).pipe(Effect.map((manifest) => manifest?.hash), Effect.orDie),
+            replicationEnabled: Effect.gen(function* () {
+                const [presence] = yield* sql.unsafe("SELECT to_regclass('_ontology_replication') IS NOT NULL AS present");
+                if (!presence?.present) {
+                    return false;
+                }
+                const [state] = yield* sql.unsafe("SELECT enabled FROM _ontology_replication WHERE id");
+                return state?.enabled === true;
+            }).pipe(Effect.orDie),
             oauth: sqlOAuthStore(ontology, sql),
         });
     }));

package/dist/runtime/sql-runtime.js

@@ -1,4 +1,5 @@
 import { Effect } from "effect";
+import { beginReplicationWrite } from "../replication/transaction.js";
 import { isSqlError } from "effect/unstable/sql/SqlError";
 import { Conflict } from "../errors.js";
 import { InvalidQuery, WriteOutsideAction } from "./errors.js";
@@ -67,7 +68,7 @@ export function makeOntologyRuntime(ontology, sql, scope) {
         }
         const edits = [];
         const result = yield* sql
-            .withTransaction(operation({ ...context, scope, edits }, entity))
+            .withTransaction(beginReplicationWrite(sql).pipe(Effect.andThen(operation({ ...context, scope, edits }, entity))))
             .pipe(Effect.catchIf(isSqlError, (error) => uniqueConflict(entity, error)));
         for (const edit of edits) {
             scope.onEdit?.(edit);

package/dist/service/aggregate-platform.d.ts

@@ -0,0 +1,60 @@
+import { Effect, Redacted } from "effect";
+import { AggregateUpstreamUnavailable } from "./errors.js";
+/**
+ * How long one aggregate may take end to end, the response included. BigQuery answers most within
+ * seconds; the margin covers a cold warehouse and the hops to and from SegmentStream.
+ */
+export declare const AGGREGATE_TIMEOUT_MS = 60000;
+/**
+ * The hash of the manifest the service resolved and checked the request against. SegmentStream
+ * answers `409 ontology_version_mismatch` when its deployed manifest differs, so a request checked
+ * against one ontology never runs against another.
+ */
+export declare const MANIFEST_HASH_HEADER = "x-ontology-manifest-hash";
+/**
+ * The entities the service checked `<entity>:read` on, sorted and comma-separated. SegmentStream
+ * answers `409 aggregate_scope_mismatch` when it would read any other, so the service's resolution
+ * and SegmentStream's can never drift apart unnoticed.
+ */
+export declare const TOUCHED_HEADER = "x-ontology-touched";
+/** SegmentStream would read beyond what the service checked: this deployment's SDK is behind. */
+export declare const SCOPE_MISMATCH = "aggregate_scope_mismatch";
+/** SegmentStream's answer: its status, its JSON body as sent, and that body parsed. */
+export interface PlatformAnswer {
+    readonly status: number;
+    readonly text: string;
+    readonly body: Readonly<Record<string, unknown>>;
+    /** SegmentStream's `Retry-After`, on a 429 or a 503 that sent one. */
+    readonly retryAfter?: string;
+}
+/**
+ * Sends a checked aggregate to SegmentStream: model-level, or the per-entity shorthand of `entity`.
+ * `touched` is every entity the service required `<entity>:read` on.
+ */
+export type AggregateTransport = (input: {
+    readonly entity?: string;
+    readonly body: unknown;
+    readonly requestId: string;
+    readonly touched: ReadonlyArray<string>;
+}) => Effect.Effect<PlatformAnswer, AggregateUpstreamUnavailable>;
+/**
+ * Where a deployment sends aggregates: its platform, its workspace and its app credential, with the
+ * hash of the manifest it serves.
+ */
+export interface AggregateEnvironment {
+    readonly segmentstreamUrl: string;
+    readonly workspaceId: string;
+    readonly appCredential?: Redacted.Redacted<string>;
+    readonly manifestHash: string;
+}
+/**
+ * SegmentStream's `POST /api/v1/workspaces/{id}/aggregate`, and its per-entity
+ * `…/entities/{entity}/aggregate`, called with the workspace app credential. `undefined` when the deployment has none, or when SegmentStream's origin is plain
+ * http to another machine, which is logged once. Every request names the manifest it was checked
+ * against, and a redirect is refused, so the credential never leaves for another origin.
+ *
+ * Every answer but a 2xx is logged with both request ids, the status and the start of the body; a
+ * failed call with its message, stack and causes. The credential is redacted from all of them. A
+ * caller who goes away interrupts the call, which is not SegmentStream's failure.
+ */
+export declare function platformAggregates(environment: AggregateEnvironment, fetchImpl?: typeof fetch): AggregateTransport | undefined;