@segmentstream-cloud/ontology 0.21.0 → 0.22.0

package/dist/migrate/apply.js

@@ -1,13 +1,12 @@
-import { randomUUID } from "node:crypto";
 import { Duration, Effect, Schema } from "effect";
 import { canonicalJson, decodeManifestDocument, encodeManifest, manifestHash, } from "../manifest.js";
 import { CatalogInvalid, CatalogMismatch, MigrationFailed, MigrationRefused } from "./errors.js";
 import { isNoopMigration } from "./model.js";
 import { planMigration } from "./plan.js";
 import { REPLICATION_DDL, REPLICATION_SCHEMA_VERSION, replicationEntityDdl, } from "../replication/schema.js";
-import { createReplicationBaseline } from "../replication/baseline.js";
+import { replicationMigrationRefusals } from "../replication/migrate.js";
 import { ATTACHMENTS_TABLE } from "../storage/tables.js";
-import { resolveReplicationSelection } from "../replication/model.js";
+import { resolveReplicationSelection, } from "../replication/model.js";
 import { AGENT_TABLES, API_KEYS_TABLE, CATALOG_TABLE, OAUTH_TABLES, USER_ROLES_TABLE, quoteIdentifier, quoteLiteral, } from "./storage.js";
 const CATALOG = quoteIdentifier(CATALOG_TABLE);
 /** Serializes migrations of one database; transaction-scoped, so it ends with the transaction. */
@@ -122,8 +121,25 @@ export const applyMigration = (plan, options) => Effect.gen(function* () {
         if (presence?.present) {
             const [state] = yield* sql.unsafe("SELECT * FROM _ontology_replication WHERE id FOR UPDATE");
             if (state?.enabled) {
-                refresh = state.exclusions ?? [];
-                // Writers wait on this row. Capture, schema changes and the new baseline commit together.
+                refresh = yield* Effect.try({
+                    try: () => resolveReplicationSelection(plan.target, state.exclusions ?? []),
+                    catch: (cause) => new CatalogInvalid({ message: "The replication exclusions are invalid.", cause }),
+                });
+                const [target] = yield* sql.unsafe("SELECT * FROM _ontology_replication_targets WHERE generation = $1::uuid", [state.active_generation]);
+                if (!target) {
+                    return yield* new CatalogInvalid({
+                        message: "The active replication target is missing; pause and rebuild replication.",
+                        cause: { generation: state.active_generation },
+                    });
+                }
+                const refusals = replicationMigrationRefusals(target, plan.target, refresh);
+                if (refusals.length > 0) {
+                    return yield* new MigrationRefused({
+                        message: refusals.map((refusal) => `${refusal.path}: ${refusal.message}`).join(" "),
+                        refusals,
+                    });
+                }
+                // Writers wait on this row. Capture and schema changes commit together on the same stream.
                 yield* sql.unsafe("UPDATE _ontology_replication SET enabled = false WHERE id");
             }
         }
@@ -148,14 +164,9 @@ export const applyMigration = (plan, options) => Effect.gen(function* () {
               "applied_at" = now()
           `, [plan.target.version, plan.target.hash, canonicalJson(encodeManifest(plan.target))]);
         if (refresh !== undefined) {
-            const selection = yield* Effect.sync(() => resolveReplicationSelection(plan.target, refresh));
-            yield* createReplicationBaseline(sql, {
-                generation: randomUUID(),
-                manifest: plan.target,
-                exclusions: refresh,
-                selection,
-                newSource: true,
-            });
+            yield* sql.unsafe(`UPDATE _ontology_replication_targets SET manifest = $1::jsonb, selection = $2::jsonb
+             WHERE generation = (SELECT active_generation FROM _ontology_replication WHERE id)`, [JSON.stringify(plan.target), JSON.stringify(refresh)]);
+            yield* sql.unsafe("UPDATE _ontology_replication SET selection = $1::jsonb, enabled = true WHERE id", [JSON.stringify(Object.fromEntries(refresh.map((item) => [item.entityId, item])))]);
         }
     }));
     return { hash: plan.target.hash, appliedSteps: plan.steps.length, changed: true };

package/dist/migrate/errors.d.ts

@@ -2,7 +2,7 @@ import { Schema } from "effect";
 declare const MigrationRefused_base: Schema.Class<MigrationRefused, Schema.TaggedStruct<"MigrationRefused", {
     readonly message: Schema.String;
     readonly refusals: Schema.$Array<Schema.Struct<{
-        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict"]>;
+        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required"]>;
         readonly path: Schema.String;
         readonly message: Schema.String;
     }>>;

package/dist/migrate/model.d.ts

@@ -25,11 +25,11 @@ export declare const MigrationStep: Schema.Struct<{
     readonly sql: Schema.$Array<Schema.String>;
 }>;
 export type MigrationStep = typeof MigrationStep.Type;
-export declare const MIGRATION_REFUSAL_CODES: readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict"];
+export declare const MIGRATION_REFUSAL_CODES: readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required"];
 export type MigrationRefusalCode = (typeof MIGRATION_REFUSAL_CODES)[number];
 /** A change the planner will not make. A plan with refusals cannot be applied. */
 export declare const MigrationRefusal: Schema.Struct<{
-    readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict"]>;
+    readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required"]>;
     /** Where, in the style of `check` diagnostics: `lead.properties.status`. */
     readonly path: Schema.String;
     readonly message: Schema.String;
@@ -284,7 +284,7 @@ export declare const MigrationPlan: Schema.Struct<{
         readonly sql: Schema.$Array<Schema.String>;
     }>>;
     readonly refusals: Schema.$Array<Schema.Struct<{
-        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict"]>;
+        readonly code: Schema.Literals<readonly ["invalid-manifest", "type-change", "relationship-change", "drop-refused", "name-collision", "identifier-too-long", "rename-conflict", "replication-rebuild-required"]>;
         /** Where, in the style of `check` diagnostics: `lead.properties.status`. */
         readonly path: Schema.String;
         readonly message: Schema.String;

package/dist/migrate/model.js

@@ -55,6 +55,7 @@ export const MIGRATION_REFUSAL_CODES = [
     "name-collision",
     "identifier-too-long",
     "rename-conflict",
+    "replication-rebuild-required",
 ];
 /** A change the planner will not make. A plan with refusals cannot be applied. */
 export const MigrationRefusal = Schema.Struct({

package/dist/replication/baseline.d.ts

@@ -1,4 +1,4 @@
-import { Effect } from "effect";
+import { Effect, Schema } from "effect";
 import type { SqlClient } from "effect/unstable/sql";
 import type { OntologyManifest } from "../manifest.js";
 import { type ReplicationSelection, type ReplicationExclusion } from "./model.js";
@@ -9,6 +9,13 @@ export interface ReplicationTarget {
     readonly manifest: OntologyManifest;
     readonly selection: ReadonlyArray<ReplicationSelection>;
 }
+declare const ReplicationBaselineFailed_base: Schema.Class<ReplicationBaselineFailed, Schema.TaggedStruct<"ReplicationBaselineFailed", {
+    readonly message: Schema.String;
+    readonly cause: Schema.optionalKey<Schema.Unknown>;
+}>, import("effect/Cause").YieldableError>;
+/** An initial snapshot cannot be created with the requested state or record projection. */
+export declare class ReplicationBaselineFailed extends ReplicationBaselineFailed_base {
+}
 /** All baseline rows are committed with capture enablement; interrupted setup can safely resume. */
 export declare const createReplicationBaseline: (sql: SqlClient.SqlClient, input: {
     readonly generation: string;
@@ -16,4 +23,5 @@ export declare const createReplicationBaseline: (sql: SqlClient.SqlClient, input
     readonly exclusions?: ReadonlyArray<ReplicationExclusion>;
     readonly selection: ReadonlyArray<ReplicationSelection>;
     readonly newSource?: boolean;
-}) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;
+}) => Effect.Effect<ReplicationTarget, ReplicationBaselineFailed | import("effect/unstable/sql/SqlError").SqlError, never>;
+export {};

package/dist/replication/baseline.js

@@ -1,26 +1,35 @@
 import { randomUUID } from "node:crypto";
-import { Effect } from "effect";
+import { Effect, Schema } from "effect";
 import { joinColumnNames, joinTableName, quoteIdentifier as qi, quoteLiteral as ql, tableName, } from "../migrate/storage.js";
 import { REPLICATION_RECORD_BYTES, validateReplicationSelection, } from "./model.js";
 import { snapshotExpression } from "./schema.js";
-const MAX_BASELINE_RECORDS = 50_000;
+/** An initial snapshot cannot be created with the requested state or record projection. */
+export class ReplicationBaselineFailed extends Schema.TaggedError()("ReplicationBaselineFailed", { message: Schema.String, cause: Schema.optionalKey(Schema.Unknown) }) {
+}
 /** All baseline rows are committed with capture enablement; interrupted setup can safely resume. */
 export const createReplicationBaseline = (sql, input) => sql.withTransaction(Effect.gen(function* () {
     yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
     yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
     const [state] = yield* sql.unsafe("SELECT source_id::text, revision::text, enabled FROM _ontology_replication WHERE id FOR UPDATE");
     if (!state) {
-        return yield* Effect.die(new Error("Run the ontology SDK migration before enabling replication."));
+        return yield* new ReplicationBaselineFailed({
+            message: "Run the ontology SDK migration before enabling replication.",
+        });
     }
     const existing = yield* sql.unsafe("SELECT * FROM _ontology_replication_targets WHERE generation = $1::uuid", [input.generation]);
     if (existing[0]) {
         return existing[0];
     }
     if (state.enabled) {
-        return yield* Effect.die(new Error("Pause replication before starting a new generation."));
+        return yield* new ReplicationBaselineFailed({
+            message: "Pause replication before starting a new generation.",
+        });
     }
     const manifest = input.manifest;
-    yield* Effect.sync(() => validateReplicationSelection(manifest, input.selection));
+    yield* Effect.try({
+        try: () => validateReplicationSelection(manifest, input.selection),
+        catch: (cause) => new ReplicationBaselineFailed({ message: "Invalid replication projection.", cause }),
+    });
     // Drain older SDK transactions that started before the compatibility fence was enabled.
     const tables = manifest.entities
         .filter((e) => e.source === undefined && !e.builtIn)
@@ -57,17 +66,17 @@ export const createReplicationBaseline = (sql, input) => sql.withTransaction(Eff
         JSON.stringify(manifest),
         JSON.stringify(input.selection),
     ]);
-    let total = 0;
     const baseline = (entityId, kind, records) => Effect.gen(function* () {
         // Keep the snapshot in PostgreSQL; large baselines must not fill a serverless process's heap.
-        const [size] = yield* sql.unsafe(`SELECT count(*)::int AS count, max(octet_length(payload::text)) AS largest
-          FROM (${records} LIMIT ${MAX_BASELINE_RECORDS + 1}) snapshot`);
-        total += size.count;
-        if (total > MAX_BASELINE_RECORDS) {
-            return yield* Effect.die(new Error("The replication baseline exceeds 50,000 records; use a bulk bootstrap."));
-        }
-        if ((size.largest ?? 0) > REPLICATION_RECORD_BYTES) {
-            return yield* Effect.die(new Error("A projected baseline record exceeds the replication size limit."));
+        const [size] = yield* sql.unsafe(`SELECT EXISTS (
+              SELECT 1 FROM (${records}) snapshot
+              WHERE octet_length(payload::text) > ${REPLICATION_RECORD_BYTES}
+            ) AS oversized`);
+        if (size.oversized) {
+            return yield* new ReplicationBaselineFailed({
+                message: `A projected baseline record exceeds ${REPLICATION_RECORD_BYTES} bytes; ` +
+                    "exclude large fields before enabling replication.",
+            });
         }
         yield* sql.unsafe(`INSERT INTO _ontology_replication_baselines (generation, event)
       SELECT $1::uuid, jsonb_build_object('source_id', $2::text, 'entity_id', $3::text,

package/dist/replication/migrate.d.ts

@@ -0,0 +1,6 @@
+import type { OntologyManifest } from "../manifest.js";
+import type { MigrationRefusal } from "../migrate/model.js";
+import type { ReplicationTarget } from "./baseline.js";
+import type { ReplicationSelection } from "./model.js";
+/** A live stream can change its projection only when retained JSON still describes current rows. */
+export declare function replicationMigrationRefusals(current: ReplicationTarget, manifest: OntologyManifest, selection: ReadonlyArray<ReplicationSelection>): ReadonlyArray<MigrationRefusal>;

package/dist/replication/migrate.js

@@ -0,0 +1,64 @@
+import { toSnakeCase } from "../names.js";
+import { queryFields } from "../query.js";
+/** A live stream can change its projection only when retained JSON still describes current rows. */
+export function replicationMigrationRefusals(current, manifest, selection) {
+    const refusals = [];
+    const refuse = (path, reason) => {
+        refusals.push({
+            code: "replication-rebuild-required",
+            path,
+            message: `${reason} Pause replication, apply the migration, then explicitly rebuild replication.`,
+        });
+    };
+    for (const previous of current.selection) {
+        const entity = current.manifest.entities.find((item) => item.id === previous.entityId);
+        const next = selection.find((item) => item.entityId === entity.id);
+        if (!next) {
+            refuse(entity.name, "Removing a replicated entity would discard its pending history.");
+            continue;
+        }
+        for (const field of previous.fields) {
+            if (!next.fields.includes(field)) {
+                refuse(`${entity.name}.${field}`, "Removing or renaming a replicated field would leave retained JSON under its old name.");
+            }
+        }
+        for (const relationship of previous.relationships ?? []) {
+            if (!next.relationships?.includes(relationship)) {
+                refuse(`${entity.name}.${relationship}`, "Removing a replicated relationship would leave its old edges in retained JSON.");
+            }
+        }
+    }
+    for (const next of selection) {
+        const entity = manifest.entities.find((item) => item.id === next.entityId);
+        const previousEntity = current.manifest.entities.find((item) => item.id === entity.id);
+        // A newly created table starts empty and all subsequent writes are captured normally.
+        if (!previousEntity) {
+            continue;
+        }
+        const previous = current.selection.find((item) => item.entityId === entity.id);
+        if (!previous) {
+            refuse(entity.name, "Previously excluded records are absent from retained JSON.");
+            continue;
+        }
+        const previousFields = new Set(queryFields(previousEntity).map((field) => field.name));
+        for (const field of next.fields.filter((name) => !previous.fields.includes(name))) {
+            const property = entity.properties.find((item) => item.name === field);
+            const renamed = property?.renamedFrom;
+            if (previousFields.has(field) ||
+                (renamed !== undefined &&
+                    (previousFields.has(renamed) || previousFields.has(toSnakeCase(renamed))))) {
+                refuse(`${entity.name}.${field}`, "Existing values are absent from this JSON field.");
+            }
+            else if (property?.default !== undefined && property.default !== null) {
+                refuse(`${entity.name}.${field}`, "Adding a field with a non-null default changes existing rows without captured writes.");
+            }
+        }
+        for (const relationship of next.relationships ?? []) {
+            if (!previous.relationships?.includes(relationship) &&
+                previousEntity.relationships.some((item) => item.name === relationship)) {
+                refuse(`${entity.name}.${relationship}`, "Previously excluded relationship edges are absent from retained JSON.");
+            }
+        }
+    }
+    return refusals;
+}

package/dist/replication/model.d.ts

@@ -1,5 +1,5 @@
 import { type OntologyManifest } from "../manifest.js";
-/** The resolved projection for one immutable generation. `id` is always included. */
+/** The current projection of a continuous replication stream. `id` is always included. */
 export interface ReplicationSelection {
     readonly entityId: string;
     readonly fields: ReadonlyArray<string>;

package/dist/replication/store.d.ts

@@ -1,14 +1,14 @@
 import { Effect } from "effect";
 import type { SqlClient } from "effect/unstable/sql";
 import { type ReplicationEvent, type ReplicationSelection, type ReplicationExclusion } from "./model.js";
-import { type ReplicationTarget } from "./baseline.js";
+import { ReplicationBaselineFailed, type ReplicationTarget } from "./baseline.js";
 export type { ReplicationTarget } from "./baseline.js";
 export declare const startReplication: (sql: SqlClient.SqlClient, input: {
     readonly generation: string;
     readonly exclusions?: ReadonlyArray<ReplicationExclusion>;
     readonly selection?: ReadonlyArray<ReplicationSelection>;
     readonly newSource?: boolean;
-}) => Effect.Effect<ReplicationTarget, import("../index.js").CatalogInvalid | import("effect/unstable/sql/SqlError").SqlError, never>;
+}) => Effect.Effect<ReplicationTarget, import("../index.js").CatalogInvalid | ReplicationBaselineFailed | import("effect/unstable/sql/SqlError").SqlError, never>;
 export declare const activeReplicationTarget: (sql: SqlClient.SqlClient) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;
 export declare const pauseReplication: (sql: SqlClient.SqlClient) => import("effect/unstable/sql/Statement").Statement<object>;
 export declare const replicationTarget: (sql: SqlClient.SqlClient, generation: string) => Effect.Effect<ReplicationTarget, import("effect/unstable/sql/SqlError").SqlError, never>;

package/dist/replication/store.js

@@ -2,17 +2,22 @@ import { Effect } from "effect";
 import { readCatalog } from "../migrate/apply.js";
 import { historyTableName, quoteIdentifier as qi } from "../migrate/storage.js";
 import { REPLICATION_BATCH_BYTES, REPLICATION_BATCH_SIZE, publishMessage, replicationEventId, resolveReplicationSelection, } from "./model.js";
-import { createReplicationBaseline } from "./baseline.js";
+import { createReplicationBaseline, ReplicationBaselineFailed, } from "./baseline.js";
 export const startReplication = (sql, input) => sql.withTransaction(Effect.gen(function* () {
     yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
     yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
     yield* sql.unsafe("SELECT id FROM _ontology_replication WHERE id FOR UPDATE");
     const manifest = yield* readCatalog({ sql });
     if (!manifest) {
-        return yield* Effect.die(new Error("The ontology has not been migrated."));
+        return yield* new ReplicationBaselineFailed({
+            message: "The ontology has not been migrated.",
+        });
     }
     const selection = input.selection ??
-        (yield* Effect.sync(() => resolveReplicationSelection(manifest, input.exclusions)));
+        (yield* Effect.try({
+            try: () => resolveReplicationSelection(manifest, input.exclusions),
+            catch: (cause) => new ReplicationBaselineFailed({ message: "Invalid replication exclusions.", cause }),
+        }));
     return yield* createReplicationBaseline(sql, { ...input, manifest, selection });
 }));
 export const activeReplicationTarget = (sql) => sql

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@segmentstream-cloud/ontology",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "description": "Define a SegmentStream workspace app's ontology: entities, relationships, roles and actions, written with Effect.",
   "license": "Apache-2.0",
   "author": "SegmentStream",