@segmentstream/ontology 0.0.0-stage → 0.92.0

package/dist/replication/baseline.js

@@ -0,0 +1,215 @@
+import { enableShadowCapture } from "./shadow.js";
+import { randomUUID } from "node:crypto";
+import { Effect, Schema } from "effect";
+import { joinColumnNames, joinTableName, quoteIdentifier as qi, quoteLiteral as ql, tableName, } from "../migrate/storage.js";
+import { REPLICATION_RECORD_BYTES, validateReplicationSelection, } from "./model.js";
+import { REPLICATION_TARGET_FILLS, REPLICATION_TARGET_METADATA, snapshotExpression, } from "./schema.js";
+import { lockReplicationLifecycle, shareReplicationLifecycle } from "./transaction.js";
+import { initialReplicationLineage } from "./lineage.js";
+import { scheduleReplicationScans, replicationScanScopes, advanceReplicationScan, } from "./scans.js";
+/** An initial scan cannot be created with the requested state or record projection. */
+export class ReplicationBaselineFailed extends Schema.TaggedError()("ReplicationBaselineFailed", { message: Schema.String, cause: Schema.optionalKey(Schema.Unknown) }) {
+}
+const targetSql = `SELECT generation::text, source_id::text, from_revision::text, manifest,
+  selection, protocol, baseline_complete, backfill_complete, initial_revision::text,
+  ${REPLICATION_TARGET_FILLS}, ${REPLICATION_TARGET_METADATA}, replay_complete, baseline_cursor
+  FROM _ontology_replication_targets WHERE generation = $1::uuid`;
+/** Enable capture in a short fenced transaction before scanning any business records. */
+export const prepareReplicationBaseline = (sql, input) => sql.withTransaction(Effect.gen(function* () {
+    yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
+    yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
+    yield* lockReplicationLifecycle(sql);
+    const [state] = yield* sql.unsafe("SELECT source_id::text, revision::text, enabled, version FROM _ontology_replication WHERE id");
+    if (!state || state.version < 4) {
+        return yield* new ReplicationBaselineFailed({
+            message: "Run the ontology SDK migration before enabling replication.",
+        });
+    }
+    const [existing] = yield* sql.unsafe(targetSql, [input.generation]);
+    if (existing) {
+        return existing;
+    }
+    const [catalog] = yield* sql.unsafe("SELECT hash FROM _ontology WHERE id");
+    if (catalog?.hash !== input.manifest.hash) {
+        return yield* new ReplicationBaselineFailed({
+            message: "The ontology changed before replication started; retry with the current manifest.",
+        });
+    }
+    yield* Effect.try({
+        try: () => validateReplicationSelection(input.manifest, input.selection),
+        catch: (cause) => new ReplicationBaselineFailed({ message: "Invalid replication projection.", cause }),
+    });
+    // Drain raw/pre-upgrade writers only at enablement; online pages never lock business tables.
+    const tables = input.manifest.entities
+        .filter((e) => e.source === undefined && !e.builtIn)
+        .flatMap((e) => [
+        tableName(e.name),
+        ...e.relationships
+            .filter((r) => r.cardinality === "many")
+            .map((r) => joinTableName(e.name, r.name)),
+    ]);
+    if (tables.length > 0) {
+        yield* sql.unsafe(`LOCK TABLE ${tables.map(qi).join(", ")} IN SHARE MODE`);
+    }
+    if (!state.enabled && state.version >= 6) {
+        yield* sql.unsafe("UPDATE _ontology_replication_targets SET capturing=false");
+    }
+    const source = input.newSource ? randomUUID() : state.source_id;
+    yield* sql.unsafe(`INSERT INTO _ontology_replication_targets
+          (generation, source_id, from_revision, manifest, selection, protocol, baseline_complete)
+         VALUES ($1::uuid, $2::uuid, 0, $3::jsonb, $4::jsonb, 4, false)`, [input.generation, source, JSON.stringify(input.manifest), JSON.stringify(input.selection)]);
+    yield* sql.unsafe(`UPDATE _ontology_replication SET source_id = $1::uuid, selection = $2::jsonb,
+          enabled = true, exclusions = $3::jsonb, active_generation = $4::uuid WHERE id`, [
+        source,
+        JSON.stringify(Object.fromEntries(input.selection.map((s) => [s.entityId, s]))),
+        JSON.stringify(input.exclusions ?? []),
+        input.generation,
+    ]);
+    if (state.version >= 5) {
+        yield* sql.unsafe(`UPDATE _ontology_replication_targets SET lineage = $2::jsonb, scans_complete = false
+           WHERE generation = $1::uuid`, [input.generation, JSON.stringify(initialReplicationLineage(input))]);
+        yield* scheduleReplicationScans(sql, input.generation, replicationScanScopes(input), {
+            initial: true,
+        });
+    }
+    const [target] = yield* sql.unsafe(targetSql, [input.generation]);
+    if (state.version >= 6) {
+        yield* enableShadowCapture(sql, target, replicationScanScopes(target));
+    }
+    return target;
+}));
+const BASELINE_PAGE_SIZE = 250;
+function baselineSources(target) {
+    return target.selection.flatMap((item) => {
+        const entity = target.manifest.entities.find((e) => e.id === item.entityId);
+        return [
+            {
+                entityId: entity.id,
+                kind: "entity",
+                records: `SELECT r.id::text AS record_key,
+          (SELECT jsonb_object_agg(key, value) FROM jsonb_each(${snapshotExpression(entity, "r")})
+           WHERE key = 'id' OR ${ql(JSON.stringify(item.fields))}::jsonb ? key) AS payload
+         FROM ${qi(tableName(entity.name))} r
+         WHERE r.id >= COALESCE(NULLIF($2::text, '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid)
+           AND ($2::text = '' OR r.id > NULLIF($2::text, '')::uuid)
+         ORDER BY r.id LIMIT ${BASELINE_PAGE_SIZE}`,
+            },
+            ...entity.relationships
+                .filter((r) => item.relationships?.includes(r.name))
+                .map((relationship) => {
+                const columns = joinColumnNames({
+                    entity: entity.name,
+                    relationship: relationship.name,
+                    target: relationship.to,
+                });
+                return {
+                    entityId: entity.id,
+                    kind: `edge:${relationship.name}`,
+                    records: `SELECT ${qi(columns.source)}::text || ':' || ${qi(columns.target)}::text AS record_key,
+            jsonb_build_object('source_id', ${qi(columns.source)}::text,
+              'target_id', ${qi(columns.target)}::text) AS payload
+           FROM ${qi(joinTableName(entity.name, relationship.name))}
+           WHERE (${qi(columns.source)}, ${qi(columns.target)}) >= (
+             COALESCE(NULLIF(split_part($2::text, ':', 1), '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid),
+             COALESCE(NULLIF(split_part($2::text, ':', 2), '')::uuid, '00000000-0000-0000-0000-000000000000'::uuid))
+             AND ($2::text = '' OR (${qi(columns.source)}, ${qi(columns.target)}) > (
+               NULLIF(split_part($2::text, ':', 1), '')::uuid, NULLIF(split_part($2::text, ':', 2), '')::uuid))
+           ORDER BY ${qi(columns.source)}, ${qi(columns.target)} LIMIT ${BASELINE_PAGE_SIZE}`,
+                };
+            }),
+        ];
+    });
+}
+/**
+ * Commit one keyset page and its cursor together. Schema 5 locks source rows before allocating
+ * snapshot revisions; older schemas retain their revision-zero baseline protocol. A failed page
+ * retries without changing earlier pages, and capture covers inserts/deletes across pages.
+ */
+export const advanceReplicationBaseline = (sql, generation) => Effect.gen(function* () {
+    const [version] = yield* sql.unsafe("SELECT version FROM _ontology_replication WHERE id");
+    if ((version?.version ?? 4) >= 5) {
+        yield* advanceReplicationScan(sql, generation);
+        const [target] = yield* sql.unsafe(targetSql, [generation]);
+        return target;
+    }
+    const progress = yield* sql.withTransaction(Effect.gen(function* () {
+        yield* sql.unsafe("SET LOCAL statement_timeout = '30s'");
+        yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
+        yield* shareReplicationLifecycle(sql);
+        const [target] = yield* sql.unsafe(`${targetSql} FOR NO KEY UPDATE`, [generation]);
+        if (!target) {
+            return yield* new ReplicationBaselineFailed({
+                message: "The replication target is missing.",
+            });
+        }
+        if (target.baseline_complete !== false) {
+            return { target, finish: false };
+        }
+        const [state] = yield* sql.unsafe("SELECT enabled, active_generation::text FROM _ontology_replication WHERE id");
+        if (!state?.enabled || state.active_generation !== generation) {
+            return yield* new ReplicationBaselineFailed({
+                message: "Resume the active replication generation before scanning its baseline.",
+            });
+        }
+        const index = target.baseline_cursor.index ?? 0;
+        const source = baselineSources(target)[index];
+        if (!source) {
+            return { target, finish: true };
+        }
+        // Materialize once in PostgreSQL; only the small progress result crosses the transport.
+        const [page] = yield* sql.unsafe(`WITH page AS MATERIALIZED (
+          SELECT * FROM (${source.records}) records
+        ), valid AS (
+          SELECT NOT EXISTS(SELECT 1 FROM page WHERE octet_length(payload::text) > ${REPLICATION_RECORD_BYTES}) AS ok
+        ), inserted AS (
+          INSERT INTO _ontology_replication_baselines (generation, event)
+          SELECT $1::uuid, jsonb_build_object('source_id', $3::text, 'entity_id', $4::text,
+            'record_key', record_key, 'record_kind', $5::text, 'revision', '0',
+            'deleted', false, 'schema_hash', $6::text, 'payload', payload::text)
+          FROM page WHERE (SELECT ok FROM valid) RETURNING id
+        ), queued AS (
+          INSERT INTO _ontology_replication_delivery (generation, event_id, revision, is_baseline)
+          SELECT $1::uuid, id, 0, true FROM inserted RETURNING event_id
+        ) SELECT (SELECT count(*)::integer FROM queued) AS count,
+          (SELECT max(record_key) FROM page) AS key, NOT (SELECT ok FROM valid) AS oversized`, [
+            generation,
+            target.baseline_cursor.key ?? "",
+            target.source_id,
+            source.entityId,
+            source.kind,
+            target.manifest.hash,
+        ]);
+        if (page.oversized) {
+            return yield* new ReplicationBaselineFailed({
+                message: `A projected baseline record exceeds ${REPLICATION_RECORD_BYTES} bytes; ` +
+                    "exclude large fields before enabling replication.",
+            });
+        }
+        const cursor = page.count < BASELINE_PAGE_SIZE ? { index: index + 1 } : { index, key: page.key };
+        yield* sql.unsafe("UPDATE _ontology_replication_targets SET baseline_cursor = $2::jsonb WHERE generation = $1::uuid", [generation, JSON.stringify(cursor)]);
+        return { target, finish: false };
+    }));
+    if (!progress.finish) {
+        return progress.target;
+    }
+    // Do not upgrade a shared lock: start a fresh transaction with the legacy singleton fence.
+    return yield* sql.withTransaction(Effect.gen(function* () {
+        yield* sql.unsafe("SET LOCAL lock_timeout = '5s'");
+        yield* lockReplicationLifecycle(sql);
+        const [state] = yield* sql.unsafe("SELECT enabled, active_generation::text FROM _ontology_replication WHERE id");
+        if (!state?.enabled || state.active_generation !== generation) {
+            return yield* new ReplicationBaselineFailed({
+                message: "Resume the active replication generation before completing its baseline.",
+            });
+        }
+        yield* sql.unsafe(`UPDATE _ontology_replication_targets SET baseline_complete = true,
+        initial_revision = (SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM _ontology_replication_revision)
+        WHERE generation = $1::uuid AND NOT baseline_complete`, [generation]);
+        const [target] = yield* sql.unsafe(targetSql, [generation]);
+        return target;
+    }));
+});
+/** Prepare once, then advance a bounded page; subsequent calls resume the same durable target. */
+export const createReplicationBaseline = (sql, input) => prepareReplicationBaseline(sql, input).pipe(Effect.flatMap((target) => target.baseline_complete === false
+    ? advanceReplicationBaseline(sql, target.generation)
+    : Effect.succeed(target)));

package/dist/replication/bigquery.d.ts

@@ -0,0 +1,37 @@
+import type { ReplicationTarget } from "./store.js";
+export declare const REPLICATION_TABLE_SCHEMA: readonly [...{
+    name: string;
+    type: string;
+    mode: string;
+}[], {
+    readonly name: "revision";
+    readonly type: "NUMERIC";
+    readonly mode: "REQUIRED";
+}, {
+    readonly name: "deleted";
+    readonly type: "BOOLEAN";
+    readonly mode: "REQUIRED";
+}, {
+    readonly name: "payload";
+    readonly type: "JSON";
+    readonly mode: "REQUIRED";
+}];
+/**
+ * Project typed fields before ranking, so a caller reading a few columns can prune JSON subfields
+ * instead of carrying whole historical payloads through the window. Keep tombstones until after
+ * ranking. The struct separates business field names from the private ranking/deletion columns.
+ * Events captured before a field was added lack its key and read as its fill.
+ */
+export declare function replicationViews(project: string, target: ReplicationTarget): {
+    readonly name: string;
+    readonly query: string;
+}[];
+/**
+ * A single BigQuery statement observes one selector snapshot across every entity/edge join.
+ * Each branch has a literal generation predicate for clustered pruning. Unsupported new columns
+ * fail honestly until their complete epoch is selected; they never synthesize null coverage.
+ */
+export declare function shadowReplicationViews(project: string, epochs: readonly import("./shadow.js").ShadowEpoch[], policy?: ReplicationTarget | undefined): {
+    name: string;
+    query: string;
+}[];

package/dist/replication/bigquery.js

@@ -0,0 +1,211 @@
+import { permittedReplicationField } from "./identities.js";
+import { queryFields } from "../query.js";
+export const REPLICATION_TABLE_SCHEMA = [
+    ...[
+        "event_id",
+        "source_id",
+        "generation",
+        "entity_id",
+        "record_key",
+        "record_kind",
+        "schema_hash",
+    ].map((name) => ({ name, type: "STRING", mode: "REQUIRED" })),
+    { name: "revision", type: "NUMERIC", mode: "REQUIRED" },
+    { name: "deleted", type: "BOOLEAN", mode: "REQUIRED" },
+    { name: "payload", type: "JSON", mode: "REQUIRED" },
+];
+// A quoted BigQuery string cannot contain a raw line break.
+const literal = (value) => `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'").replaceAll("\n", "\\n").replaceAll("\r", "\\r")}'`;
+const identifier = (value) => {
+    if (!/^[A-Za-z0-9_-]+$/.test(value)) {
+        throw new Error("Invalid BigQuery replication identifier.");
+    }
+    return `\`${value}\``;
+};
+/** A fill in the form the captured branch takes before its cast: JSON, or JSON_VALUE's string. */
+const fill = (storage, value) => storage === "json"
+    ? `PARSE_JSON(${literal(JSON.stringify(value))})`
+    : literal(typeof value === "string" ? value : JSON.stringify(value));
+const projectedValue = (storage, binding) => {
+    const fallback = Object.hasOwn(binding, "fill")
+        ? fill(storage, binding.fill)
+        : `CAST(NULL AS ${storage === "json" ? "JSON" : "STRING"})`;
+    if (binding.key === null) {
+        return fallback;
+    }
+    const path = literal(`$.${binding.key}`);
+    const captured = storage === "json" ? `JSON_QUERY(payload, ${path})` : `JSON_VALUE(payload, ${path})`;
+    return Object.hasOwn(binding, "fill")
+        ? `IF(JSON_QUERY(payload, ${path}) IS NULL, ${fallback}, ${captured})`
+        : captured;
+};
+/** CASE chooses one schema incarnation; explicit JSON null never falls back to an older key. */
+function lineageValue(target, entity, field, storage) {
+    const epochs = target.lineage?.epochs;
+    if (!epochs?.length) {
+        const fills = target.fills?.[entity] ?? {};
+        return projectedValue(storage, {
+            key: field,
+            ...(Object.hasOwn(fills, field) ? { fill: fills[field] } : {}),
+        });
+    }
+    const groups = [];
+    for (const epoch of epochs) {
+        const value = projectedValue(storage, epoch.fields[entity]?.[field] ?? { key: null });
+        const previous = groups.at(-1);
+        if (previous?.value === value) {
+            previous.through = epoch.throughRevision;
+        }
+        else {
+            groups.push({ value, through: epoch.throughRevision });
+        }
+    }
+    if (groups.length === 1) {
+        return groups[0].value;
+    }
+    return `CASE ${groups
+        .slice(0, -1)
+        .map((group) => `WHEN revision <= CAST(${literal(group.through)} AS NUMERIC) THEN ${group.value}`)
+        .join(" ")} ELSE ${groups.at(-1).value} END`;
+}
+/** Field storage read through SAFE_CAST. */
+const castTypes = {
+    integer: "INT64",
+    number: "FLOAT64",
+    boolean: "BOOL",
+    date: "DATE",
+    timestamp: "TIMESTAMP",
+};
+/** The BigQuery type a projected field reads as: JSON_QUERY yields JSON, JSON_VALUE yields STRING. */
+const columnType = (storage) => storage === "json" ? "JSON" : (castTypes[storage] ?? "STRING");
+/**
+ * Project typed fields before ranking, so a caller reading a few columns can prune JSON subfields
+ * instead of carrying whole historical payloads through the window. Keep tombstones until after
+ * ranking. The struct separates business field names from the private ranking/deletion columns.
+ * Events captured before a field was added lack its key and read as its fill.
+ */
+export function replicationViews(project, target) {
+    identifier(project);
+    const result = [];
+    const latest = (entity, kind, columns) => {
+        const minimum = target.lineage?.minimumRevisions[entity]?.[kind];
+        return `SELECT record.* FROM (
+      SELECT STRUCT(${columns}) AS record, deleted,
+        ROW_NUMBER() OVER (PARTITION BY record_key ORDER BY revision DESC, event_id DESC) AS position
+      FROM \`${project}.ontology_internal.events\`
+      WHERE generation = ${literal(target.generation)} AND source_id = ${literal(target.source_id)}
+        AND entity_id = ${literal(entity)} AND record_kind = ${literal(kind)}
+        ${minimum === undefined ? "" : `AND revision > CAST(${literal(minimum)} AS NUMERIC)`}
+    ) WHERE position = 1 AND NOT deleted`;
+    };
+    for (const selection of target.selection) {
+        const entity = target.manifest.entities.find((e) => e.id === selection.entityId);
+        const columns = queryFields(entity)
+            .filter((field) => field.name === "id" || selection.fields.includes(field.name))
+            .map((field) => {
+            const type = castTypes[field.storage];
+            const value = lineageValue(target, entity.id, field.name, field.storage);
+            return `${type ? `SAFE_CAST(${value} AS ${type})` : value} AS ${identifier(field.name)}`;
+        });
+        result.push({ name: entity.name, query: latest(entity.id, "entity", columns.join(", ")) });
+        for (const relationship of selection.relationships ?? []) {
+            result.push({
+                name: `${entity.name}_${relationship}`,
+                query: latest(entity.id, `edge:${relationship}`, "JSON_VALUE(payload, '$.source_id') AS source_id, JSON_VALUE(payload, '$.target_id') AS target_id"),
+            });
+        }
+    }
+    if (new Set(result.map((v) => v.name.toLowerCase())).size !== result.length) {
+        throw new Error("Replicated entity and relationship view names collide.");
+    }
+    return result;
+}
+/**
+ * A single BigQuery statement observes one selector snapshot across every entity/edge join.
+ * Each branch has a literal generation predicate for clustered pruning. Unsupported new columns
+ * fail honestly until their complete epoch is selected; they never synthesize null coverage.
+ */
+export function shadowReplicationViews(project, epochs, policy = epochs.at(-1)?.target) {
+    identifier(project);
+    const scopes = new Map();
+    // A placeholder is typed like the column it stands in for; an untyped NULL reads as INT64.
+    const types = new Map();
+    for (const epoch of epochs) {
+        for (const route of epoch.routes) {
+            const entity = route.target.manifest.entities.find((item) => item.id === route.entityId);
+            const name = route.kind === "entity" ? entity.name : `${entity.name}_${route.kind.slice(5)}`;
+            const view = replicationViews(project, route.target).find((view) => view.name === name);
+            if (!view) {
+                throw new Error("Replication route has no projection.");
+            }
+            const fields = route.kind === "entity"
+                ? queryFields(entity).filter((field) => field.name === "id" ||
+                    route.target.selection
+                        .find((item) => item.entityId === entity.id)
+                        ?.fields.includes(field.name))
+                : [];
+            const columns = route.kind === "entity" ? fields.map((field) => field.name) : ["source_id", "target_id"];
+            for (const field of fields) {
+                // Use the latest incarnation's type; older property identities remain policy-gated.
+                types.set(`${name}.${field.name}`, columnType(field.storage));
+            }
+            const entries = scopes.get(name) ?? [];
+            entries.push({
+                epoch: epoch.id,
+                query: view.query,
+                columns,
+                entityId: route.entityId,
+                kind: route.kind,
+                generation: route.target.generation,
+                target: route.target,
+            });
+            scopes.set(name, entries);
+        }
+    }
+    return [...scopes].map(([name, branches]) => {
+        const columns = [...new Set(branches.flatMap((branch) => branch.columns))];
+        const empty = (column) => `CAST(NULL AS ${types.get(`${name}.${column}`) ?? "STRING"})`;
+        const missing = `NOT EXISTS(SELECT 1 FROM \`${project}.ontology_internal.serving_selector\` selector
+      JOIN \`${project}.ontology_internal.epoch_routes\` route ON route.epoch=selector.epoch
+      WHERE selector.singleton AND (${branches
+            .map((branch) => `(selector.epoch=${literal(branch.epoch)} AND route.entity_id=${literal(branch.entityId)}
+          AND route.record_kind=${literal(branch.kind)} AND route.generation=${literal(branch.generation)})`)
+            .join(" OR ")}))`;
+        // COUNT(*) can prune projected errors; the fallback row predicate must fail too.
+        const unavailableColumns = columns
+            .map((column) => `IF(${missing}, ERROR('ontology_unavailable'), ${empty(column)}) AS ${identifier(column)}`)
+            .join(", ");
+        const permitted = (branch, column) => {
+            const selected = policy?.selection.find((item) => item.entityId === branch.entityId);
+            if (!selected) {
+                return false;
+            }
+            if (branch.kind !== "entity") {
+                return (policy !== undefined &&
+                    permittedReplicationField(branch.target, policy, branch.entityId, branch.kind));
+            }
+            return (policy !== undefined &&
+                permittedReplicationField(branch.target, policy, branch.entityId, column));
+        };
+        return {
+            name,
+            query: branches
+                .map((branch) => `SELECT ${columns
+                .map((column) => branch.columns.includes(column) && permitted(branch, column)
+                ? `data.${identifier(column)}`
+                : `IF(EXISTS(SELECT 1 FROM \`${project}.ontology_internal.serving_selector\`
+                      WHERE singleton AND epoch=${literal(branch.epoch)}),
+                    ERROR('ontology_coverage_unavailable'), ${empty(column)}) AS ${identifier(column)}`)
+                .join(", ")}
+      FROM (${branch.query}) data WHERE EXISTS (SELECT 1 FROM \`${project}.ontology_internal.serving_selector\` selector
+          JOIN \`${project}.ontology_internal.epoch_routes\` route ON route.epoch=selector.epoch
+        WHERE selector.singleton AND selector.epoch=${literal(branch.epoch)}
+          AND route.entity_id=${literal(branch.entityId)} AND route.record_kind=${literal(branch.kind)}
+          AND route.generation=${literal(branch.generation)})`)
+                .join("\nUNION ALL\n") +
+                `\nUNION ALL SELECT ${unavailableColumns}
+          FROM (SELECT 1) unavailable
+          WHERE IF(${missing}, ERROR('ontology_unavailable'), FALSE)`,
+        };
+    });
+}

package/dist/replication/coverage.d.ts

@@ -0,0 +1,14 @@
+import type { ReplicationTarget } from "./baseline.js";
+import type { ShadowEpoch } from "./shadow.js";
+/** Coverage is proven by identity against the selected epoch and the current export policy. */
+export declare function shadowQueryCoverage(project: string, epoch: ShadowEpoch, policy: ReplicationTarget, requested: ReplicationTarget): {
+    entities: {
+        entityId: string;
+        name: string;
+        fields: string[];
+    }[];
+    views: {
+        name: string;
+        query: string;
+    }[];
+};

package/dist/replication/coverage.js

@@ -0,0 +1,53 @@
+import { replicationViews } from "./bigquery.js";
+import { replicationIdentities } from "./identities.js";
+const identifier = (name) => {
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
+        throw new Error("replication_coverage_identifier_invalid");
+    }
+    return `\`${name}\``;
+};
+/** Coverage is proven by identity against the selected epoch and the current export policy. */
+export function shadowQueryCoverage(project, epoch, policy, requested) {
+    const requestIds = replicationIdentities(requested);
+    const policyIds = replicationIdentities(policy);
+    const entities = [];
+    const views = [];
+    for (const selection of requested.selection) {
+        const route = epoch.routes.find((item) => item.entityId === selection.entityId && item.kind === "entity");
+        const declaration = requested.manifest.entities.find((item) => item.id === selection.entityId);
+        if (!route || !declaration) {
+            continue;
+        }
+        const actualIds = replicationIdentities(route.target)[selection.entityId] ?? {};
+        const actualSelection = route.target.selection.find((item) => item.entityId === selection.entityId);
+        const fields = [...new Set(["id", ...selection.fields])].flatMap((field) => {
+            const identity = requestIds[selection.entityId]?.[field];
+            const actual = ["id", ...(actualSelection?.fields ?? [])].find((name) => identity !== undefined &&
+                actualIds[name] === identity &&
+                [
+                    "id",
+                    ...(policy.selection.find((item) => item.entityId === selection.entityId)?.fields ??
+                        []),
+                ].some((permitted) => policyIds[selection.entityId]?.[permitted] === identity) &&
+                policy.selection.some((item) => item.entityId === selection.entityId));
+            return actual ? [{ field, actual }] : [];
+        });
+        const actualName = route.target.manifest.entities.find((item) => item.id === selection.entityId).name;
+        const query = replicationViews(project, route.target).find((item) => item.name === actualName).query;
+        if (!fields.length) {
+            continue;
+        }
+        entities.push({
+            entityId: selection.entityId,
+            name: declaration.name,
+            fields: fields.map((item) => item.field),
+        });
+        views.push({
+            name: declaration.name,
+            query: `SELECT ${fields
+                .map(({ field, actual }) => `data.${identifier(actual)} AS ${identifier(field)}`)
+                .join(", ")} FROM (${query}) data`,
+        });
+    }
+    return { entities, views };
+}

package/dist/replication/evolution.d.ts

@@ -0,0 +1,25 @@
+import { Effect } from "effect";
+import type { SqlClient } from "effect/unstable/sql";
+import type { RetainedStorage } from "../migrate/model.js";
+import type { ReplicationTarget } from "./baseline.js";
+import { type ReplicationScanScope } from "./scans.js";
+/** Freeze an import's source order before migrations change declaration order or storage names. */
+export declare const prepareReplicationEvolution: (sql: SqlClient.SqlClient) => Effect.Effect<void, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare function replicationRecoveryScopes(current: ReplicationTarget, target: ReplicationTarget, retained?: RetainedStorage): {
+    scopes: ReplicationScanScope[];
+    resets: ReplicationScanScope[];
+};
+export declare const refreshReplicationProjection: (sql: SqlClient.SqlClient, current: ReplicationTarget, target: ReplicationTarget, retained?: RetainedStorage, options?: {
+    readonly resuming?: boolean;
+}) => Effect.Effect<void, import("effect/unstable/sql/SqlError").SqlError | import("./baseline.js").ReplicationBaselineFailed, never>;
+/** Pending and replayable envelopes outlive storage explicitly dropped by a supported migration. */
+export declare const retainReplicationHistory: (sql: SqlClient.SqlClient, table: string) => Effect.Effect<void, import("effect/unstable/sql/SqlError").SqlError, never>;
+export declare const renameReplicationHistory: (sql: SqlClient.SqlClient, previous: string, next: string) => Effect.Effect<void, import("effect/unstable/sql/SqlError").SqlError, never>;
+/** Physical cleanup may proceed after cutover; only storage still needed by serving is protected. */
+export declare const dropsServingReplicationStorage: (sql: SqlClient.SqlClient, input: {
+    readonly current: import("../manifest.js").OntologyManifest;
+    readonly target: ReplicationTarget;
+    readonly version: number;
+    readonly retained?: RetainedStorage;
+    readonly steps: readonly import("../migrate/model.js").MigrationStep[];
+}) => Effect.Effect<boolean, import("effect/unstable/sql/SqlError").SqlError, never>;