@smart-data-engines/sde 0.1.0-dev.0

package/dist/index.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Smart Data Engine - client library for TypeScript.
+ *
+ * You declare entities and relations. We decide which database engine each colocation group lives in,
+ * what its physical layout is there, and when it should move - and your code never names a table or
+ * an engine, which is exactly what lets us change both without touching it.
+ *
+ * This implementation reaches Tier 2 of the capability tiers, for PostgreSQL and ClickHouse: the
+ * model, canonical IR and version, colocation groups, operation shapes and ids, placement maps with
+ * signature verification and the refusals that go with them, routing, telemetry, schema rendering
+ * and application, dual write, and backfill with verification.
+ *
+ * The shared vectors for Tier 1 and Tier 2 were written **before** this library claimed those
+ * tiers, which is what §10 of the format contract requires and is the interesting fact about the
+ * claim. Closing that gap found five defects in the reference implementation.
+ *
+ * **Everything below Tier 2 is synchronous and touches no socket; Tier 2 is asynchronous.** That is
+ * the runtime's shape rather than a preference: a synchronous wrapper around Node's drivers means
+ * blocking the event loop. The engine adapters are behind their own subpath exports, so importing
+ * this module resolves no driver - pinned by a test over the import closure, because that absence
+ * is what makes the no-account mode free.
+ *
+ * It also implements hashed identifiers (section 2a), which is a mode rather than a tier: a complete
+ * Tier 0 library may omit it, but one that offers it has to derive the same digests as every other, or
+ * two services on one model compute two model versions and each refuses the other's map.
+ *
+ * Unlike Python, the model is declared explicitly rather than read from annotations - TypeScript's
+ * types are erased before the code runs. That is not a workaround; it is why this was the right second
+ * implementation. Anything the format contract left implicit had nowhere to hide.
+ */
+export { CanonicalError, canonicalBytes, canonicalString, compareCodePoints, digest16 } from './canonical.js';
+export { DeclarationError, BulkWriteRefused, EngineError, MapError, MapRolledBack, MigrationRefused, ModelPlanningError, SdeError, ResourceBusy, ResourceClosed, } from './errors.js';
+export type { NameMap } from './hashing.js';
+export { DIGEST_CHARS, hashIdentifiers } from './hashing.js';
+export type { Group } from './groups.js';
+export { colocationGroups, groupOf } from './groups.js';
+export type { CostCeiling, Entity, EntityDeclaration, EntitySpec, FieldSpec, LogicalModel, RelationSpec, } from './model.js';
+export { assemble, buildModel, CONTRACT, entity, entityOf, irBytes, neutralDeclaration, ref, } from './model.js';
+export type { DeclaredIndex, DeclaredTable, PartitionSpec, PhysicalFinding } from './physical.js';
+export { PHYSICAL_DESIGN_SINCE, capabilities as physicalCapabilities, describeFinding, } from './physical.js';
+export type { GroupPlacement, LoadOptions, Materialization, PhysicalLayout, PlacementMap, } from './placement.js';
+export { ALSO_WRITE_SINCE, BACKFILL_TABLE, MAP_CONTRACT, MAP_CONTRACT_FLOOR, RESERVED_TABLES, WATERMARK_TABLE, loadMap, materializationById, placementOf, } from './placement.js';
+export { groupColumns } from './layout.js';
+export type { BackfillOptions, BackfillProgress, Difference, EntityProgress, Migratable, VerifyOptions, VerifyReport, } from './migration.js';
+export { backfill, backfillForAHuman, backfillRecord, CHUNK_ROWS, DIALECT_PRECISION, entityProgressRecord, keyColumns, MIGRATABLE_IS_TOTAL, PRECISION_INDEPENDENT, sameWidth, verify, verifyForAHuman, verifyRecord, } from './migration.js';
+export type { Engine, ManagedEngine, Row, SessionOptions } from './session.js';
+export { Session, tableFor } from './session.js';
+export type { ScanOptions, CountOptions, SummaryOptions } from './session.js';
+export { MAX_PAGE_ROWS, QueryRefused } from './query.js';
+export type { Range, ScanPage, Queryable, ReadColumn, ReadFilter, ReadPlan, NumericSummary, Summarizable } from './query.js';
+export type { BulkWritable } from './bulk.js';
+export { MAX_BATCH_ROWS, MAX_BATCH_VALUES } from './bulk.js';
+export type { Protection, WatermarkCheck, WatermarkStore } from './watermark.js';
+export { enforceForwardOnly, WATERMARK_STORE_IS_TOTAL, watermarkRecord, } from './watermark.js';
+export { MIGRATABLE_MEMBERS, satisfies, WATERMARK_MEMBERS } from './capabilities.js';
+export type { ResolveOptions } from './routing.js';
+export { resolve } from './routing.js';
+export type { CompatibilityViews, Dialect, SchemaOptions, ViewOptions } from './schema.js';
+export { compatibilityViews, DIALECTS, FIXED_SCHEMA, QUOTE, schemaIsFixed, schemaStatements, } from './schema.js';
+export type { OperationShape, ShapeKind } from './shapes.js';
+export { enumerateShapes, SHAPE_KINDS, shapeId, shapeIr, WRITE_KINDS } from './shapes.js';
+export { guard, internalFailures, resetInternalFailures } from './internal.js';
+export type { CopyFreshness, FanOutOptions, FeatureOptions, GroupFeatures, ReadPredicates, RecordOptions, StorageMeasurement, StorageOptions, StorageSample, StorageSize, StorageUnavailable, Window, } from './telemetry.js';
+export { BUCKET_BASE_NS, BUCKET_COUNT, copyFreshnessRecord, DAY_NS, exactBytes, FanOutStats, featuresRecord, FIELD_LIST_IS_TOTAL, FILTERED_KINDS, GROWTH_MIN_NS, hasTimeDimension, Histogram, MEASURED_FIELDS, Recorder, SECOND_NS, ShapeStats, STORAGE_UNAVAILABLE, timeFields, windowCopies, windowFeatures, windowRecord, windowShapes, } from './telemetry.js';
+export type { FieldType, NeutralType } from './types.js';
+export { checkType, NEUTRAL_TYPES, T } from './types.js';
+export { Timestamp } from './timestamp.js';
+/** The format contract version this library implements. */
+export declare const CONTRACT_VERSION = 1;
+/** The capability tier this library reaches. See docs/format-contract.md, section 9. */
+export declare const TIER = 2;
+export { VerificationRequest, verificationRequest } from './verification.js';
+export { EPOCH_COLUMN as WRITE_EPOCH_COLUMN, FenceState, WriteFence } from './write-fence.js';
+export { prepareSchema } from './provisioning.js';
+export { InspectionContext } from './inspection.js';
+export { verifyFrozen, frozenVerifyRecord, type FrozenVerifyReport, type FrozenTable } from './frozen-verification.js';
+export { CUTOVER_PROTOCOL, CUTOVER_RELAYOUT_PROTOCOL, CutoverPlan, loadCutoverPlan } from './cutover.js';
+export { STAGING_PROTOCOL, STAGING_RELAYOUT_PROTOCOL, StagingPlan, loadStagingPlan, stagingTableName } from './staging.js';
+export { INDEX_CHANGE_PROTOCOL, INDEX_PROTOCOL, IndexPlan, indexBuildName, loadIndexPlan } from './in-place-index.js';

package/dist/index.js

@@ -0,0 +1,64 @@
+/**
+ * Smart Data Engine - client library for TypeScript.
+ *
+ * You declare entities and relations. We decide which database engine each colocation group lives in,
+ * what its physical layout is there, and when it should move - and your code never names a table or
+ * an engine, which is exactly what lets us change both without touching it.
+ *
+ * This implementation reaches Tier 2 of the capability tiers, for PostgreSQL and ClickHouse: the
+ * model, canonical IR and version, colocation groups, operation shapes and ids, placement maps with
+ * signature verification and the refusals that go with them, routing, telemetry, schema rendering
+ * and application, dual write, and backfill with verification.
+ *
+ * The shared vectors for Tier 1 and Tier 2 were written **before** this library claimed those
+ * tiers, which is what §10 of the format contract requires and is the interesting fact about the
+ * claim. Closing that gap found five defects in the reference implementation.
+ *
+ * **Everything below Tier 2 is synchronous and touches no socket; Tier 2 is asynchronous.** That is
+ * the runtime's shape rather than a preference: a synchronous wrapper around Node's drivers means
+ * blocking the event loop. The engine adapters are behind their own subpath exports, so importing
+ * this module resolves no driver - pinned by a test over the import closure, because that absence
+ * is what makes the no-account mode free.
+ *
+ * It also implements hashed identifiers (section 2a), which is a mode rather than a tier: a complete
+ * Tier 0 library may omit it, but one that offers it has to derive the same digests as every other, or
+ * two services on one model compute two model versions and each refuses the other's map.
+ *
+ * Unlike Python, the model is declared explicitly rather than read from annotations - TypeScript's
+ * types are erased before the code runs. That is not a workaround; it is why this was the right second
+ * implementation. Anything the format contract left implicit had nowhere to hide.
+ */
+export { CanonicalError, canonicalBytes, canonicalString, compareCodePoints, digest16 } from './canonical.js';
+export { DeclarationError, BulkWriteRefused, EngineError, MapError, MapRolledBack, MigrationRefused, ModelPlanningError, SdeError, ResourceBusy, ResourceClosed, } from './errors.js';
+export { DIGEST_CHARS, hashIdentifiers } from './hashing.js';
+export { colocationGroups, groupOf } from './groups.js';
+export { assemble, buildModel, CONTRACT, entity, entityOf, irBytes, neutralDeclaration, ref, } from './model.js';
+export { PHYSICAL_DESIGN_SINCE, capabilities as physicalCapabilities, describeFinding, } from './physical.js';
+export { ALSO_WRITE_SINCE, BACKFILL_TABLE, MAP_CONTRACT, MAP_CONTRACT_FLOOR, RESERVED_TABLES, WATERMARK_TABLE, loadMap, materializationById, placementOf, } from './placement.js';
+export { groupColumns } from './layout.js';
+export { backfill, backfillForAHuman, backfillRecord, CHUNK_ROWS, DIALECT_PRECISION, entityProgressRecord, keyColumns, MIGRATABLE_IS_TOTAL, PRECISION_INDEPENDENT, sameWidth, verify, verifyForAHuman, verifyRecord, } from './migration.js';
+export { Session, tableFor } from './session.js';
+export { MAX_PAGE_ROWS, QueryRefused } from './query.js';
+export { MAX_BATCH_ROWS, MAX_BATCH_VALUES } from './bulk.js';
+export { enforceForwardOnly, WATERMARK_STORE_IS_TOTAL, watermarkRecord, } from './watermark.js';
+export { MIGRATABLE_MEMBERS, satisfies, WATERMARK_MEMBERS } from './capabilities.js';
+export { resolve } from './routing.js';
+export { compatibilityViews, DIALECTS, FIXED_SCHEMA, QUOTE, schemaIsFixed, schemaStatements, } from './schema.js';
+export { enumerateShapes, SHAPE_KINDS, shapeId, shapeIr, WRITE_KINDS } from './shapes.js';
+export { guard, internalFailures, resetInternalFailures } from './internal.js';
+export { BUCKET_BASE_NS, BUCKET_COUNT, copyFreshnessRecord, DAY_NS, exactBytes, FanOutStats, featuresRecord, FIELD_LIST_IS_TOTAL, FILTERED_KINDS, GROWTH_MIN_NS, hasTimeDimension, Histogram, MEASURED_FIELDS, Recorder, SECOND_NS, ShapeStats, STORAGE_UNAVAILABLE, timeFields, windowCopies, windowFeatures, windowRecord, windowShapes, } from './telemetry.js';
+export { checkType, NEUTRAL_TYPES, T } from './types.js';
+export { Timestamp } from './timestamp.js';
+/** The format contract version this library implements. */
+export const CONTRACT_VERSION = 1;
+/** The capability tier this library reaches. See docs/format-contract.md, section 9. */
+export const TIER = 2;
+export { VerificationRequest, verificationRequest } from './verification.js';
+export { EPOCH_COLUMN as WRITE_EPOCH_COLUMN, FenceState, WriteFence } from './write-fence.js';
+export { prepareSchema } from './provisioning.js';
+export { InspectionContext } from './inspection.js';
+export { verifyFrozen, frozenVerifyRecord } from './frozen-verification.js';
+export { CUTOVER_PROTOCOL, CUTOVER_RELAYOUT_PROTOCOL, CutoverPlan, loadCutoverPlan } from './cutover.js';
+export { STAGING_PROTOCOL, STAGING_RELAYOUT_PROTOCOL, StagingPlan, loadStagingPlan, stagingTableName } from './staging.js';
+export { INDEX_CHANGE_PROTOCOL, INDEX_PROTOCOL, IndexPlan, indexBuildName, loadIndexPlan } from './in-place-index.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAC7G,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,QAAQ,EACR,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,QAAQ,EACR,YAAY,EACZ,cAAc,GACf,MAAM,aAAa,CAAA;AAEpB,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAE5D,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AAUvD,OAAO,EACL,QAAQ,EACR,UAAU,EACV,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,OAAO,EACP,kBAAkB,EAClB,GAAG,GACJ,MAAM,YAAY,CAAA;AAEnB,OAAO,EACL,qBAAqB,EACrB,YAAY,IAAI,oBAAoB,EACpC,eAAe,GAChB,MAAM,eAAe,CAAA;AAQtB,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,OAAO,EACP,mBAAmB,EACnB,WAAW,GACZ,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAU1C,OAAO,EACL,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,iBAAiB,EACjB,oBAAoB,EACpB,UAAU,EACV,mBAAmB,EACnB,qBAAqB,EACrB,SAAS,EACT,MAAM,EACN,eAAe,EACf,YAAY,GACb,MAAM,gBAAgB,CAAA;AAEvB,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AAEhD,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAGxD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAE5D,OAAO,EACL,kBAAkB,EAClB,wBAAwB,EACxB,eAAe,GAChB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAEpF,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,OAAO,EACL,kBAAkB,EAClB,QAAQ,EACR,YAAY,EACZ,KAAK,EACL,aAAa,EACb,gBAAgB,GACjB,MAAM,aAAa,CAAA;AAEpB,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzF,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAA;AAe9E,OAAO,EACL,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,MAAM,EACN,UAAU,EACV,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,QAAQ,EACR,SAAS,EACT,UAAU,EACV,mBAAmB,EACnB,UAAU,EACV,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,YAAY,GACb,MAAM,gBAAgB,CAAA;AAEvB,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,EAAE,MAAM,YAAY,CAAA;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAE1C,2DAA2D;AAC3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAEjC,wFAAwF;AACxF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAA;AAErB,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAA;AAE5E,OAAO,EAAE,YAAY,IAAI,kBAAkB,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAE7F,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAEjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAA6C,MAAM,0BAA0B,CAAA;AAEtH,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAExG,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC1H,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA"}

package/dist/inspection.d.ts

@@ -0,0 +1,19 @@
+import type { LogicalModel } from './model.js';
+import type { PlacementMap } from './placement.js';
+import type { Engine } from './session.js';
+export interface MigrationView {
+    readonly model: LogicalModel;
+    readonly placement: PlacementMap;
+    readonly projectId: string | undefined;
+    engineNamed(name: string): Engine;
+}
+export declare class InspectionContext implements MigrationView {
+    readonly model: LogicalModel;
+    readonly placement: PlacementMap;
+    private readonly engines;
+    readonly projectId: string;
+    constructor(model: LogicalModel, placement: PlacementMap, engines: Readonly<Record<string, Engine>>, options: {
+        projectId: string;
+    });
+    engineNamed(name: string): Engine;
+}

package/dist/inspection.js

@@ -0,0 +1,31 @@
+/** A local operator's data view, without runtime writes or watermark adoption. */
+import { MigrationRefused } from './errors.js';
+import { checkMapProject } from './generation.js';
+export class InspectionContext {
+    model;
+    placement;
+    engines;
+    projectId;
+    constructor(model, placement, engines, options) {
+        this.model = model;
+        this.placement = placement;
+        if (placement.contract < 4)
+            throw new MigrationRefused('operator inspection requires a generation-bearing placement map');
+        checkMapProject(placement, options.projectId);
+        if (model.version !== placement.modelVersion)
+            throw new MigrationRefused('operator inspection needs the model named by its map');
+        const missing = [...new Set(Object.values(placement.groups).flatMap((spot) => [spot.source, ...spot.derived].map((material) => material.engine)))].filter((name) => engines[name] === undefined).sort();
+        if (missing.length > 0)
+            throw new MigrationRefused(`operator inspection is missing engines ${JSON.stringify(missing)}`);
+        this.engines = Object.freeze({ ...engines });
+        this.projectId = options.projectId;
+        Object.freeze(this);
+    }
+    engineNamed(name) {
+        const engine = this.engines[name];
+        if (engine === undefined)
+            throw new MigrationRefused(`operator inspection has no engine ${name}`);
+        return engine;
+    }
+}
+//# sourceMappingURL=inspection.js.map

package/dist/inspection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inspection.js","sourceRoot":"","sources":["../src/inspection.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AAYjD,MAAM,OAAO,iBAAiB;IAGP;IAA8B;IAFlC,OAAO,CAAkC;IACjD,SAAS,CAAQ;IAC1B,YAAqB,KAAmB,EAAW,SAAuB,EACxE,OAAyC,EAAE,OAA8B;QADtD,UAAK,GAAL,KAAK,CAAc;QAAW,cAAS,GAAT,SAAS,CAAc;QAExE,IAAI,SAAS,CAAC,QAAQ,GAAG,CAAC;YAAE,MAAM,IAAI,gBAAgB,CAAC,iEAAiE,CAAC,CAAA;QACzH,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;QAC7C,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,YAAY;YAAE,MAAM,IAAI,gBAAgB,CAAC,sDAAsD,CAAC,CAAA;QAChI,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAC3E,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,IAAI,EAAE,CAAA;QAC3H,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,IAAI,gBAAgB,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACvH,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;QAC5C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACrB,CAAC;IACD,WAAW,CAAC,IAAY;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,IAAI,gBAAgB,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAA;QACjG,OAAO,MAAM,CAAA;IACf,CAAC;CACF"}

package/dist/internal.d.ts

@@ -0,0 +1,42 @@
+/**
+ * The boundary between our bugs and the client's uptime.
+ *
+ * This library runs inside somebody else's application. A defect in our telemetry or our
+ * aggregation must not take down their request, and the honest way to guarantee that is to route
+ * every purely-internal side effect through one function that cannot throw.
+ *
+ * The hard part is not the try/catch. It is deciding what counts as internal, and getting that
+ * wrong in either direction is bad: swallow too little and a bug in a counter becomes a customer's
+ * outage; swallow too much and a write that never happened is reported as success, which is the
+ * worst thing this library could do.
+ *
+ * So the rule is narrow and stated once here: **internal means it cannot change whether the
+ * client's operation was performed correctly.** Aggregating a telemetry window is internal.
+ * Choosing which materialisation a read goes to is *not* - a wrong route returns wrong data.
+ * Writing a row is not. Verifying a placement map's signature is not, because the map decides where
+ * data is written.
+ *
+ * A failure is **counted**, and that counter is what makes this honest: a library that swallows
+ * silently is indistinguishable from one that works. It is not logged, unlike the reference
+ * implementation, and the difference is deliberate - this library has no logging channel at all
+ * (see `tests/silence.test.ts`), so a line here would go to whatever the client's process prints
+ * with no handler to turn it off. The counter is readable instead.
+ */
+/**
+ * How many times each guarded operation has failed, by name.
+ *
+ * Exposed rather than hidden. Swallowing a failure and leaving no trace of it would make this
+ * library indistinguishable from one that works, and a client should be able to see that our code
+ * is misbehaving inside their process even when we cannot tell them.
+ */
+export declare function internalFailures(): Record<string, number>;
+/** For tests. */
+export declare function resetInternalFailures(): void;
+/**
+ * Run a purely-internal operation. Never throws.
+ *
+ * `what` names the operation and becomes the key in {@link internalFailures}, so it should be
+ * stable across releases - a client may be alerting on it. The names match the reference
+ * implementation's, because a client running both languages has one dashboard.
+ */
+export declare function guard<T>(what: string, operation: () => T): T | undefined;

package/dist/internal.js

@@ -0,0 +1,56 @@
+/**
+ * The boundary between our bugs and the client's uptime.
+ *
+ * This library runs inside somebody else's application. A defect in our telemetry or our
+ * aggregation must not take down their request, and the honest way to guarantee that is to route
+ * every purely-internal side effect through one function that cannot throw.
+ *
+ * The hard part is not the try/catch. It is deciding what counts as internal, and getting that
+ * wrong in either direction is bad: swallow too little and a bug in a counter becomes a customer's
+ * outage; swallow too much and a write that never happened is reported as success, which is the
+ * worst thing this library could do.
+ *
+ * So the rule is narrow and stated once here: **internal means it cannot change whether the
+ * client's operation was performed correctly.** Aggregating a telemetry window is internal.
+ * Choosing which materialisation a read goes to is *not* - a wrong route returns wrong data.
+ * Writing a row is not. Verifying a placement map's signature is not, because the map decides where
+ * data is written.
+ *
+ * A failure is **counted**, and that counter is what makes this honest: a library that swallows
+ * silently is indistinguishable from one that works. It is not logged, unlike the reference
+ * implementation, and the difference is deliberate - this library has no logging channel at all
+ * (see `tests/silence.test.ts`), so a line here would go to whatever the client's process prints
+ * with no handler to turn it off. The counter is readable instead.
+ */
+const failures = new Map();
+/**
+ * How many times each guarded operation has failed, by name.
+ *
+ * Exposed rather than hidden. Swallowing a failure and leaving no trace of it would make this
+ * library indistinguishable from one that works, and a client should be able to see that our code
+ * is misbehaving inside their process even when we cannot tell them.
+ */
+export function internalFailures() {
+    return Object.fromEntries([...failures.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
+}
+/** For tests. */
+export function resetInternalFailures() {
+    failures.clear();
+}
+/**
+ * Run a purely-internal operation. Never throws.
+ *
+ * `what` names the operation and becomes the key in {@link internalFailures}, so it should be
+ * stable across releases - a client may be alerting on it. The names match the reference
+ * implementation's, because a client running both languages has one dashboard.
+ */
+export function guard(what, operation) {
+    try {
+        return operation();
+    }
+    catch {
+        failures.set(what, (failures.get(what) ?? 0) + 1);
+        return undefined;
+    }
+}
+//# sourceMappingURL=internal.js.map

package/dist/internal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"internal.js","sourceRoot":"","sources":["../src/internal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE1C;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACrG,CAAC;AAED,iBAAiB;AACjB,MAAM,UAAU,qBAAqB;IACnC,QAAQ,CAAC,KAAK,EAAE,CAAA;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,KAAK,CAAI,IAAY,EAAE,SAAkB;IACvD,IAAI,CAAC;QACH,OAAO,SAAS,EAAE,CAAA;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACjD,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC"}

package/dist/layout.d.ts

@@ -0,0 +1,36 @@
+/**
+ * The one derivation this library needs from a model that a map does not already carry.
+ *
+ * A placement map states a layout - table names and **dialect** column types - so nothing here has
+ * to work out what a column is called or how PostgreSQL spells a timestamp. That is the planner's
+ * job and it is in the control plane.
+ *
+ * What a map does not carry is the **neutral** type of a column, and one gate needs it: the
+ * migration precision check. `timestamptz` is microseconds in PostgreSQL and milliseconds in
+ * ClickHouse, so copying that column in that direction truncates every value that has more
+ * precision - silently, because the insert succeeds and the value comes back changed. Deciding that
+ * from the dialect spellings would mean parsing `DateTime64(3, 'UTC')`, which is a second
+ * implementation of the type mapping; deciding it from the neutral vocabulary is a lookup.
+ *
+ * So this file is deliberately one function and not a layout module. The reference implementation
+ * has a much larger one because it also derives the layout itself for the control plane to issue;
+ * a client library never does that, and porting it here would be code with no caller and a rule to
+ * keep in sync for nothing.
+ */
+import type { Group } from './groups.js';
+import type { LogicalModel } from './model.js';
+/**
+ * Every column a group's tables need, per entity, in the neutral vocabulary.
+ *
+ * A group has a foreign-key column for every relation whose source is a member, named
+ * `<relation>_<target key field>` - one column per key field, for a single-field key and a
+ * composite one alike. Today those add no *type* the group did not already have, because a relation
+ * unions its two ends into one colocation group, so the target's key fields are declared fields of
+ * a member. That is a fact about how groups are formed rather than about layouts, and this
+ * derivation does not rely on it.
+ *
+ * Declared field order first, then relations in name order - the same order the reference produces,
+ * because the migration gate iterates these and a refusal has to name the same column first in both
+ * languages.
+ */
+export declare function groupColumns(model: LogicalModel, group: Group): Readonly<Record<string, Readonly<Record<string, string>>>>;

package/dist/layout.js

@@ -0,0 +1,62 @@
+/**
+ * The one derivation this library needs from a model that a map does not already carry.
+ *
+ * A placement map states a layout - table names and **dialect** column types - so nothing here has
+ * to work out what a column is called or how PostgreSQL spells a timestamp. That is the planner's
+ * job and it is in the control plane.
+ *
+ * What a map does not carry is the **neutral** type of a column, and one gate needs it: the
+ * migration precision check. `timestamptz` is microseconds in PostgreSQL and milliseconds in
+ * ClickHouse, so copying that column in that direction truncates every value that has more
+ * precision - silently, because the insert succeeds and the value comes back changed. Deciding that
+ * from the dialect spellings would mean parsing `DateTime64(3, 'UTC')`, which is a second
+ * implementation of the type mapping; deciding it from the neutral vocabulary is a lookup.
+ *
+ * So this file is deliberately one function and not a layout module. The reference implementation
+ * has a much larger one because it also derives the layout itself for the control plane to issue;
+ * a client library never does that, and porting it here would be code with no caller and a rule to
+ * keep in sync for nothing.
+ */
+import { compareCodePoints } from './canonical.js';
+/**
+ * Every column a group's tables need, per entity, in the neutral vocabulary.
+ *
+ * A group has a foreign-key column for every relation whose source is a member, named
+ * `<relation>_<target key field>` - one column per key field, for a single-field key and a
+ * composite one alike. Today those add no *type* the group did not already have, because a relation
+ * unions its two ends into one colocation group, so the target's key fields are declared fields of
+ * a member. That is a fact about how groups are formed rather than about layouts, and this
+ * derivation does not rely on it.
+ *
+ * Declared field order first, then relations in name order - the same order the reference produces,
+ * because the migration gate iterates these and a refusal has to name the same column first in both
+ * languages.
+ */
+export function groupColumns(model, group) {
+    const out = {};
+    const relations = [...model.relations].sort((a, b) => compareCodePoints(a.name, b.name));
+    for (const member of group.members) {
+        const spec = model.entities.find((entity) => entity.name === member);
+        if (spec === undefined)
+            continue;
+        const columns = {};
+        for (const field of spec.fields)
+            columns[field.name] = field.type;
+        for (const relation of relations) {
+            if (relation.source !== member)
+                continue;
+            const target = model.entities.find((entity) => entity.name === relation.target);
+            if (target === undefined)
+                continue;
+            for (const keyField of target.key) {
+                const declared = target.fields.find((field) => field.name === keyField);
+                if (declared === undefined)
+                    continue;
+                columns[`${relation.name}_${keyField}`] = declared.type;
+            }
+        }
+        out[member] = columns;
+    }
+    return out;
+}
+//# sourceMappingURL=layout.js.map

package/dist/layout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"layout.js","sourceRoot":"","sources":["../src/layout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAIlD;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAmB,EACnB,KAAY;IAEZ,MAAM,GAAG,GAA2C,EAAE,CAAA;IACtD,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACxF,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;QACpE,IAAI,IAAI,KAAK,SAAS;YAAE,SAAQ;QAChC,MAAM,OAAO,GAA2B,EAAE,CAAA;QAC1C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAA;QACjE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM;gBAAE,SAAQ;YACxC,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAA;YAC/E,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAQ;YAClC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;gBAClC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAA;gBACvE,IAAI,QAAQ,KAAK,SAAS;oBAAE,SAAQ;gBACpC,OAAO,CAAC,GAAG,QAAQ,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAA;YACzD,CAAC;QACH,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAA;IACvB,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC"}

package/dist/migration.d.ts

@@ -0,0 +1,197 @@
+import type { MigrationView } from './inspection.js';
+import type { VerificationRequest } from './verification.js';
+import { MIGRATABLE_MEMBERS } from './capabilities.js';
+import type { Row, Session } from './session.js';
+/**
+ * Rows per chunk, by default.
+ *
+ * A thousand rather than a round ten thousand: a chunk is held in memory twice during `verify` -
+ * the source's rows and the target's - and the number that matters is not throughput but how much
+ * work a crash discards, which is one chunk.
+ */
+export declare const CHUNK_ROWS = 1000;
+/**
+ * Sub-second digits each dialect keeps, for the neutral types where dialects differ.
+ *
+ * Both current SQL timestamp types retain six digits. This table guards future dialects; the
+ * runtime value must preserve those digits too, which is why the adapters return Timestamp.
+ */
+export declare const DIALECT_PRECISION: Readonly<Record<string, number>>;
+/**
+ * Neutral types a copy between dialects does not silently change.
+ *
+ * Not the same claim as "every value survives". PostgreSQL's `date` has a wider range than
+ * ClickHouse's `Date32`, so a date in the year 1800 does not survive that move - but it fails
+ * *loudly*, on the insert or as a mismatch in `verify`. The line drawn here is silent-and-universal
+ * loss, which is a much smaller set than lossy, and it is the set worth a refusal that arrives
+ * before the work.
+ */
+export declare const PRECISION_INDEPENDENT: ReadonlySet<string>;
+/**
+ * What an engine adapter has to offer for a group to be migrated into or out of it.
+ *
+ * Optional, exactly like the watermark store and for the same reason: requiring these of the engine
+ * interface would break every adapter anybody has written, including fakes in someone else's test
+ * suite, for a capability an engine with a fixed schema cannot provide. Non-participation is a
+ * **named refusal** rather than a silent skip, because a migration that quietly copies nothing is
+ * the worst outcome available here.
+ */
+export interface Migratable {
+    readonly dialect: string;
+    keyRange(table: string, order: readonly string[], options?: {
+        readonly after?: readonly unknown[];
+        readonly upto?: readonly unknown[];
+        readonly limit?: number;
+    }): Promise<Row[]>;
+    nthKey(table: string, order: readonly string[], options: {
+        readonly position: number;
+    }): Promise<unknown[] | null>;
+    copyIn(table: string, rows: readonly Row[]): Promise<void>;
+    count(table: string): Promise<number>;
+    get(table: string, key: Readonly<Row>): Promise<Row | null>;
+    backfillMarker(options: {
+        readonly materialization: string;
+        readonly entity: string;
+    }): Promise<number>;
+    recordBackfillMarker(options: {
+        readonly materialization: string;
+        readonly entity: string;
+        readonly rows: number;
+    }): Promise<void>;
+}
+/** A compile-time ratchet between the interface and the list the runtime check uses. */
+export declare const MIGRATABLE_IS_TOTAL: (typeof MIGRATABLE_MEMBERS)[number] extends keyof Migratable ? true : never;
+/**
+ * The ordering columns for a keyset scan, refusing an empty one.
+ *
+ * Here rather than in each adapter so that the two cannot disagree about it, and exported because
+ * an adapter written outside this repository has the same argument to check. An empty order is not
+ * a scan of everything in an unspecified order - it is a paginated scan with no pagination, which
+ * returns the same first page forever.
+ */
+export declare function keyColumns(order: readonly string[], table: string): readonly string[];
+/** A bound has one value per ordering column, or the comparison is not the one intended. */
+export declare function sameWidth(bound: readonly unknown[], cols: readonly string[], name: string): void;
+/** How far one entity's copy into one target has got. */
+export interface EntityProgress {
+    readonly entity: string;
+    readonly engine: string;
+    readonly table: string;
+    /** The marker: rows of this entity copied into this target, across every run. */
+    readonly rowsCopied: number;
+    readonly rowsThisRun: number;
+    readonly chunks: number;
+    /** Whether the last chunk came back short, which is what "the tail is the fan-out's now" means. */
+    readonly complete: boolean;
+}
+export declare function entityProgressRecord(progress: EntityProgress): Record<string, unknown>;
+/** What one call to {@link backfill} did, per entity and per target. */
+export interface BackfillProgress {
+    readonly group: string;
+    readonly entities: readonly EntityProgress[];
+    /** Every entity of every target has reached the end of its table at least once. */
+    readonly complete: boolean;
+    readonly rowsThisRun: number;
+}
+export declare function backfillRecord(progress: BackfillProgress): Record<string, unknown>;
+export declare function backfillForAHuman(progress: BackfillProgress): string;
+/**
+ * One source row the target does not have, or has differently.
+ *
+ * **This holds the client's own data and it is the reason the verify record does not.** The key is
+ * here because "which row" is the first thing an operator needs and a count cannot say it; it stays
+ * on their machine because a row is the one thing that must never travel to us. The two facts are
+ * the same decision seen from two sides.
+ */
+export interface Difference {
+    readonly entity: string;
+    readonly table: string;
+    readonly key: Readonly<Row>;
+    /** Columns whose values differ, or empty when the row is absent from the target altogether. */
+    readonly columns: readonly string[];
+}
+/**
+ * What the comparison found, in the shape the gate needs and nothing wider.
+ *
+ * {@link verifyRecord} is the boundary. It carries seven counts, and the control plane reads exactly
+ * those - so there is no field in which a value of the client's could travel, and adding one would
+ * be a visible change to that function rather than an accident somewhere in a call chain.
+ * `differences` is the other half of that: the detail that makes a mismatch fixable, kept here and
+ * deliberately absent from the record.
+ */
+export interface VerifyReport {
+    readonly request?: VerificationRequest;
+    readonly at: string;
+    readonly group: string;
+    readonly chunksCompared: number;
+    readonly chunksMismatched: number;
+    readonly tailRowsRead: number;
+    readonly tailRowsMissingInTarget: number;
+    readonly rowsSource: number;
+    readonly rowsTarget: number;
+    readonly differences: readonly Difference[];
+    readonly differencesSuppressed: number;
+    /**
+     * Whether the target holds everything the source holds. Zero tolerance, both terms.
+     *
+     * Zero rather than a threshold, and the reason is arithmetic rather than principled: any non-zero
+     * threshold is an answer to "how many of your rows may we lose", and there is no number to say
+     * out loud there.
+     */
+    readonly matched: boolean;
+}
+/** The seven counts the gate reads. **Numbers, never rows** - see {@link VerifyReport}. */
+export declare function verifyRecord(report: VerifyReport): Record<string, unknown>;
+export declare function verifyForAHuman(report: VerifyReport): string;
+/**
+ * Why a copy of these columns between these two dialects would change values, or `null`.
+ *
+ * A string rather than a throw, and dialect names rather than engines, because two doors ask this
+ * question and only one of them used to. `backfill` refuses a copy that would truncate; the write
+ * fan-out in `Session` was doing the same truncation one row at a time, for as long as an
+ * `also_write` map was in force. Measured on live servers before it was fixed: a `timestamptz`
+ * written as `09:30:15.123456` came back from PostgreSQL unchanged and from ClickHouse as
+ * `09:30:15.123`, with no error on either side.
+ */
+export declare function precisionRefusal(group: string, entity: string, columns: Readonly<Record<string, string>>, sourceDialect: string, targetDialect: string): string | null;
+export interface BackfillOptions {
+    readonly chunkRows?: number;
+    /**
+     * Bound the work to that many chunks per entity, for an operator who wants to copy for a while
+     * and stop, and for a test that needs to interrupt at a known point. Absent runs each entity to
+     * the end of its table.
+     */
+    readonly stopAfter?: number;
+}
+/**
+ * Copy a group's existing rows into every fan-out target the map names. Resumable.
+ *
+ * Called again after an interruption it picks up from the marker, and called again after completion
+ * it does nothing - both because the marker is durable and lives in the target engine, next to the
+ * rows it describes. That is the correct coupling: a target dropped and recreated loses its marker
+ * with its data, and a marker kept anywhere else would claim work that no longer exists.
+ */
+export declare function backfill(session: Session, group: string, options?: BackfillOptions): Promise<BackfillProgress>;
+export interface VerifyOptions {
+    readonly request?: VerificationRequest;
+    readonly chunkRows?: number;
+    /**
+     * The instant the report is stamped with. Absent reads the calendar clock.
+     *
+     * An argument at all because this library reads a wall clock in exactly two places and both are
+     * pinned by a test - a map carries no date, so the only thing a clock here could do is stamp a
+     * report. Passing one makes the report reproducible for a caller who wants that.
+     */
+    readonly at?: string;
+}
+/**
+ * Compare both copies of a group and report counts. The gate reads the counts, not the rows.
+ *
+ * Reads the **source first and the target second**, always, and the order is load-bearing. A write
+ * is in the source before it is in the copy, so a row read from the source may not have reached the
+ * copy yet - but the copy is read after the whole source window, so the window has already elapsed.
+ * Anything still missing is then looked up once more by point read, on what is normally an empty
+ * set. Reversing the two reads would make this flaky in the direction that stops a healthy
+ * migration.
+ */
+export declare function verify(session: MigrationView, group: string, options?: VerifyOptions): Promise<VerifyReport>;