effect-dynamodb 1.5.0 → 1.7.0

package/dist/Entity.js

@@ -6,11 +6,11 @@
  * carries derived schemas for 7 type extractors (Model, Record, Input, Update,
  * Key, Item, Marshalled).
  */
-import { DateTime, Duration, Effect, Option, Schema, Stream } from "effect";
+import { DateTime, Duration, Effect, Option, Schema, SchemaAST, Stream } from "effect";
 import { DynamoClient } from "./DynamoClient.js";
 import { getIdentifierField, getSparseFields, isConfiguredModel, isHidden, isRecordAst, isRecordSchema, isRef, isRefField, } from "./DynamoModel.js";
 import * as DynamoSchema from "./DynamoSchema.js";
-import { CascadePartialFailure, ConditionalCheckFailed, ItemNotFound, isAwsConditionalCheckFailed, isAwsTransactionCancelled, OptimisticLockError, RefNotFound, StaleAppend, TransactionOverflow, UniqueConstraintViolation, ValidationError, } from "./Errors.js";
+import { CascadePartialFailure, CompositeKeyHoleError, ConditionalCheckFailed, ItemNotFound, isAwsConditionalCheckFailed, isAwsTransactionCancelled, makeCompositeNullableError, OptimisticLockError, RefNotFound, StaleAppend, TransactionOverflow, UniqueConstraintViolation, ValidationError, } from "./Errors.js";
 import { makeBoundAppend, makeBoundDelete, makeBoundPut, makeBoundUpdate, } from "./internal/BoundCrud.js";
 import { BoundQueryImpl } from "./internal/BoundQuery.js";
 import { compileExpr, createConditionOps, isExpr, parseShorthand, parseSimpleShorthand, } from "./internal/Expr.js";
@@ -206,6 +206,108 @@ const makeImpl = (config) => {
         }
     }
     // ---------------------------------------------------------------------------
+    // EDD-9025: composite attribute schemas must not include `null`.
+    //
+    // Composites participate in string composition (`acc#X#alert#Y`); `null` is
+    // not a meaningful slot value. Allowing `Schema.NullOr` / `NullishOr` /
+    // `Schema.Union([..., Schema.Null])` on a composite would let
+    // `set({ composite: null })` typecheck (after the v1.6 update-payload
+    // widening was reverted), then either blow up at runtime or silently
+    // produce a key with the literal string `"null"` as a slot.
+    //
+    // Sparse pattern is still expressible via `Schema.optional(...)` (T |
+    // undefined), which produces no Null AST node. See DESIGN.md §7.
+    // ---------------------------------------------------------------------------
+    // Resolve the AST for a domain composite name. For ref-derived `${name}Id`
+    // composites, the schema is the referenced entity's identifier schema (not
+    // the model field). For all other composites, it's the model field's
+    // schema. Returns undefined when the composite isn't declared on the model
+    // (already caught by EDD-9002 above, so this is defensive).
+    const resolveCompositeSchemaForNullCheck = (compositeAttr) => {
+        if (compositeAttr in modelFields) {
+            return { schema: modelFields[compositeAttr], source: `model field` };
+        }
+        // Ref-derived ${name}Id composite — look up the identifier schema on the
+        // referenced entity's model. We can't use the resolvedRefs array here
+        // (it's built later); recompute inline against config.refs.
+        if (config.refs) {
+            for (const [refFieldName, refValue] of Object.entries(config.refs)) {
+                if (`${refFieldName}Id` !== compositeAttr)
+                    continue;
+                const refEntity = refValue.entity;
+                const idField = getIdentifierField(refEntity.model);
+                if (!idField)
+                    return undefined;
+                return { schema: idField.schema, source: `ref "${refFieldName}" identifier` };
+            }
+        }
+        return undefined;
+    };
+    // Recursively walk an AST and return a path-string if `null` is reachable
+    // in the type union (anywhere — direct, nested in Union, behind Suspend).
+    // `Schema.optional(X)` produces `T | undefined` (no Null) and is allowed.
+    // `Schema.NullOr(X)` produces `T | null` and is rejected.
+    const findNullInAst = (ast, path, seen = new Set()) => {
+        if (seen.has(ast))
+            return undefined;
+        seen.add(ast);
+        if (SchemaAST.isNull(ast))
+            return `${path}.Null`;
+        if (SchemaAST.isUnion(ast)) {
+            for (let i = 0; i < ast.types.length; i++) {
+                const found = findNullInAst(ast.types[i], `${path}.Union[${i}]`, seen);
+                if (found)
+                    return found;
+            }
+        }
+        // Other AST shapes (Declaration, String, Number, Suspend, ...) cannot
+        // introduce `null` themselves. Suspend wrappers are handled
+        // conservatively — we don't recurse into thunks because the EDD-9025
+        // check is about static type unions, and recursive schemas wrapping
+        // composite-eligible scalars are pathological for keys anyway.
+        return undefined;
+    };
+    const checkCompositeForNull = (surface, compositeAttr) => {
+        const resolved = resolveCompositeSchemaForNullCheck(compositeAttr);
+        if (!resolved)
+            return; // EDD-9002 already caught the unknown-attribute case
+        const found = findNullInAst(resolved.schema.ast, resolved.source);
+        if (found) {
+            throw makeCompositeNullableError({
+                entityType: config.entityType,
+                surface,
+                compositeAttribute: compositeAttr,
+                schemaPath: found,
+            });
+        }
+    };
+    // primaryKey composites
+    for (const attr of [
+        ...config.indexes.primary.pk.composite,
+        ...config.indexes.primary.sk.composite,
+    ]) {
+        checkCompositeForNull("primaryKey", attr);
+    }
+    // GSI composites (each index)
+    for (const [indexName, indexDef] of Object.entries(config.indexes)) {
+        if (indexName === "primary")
+            continue;
+        for (const attr of [...indexDef.pk.composite, ...indexDef.sk.composite]) {
+            checkCompositeForNull(`index "${indexName}"`, attr);
+        }
+    }
+    // Unique constraint composites — `unique` is keyed by constraint name,
+    // value is a single field name OR an array of field names OR a config
+    // object. resolveUniqueFields normalises all three to a list of field names.
+    if (config.unique) {
+        for (const [constraintName, constraintDef] of Object.entries(config.unique)) {
+            const fields = resolveUniqueFields(constraintDef);
+            for (const f of fields) {
+                checkCompositeForNull(`unique:${constraintName}`, f);
+            }
+        }
+    }
+    // ---------------------------------------------------------------------------
     // Validate timeSeries config (EDD-9010..9016)
     // ---------------------------------------------------------------------------
     if (config.timeSeries !== undefined && config.timeSeries !== null) {
@@ -1173,13 +1275,20 @@ const makeImpl = (config) => {
             // rather than whole-field replace — concurrent writers to disjoint
             // buckets must coexist (the version CAS protects against same-bucket
             // races). Non-sparse fields use the existing replace semantics.
+            //
+            // null and undefined collapse: both mean "absent" under v3's
+            // two-way classification (DESIGN.md §7). For the in-memory item
+            // we DELETE the attribute (mirrors a DynamoDB REMOVE), so
+            // subsequent decode sees the field as absent rather than as a
+            // typed null that the model schema may not permit. The composer
+            // below sees the attr as absent in `newItem` and applies the
+            // structural rule.
             for (const [attr, val] of Object.entries(hydratedUpdates)) {
-                if (val === undefined)
+                if (val === null || val === undefined) {
+                    delete newItem[attr];
                     continue;
-                if (hasSparseFields &&
-                    attr in sparseFields &&
-                    val !== null &&
-                    typeof val === "object") {
+                }
+                if (hasSparseFields && attr in sparseFields && typeof val === "object") {
                     const existing = newItem[attr] ?? {};
                     newItem[attr] = { ...existing, ...val };
                 }
@@ -1259,8 +1368,55 @@ const makeImpl = (config) => {
             // `newItem` is already in domain names (the merge built it from
             // `currentDomainItem` + domain-keyed updates) — no rename needed
             // here. Recompose all keys with the updated attributes.
-            const newKeys = composeAllKeys(newItem);
-            Object.assign(newItem, newKeys);
+            //
+            // Primary key always recomposes from `newItem` (Put-style).
+            const primaryKeyMap = composePrimaryKey(newItem);
+            Object.assign(newItem, primaryKeyMap);
+            // GSI keys: route through the policy-aware composer so the v3
+            // structural rule + `Entity.remove` cascade match the standard
+            // update path. `newItem` carries stored values for any composite
+            // the user did not touch — passing it as `keyRecord` lets the
+            // composer treat omissions as no-ops (the half stays untouched
+            // because the stored value is present), while still firing the
+            // cascade and the structural truncation rule. See DESIGN.md §7.
+            const retainRemovedSet = uState.remove ? new Set(uState.remove) : undefined;
+            try {
+                const gsiUpdate = KeyComposer.composeGsiKeysForUpdatePolicyAware(schema, entityType, entityVersion, allIndexes, hydratedUpdates, newItem, { removedSet: retainRemovedSet });
+                for (const [field, value] of Object.entries(gsiUpdate.sets)) {
+                    newItem[field] = value;
+                }
+                for (const field of gsiUpdate.removes) {
+                    delete newItem[field];
+                }
+                // Untouched GSIs (no policy + no payload + no cascade): retain
+                // path semantics are Put-style — recompose from `newItem` and
+                // drop both keys when any composite is missing. Matches the
+                // existing `composeAllKeys` behavior for those GSIs.
+                const addressed = new Set([
+                    ...Object.keys(gsiUpdate.sets),
+                    ...gsiUpdate.removes,
+                ]);
+                for (const [indexName, indexDef] of Object.entries(allIndexes)) {
+                    if (indexName === "primary")
+                        continue;
+                    if (addressed.has(indexDef.pk.field) || addressed.has(indexDef.sk.field))
+                        continue;
+                    const keys = KeyComposer.tryComposeIndexKeys(schema, entityType, entityVersion, indexDef, newItem);
+                    if (keys) {
+                        Object.assign(newItem, keys);
+                    }
+                    else {
+                        delete newItem[indexDef.pk.field];
+                        delete newItem[indexDef.sk.field];
+                    }
+                }
+            }
+            catch (e) {
+                if (e instanceof CompositeKeyHoleError) {
+                    return yield* e;
+                }
+                throw e;
+            }
             const sentinelRotations = [];
             if (touchesUniqueFields) {
                 const currentRawRecord = currentRaw;
@@ -1443,18 +1599,32 @@ const makeImpl = (config) => {
         const updateSystemColliders = new Set();
         if (systemFields.updatedAtCollision && systemFields.updatedAt)
             updateSystemColliders.add(systemFields.updatedAt);
-        // Add user-provided updates
+        // Add user-provided updates. Buffered REMOVE clauses for null
+        // payload entries land in `removeClauses` below — under v3,
+        // `set({ attr: null | undefined })` REMOVEs the attribute from the
+        // item but does NOT separately cascade-drop GSI keys. The structural
+        // composer treats the cleared attribute as absent and recomposes
+        // (or truncates / drops / no-ops) per the half's policy.
+        // EDD-9025 guarantees no composite is nullable, so `set` of `null`
+        // is only reachable for non-composite, model-declared-nullable
+        // attributes. See DESIGN.md §7.
+        const nullClears = [];
         for (const [attr, val] of Object.entries(encodedUpdatesMap)) {
-            if (val === undefined)
-                continue;
             if (updateSystemColliders.has(attr))
                 continue;
+            if (val === null || val === undefined) {
+                // Sparse fields: clearing a sparse field with null is a no-op
+                // here — the `.removeEntries` API is the explicit per-key
+                // remove. Skip to avoid REMOVE'ing the whole prefix erroneously.
+                if (hasSparseFields && attr in sparseFields)
+                    continue;
+                nullClears.push(attr);
+                continue;
+            }
             // Sparse fields: emit one SET per bucket. Whole-bucket replace
             // semantics — concurrent writers to disjoint buckets are safe.
-            // `null` is NOT REMOVE — that's footgunny; explicit removal goes
-            // through `.removeEntries(field, keys)`.
             if (hasSparseFields && attr in sparseFields) {
-                if (val === null || typeof val !== "object")
+                if (typeof val !== "object")
                     continue;
                 const sparse = sparseFields[attr];
                 for (const [k, v] of Object.entries(val)) {
@@ -1530,6 +1700,17 @@ const makeImpl = (config) => {
             }
         }
         // REMOVE
+        // Two channels feed REMOVE clauses:
+        //  1. `Entity.remove([attr])` — explicit per-attribute drop. Cascades
+        //     to drop the GSI keys for any GSI containing the attr as a
+        //     composite (one of v3's two drop triggers — see DESIGN.md §7).
+        //     Tracked in `removedSet` so the composer can apply the cascade
+        //     rule.
+        //  2. Null payload entries (`set({ attr: null })`). These REMOVE the
+        //     attribute from the item only. They do NOT cascade GSI drops —
+        //     under v3 two-way classification, the structural composer just
+        //     treats the attribute as absent. (EDD-9025 guarantees these
+        //     are never composites.)
         const removedSet = uState.remove ? new Set(uState.remove) : undefined;
         if (uState.remove) {
             for (const attr of uState.remove) {
@@ -1538,11 +1719,31 @@ const makeImpl = (config) => {
                 removeClauses.push(nameKey);
             }
         }
-        // Policy-aware GSI key composition. One call covers SETs (full
-        // recompose or half-wise), sparse dropout REMOVEs, and cascade
-        // REMOVEs when an Entity.remove() targets a GSI composite.
-        // See DESIGN.md §7 Policy-Aware GSI Composition.
-        const gsiUpdate = KeyComposer.composeGsiKeysForUpdatePolicyAware(schema, entityType, entityVersion, allIndexes, hydratedUpdates, decodedKey, { removedSet });
+        for (const attr of nullClears) {
+            const nameKey = `#r${removeClauses.length}`;
+            names[nameKey] = resolveDbName(attr);
+            removeClauses.push(nameKey);
+        }
+        // Policy-aware GSI key composition (v3 — per-half structural rule).
+        // One call covers SETs (full recompose, truncated leading prefix,
+        // or no-op), sparse whole-half-empty REMOVEs, and cascade REMOVEs
+        // when an Entity.remove() targets a GSI composite. See DESIGN.md
+        // §7 Policy-Aware GSI Composition.
+        //
+        // The composer throws CompositeKeyHoleError (EDD-9024) when the
+        // merged payload describes a hole pattern AND the relevant half's
+        // policy is `'preserve'`. Under `'sparse'`, holes silently truncate.
+        // Catch the error synchronously and surface as a typed Effect failure.
+        let gsiUpdate;
+        try {
+            gsiUpdate = KeyComposer.composeGsiKeysForUpdatePolicyAware(schema, entityType, entityVersion, allIndexes, hydratedUpdates, decodedKey, { removedSet });
+        }
+        catch (e) {
+            if (e instanceof CompositeKeyHoleError) {
+                return yield* e;
+            }
+            throw e;
+        }
         for (const [field, value] of Object.entries(gsiUpdate.sets)) {
             const nameKey = `#u${counter}`;
             const valKey = `:u${counter}`;
@@ -2267,48 +2468,33 @@ const makeImpl = (config) => {
             setClauses.push(`${nameKey} = ${valKey}`);
             counter++;
         }
-        // GSI key recomposition: any GSI whose composites are touched by input.
-        // Reuses the policy-aware helper from `.update()`. PK composites are
-        // excluded from the update payload — they're already in the key and
-        // never change during an append. Non-PK composites in appendInput that
-        // overlap a GSI's composites trigger per-GSI resolution via indexPolicy.
+        // GSI key recomposition. v3 unifies append with update — both call
+        // the same composer. PK composites are excluded from the update
+        // payload — they're already in the key and never change during an
+        // append. The composer treats composites outside `appendInput` as
+        // absent under the structural rule (per-half policy decides whether
+        // a half empties to drop or no-op). See DESIGN.md §7 — "append uses
+        // the same composer."
         const nonPkAppendFields = {};
         for (const [attr, val] of Object.entries(encoded)) {
             if (!pkCompositeSet.has(attr)) {
                 nonPkAppendFields[attr] = val;
             }
         }
-        // At append-time, indexPolicy is restricted to composites that are
-        // members of appendInput. Non-appendInput composites are owned by
-        // other writers and are by contract never touched by .append(); their
-        // policy cannot fire at append-time. Each GSI's indexPolicy return is
-        // filtered to appendInput attrs only — non-appendInput keys fall through
-        // to the implicit `"preserve"` default, leaving the half containing
-        // them untouched.
-        const appendInputFieldSet = new Set(Object.keys(appendInputSchema
-            .fields ?? {}));
-        const policyAwareIndexes = {};
-        for (const [indexName, indexDef] of Object.entries(allIndexes)) {
-            if (!indexDef.indexPolicy) {
-                policyAwareIndexes[indexName] = indexDef;
-                continue;
+        // The composer can throw CompositeKeyHoleError (EDD-9024) when a hole
+        // pattern is encountered AND the relevant half's policy is `'preserve'`.
+        // Under `'sparse'`, holes silently truncate. Surface the error as a
+        // typed Effect failure.
+        let gsiUpdate;
+        try {
+            gsiUpdate = KeyComposer.composeGsiKeysForUpdatePolicyAware(schema, entityType, entityVersion, allIndexes, nonPkAppendFields, encoded);
+        }
+        catch (e) {
+            if (e instanceof CompositeKeyHoleError) {
+                return yield* e;
             }
-            const originalPolicy = indexDef.indexPolicy;
-            policyAwareIndexes[indexName] = {
-                ...indexDef,
-                indexPolicy: (item) => {
-                    const raw = originalPolicy(item);
-                    const filtered = {};
-                    for (const [attr, policy] of Object.entries(raw)) {
-                        if (appendInputFieldSet.has(attr) && policy !== undefined) {
-                            filtered[attr] = policy;
-                        }
-                    }
-                    return filtered;
-                },
-            };
+            throw e;
         }
-        const gsiUpdate = KeyComposer.composeGsiKeysForUpdatePolicyAware(schema, entityType, entityVersion, policyAwareIndexes, nonPkAppendFields, encoded);
         for (const [field, value] of Object.entries(gsiUpdate.sets)) {
             const nameKey = `#a${counter}`;
             const valKey = `:a${counter}`;