effect-dynamodb 1.25.0 → 1.26.0

package/dist/Entity.js

@@ -12,6 +12,7 @@ import { Embedder } from "@effect-dynamodb/schema/Embedder.js";
 import { CascadePartialFailure, ConcurrentModification, ConditionalCheckFailed, DeleteAppliedButUnreadable, DynamoError, EmbeddingError, ItemNotDeleted, ItemNotFound, isAwsConditionalCheckFailed, isAwsTransactionCancelled, OptimisticLockError, RefNotFound, StaleAppend, TransactionOverflow, UniqueConstraintViolation, UpdateAppliedButUnreadable, ValidationError, } from "@effect-dynamodb/schema/Errors.js";
 import { makeCompositeKeyForm, toCompositeKeyRecord, } from "@effect-dynamodb/schema/internal/CompositeCodec.js";
 import { makeDefaultCrypto } from "@effect-dynamodb/schema/internal/DefaultCrypto.js";
+import { packedIndexRefusal } from "@effect-dynamodb/schema/internal/EntityPacked.js";
 import * as KeyComposer from "@effect-dynamodb/schema/KeyComposer.js";
 import { normalizeGsiConfig } from "@effect-dynamodb/schema/KeyComposer.js";
 import { deriveSourceText } from "@effect-dynamodb/schema/VectorIndex.js";
@@ -22,6 +23,7 @@ import { BoundQueryImpl, entityNaming, } from "./internal/BoundQuery.js";
 import { compileExpr, createConditionOps, emptyPartProblem, isExpr, nonEmptyCondition, parseShorthand, parseSimpleShorthand, toExpr, } from "./internal/Expr.js";
 import { countOperators, expressionFits } from "./internal/ExpressionLimits.js";
 import { BASE_DELAY_MS, MAX_RETRIES } from "./internal/Hydrate.js";
+import { decodeImage, imageEntityType, ownRowsRawRow, packAttributeValueSync, packedExprRefusal, packedProjectionRefusal, packItemSync, unpackRow, withRegistry, } from "./internal/Packed.js";
 import { compilePath, createPathBuilder } from "./internal/PathBuilder.js";
 import { freshIncarnationToken, generateTimestampPrimitive, INCARNATION_TOKEN, UNSENTINELED_DEFAULTS, } from "./internal/TransactableOps.js";
 import { batchEntries, diagnosedValidationError, diagnoseFailure, keyAttributesOf, targetsFor, transactItemTarget, } from "./internal/WriteDiagnosis.js";
@@ -614,17 +616,46 @@ const makeImpl = (config, precomputedData) => {
             if (encoded.issue !== undefined) {
                 return `.set() of immutable "${field}": ${String(encoded.issue)}`;
             }
-            out.push({ field, attr: resolveDbName(field), value: toAttributeValue(encoded.encoded) });
+            out.push({
+                field,
+                attr: resolveDbName(field),
+                value: toAttributeValue(encoded.encoded),
+                wire: encoded.encoded,
+                packed: field in packedFields,
+            });
         }
         return out;
     };
+    const immutableChangedMessage = (field) => `.set() of "${field}": "${field}" is immutable and the update's value ` +
+        "differs from the stored one.";
+    /** A restated immutable value that differs from the stored item's (packed fields excepted). */
     const immutableMismatch = (asserted, stored) => {
-        const changed = asserted.find(({ attr, value }) => !attributeValueEquals(stored[attr], value));
-        return changed === undefined
-            ? undefined
-            : `.set() of "${changed.field}": "${changed.field}" is immutable and the update's value ` +
-                "differs from the stored one.";
+        const changed = asserted.find(({ attr, value, packed }) => !packed && !attributeValueEquals(stored[attr], value));
+        return changed === undefined ? undefined : immutableChangedMessage(changed.field);
     };
+    /**
+     * A restated immutable PACKED value that differs from the stored item's,
+     * compared by the stored value's unpacked JSON (#147) — whatever codec or
+     * threshold it was written under.
+     */
+    const packedImmutableMismatch = (asserted, stored) => Effect.gen(function* () {
+        for (const { field, attr, wire, packed } of asserted) {
+            if (!packed)
+                continue;
+            const storedValue = stored[attr];
+            if (storedValue === undefined) {
+                if (wire === undefined || wire === null)
+                    continue;
+                return immutableChangedMessage(field);
+            }
+            const raw = fromAttributeMap({ [attr]: storedValue });
+            yield* unpackFields(raw, "update.decode");
+            if (!attributeValueEquals(toAttributeValue(raw[attr]), toAttributeValue(wire))) {
+                return immutableChangedMessage(field);
+            }
+        }
+        return undefined;
+    });
     /**
      * Whether an update can safely create the item it names (#133): its payload
      * supplies every required model field and every primary-key composite, so
@@ -680,6 +711,136 @@ const makeImpl = (config, precomputedData) => {
             return;
         decodeSparseFields(raw, sparseFields);
     };
+    // ---------------------------------------------------------------------------
+    // Packed fields (`storedAs: DynamoModel.Packed`) — see
+    // `docs/designs/storage-encoding.md` §6. Packing runs after renames and
+    // sparse flattening, before `toAttributeMap`; unpacking after
+    // `fromAttributeMap`, before sparse rebuild and renames — the same points as
+    // the sparse layout, keyed by stored attribute names.
+    // ---------------------------------------------------------------------------
+    const { packedFields, packedAttributes, dispatchAttributes, hasPackedFields } = data;
+    /**
+     * Pack, in place, the packed attributes of a stored-name item. `only`
+     * restricts it to those stored attributes (the read-merge update path packs
+     * only the fields it names: the others still hold their stored value).
+     */
+    const packFields = (item, operation, only) => {
+        if (!hasPackedFields)
+            return Effect.void;
+        const targets = only === undefined
+            ? packedAttributes
+            : Object.fromEntries(Object.entries(packedAttributes).filter(([attr]) => only.has(attr)));
+        if (Object.keys(targets).length === 0)
+            return Effect.void;
+        return withRegistry((registry) => packItemSync(item, targets, registry, { entityType, operation: `${operation}.packed` }));
+    };
+    /** Pack one attribute's wire value straight to its `AttributeValue` (an `UpdateItem` SET). */
+    const packAttribute = (attribute, wire, operation) => withRegistry((registry) => packAttributeValueSync(packedAttributes[attribute], wire, registry, { entityType, operation: `${operation}.packed` }, attribute));
+    /**
+     * Read, in place, the dispatch-eligible attributes of a stored-name row by
+     * their stored form. A row with no string or binary value on any of them
+     * (every native row) costs one `typeof` per attribute.
+     */
+    const unpackFields = (raw, operation = "decode") => dispatchAttributes.length === 0
+        ? Effect.void
+        : unpackRow(raw, dispatchAttributes, { entityType, operation });
+    /**
+     * A domain-named copy of `record` with its packed (or once-packed) attributes
+     * read by their stored form. The update paths derive vector source text from
+     * it: an untouched packed field still holds its stored value there (a JSON
+     * string or compressed bytes), not the value the source `compose` expects.
+     * `record` itself is left as is, so the write still carries the stored bytes.
+     */
+    const unpackedDomainCopy = (record, operation) => {
+        if (dispatchAttributes.length === 0)
+            return Effect.succeed(record);
+        const stored = { ...record };
+        renameToDynamo(stored);
+        return Effect.map(unpackFields(stored, operation), () => {
+            renameFromDynamo(stored);
+            return stored;
+        });
+    };
+    /** The packed attributes an expression or projection may not look inside (EDD-9076). */
+    const packedGuards = Object.entries(packedFields).flatMap(([field, storedAs]) => {
+        const guard = (attribute) => ({
+            attribute,
+            owner: `Entity "${entityType}"`,
+            storedAs,
+            alternative: () => `To filter on it, use .filterBy((item) => ...), which runs on the decoded item; to ` +
+                `condition a write on it, read the item and compare in code, then write the whole ` +
+                `value with .set({ ${field} }).`,
+        });
+        const stored = resolveDbName(field);
+        return stored === field ? [guard(field)] : [guard(field), guard(stored)];
+    });
+    /** Throw `[EDD-9076]` when an expression looks inside a packed field. */
+    const refusePackedExpr = (expr, usage) => {
+        const refused = packedExprRefusal(expr, packedGuards, usage);
+        if (refused !== undefined)
+            throw new Error(refused);
+    };
+    /**
+     * The `Query.make` options of this entity's queries: the packed guards, and
+     * the raw-row hook that reads packed attributes of a projected row of THIS
+     * entity type by their stored form (parsed JSON, not schema-decoded) — rows
+     * of other types (`ignoreOwnership()`) are left untouched.
+     */
+    const packedQueryOptions = {
+        packed: packedGuards,
+        rawRow: ownRowsRawRow(entityType, dispatchAttributes, (raw) => unpackFields(raw)),
+    };
+    /**
+     * `[EDD-9077]`: an update applies anything but a whole-value `.set()` /
+     * `.remove()` to a packed field — DynamoDB cannot compute inside one opaque
+     * value. `undefined` when the update is fine.
+     */
+    const refusedPackedUpdate = (uState) => {
+        if (!hasPackedFields)
+            return undefined;
+        const refuse = (op, field) => `[EDD-9077] Entity "${entityType}": ${op} on "${field}", but "${field}" is stored packed ` +
+            `(format "${packedFields[field].format}"): DynamoDB holds it as one opaque string or ` +
+            `binary value and cannot apply ${op} inside it. Write the whole value with ` +
+            `.set({ ${field} }) (or .remove(["${field}"])).`;
+        const records = [
+            [".add()", uState.add],
+            [".subtract()", uState.subtract],
+            [".append()", uState.append],
+            [".deleteFromSet()", uState.deleteFromSet],
+        ];
+        for (const [op, values] of records) {
+            for (const field of Object.keys(values ?? {})) {
+                if (field in packedFields)
+                    return refuse(op, field);
+            }
+        }
+        const pathOps = [
+            ["pathSet", uState.pathSets ?? []],
+            ["pathRemove", (uState.pathRemoves ?? []).map((segments) => ({ segments }))],
+            ["pathAdd", uState.pathAdds ?? []],
+            ["pathSubtract", uState.pathSubtracts ?? []],
+            ["pathAppend", uState.pathAppends ?? []],
+            ["pathPrepend", uState.pathPrepends ?? []],
+            ["pathIfNotExists", uState.pathIfNotExists ?? []],
+            ["pathDelete", uState.pathDeletes ?? []],
+        ];
+        for (const [op, ops] of pathOps) {
+            for (const { segments } of ops) {
+                const head = segments[0];
+                if (typeof head === "string" && head in packedFields)
+                    return refuse(op, head);
+            }
+        }
+        for (const { field } of uState.sparseRemoveEntries ?? []) {
+            if (field in packedFields)
+                return refuse(".removeEntries()", field);
+        }
+        for (const field of uState.sparseClearFields ?? []) {
+            if (field in packedFields)
+                return refuse(".clearMap()", field);
+        }
+        return undefined;
+    };
     /**
      * A DOMAIN-keyed VIEW of a row that came off the wire.
      *
@@ -973,43 +1134,68 @@ const makeImpl = (config, precomputedData) => {
         const corruption = versionCorruption(raw, "decode");
         if (corruption !== undefined)
             return Effect.fail(corruption);
-        deserializeSparseFields(raw);
-        renameFromDynamo(raw);
-        return Schema.decodeUnknownEffect(schemas.recordSchema)(raw).pipe(Effect.map(attachPrototype), Effect.mapError((cause) => new ValidationError({
-            entityType,
-            operation: "decode",
-            cause,
-        })));
+        return Effect.flatMap(unpackFields(raw), () => {
+            deserializeSparseFields(raw);
+            renameFromDynamo(raw);
+            return Schema.decodeUnknownEffect(schemas.recordSchema)(raw).pipe(Effect.map(attachPrototype), Effect.mapError((cause) => new ValidationError({
+                entityType,
+                operation: "decode",
+                cause,
+            })));
+        });
     };
     /** Decode a raw item using the schema selected by mode */
     const decodeAs = (raw, marshalled, mode) => {
         const corruption = versionCorruption(raw, "decode");
         if (corruption !== undefined)
             return Effect.fail(corruption);
+        // Native: the item as stored — packed values stay their `S` / `B`.
         if (mode === "native")
             return Effect.succeed(marshalled);
-        // Collect flattened sparse-map attributes back into the domain Record.
-        // Must happen before rename so the rebuilt Record sits at the domain field
-        // name (sparse field renaming is not supported).
+        return Effect.flatMap(unpackFields(raw), () => {
+            // Collect flattened sparse-map attributes back into the domain Record.
+            // Must happen before rename so the rebuilt Record sits at the domain field
+            // name (sparse field renaming is not supported).
+            deserializeSparseFields(raw);
+            // Rename DynamoDB attributes back to domain field names. Substituted
+            // schemas handle wire→domain conversion (date primitives, Redacted, etc.).
+            renameFromDynamo(raw);
+            const targetSchema = mode === "model"
+                ? schemas.modelSchema
+                : mode === "record"
+                    ? schemas.recordSchema
+                    : schemas.itemSchema;
+            return Schema.decodeUnknownEffect(targetSchema)(raw).pipe(Effect.map((decoded) => mode === "model" && isSchemaClass && !hasHiddenFields
+                ? new rawModel(decoded)
+                : mode !== "item"
+                    ? attachPrototype(decoded)
+                    : decoded), Effect.mapError((cause) => new ValidationError({
+                entityType,
+                operation: "decode",
+                cause,
+            })));
+        });
+    };
+    /**
+     * Decode a DynamoDB Streams image into an item record (`Entity.Item`): the
+     * image normalised (DynamoDB JSON or unmarshalled; `B` as bytes or base64),
+     * its dispatch-eligible attributes read by stored form, then sparse rebuild,
+     * renames and the item-schema decode — exactly what a table read applies.
+     */
+    const decodeImageItem = (image) => Effect.gen(function* () {
+        // An image of another entity type is decoded as 1.25.0 decoded it: its
+        // attributes are never read as packed values (a plain string of that
+        // type is not this entity's JSON). The type is read off the raw image.
+        const owner = imageEntityType(image);
+        const raw = yield* decodeImage(image, owner === undefined || owner === entityType ? dispatchAttributes : [], { entityType, operation: "decodeMarshalledItem" }, { probe: config.indexes.primary.pk.field });
+        const corruption = versionCorruption(raw, "decodeMarshalledItem");
+        if (corruption !== undefined)
+            return yield* corruption;
         deserializeSparseFields(raw);
-        // Rename DynamoDB attributes back to domain field names. Substituted
-        // schemas handle wire→domain conversion (date primitives, Redacted, etc.).
         renameFromDynamo(raw);
-        const targetSchema = mode === "model"
-            ? schemas.modelSchema
-            : mode === "record"
-                ? schemas.recordSchema
-                : schemas.itemSchema;
-        return Schema.decodeUnknownEffect(targetSchema)(raw).pipe(Effect.map((decoded) => mode === "model" && isSchemaClass && !hasHiddenFields
-            ? new rawModel(decoded)
-            : mode !== "item"
-                ? attachPrototype(decoded)
-                : decoded), Effect.mapError((cause) => new ValidationError({
-            entityType,
-            operation: "decode",
-            cause,
-        })));
-    };
+        const decoded = yield* Schema.decodeUnknownEffect(schemas.itemSchema)(raw).pipe(Effect.mapError((cause) => new ValidationError({ entityType, operation: "decodeMarshalledItem", cause })));
+        return attachPrototype(decoded);
+    });
     const nowIso = (now) => DateTime.formatIso(now);
     /**
      * Generate a timestamp value as a wire primitive ready for marshalling.
@@ -1314,6 +1500,9 @@ const makeImpl = (config, precomputedData) => {
             const current = yield* client.getItem({ TableName: tableName, Key: marshalledKey });
             if (current.Item) {
                 const currentDomain = fromAttributeMap(current.Item);
+                // Packed fields are read by their stored form: the source text is
+                // derived from the value, not its JSON string or compressed bytes.
+                yield* unpackFields(currentDomain, "update.vector");
                 renameFromDynamo(currentDomain);
                 merged = { ...currentDomain, ...merged };
             }
@@ -1680,16 +1869,18 @@ const makeImpl = (config, precomputedData) => {
         if (mode === "native")
             return Effect.succeed(projected);
         const raw = fromAttributeMap(projected);
-        deserializeSparseFields(raw);
-        renameFromDynamo(raw);
-        const target = mode === "model"
-            ? schemas.modelSchema
-            : mode === "record"
-                ? schemas.recordSchema
-                : schemas.itemSchema;
-        const fields = target.fields;
-        const present = Object.fromEntries(Object.entries(fields).filter(([name]) => name in raw));
-        return Schema.decodeUnknownEffect(Schema.Struct(present))(raw).pipe(Effect.mapError((cause) => new ValidationError({ entityType, operation: "decode", cause })));
+        return Effect.flatMap(unpackFields(raw), () => {
+            deserializeSparseFields(raw);
+            renameFromDynamo(raw);
+            const target = mode === "model"
+                ? schemas.modelSchema
+                : mode === "record"
+                    ? schemas.recordSchema
+                    : schemas.itemSchema;
+            const fields = target.fields;
+            const present = Object.fromEntries(Object.entries(fields).filter(([name]) => name in raw));
+            return Schema.decodeUnknownEffect(Schema.Struct(present))(raw).pipe(Effect.mapError((cause) => new ValidationError({ entityType, operation: "decode", cause })));
+        });
     };
     /**
      * The value an update returns for its `returnValues` mode, from the item
@@ -2737,6 +2928,11 @@ const makeImpl = (config, precomputedData) => {
                 cause: e instanceof Error ? e.message : String(e),
             });
         }
+        // Pack `storedAs: DynamoModel.Packed` fields to their stored form
+        // (JSON string, or compressed binary over the threshold) — after
+        // renames and sparse flattening, before marshalling, so every size
+        // measured from here on is the stored size.
+        yield* packFields(item, operation);
         // The caller's condition, ANDed onto `create`'s not-exists guard —
         // a guard no `.condition()` can replace (#133). `upsert`'s create
         // attempt brings its own (`runGuardedPut` requires the item missing).
@@ -2823,6 +3019,14 @@ const makeImpl = (config, precomputedData) => {
         let projectionExpression;
         let projectionNames;
         if (opts.projection && opts.projection.length > 0) {
+            const packedProjection = packedProjectionRefusal(opts.projection.map((attribute) => [attribute]), packedGuards);
+            if (packedProjection !== undefined) {
+                return yield* new ValidationError({
+                    entityType,
+                    operation: "get.projection",
+                    cause: packedProjection,
+                });
+            }
             const proj = Projection.projection(opts.projection);
             projectionExpression = proj.expression;
             projectionNames = proj.names;
@@ -2839,9 +3043,12 @@ const makeImpl = (config, precomputedData) => {
         if (!result.Item) {
             return yield* new ItemNotFound({ entityType, key: encodedKey });
         }
-        // Raw mode: return unmarshalled record without schema decode
+        // Raw mode: return unmarshalled record without schema decode — a
+        // packed attribute as its parsed JSON (#147).
         if (mode === "raw") {
-            return fromAttributeMap(result.Item);
+            const row = fromAttributeMap(result.Item);
+            yield* unpackFields(row);
+            return row;
         }
         // Unmarshall and decode using selected mode
         const raw = fromAttributeMap(result.Item);
@@ -2856,6 +3063,15 @@ const makeImpl = (config, precomputedData) => {
      * Used instead of reading it again: the write is guarded on it either way.
      */
     preRead) => Effect.gen(function* () {
+        // A packed field takes only a whole-value .set() / .remove() (#147).
+        const packedRefusal = refusedPackedUpdate(requested);
+        if (packedRefusal !== undefined) {
+            return yield* new ValidationError({
+                entityType,
+                operation: "update",
+                cause: packedRefusal,
+            });
+        }
         // Path operations on a field that feeds a key or a unique sentinel
         // become the record operations they are equivalent to, or are
         // refused (#133) — before anything reads the state.
@@ -3248,7 +3464,10 @@ const makeImpl = (config, precomputedData) => {
                 const embedFor = new Set(vectorIndexesNeedingUpdate(collectTouchedFields(hydratedUpdates, uState), uState.withVectors).map(([logicalName]) => logicalName));
                 // `newItem` is the fully merged post-update item (removals already
                 // applied above), so a put-style recompute is exact.
-                const vectorWrite = yield* computeVectorAttributes(newItem, uState.withVectors, embedFor);
+                // Untouched packed fields still hold their stored form on `newItem`
+                // (kept byte-identical for the write diff), so the source text is
+                // derived from an unpacked copy.
+                const vectorWrite = yield* computeVectorAttributes(yield* unpackedDomainCopy(newItem, "update.vector"), uState.withVectors, embedFor);
                 Object.assign(newItem, vectorWrite.sets);
                 for (const field of vectorWrite.removes)
                     delete newItem[field];
@@ -3266,9 +3485,17 @@ const makeImpl = (config, precomputedData) => {
                     cause: e instanceof Error ? e.message : String(e),
                 });
             }
+            // Pack only the packed fields this update names: the others still
+            // hold the stored value read above (a string or bytes), carried
+            // byte-identical so the "write only what differs" diff below leaves
+            // them alone (#147).
+            yield* packFields(newItem, "update", new Set(Object.entries(hydratedUpdates)
+                .filter(([field, value]) => field in packedFields && value != null)
+                .map(([field]) => resolveDbName(field))));
             const currentItem = currentResult.Item;
             yield* checkVersion(currentItem, "update");
-            const immutableChange = immutableMismatch(immutables, currentItem);
+            const immutableChange = immutableMismatch(immutables, currentItem) ??
+                (yield* packedImmutableMismatch(immutables, currentItem));
             if (immutableChange !== undefined) {
                 return yield* new ValidationError({
                     entityType,
@@ -3609,7 +3836,8 @@ const makeImpl = (config, precomputedData) => {
                 return yield* missingItem();
             }
             yield* checkVersion(current.Item, "update");
-            const immutableChange = immutableMismatch(immutables, current.Item);
+            const immutableChange = immutableMismatch(immutables, current.Item) ??
+                (yield* packedImmutableMismatch(immutables, current.Item));
             if (immutableChange !== undefined) {
                 return yield* new ValidationError({
                     entityType,
@@ -3662,6 +3890,28 @@ const makeImpl = (config, precomputedData) => {
             [uState.remove, uState.sparseRemoveEntries, uState.sparseClearFields].every((ops) => ops === undefined || ops.length === 0) &&
             [uState.add, uState.subtract, uState.append, uState.deleteFromSet].every((ops) => ops === undefined || Object.keys(ops).length === 0) &&
             !hasPathOps(uState);
+        // A restated packed immutable cannot be asserted by a condition on its
+        // stored bytes (they depend on the codec and threshold it was written
+        // under): an unread write reads it and compares the unpacked JSON
+        // (#147). The write still requires the item (`attribute_exists`), and
+        // no library write changes an immutable value in between.
+        if (plainWrite && immutables.some((asserted) => asserted.packed)) {
+            const stored = yield* client.getItem({
+                TableName: tableName,
+                Key: marshalledKey,
+                ConsistentRead: true,
+            });
+            if (stored.Item !== undefined) {
+                const packedChange = yield* packedImmutableMismatch(immutables, stored.Item);
+                if (packedChange !== undefined) {
+                    return yield* new ValidationError({
+                        entityType,
+                        operation: "update",
+                        cause: packedChange,
+                    });
+                }
+            }
+        }
         // Build UpdateExpression
         const setClauses = [];
         const names = {};
@@ -3723,8 +3973,13 @@ const makeImpl = (config, precomputedData) => {
             }
             const nameKey = `#u${counter}`;
             const valKey = `:u${counter}`;
-            names[nameKey] = resolveDbName(attr);
-            values[valKey] = toAttributeValue(val);
+            const storedName = resolveDbName(attr);
+            names[nameKey] = storedName;
+            // A packed field is written whole, as its stored form (#147).
+            values[valKey] =
+                attr in packedFields
+                    ? (yield* packAttribute(storedName, val, "update"))
+                    : toAttributeValue(val);
             setClauses.push(`${nameKey} = ${valKey}`);
             counter++;
         }
@@ -4123,7 +4378,9 @@ const makeImpl = (config, precomputedData) => {
         }
         if (plainWrite) {
             // Restated immutable values must match the item's.
-            for (const [i, { attr, value }] of immutables.entries()) {
+            for (const [i, { attr, value, packed }] of immutables.entries()) {
+                if (packed)
+                    continue; // read and compared above (#147)
                 names[`#imm${i}`] = attr;
                 values[`:imm${i}`] = value;
                 condParts.push(`#imm${i} = :imm${i}`);
@@ -4870,8 +5127,13 @@ const makeImpl = (config, precomputedData) => {
             }
             const nameKey = `#u${counter}`;
             const valKey = `:u${counter}`;
-            names[nameKey] = resolveDbName(attr);
-            values[valKey] = toAttributeValue(val);
+            const storedName = resolveDbName(attr);
+            names[nameKey] = storedName;
+            // A packed field is written whole, as its stored form (#147).
+            values[valKey] =
+                attr in packedFields
+                    ? (yield* packAttribute(storedName, val, "upsert"))
+                    : toAttributeValue(val);
             // Immutable fields use if_not_exists — only set on first create
             if (immutableFields.has(attr)) {
                 setClauses.push(`${nameKey} = if_not_exists(${nameKey}, ${valKey})`);
@@ -5102,6 +5364,16 @@ const makeImpl = (config, precomputedData) => {
         // `append` takes its condition as an argument — no `condition`
         // combinator — so an empty one becomes none here (#133).
         const userCondition = nonEmptyCondition(givenCondition);
+        if (userCondition !== undefined) {
+            const packedRefusal = packedExprRefusal(toExpr(userCondition), packedGuards, "condition");
+            if (packedRefusal !== undefined) {
+                return yield* new ValidationError({
+                    entityType,
+                    operation: "append.condition",
+                    cause: packedRefusal,
+                });
+            }
+        }
         const emptyPart = userCondition === undefined ? undefined : emptyPartProblem(toExpr(userCondition));
         if (emptyPart !== undefined) {
             return yield* new ValidationError({
@@ -5199,6 +5471,15 @@ const makeImpl = (config, precomputedData) => {
         });
         // `encoded` is already in wire-form via Schema.encode — used directly.
         const serialisedInput = { ...encoded };
+        // Packed `appendInput` fields (#147), packed ONCE: the same stored value
+        // is SET on the current item and written on the event item.
+        const packedStored = {};
+        for (const field of Object.keys(packedFields)) {
+            if (serialisedInput[field] !== undefined) {
+                packedStored[resolveDbName(field)] = serialisedInput[field];
+            }
+        }
+        yield* packFields(packedStored, "append");
         // ---------- Build UpdateItem (scoped SET + CAS) ----------
         const setClauses = [];
         const names = {};
@@ -5215,8 +5496,9 @@ const makeImpl = (config, precomputedData) => {
                 continue;
             const nameKey = `#a${counter}`;
             const valKey = `:a${counter}`;
-            names[nameKey] = resolveDbName(attr);
-            values[valKey] = toAttributeValue(val);
+            const storedName = resolveDbName(attr);
+            names[nameKey] = storedName;
+            values[valKey] = toAttributeValue(attr in packedFields ? packedStored[storedName] : val);
             setClauses.push(`${nameKey} = ${valKey}`);
             counter++;
         }
@@ -5315,6 +5597,8 @@ const makeImpl = (config, precomputedData) => {
         const eventItem = { ...encoded };
         // Rename domain → DB names for the Put (matches put path)
         renameToDynamo(eventItem);
+        // The packed fields' stored values — the same ones SET above (#147).
+        Object.assign(eventItem, packedStored);
         // pk + sk (event)
         eventItem[primary.pk.field] = pkValue;
         eventItem[primary.sk.field] = eventSk;
@@ -5484,13 +5768,15 @@ const makeImpl = (config, precomputedData) => {
             // NOT carry `<prefix>#*` attributes. `deserializeSparseFields` populates
             // empty `{}` Records under the domain field names so the historyRecord
             // schema (which still includes the sparse Record fields) decodes.
-            deserializeSparseFields(raw);
-            renameFromDynamo(raw);
-            return Schema.decodeUnknownEffect(schemas.historyRecordSchema)(raw).pipe(Effect.map(attachPrototype), Effect.mapError((cause) => new ValidationError({
-                entityType,
-                operation: "history.decode",
-                cause,
-            })));
+            return Effect.flatMap(unpackFields(raw, "history.decode"), () => {
+                deserializeSparseFields(raw);
+                renameFromDynamo(raw);
+                return Schema.decodeUnknownEffect(schemas.historyRecordSchema)(raw).pipe(Effect.map(attachPrototype), Effect.mapError((cause) => new ValidationError({
+                    entityType,
+                    operation: "history.decode",
+                    cause,
+                })));
+            });
         };
         return Query.make({
             tableName: "",
@@ -5500,6 +5786,7 @@ const makeImpl = (config, precomputedData) => {
             skField: primary.sk.field,
             entityTypes: [entityType],
             decoder: (raw) => decodeHistory(raw),
+            ...packedQueryOptions,
             resolveTableName: tableTag.useSync((tc) => tc.name),
             keyFields: [primary.pk.field, primary.sk.field],
         }).pipe(Query.where({ beginsWith: prefix }));
@@ -5526,6 +5813,8 @@ const makeImpl = (config, precomputedData) => {
                 skField: indexDef.sk.field,
                 entityTypes: [entityType],
                 decoder: (raw) => decodeRecord(raw),
+                ...packedQueryOptions,
+                ...packedQueryOptions,
                 resolveTableName: tableTag.useSync((tc) => tc.name),
                 keyFields: [
                     indexDef.pk.field,
@@ -5555,6 +5844,7 @@ const makeImpl = (config, precomputedData) => {
         indexName: undefined,
         entityTypes: [entityType],
         decoder: (raw) => decodeRecord(raw),
+        ...packedQueryOptions,
         resolveTableName: tableTag.useSync((tc) => tc.name),
         keyFields: [config.indexes.primary.pk.field, config.indexes.primary.sk.field],
         liveRows: liveRows(),
@@ -5658,6 +5948,7 @@ const makeImpl = (config, precomputedData) => {
             skField: primary.sk.field,
             entityTypes: [entityType],
             decoder: (raw) => decodeRecord(raw),
+            ...packedQueryOptions,
             resolveTableName: tableTag.useSync((tc) => tc.name),
             keyFields: [primary.pk.field, primary.sk.field],
             prepare: withLegacy,
@@ -5684,6 +5975,9 @@ const makeImpl = (config, precomputedData) => {
         // (itemSchema would fail because it expects GSI key fields that aren't present)
         if (mode === "native")
             return tombstone;
+        // Packed fields: read by their stored form (copied as-is onto the
+        // tombstone, #147).
+        yield* unpackFields(raw, "deleted.get.decode");
         // Sparse Map fields: rebuild domain Records from flattened attrs so
         // soft-deleted items decode correctly. Sparse data is preserved
         // verbatim across soft-delete (GSI keys are stripped, sparse data is
@@ -5712,16 +6006,18 @@ const makeImpl = (config, precomputedData) => {
             const corruption = versionCorruption(raw, "deleted.list.decode");
             if (corruption !== undefined)
                 return Effect.fail(corruption);
-            // Sparse fields are domain data and are preserved across soft-delete.
-            // Rebuild Records from flattened attrs before schema decode.
-            deserializeSparseFields(raw);
-            // Then attribute → domain names, same order as `decodeRecord` (#127).
-            renameFromDynamo(raw);
-            return Schema.decodeUnknownEffect(schemas.deletedRecordSchema)(raw).pipe(Effect.map(attachPrototype), Effect.mapError((cause) => new ValidationError({
-                entityType,
-                operation: "deleted.list.decode",
-                cause,
-            })));
+            return Effect.flatMap(unpackFields(raw, "deleted.list.decode"), () => {
+                // Sparse fields are domain data and are preserved across soft-delete.
+                // Rebuild Records from flattened attrs before schema decode.
+                deserializeSparseFields(raw);
+                // Then attribute → domain names, same order as `decodeRecord` (#127).
+                renameFromDynamo(raw);
+                return Schema.decodeUnknownEffect(schemas.deletedRecordSchema)(raw).pipe(Effect.map(attachPrototype), Effect.mapError((cause) => new ValidationError({
+                    entityType,
+                    operation: "deleted.list.decode",
+                    cause,
+                })));
+            });
         };
         return Query.make({
             tableName: "",
@@ -5731,6 +6027,7 @@ const makeImpl = (config, precomputedData) => {
             skField: primary.sk.field,
             entityTypes: [],
             decoder: (raw) => decodeDeleted(raw),
+            ...packedQueryOptions,
             resolveTableName: tableTag.useSync((tc) => tc.name),
             keyFields: [primary.pk.field, primary.sk.field],
         }).pipe(Query.where({ beginsWith: deletedPrefix }));
@@ -6151,6 +6448,7 @@ const makeImpl = (config, precomputedData) => {
         const expr = typeof cbOrShorthand === "function"
             ? cbOrShorthand(entityPathBuilder, entityConditionOps)
             : parseSimpleShorthand(cbOrShorthand);
+        refusePackedExpr(expr, "condition");
         return (self) => conditionCombinator(self, expr);
     };
     /**
@@ -6161,17 +6459,25 @@ const makeImpl = (config, precomputedData) => {
         const expr = typeof cbOrShorthand === "function"
             ? cbOrShorthand(entityPathBuilder, entityConditionOps)
             : parseSimpleShorthand(cbOrShorthand);
+        refusePackedExpr(expr, "filter");
         return (q) => filterExpr(q, expr);
     };
     /**
      * Build a select (projection) combinator from callback or string array.
      */
     const entitySelect = (cbOrAttrs) => {
+        const refuse = (paths) => {
+            const refused = packedProjectionRefusal(paths, packedGuards);
+            if (refused !== undefined)
+                throw new Error(refused);
+        };
         if (typeof cbOrAttrs === "function") {
             const paths = cbOrAttrs(entityPathBuilder);
             const segments = paths.map((p) => p.segments);
+            refuse(segments);
             return (q) => selectPaths(q, segments);
         }
+        refuse(cbOrAttrs.map((attribute) => [attribute]));
         return (q) => Query.select(q, cbOrAttrs);
     };
     // ---------------------------------------------------------------------------
@@ -6199,8 +6505,14 @@ const makeImpl = (config, precomputedData) => {
         generatedId: config.generatedId,
         vectorIndexes: config.vectorIndexes,
         _vectorIndexes: vectorIndexes,
+        _packedFields: Object.keys(packedFields),
+        _packFields: packFields,
+        _unpackFields: unpackFields,
+        _dispatchAttributes: dispatchAttributes,
+        _packedGuards: packedGuards,
+        _decodeImage: decodeImageItem,
         _resolvedRefs: resolvedRefs,
-        /** @internal Full decode pipeline: rename + schema decode. Used by Batch/Aggregate. */
+        /** @internal Full decode pipeline: unpack + sparse + rename + schema decode. Used by Batch/Aggregate. */
         _decodeRecord: decodeRecord,
         /** @internal Domain field name → stored attribute name (`storedAs` renames). */
         _resolveDbName: resolveDbName,
@@ -6222,6 +6534,9 @@ const makeImpl = (config, precomputedData) => {
             tableTag = injectedTableTag;
         },
         _injectIndex: (name, def) => {
+            const refused = packedIndexRefusal(entityType, packedFields, name, def);
+            if (refused !== undefined)
+                throw new Error(refused);
             allIndexes = { ...allIndexes, [name]: def };
         },
         get _schema() {
@@ -6636,12 +6951,27 @@ export const itemSchema = (entity) => entity.schemas.itemSchema;
 /**
  * Decode a marshalled DynamoDB item (AttributeValue format) into a typed Entity.Record.
  *
- * Unmarshalls the AttributeValue map to plain JS, then decodes via the entity's item schema.
- * Useful for consuming DynamoDB Streams events or raw SDK responses.
+ * Built for DynamoDB Streams records (`NewImage` / `OldImage`) and raw SDK
+ * responses. Applies exactly what a table read applies:
+ *
+ * - **Packed fields** (`storedAs: DynamoModel.Packed`) are read by their
+ *   stored form — a legacy native map, a JSON string (`S`), or compressed
+ *   binary (`B`). Lambda and EventBridge Pipes deliver `B` / `BS` members as
+ *   base64 text; they are decoded at any depth.
+ * - Sparse-map fields are rebuilt and `field:` renames reversed.
+ *
+ * Accepts DynamoDB JSON (`{ pk: { S: "…" }, … }`, `B` as bytes or base64) or
+ * an already-unmarshalled item. Fails with `ValidationError` if the item does
+ * not match the schema, or when a compressed value's codec is not registered —
+ * its `cause` is then a `CompressorNotRegistered` (register the codec with a
+ * `Compression.layer([...])` provided where the decode runs).
  *
- * Returns an Effect that fails with `ValidationError` if the item doesn't match the schema.
+ * Note: {@link itemSchema} is a pure schema over unmarshalled items and does
+ * not unpack — use this for entities with packed fields.
  */
 export const decodeMarshalledItem = (entity, marshalledItem) => {
+    if (entity._decodeImage !== undefined)
+        return entity._decodeImage(marshalledItem);
     const corruption = entity._versionCorruption?.(marshalledItem, "decodeMarshalledItem");
     if (corruption !== undefined)
         return Effect.fail(corruption);