effect-dynamodb 1.9.5 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/Aggregate.d.ts.map +1 -1
  2. package/dist/Aggregate.js +5 -42
  3. package/dist/Aggregate.js.map +1 -1
  4. package/dist/DynamoClient.d.ts +88 -5
  5. package/dist/DynamoClient.d.ts.map +1 -1
  6. package/dist/DynamoClient.js +158 -4
  7. package/dist/DynamoClient.js.map +1 -1
  8. package/dist/Entity.d.ts +35 -17
  9. package/dist/Entity.d.ts.map +1 -1
  10. package/dist/Entity.js +503 -16
  11. package/dist/Entity.js.map +1 -1
  12. package/dist/Table.d.ts +78 -0
  13. package/dist/Table.d.ts.map +1 -1
  14. package/dist/Table.js +153 -0
  15. package/dist/Table.js.map +1 -1
  16. package/dist/VectorSearchEmulation.d.ts +101 -0
  17. package/dist/VectorSearchEmulation.d.ts.map +1 -0
  18. package/dist/VectorSearchEmulation.js +290 -0
  19. package/dist/VectorSearchEmulation.js.map +1 -0
  20. package/dist/index.d.ts +9 -2
  21. package/dist/index.d.ts.map +1 -1
  22. package/dist/index.js +4 -1
  23. package/dist/index.js.map +1 -1
  24. package/dist/internal/BoundCrud.d.ts +62 -48
  25. package/dist/internal/BoundCrud.d.ts.map +1 -1
  26. package/dist/internal/BoundCrud.js +7 -1
  27. package/dist/internal/BoundCrud.js.map +1 -1
  28. package/dist/internal/BoundVectorQuery.d.ts +130 -0
  29. package/dist/internal/BoundVectorQuery.d.ts.map +1 -0
  30. package/dist/internal/BoundVectorQuery.js +257 -0
  31. package/dist/internal/BoundVectorQuery.js.map +1 -0
  32. package/dist/internal/EntityCombinators.d.ts +16 -0
  33. package/dist/internal/EntityCombinators.d.ts.map +1 -1
  34. package/dist/internal/EntityCombinators.js +25 -1
  35. package/dist/internal/EntityCombinators.js.map +1 -1
  36. package/dist/internal/EntityOps.d.ts +19 -6
  37. package/dist/internal/EntityOps.d.ts.map +1 -1
  38. package/dist/internal/EntityOps.js +5 -2
  39. package/dist/internal/EntityOps.js.map +1 -1
  40. package/dist/internal/TransactableOps.d.ts +1 -1
  41. package/dist/internal/TransactableOps.d.ts.map +1 -1
  42. package/package.json +6 -6
package/dist/Entity.js CHANGED
@@ -8,10 +8,12 @@
8
8
  */
9
9
  import { getIdentifierField, getSparseFields, isRefField, } from "@effect-dynamodb/schema/DynamoModel.js";
10
10
  import * as DynamoSchema from "@effect-dynamodb/schema/DynamoSchema.js";
11
- import { CascadePartialFailure, ConditionalCheckFailed, ItemNotFound, isAwsConditionalCheckFailed, isAwsTransactionCancelled, OptimisticLockError, RefNotFound, StaleAppend, TransactionOverflow, UniqueConstraintViolation, ValidationError, } from "@effect-dynamodb/schema/Errors.js";
11
+ import { Embedder } from "@effect-dynamodb/schema/Embedder.js";
12
+ import { CascadePartialFailure, ConditionalCheckFailed, EmbeddingError, ItemNotFound, isAwsConditionalCheckFailed, isAwsTransactionCancelled, OptimisticLockError, RefNotFound, StaleAppend, TransactionOverflow, UniqueConstraintViolation, ValidationError, } from "@effect-dynamodb/schema/Errors.js";
12
13
  import { makeDefaultCrypto } from "@effect-dynamodb/schema/internal/DefaultCrypto.js";
13
14
  import * as KeyComposer from "@effect-dynamodb/schema/KeyComposer.js";
14
15
  import { normalizeGsiConfig } from "@effect-dynamodb/schema/KeyComposer.js";
16
+ import { deriveSourceText } from "@effect-dynamodb/schema/VectorIndex.js";
15
17
  import { Context, Crypto, DateTime, Effect, Option, Schema, Stream } from "effect";
16
18
  import { DynamoClient } from "./DynamoClient.js";
17
19
  import { makeBoundAppend, makeBoundDelete, makeBoundPut, makeBoundUpdate, } from "./internal/BoundCrud.js";
@@ -400,6 +402,258 @@ const makeImpl = (config, precomputedData) => {
400
402
  }
401
403
  return fields;
402
404
  };
405
+ // ---------------------------------------------------------------------------
406
+ // Vector search — write-path helpers (see `DESIGN.md §14`)
407
+ // ---------------------------------------------------------------------------
408
+ const vectorIndexes = data.vectorIndexes;
409
+ const hasVectorIndexes = data.hasVectorIndexes;
410
+ const vectorIndexEntries = Object.entries(vectorIndexes);
411
+ /**
412
+ * Library-managed vector attribute names (embedding + composed partition).
413
+ *
414
+ * These join `gsiKeyFields()` in every strip set — version snapshots,
415
+ * soft-delete tombstones, and time-series event items. DynamoDB's sparse
416
+ * vector index semantics do the rest: an item without the vector attribute
417
+ * (or without the HASH attribute) is simply not in the index, so stripping IS
418
+ * the delete. See `DESIGN.md §14 Lifecycle integration`.
419
+ */
420
+ const vectorKeyFields = () => {
421
+ const fields = [];
422
+ for (const [, definition] of vectorIndexEntries) {
423
+ fields.push(definition.vectorField, definition.partitionField);
424
+ }
425
+ return fields;
426
+ };
427
+ /**
428
+ * Resolve the `Embedder` service from context.
429
+ *
430
+ * Deliberately resolved with `Effect.serviceOption` rather than a plain
431
+ * `yield* Embedder`: only entities that declare `vectorIndexes` need one, and
432
+ * widening every entity operation's `R` to include `Embedder` would force the
433
+ * service on consumers who will never call it. A missing embedder surfaces as
434
+ * a tagged {@link EmbeddingError} naming the index, not a type error at the
435
+ * far end of the program.
436
+ */
437
+ const resolveEmbedder = (indexName) => Effect.flatMap(Effect.serviceOption(Embedder), (maybe) => Option.isSome(maybe)
438
+ ? Effect.succeed(maybe.value)
439
+ : Effect.fail(new EmbeddingError({
440
+ entityType,
441
+ index: indexName,
442
+ reason: `No Embedder service is available. Provide one via ` +
443
+ `DynamoClient.make({ embedder }) or Effect.provide(Embedder.layerTest(...)), ` +
444
+ `or supply a pre-computed vector with .withVector("${indexName}", [...]).`,
445
+ })));
446
+ /**
447
+ * Reject `.withVector("typo", ...)` before it silently does nothing.
448
+ *
449
+ * The combinator takes a plain string at runtime, and an unknown logical name
450
+ * would otherwise be dropped by the declared-name lookup — the Embedder would
451
+ * run anyway and the caller's supplied vector would vanish without a trace.
452
+ * Fails fast with the list of declared names instead.
453
+ */
454
+ const checkWithVectorNames = (withVectors, operation) => {
455
+ if (withVectors === undefined)
456
+ return Effect.void;
457
+ for (const name of Object.keys(withVectors)) {
458
+ if (name in vectorIndexes)
459
+ continue;
460
+ const declared = Object.keys(vectorIndexes).sort().join(", ");
461
+ return Effect.fail(new ValidationError({
462
+ entityType,
463
+ operation: `${operation}.withVector`,
464
+ cause: `Unknown vector index "${name}". Declared vector indexes: ` +
465
+ `${declared.length > 0 ? declared : "(none)"}. ` +
466
+ `\`.withVector()\` takes the LOGICAL name from Entity.make({ vectorIndexes }).`,
467
+ }));
468
+ }
469
+ return Effect.void;
470
+ };
471
+ /** @internal Validate an embedding against the index's declared dimensionality. */
472
+ const checkDimensions = (indexName, definition, vector) => vector.length === definition.dimensions
473
+ ? Effect.succeed(vector)
474
+ : Effect.fail(new EmbeddingError({
475
+ entityType,
476
+ index: indexName,
477
+ reason: `Embedding has ${vector.length} dimensions but vector index "${definition.index}" ` +
478
+ `declares ${definition.dimensions}. Dimensions are immutable on a DynamoDB vector index.`,
479
+ }));
480
+ /**
481
+ * Compute the vector + partition attributes for a full (put-style) record.
482
+ *
483
+ * Each index is independent and sparse: an index whose partition composites
484
+ * or source fields are absent from the record contributes nothing, and the
485
+ * item is simply not in that index.
486
+ *
487
+ * `embedFor` restricts which indexes are (re-)embedded — `"all"` on the put
488
+ * path, a gated subset on the update path. The partition value is always
489
+ * recomposed when it can be: it is idempotent, and writing it is what keeps
490
+ * an item in the index.
491
+ */
492
+ const computeVectorAttributes = (record, withVectors, embedFor) => Effect.gen(function* () {
493
+ const sets = {};
494
+ const removes = [];
495
+ for (const [logicalName, definition] of vectorIndexEntries) {
496
+ const shouldEmbed = embedFor === "all" || embedFor.has(logicalName);
497
+ const partition = KeyComposer.tryComposeVectorPartition(schema, entityType, definition, record);
498
+ if (partition === undefined) {
499
+ // No partition value ⇒ the item cannot be in the index at all. Drop
500
+ // both attributes so a previously-indexed item leaves the index
501
+ // rather than lingering under a stale partition.
502
+ removes.push(definition.partitionField, definition.vectorField);
503
+ continue;
504
+ }
505
+ sets[definition.partitionField] = partition;
506
+ if (!shouldEmbed)
507
+ continue;
508
+ const explicit = withVectors?.[logicalName];
509
+ if (explicit !== undefined) {
510
+ sets[definition.vectorField] = yield* checkDimensions(logicalName, definition, explicit);
511
+ continue;
512
+ }
513
+ const text = deriveSourceText(definition, record);
514
+ if (text === undefined) {
515
+ // Every source field is gone. Sparse semantics say the item should
516
+ // leave the index — keeping the previous embedding would make it
517
+ // findable by a description it no longer has.
518
+ removes.push(definition.vectorField, definition.partitionField);
519
+ delete sets[definition.partitionField];
520
+ continue;
521
+ }
522
+ const embedder = yield* resolveEmbedder(logicalName);
523
+ const embedded = yield* embedder.embed(text);
524
+ sets[definition.vectorField] = yield* checkDimensions(logicalName, definition, embedded);
525
+ }
526
+ return { sets, removes };
527
+ });
528
+ /**
529
+ * Which vector indexes does this update require work for?
530
+ *
531
+ * Re-embedding fires when the write touches a `source.fields` member by ANY
532
+ * channel — a `set()` payload entry, an `Entity.remove([...])` clearing it, or
533
+ * a path-based operation whose root segment names it — or when the caller
534
+ * supplied an explicit vector.
535
+ *
536
+ * Enumerating every channel (rather than only the SET payload) is the same
537
+ * discipline the policy-aware GSI composer applies to `removedSet` (§7): a
538
+ * clear is a change, and a writer that clears the source must not leave the
539
+ * old embedding behind. Writers that touch none of it neither pay for an
540
+ * embedding call nor clobber a vector another writer owns.
541
+ */
542
+ const vectorIndexesNeedingUpdate = (touchedFields, withVectors) => vectorIndexEntries.filter(([logicalName, definition]) => withVectors?.[logicalName] !== undefined ||
543
+ definition.sourceFields.some((field) => touchedFields.has(field)));
544
+ /**
545
+ * Every top-level attribute this update touches, by any channel.
546
+ *
547
+ * `touchesSource` alone only sees the SET payload; `remove([...])`, null
548
+ * clears and path-based operations are just as capable of invalidating an
549
+ * embedding, so they all feed the same set.
550
+ */
551
+ const collectTouchedFields = (updatePayload, uState) => {
552
+ const touched = new Set(Object.keys(updatePayload));
553
+ for (const attr of uState.remove ?? [])
554
+ touched.add(attr);
555
+ const pathOps = [
556
+ uState.pathSets ?? [],
557
+ uState.pathAdds ?? [],
558
+ uState.pathSubtracts ?? [],
559
+ uState.pathAppends ?? [],
560
+ uState.pathPrepends ?? [],
561
+ uState.pathIfNotExists ?? [],
562
+ uState.pathDeletes ?? [],
563
+ ];
564
+ for (const ops of pathOps) {
565
+ for (const op of ops) {
566
+ const root = op.segments[0];
567
+ if (typeof root === "string")
568
+ touched.add(root);
569
+ }
570
+ }
571
+ for (const segments of uState.pathRemoves ?? []) {
572
+ const root = segments[0];
573
+ if (typeof root === "string")
574
+ touched.add(root);
575
+ }
576
+ return touched;
577
+ };
578
+ /**
579
+ * Compute vector-related SET / REMOVE attributes for the standard
580
+ * (UpdateItem) path.
581
+ *
582
+ * When re-embedding is required and the payload does not carry every source
583
+ * field, the current item is read once so the embedding is derived from the
584
+ * complete post-update source text rather than the fragment that happened to
585
+ * be in this payload. Attributes the update clears (via `remove([...])` or a
586
+ * null payload entry) are subtracted from that merged record, so "clear the
587
+ * description" produces the embedding of what actually remains — or drops the
588
+ * item out of the index when nothing does.
589
+ */
590
+ const computeVectorUpdateAttributes = (decodedKey, marshalledKey, tableName, updatePayload, uState) => Effect.gen(function* () {
591
+ if (!hasVectorIndexes)
592
+ return { sets: {}, removes: [] };
593
+ const withVectors = uState.withVectors;
594
+ const touchedFields = collectTouchedFields(updatePayload, uState);
595
+ const needing = vectorIndexesNeedingUpdate(touchedFields, withVectors);
596
+ const clearedFields = new Set(uState.remove ?? []);
597
+ for (const [attr, value] of Object.entries(updatePayload)) {
598
+ if (value === null || value === undefined)
599
+ clearedFields.add(attr);
600
+ }
601
+ // Base record: key composites plus whatever the payload supplies. Enough
602
+ // to compose partition values (their composites are normally primary-key
603
+ // members) without any extra read.
604
+ let merged = { ...decodedKey, ...updatePayload };
605
+ const needsFullSource = needing.some(([logicalName, definition]) => withVectors?.[logicalName] === undefined &&
606
+ definition.sourceFields.some((field) => !(field in merged)));
607
+ if (needsFullSource) {
608
+ const client = yield* DynamoClient;
609
+ const current = yield* client.getItem({ TableName: tableName, Key: marshalledKey });
610
+ if (current.Item) {
611
+ const currentDomain = fromAttributeMap(current.Item);
612
+ renameFromDynamo(currentDomain);
613
+ merged = { ...currentDomain, ...merged };
614
+ }
615
+ }
616
+ // Cleared attributes are absent post-update, whichever channel cleared
617
+ // them — the same two-way classification the GSI composer uses (§7).
618
+ for (const attr of clearedFields)
619
+ delete merged[attr];
620
+ const sets = {};
621
+ const removes = [];
622
+ for (const [logicalName, definition] of vectorIndexEntries) {
623
+ const isNeeded = needing.some(([name]) => name === logicalName);
624
+ // Partition value: idempotent and cheap, so recompose whenever the
625
+ // merged record can supply it (same reasoning as the PK-composites-only
626
+ // GSI shape in §7).
627
+ const partition = KeyComposer.tryComposeVectorPartition(schema, entityType, definition, merged);
628
+ if (partition === undefined) {
629
+ // Only drop the partition when THIS writer invalidated it. Otherwise
630
+ // the composite simply wasn't in scope for this call and dropping it
631
+ // would be the multi-writer clobber the §7 gate exists to prevent.
632
+ if (definition.partition.some((attr) => clearedFields.has(attr))) {
633
+ removes.push(definition.partitionField, definition.vectorField);
634
+ }
635
+ continue;
636
+ }
637
+ sets[definition.partitionField] = partition;
638
+ if (!isNeeded)
639
+ continue;
640
+ const explicit = withVectors?.[logicalName];
641
+ if (explicit !== undefined) {
642
+ sets[definition.vectorField] = yield* checkDimensions(logicalName, definition, explicit);
643
+ continue;
644
+ }
645
+ const text = deriveSourceText(definition, merged);
646
+ if (text === undefined) {
647
+ removes.push(definition.vectorField, definition.partitionField);
648
+ delete sets[definition.partitionField];
649
+ continue;
650
+ }
651
+ const embedder = yield* resolveEmbedder(logicalName);
652
+ const embedded = yield* embedder.embed(text);
653
+ sets[definition.vectorField] = yield* checkDimensions(logicalName, definition, embedded);
654
+ }
655
+ return { sets, removes };
656
+ });
403
657
  /**
404
658
  * Build a version snapshot item: same PK, version SK, stripped GSI keys,
405
659
  * keeps __edd_e__ for entity type filtering.
@@ -414,13 +668,16 @@ const makeImpl = (config, precomputedData) => {
414
668
  for (const field of gsiKeyFields()) {
415
669
  delete snapshot[field];
416
670
  }
671
+ // Same for vector + partition attributes — a snapshot is never an ANN hit.
672
+ for (const field of vectorKeyFields()) {
673
+ delete snapshot[field];
674
+ }
417
675
  // Replace SK with version SK
418
676
  snapshot[tableSkField] = DynamoSchema.composeVersionKey(schema, entityType, version);
419
677
  // Add optional TTL
420
678
  const ttl = retainTtl();
421
679
  if (ttl) {
422
- snapshot[ttlAttrName] =
423
- Math.floor(DateTime.toEpochMillis(now) / 1000) + normalizeTtlSeconds(ttl);
680
+ snapshot[ttlAttrName] = DateTime.toEpochSeconds(now) + normalizeTtlSeconds(ttl);
424
681
  }
425
682
  return snapshot;
426
683
  };
@@ -675,6 +932,7 @@ const makeImpl = (config, precomputedData) => {
675
932
  const tc = yield* tableTag;
676
933
  const tableName = tc.name;
677
934
  const ttlAttrName = resolveTtlAttributeName(tc);
935
+ yield* checkWithVectorNames(opts.withVectors, "put");
678
936
  // Auto-generated id: fill the configured primary-key field with a
679
937
  // fresh UUID BEFORE the encode, but only when the caller omitted it —
680
938
  // a caller-supplied value is always respected. The id is sourced from
@@ -730,6 +988,16 @@ const makeImpl = (config, precomputedData) => {
730
988
  item.__edd_e__ = entityType;
731
989
  // Apply composed keys
732
990
  Object.assign(item, keys);
991
+ // Vector search: embed the source text and compose the partition
992
+ // value. Sparse — an index whose source or partition composites are
993
+ // absent contributes nothing. See `DESIGN.md §14`.
994
+ if (hasVectorIndexes) {
995
+ const vectorWrite = yield* computeVectorAttributes(encoded, opts.withVectors, "all");
996
+ Object.assign(item, vectorWrite.sets);
997
+ // `put` writes a whole item, so a REMOVE is just an omission.
998
+ for (const field of vectorWrite.removes)
999
+ delete item[field];
1000
+ }
733
1001
  // Rename domain fields to DynamoDB attribute names
734
1002
  renameToDynamo(item);
735
1003
  // Flatten sparse Map fields into per-entry top-level attributes.
@@ -788,7 +1056,7 @@ const makeImpl = (config, precomputedData) => {
788
1056
  const uniqueTtl = resolveUniqueTtl(constraintDef);
789
1057
  if (uniqueTtl !== undefined) {
790
1058
  sentinelItem[ttlAttrName] =
791
- Math.floor(DateTime.toEpochMillis(now) / 1000) + normalizeTtlSeconds(uniqueTtl);
1059
+ DateTime.toEpochSeconds(now) + normalizeTtlSeconds(uniqueTtl);
792
1060
  }
793
1061
  transactItems.push({
794
1062
  Put: {
@@ -887,9 +1155,7 @@ const makeImpl = (config, precomputedData) => {
887
1155
  const pkField = config.indexes.primary.pk.field;
888
1156
  const skField = config.indexes.primary.sk.field;
889
1157
  const op = put(input);
890
- return new EntityPutImpl(op._builder, op._entity, op._input, {
891
- attributeNotExists: [pkField, skField],
892
- });
1158
+ return new EntityPutImpl(op._builder, op._entity, op._input, { attributeNotExists: [pkField, skField] }, op._withVectors);
893
1159
  };
894
1160
  // ---------------------------------------------------------------------------
895
1161
  // patch operation — update + attribute_exists condition
@@ -955,6 +1221,7 @@ const makeImpl = (config, precomputedData) => {
955
1221
  const ttlAttrName = resolveTtlAttributeName(tc);
956
1222
  // Clock-backed time source for the updatedAt timestamp + retain snapshot TTL.
957
1223
  const now = yield* DateTime.now;
1224
+ yield* checkWithVectorNames(uState.withVectors, "update");
958
1225
  // Decode key
959
1226
  const decodedKey = yield* Schema.decodeUnknownEffect(schemas.keySchema)(key).pipe(Effect.mapError((cause) => new ValidationError({
960
1227
  entityType,
@@ -1196,6 +1463,20 @@ const makeImpl = (config, precomputedData) => {
1196
1463
  });
1197
1464
  }
1198
1465
  }
1466
+ // Vector search: the retain path already holds the full merged
1467
+ // item, so a put-style recompute is exact — no extra read needed.
1468
+ // Re-embedding is still gated on the payload touching a source
1469
+ // field (`DESIGN.md §14`); untouched indexes keep their stored
1470
+ // vector, which the merge above already carried onto `newItem`.
1471
+ if (hasVectorIndexes) {
1472
+ const embedFor = new Set(vectorIndexesNeedingUpdate(collectTouchedFields(hydratedUpdates, uState), uState.withVectors).map(([logicalName]) => logicalName));
1473
+ // `newItem` is the fully merged post-update item (removals already
1474
+ // applied above), so a put-style recompute is exact.
1475
+ const vectorWrite = yield* computeVectorAttributes(newItem, uState.withVectors, embedFor);
1476
+ Object.assign(newItem, vectorWrite.sets);
1477
+ for (const field of vectorWrite.removes)
1478
+ delete newItem[field];
1479
+ }
1199
1480
  // Convert back to DynamoDB attribute names for storage
1200
1481
  renameToDynamo(newItem);
1201
1482
  // Flatten sparse Map fields into per-entry top-level attributes.
@@ -1505,6 +1786,28 @@ const makeImpl = (config, precomputedData) => {
1505
1786
  names[nameKey] = keyField;
1506
1787
  removeClauses.push(nameKey);
1507
1788
  }
1789
+ // Vector search: partition value is recomposed whenever it can be
1790
+ // (idempotent); the embedding is regenerated only when the payload
1791
+ // touched a `source.fields` member or supplied `.withVector(...)`.
1792
+ // See `DESIGN.md §14 Write path`.
1793
+ if (hasVectorIndexes) {
1794
+ const vectorWrite = yield* computeVectorUpdateAttributes(decodedKey, marshalledKey, tableName, hydratedUpdates, uState);
1795
+ for (const [field, value] of Object.entries(vectorWrite.sets)) {
1796
+ const nameKey = `#u${counter}`;
1797
+ const valKey = `:u${counter}`;
1798
+ names[nameKey] = field;
1799
+ values[valKey] = toAttributeValue(value);
1800
+ setClauses.push(`${nameKey} = ${valKey}`);
1801
+ counter++;
1802
+ }
1803
+ // Clearing every source field takes the item out of the index —
1804
+ // there is no other way to delete a vector index entry.
1805
+ for (const field of new Set(vectorWrite.removes)) {
1806
+ const nameKey = `#r${removeClauses.length}`;
1807
+ names[nameKey] = field;
1808
+ removeClauses.push(nameKey);
1809
+ }
1810
+ }
1508
1811
  // ADD (atomic numeric increment / set addition)
1509
1812
  if (uState.add) {
1510
1813
  for (const [attr, val] of Object.entries(uState.add)) {
@@ -1802,6 +2105,17 @@ const makeImpl = (config, precomputedData) => {
1802
2105
  for (const field of gsiKeyFields()) {
1803
2106
  delete deletedItem[field];
1804
2107
  }
2108
+ // Vector attributes: stash the embedding under a non-indexed name
2109
+ // before stripping. Sparse semantics drop the tombstone out of the
2110
+ // vector index immediately, and `restore()` un-stashes without
2111
+ // paying for another Embedder call. See `DESIGN.md §14`.
2112
+ for (const [, definition] of vectorIndexEntries) {
2113
+ const stored = deletedItem[definition.vectorField];
2114
+ if (stored !== undefined)
2115
+ deletedItem[definition.stashField] = stored;
2116
+ delete deletedItem[definition.vectorField];
2117
+ delete deletedItem[definition.partitionField];
2118
+ }
1805
2119
  // Replace SK with deleted sort key
1806
2120
  deletedItem[primary.sk.field] = DynamoSchema.composeDeletedKey(schema, entityType, now);
1807
2121
  // Add deletedAt
@@ -1809,8 +2123,7 @@ const makeImpl = (config, precomputedData) => {
1809
2123
  // Add optional TTL
1810
2124
  const sdTtl = softDeleteTtl();
1811
2125
  if (sdTtl) {
1812
- deletedItem[ttlAttrName] =
1813
- Math.floor(DateTime.toEpochMillis(dtNow) / 1000) + normalizeTtlSeconds(sdTtl);
2126
+ deletedItem[ttlAttrName] = DateTime.toEpochSeconds(dtNow) + normalizeTtlSeconds(sdTtl);
1814
2127
  }
1815
2128
  const transactItems = [];
1816
2129
  // Delete current entity item
@@ -1940,6 +2253,7 @@ const makeImpl = (config, precomputedData) => {
1940
2253
  const { name: tableName } = yield* tableTag;
1941
2254
  // Clock-backed time source for createdAt/updatedAt SET clauses.
1942
2255
  const now = yield* DateTime.now;
2256
+ yield* checkWithVectorNames(opts.withVectors, "upsert");
1943
2257
  // Encode user input → wire form (see `put` for strategy).
1944
2258
  const encodedInput = yield* encodeOrDecodeEncode(schemas.inputSchema, input, entityType, "upsert");
1945
2259
  // Hydrate refs: replace ID fields with full entity domain data
@@ -1952,11 +2266,21 @@ const makeImpl = (config, precomputedData) => {
1952
2266
  const marshalledKey = toAttributeMap(primaryKey);
1953
2267
  // Build UpdateExpression with if_not_exists for immutable fields + createdAt
1954
2268
  const setClauses = [];
2269
+ const upsertRemoveClauses = [];
1955
2270
  const names = {};
1956
2271
  const values = {};
1957
2272
  let counter = 0;
1958
- // Determine primary key composite attribute names
1959
- const pkComposites = new Set(primaryKeyComposites(config.indexes));
2273
+ // The physical key attributes (`pk` / `sk`) live in `Key` and cannot
2274
+ // appear in an UpdateExpression. The COMPOSITE SOURCE fields are
2275
+ // ordinary attributes and must be written like any other model field —
2276
+ // skipping them (as this path used to) left the stored item without
2277
+ // e.g. `productId`, so every read of an upserted item failed to decode
2278
+ // with `Missing key`. Their values are fixed by the key, so a plain
2279
+ // SET is idempotent.
2280
+ const keyAttributeFields = new Set([
2281
+ config.indexes.primary.pk.field,
2282
+ config.indexes.primary.sk.field,
2283
+ ]);
1960
2284
  // System-colliding fields are written below in the system-field block
1961
2285
  // (so their semantics — if_not_exists / always-set — stay consistent).
1962
2286
  const systemColliders = new Set();
@@ -1967,9 +2291,9 @@ const makeImpl = (config, precomputedData) => {
1967
2291
  if (systemFields.versionCollision && systemFields.version)
1968
2292
  systemColliders.add(systemFields.version);
1969
2293
  // `item` is already in wire-form via `Schema.encode(inputSchema)`.
1970
- // All model fields (excluding PK composites, which are in the Key)
2294
+ // All model fields (excluding the physical key attributes themselves)
1971
2295
  for (const [attr, val] of Object.entries(item)) {
1972
- if (pkComposites.has(attr))
2296
+ if (keyAttributeFields.has(resolveDbName(attr)))
1973
2297
  continue;
1974
2298
  if (systemColliders.has(attr))
1975
2299
  continue;
@@ -2031,6 +2355,30 @@ const makeImpl = (config, precomputedData) => {
2031
2355
  setClauses.push(`${nameKey} = ${valKey}`);
2032
2356
  counter++;
2033
2357
  }
2358
+ // Vector search: upsert writes the full item, so it always re-embeds
2359
+ // (same contract as `put` — see `DESIGN.md §14 Write path`). Emitted
2360
+ // as plain SETs, never `if_not_exists`: an upsert that overwrites the
2361
+ // source fields must overwrite the vector derived from them, or the
2362
+ // item would stay searchable under its previous description forever.
2363
+ if (hasVectorIndexes) {
2364
+ const vectorWrite = yield* computeVectorAttributes(item, opts.withVectors, "all");
2365
+ for (const [field, value] of Object.entries(vectorWrite.sets)) {
2366
+ const nameKey = `#u${counter}`;
2367
+ const valKey = `:u${counter}`;
2368
+ names[nameKey] = field;
2369
+ values[valKey] = toAttributeValue(value);
2370
+ setClauses.push(`${nameKey} = ${valKey}`);
2371
+ counter++;
2372
+ }
2373
+ // An upsert whose input carries no source text takes the item out
2374
+ // of the index, exactly as a `put` of the same input would.
2375
+ for (const field of new Set(vectorWrite.removes)) {
2376
+ const nameKey = `#vr${counter}`;
2377
+ names[nameKey] = field;
2378
+ upsertRemoveClauses.push(nameKey);
2379
+ counter++;
2380
+ }
2381
+ }
2034
2382
  // Add entity type discriminator
2035
2383
  {
2036
2384
  const nameKey = `#u${counter}`;
@@ -2076,7 +2424,9 @@ const makeImpl = (config, precomputedData) => {
2076
2424
  setClauses.push(`${nameKey} = if_not_exists(${nameKey}, ${zeroKey}) + :vinc`);
2077
2425
  counter++;
2078
2426
  }
2079
- const updateExpression = `SET ${setClauses.join(", ")}`;
2427
+ const updateExpression = upsertRemoveClauses.length > 0
2428
+ ? `SET ${setClauses.join(", ")} REMOVE ${upsertRemoveClauses.join(", ")}`
2429
+ : `SET ${setClauses.join(", ")}`;
2080
2430
  // Optional user condition
2081
2431
  const condParts = [];
2082
2432
  if (opts.condition) {
@@ -2395,8 +2745,7 @@ const makeImpl = (config, precomputedData) => {
2395
2745
  eventItem.__edd_e__ = entityType;
2396
2746
  // TTL — attribute name comes from TableConfig (default "_ttl")
2397
2747
  if (ttlDuration) {
2398
- eventItem[ttlAttrName] =
2399
- Math.floor(DateTime.toEpochMillis(now) / 1000) + normalizeTtlSeconds(ttlDuration);
2748
+ eventItem[ttlAttrName] = DateTime.toEpochSeconds(now) + normalizeTtlSeconds(ttlDuration);
2400
2749
  }
2401
2750
  // Events never participate in indexes: strip any GSI key fields that the
2402
2751
  // naive spread above may have carried over. gsiKeys weren't written into
@@ -2404,6 +2753,10 @@ const makeImpl = (config, precomputedData) => {
2404
2753
  for (const field of gsiKeyFields()) {
2405
2754
  delete eventItem[field];
2406
2755
  }
2756
+ // Only the current item is searchable — event items carry no embedding.
2757
+ for (const field of vectorKeyFields()) {
2758
+ delete eventItem[field];
2759
+ }
2407
2760
  // Sparse-map fields are aggregate state, not event state. They live on
2408
2761
  // the current item only — strip from event items entirely. (Same
2409
2762
  // treatment as enrichment fields outside `appendInput`. By design,
@@ -2785,6 +3138,20 @@ const makeImpl = (config, precomputedData) => {
2785
3138
  // Recompose all keys (original SK, GSI keys)
2786
3139
  const restoredKeys = composeAllKeys(restoredItem);
2787
3140
  Object.assign(restoredItem, restoredKeys);
3141
+ // Un-stash the embedding and recompose the vector partition value —
3142
+ // restoring never costs an Embedder call. See `DESIGN.md §14`.
3143
+ for (const [, definition] of vectorIndexEntries) {
3144
+ const stashed = restoredItem[definition.stashField];
3145
+ if (stashed !== undefined) {
3146
+ restoredItem[definition.vectorField] = stashed;
3147
+ delete restoredItem[definition.stashField];
3148
+ }
3149
+ if (restoredItem[definition.vectorField] === undefined)
3150
+ continue;
3151
+ const partition = KeyComposer.tryComposeVectorPartition(schema, entityType, definition, restoredItem);
3152
+ if (partition !== undefined)
3153
+ restoredItem[definition.partitionField] = partition;
3154
+ }
2788
3155
  const marshalledRestoredItem = toAttributeMap(restoredItem);
2789
3156
  const transactItems = [];
2790
3157
  // Delete the soft-deleted item
@@ -3030,9 +3397,15 @@ const makeImpl = (config, precomputedData) => {
3030
3397
  identifier: resolvedIdentifier,
3031
3398
  timeSeries: config.timeSeries,
3032
3399
  generatedId: config.generatedId,
3400
+ vectorIndexes: config.vectorIndexes,
3401
+ _vectorIndexes: vectorIndexes,
3033
3402
  _resolvedRefs: resolvedRefs,
3034
3403
  /** @internal Full decode pipeline: rename + schema decode. Used by Batch/Aggregate. */
3035
3404
  _decodeRecord: decodeRecord,
3405
+ /** @internal Domain field name → stored attribute name (`storedAs` renames). */
3406
+ _resolveDbName: resolveDbName,
3407
+ /** @internal Entity schema version baked into composed keys. */
3408
+ _entityVersion: entityVersion,
3036
3409
  _serializeSparseFields: serializeSparseFields,
3037
3410
  _attachPrototype: attachPrototype,
3038
3411
  _configure: (injectedSchema, injectedTableTag) => {
@@ -3143,8 +3516,122 @@ export const bind = (entity) => Effect.gen(function* () {
3143
3516
  };
3144
3517
  return new BoundQueryImpl(q, bqConfig);
3145
3518
  };
3519
+ /**
3520
+ * Out-of-band embedding refresh.
3521
+ *
3522
+ * DynamoDB never recomputes a vector, so changing the embedding model (or
3523
+ * the declared `source.fields`) leaves every stored vector stale until
3524
+ * something rewrites it. `reembed` is that something: scan the entity's
3525
+ * items, re-derive the source text, embed, and write the vector +
3526
+ * partition attributes back.
3527
+ *
3528
+ * Version snapshots and soft-delete tombstones carry the same `__edd_e__`
3529
+ * discriminator as live items, so they are filtered out by recomposing the
3530
+ * primary sort key from the decoded record and comparing it with what is
3531
+ * stored — a snapshot's SK never round-trips.
3532
+ *
3533
+ * See `DESIGN.md §14 Write path`.
3534
+ */
3535
+ const reembed = (options) => {
3536
+ const vectorDefs = Object.entries(entity._vectorIndexes);
3537
+ const boundSchema = entity._schema;
3538
+ const primary = entity.indexes.primary;
3539
+ const boundEntityVersion = entity._entityVersion;
3540
+ const concurrency = options?.concurrency ?? 4;
3541
+ return provide(Effect.gen(function* () {
3542
+ if (vectorDefs.length === 0)
3543
+ return 0;
3544
+ const client = yield* DynamoClient;
3545
+ const { name: tableName } = yield* entity._tableTag;
3546
+ const reembedItem = (marshalled) => Effect.gen(function* () {
3547
+ const raw = fromAttributeMap(marshalled);
3548
+ const storedSk = raw[primary.sk.field];
3549
+ const decoded = (yield* entity._decodeRecord(raw));
3550
+ // Snapshots / tombstones live in the same partition under a
3551
+ // rewritten SK — their SK cannot be recomposed from the record.
3552
+ const liveSk = KeyComposer.composeSk(boundSchema, entity.entityType, boundEntityVersion, primary, decoded);
3553
+ if (storedSk !== liveSk)
3554
+ return false;
3555
+ const attributes = {};
3556
+ for (const [logicalName, definition] of vectorDefs) {
3557
+ const partition = KeyComposer.tryComposeVectorPartition(boundSchema, entity.entityType, definition, decoded);
3558
+ if (partition === undefined)
3559
+ continue;
3560
+ const text = deriveSourceText(definition, decoded);
3561
+ if (text === undefined)
3562
+ continue;
3563
+ const maybeEmbedder = yield* Effect.serviceOption(Embedder);
3564
+ if (Option.isNone(maybeEmbedder)) {
3565
+ return yield* new EmbeddingError({
3566
+ entityType: entity.entityType,
3567
+ index: logicalName,
3568
+ reason: "No Embedder service is available. Provide one via " +
3569
+ "DynamoClient.make({ embedder }) before calling reembed().",
3570
+ });
3571
+ }
3572
+ const vector = yield* maybeEmbedder.value.embed(text);
3573
+ if (vector.length !== definition.dimensions) {
3574
+ return yield* new EmbeddingError({
3575
+ entityType: entity.entityType,
3576
+ index: logicalName,
3577
+ reason: `Embedder produced ${vector.length} dimensions but vector index ` +
3578
+ `"${definition.index}" declares ${definition.dimensions}.`,
3579
+ });
3580
+ }
3581
+ attributes[definition.partitionField] = partition;
3582
+ attributes[definition.vectorField] = vector;
3583
+ }
3584
+ if (Object.keys(attributes).length === 0)
3585
+ return false;
3586
+ const names = { "#pk": primary.pk.field };
3587
+ const values = {};
3588
+ const sets = [];
3589
+ let i = 0;
3590
+ for (const [field, value] of Object.entries(attributes)) {
3591
+ names[`#v${i}`] = field;
3592
+ values[`:v${i}`] = toAttributeValue(value);
3593
+ sets.push(`#v${i} = :v${i}`);
3594
+ i++;
3595
+ }
3596
+ // Scan → embed → update is not atomic: an item can be hard-deleted
3597
+ // between the scan page and this write, and an unconditional
3598
+ // UpdateItem would resurrect it as a key-plus-vector fragment that
3599
+ // decodes to nothing but still answers searches. Guard on the item
3600
+ // still existing and treat the rejection as "skipped".
3601
+ return yield* client
3602
+ .updateItem({
3603
+ TableName: tableName,
3604
+ Key: {
3605
+ [primary.pk.field]: marshalled[primary.pk.field],
3606
+ [primary.sk.field]: marshalled[primary.sk.field],
3607
+ },
3608
+ UpdateExpression: `SET ${sets.join(", ")}`,
3609
+ ConditionExpression: "attribute_exists(#pk)",
3610
+ ExpressionAttributeNames: names,
3611
+ ExpressionAttributeValues: values,
3612
+ })
3613
+ .pipe(Effect.as(true), Effect.catchIf((err) => isAwsConditionalCheckFailed(err.cause), () => Effect.succeed(false)));
3614
+ });
3615
+ let updated = 0;
3616
+ let exclusiveStartKey;
3617
+ do {
3618
+ const page = yield* client.scan({
3619
+ TableName: tableName,
3620
+ FilterExpression: "#et = :et",
3621
+ ExpressionAttributeNames: { "#et": "__edd_e__" },
3622
+ ExpressionAttributeValues: { ":et": { S: entity.entityType } },
3623
+ ExclusiveStartKey: exclusiveStartKey,
3624
+ });
3625
+ const results = yield* Effect.all((page.Items ?? []).map((item) => reembedItem(item)), { concurrency });
3626
+ updated += results.filter((applied) => applied).length;
3627
+ exclusiveStartKey = page.LastEvaluatedKey;
3628
+ } while (exclusiveStartKey !== undefined);
3629
+ return updated;
3630
+ }));
3631
+ };
3146
3632
  return {
3147
3633
  // CRUD — fluent bound builders (yieldable, no .run() terminal)
3634
+ reembed,
3148
3635
  get: (key) => provide(entity.get(key)._run("record")),
3149
3636
  put: (input) => makeBoundPut(entity.put(input), boundCrudConfig),
3150
3637
  create: (input) => makeBoundPut(entity.create(input), boundCrudConfig),