effect-dynamodb 1.10.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DynamoClient.d.ts +88 -5
- package/dist/DynamoClient.d.ts.map +1 -1
- package/dist/DynamoClient.js +158 -4
- package/dist/DynamoClient.js.map +1 -1
- package/dist/Entity.d.ts +35 -17
- package/dist/Entity.d.ts.map +1 -1
- package/dist/Entity.js +499 -9
- package/dist/Entity.js.map +1 -1
- package/dist/EventStore.d.ts +242 -35
- package/dist/EventStore.d.ts.map +1 -1
- package/dist/EventStore.js +516 -72
- package/dist/EventStore.js.map +1 -1
- package/dist/Table.d.ts +78 -0
- package/dist/Table.d.ts.map +1 -1
- package/dist/Table.js +153 -0
- package/dist/Table.js.map +1 -1
- package/dist/Transaction.d.ts +3 -18
- package/dist/Transaction.d.ts.map +1 -1
- package/dist/Transaction.js +13 -74
- package/dist/Transaction.js.map +1 -1
- package/dist/VectorSearchEmulation.d.ts +101 -0
- package/dist/VectorSearchEmulation.d.ts.map +1 -0
- package/dist/VectorSearchEmulation.js +290 -0
- package/dist/VectorSearchEmulation.js.map +1 -0
- package/dist/index.d.ts +9 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/internal/BoundCrud.d.ts +62 -48
- package/dist/internal/BoundCrud.d.ts.map +1 -1
- package/dist/internal/BoundCrud.js +7 -1
- package/dist/internal/BoundCrud.js.map +1 -1
- package/dist/internal/BoundVectorQuery.d.ts +130 -0
- package/dist/internal/BoundVectorQuery.d.ts.map +1 -0
- package/dist/internal/BoundVectorQuery.js +257 -0
- package/dist/internal/BoundVectorQuery.js.map +1 -0
- package/dist/internal/EntityCombinators.d.ts +16 -0
- package/dist/internal/EntityCombinators.d.ts.map +1 -1
- package/dist/internal/EntityCombinators.js +25 -1
- package/dist/internal/EntityCombinators.js.map +1 -1
- package/dist/internal/EntityOps.d.ts +19 -6
- package/dist/internal/EntityOps.d.ts.map +1 -1
- package/dist/internal/EntityOps.js +5 -2
- package/dist/internal/EntityOps.js.map +1 -1
- package/dist/internal/TransactWriteOps.d.ts +50 -0
- package/dist/internal/TransactWriteOps.d.ts.map +1 -0
- package/dist/internal/TransactWriteOps.js +102 -0
- package/dist/internal/TransactWriteOps.js.map +1 -0
- package/dist/internal/TransactableOps.d.ts +1 -1
- package/dist/internal/TransactableOps.d.ts.map +1 -1
- package/package.json +3 -3
package/dist/Entity.js
CHANGED
|
@@ -8,10 +8,12 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { getIdentifierField, getSparseFields, isRefField, } from "@effect-dynamodb/schema/DynamoModel.js";
|
|
10
10
|
import * as DynamoSchema from "@effect-dynamodb/schema/DynamoSchema.js";
|
|
11
|
-
import {
|
|
11
|
+
import { Embedder } from "@effect-dynamodb/schema/Embedder.js";
|
|
12
|
+
import { CascadePartialFailure, ConditionalCheckFailed, EmbeddingError, ItemNotFound, isAwsConditionalCheckFailed, isAwsTransactionCancelled, OptimisticLockError, RefNotFound, StaleAppend, TransactionOverflow, UniqueConstraintViolation, ValidationError, } from "@effect-dynamodb/schema/Errors.js";
|
|
12
13
|
import { makeDefaultCrypto } from "@effect-dynamodb/schema/internal/DefaultCrypto.js";
|
|
13
14
|
import * as KeyComposer from "@effect-dynamodb/schema/KeyComposer.js";
|
|
14
15
|
import { normalizeGsiConfig } from "@effect-dynamodb/schema/KeyComposer.js";
|
|
16
|
+
import { deriveSourceText } from "@effect-dynamodb/schema/VectorIndex.js";
|
|
15
17
|
import { Context, Crypto, DateTime, Effect, Option, Schema, Stream } from "effect";
|
|
16
18
|
import { DynamoClient } from "./DynamoClient.js";
|
|
17
19
|
import { makeBoundAppend, makeBoundDelete, makeBoundPut, makeBoundUpdate, } from "./internal/BoundCrud.js";
|
|
@@ -400,6 +402,258 @@ const makeImpl = (config, precomputedData) => {
|
|
|
400
402
|
}
|
|
401
403
|
return fields;
|
|
402
404
|
};
|
|
405
|
+
// ---------------------------------------------------------------------------
|
|
406
|
+
// Vector search — write-path helpers (see `DESIGN.md §14`)
|
|
407
|
+
// ---------------------------------------------------------------------------
|
|
408
|
+
const vectorIndexes = data.vectorIndexes;
|
|
409
|
+
const hasVectorIndexes = data.hasVectorIndexes;
|
|
410
|
+
const vectorIndexEntries = Object.entries(vectorIndexes);
|
|
411
|
+
/**
|
|
412
|
+
* Library-managed vector attribute names (embedding + composed partition).
|
|
413
|
+
*
|
|
414
|
+
* These join `gsiKeyFields()` in every strip set — version snapshots,
|
|
415
|
+
* soft-delete tombstones, and time-series event items. DynamoDB's sparse
|
|
416
|
+
* vector index semantics do the rest: an item without the vector attribute
|
|
417
|
+
* (or without the HASH attribute) is simply not in the index, so stripping IS
|
|
418
|
+
* the delete. See `DESIGN.md §14 Lifecycle integration`.
|
|
419
|
+
*/
|
|
420
|
+
const vectorKeyFields = () => {
|
|
421
|
+
const fields = [];
|
|
422
|
+
for (const [, definition] of vectorIndexEntries) {
|
|
423
|
+
fields.push(definition.vectorField, definition.partitionField);
|
|
424
|
+
}
|
|
425
|
+
return fields;
|
|
426
|
+
};
|
|
427
|
+
/**
|
|
428
|
+
* Resolve the `Embedder` service from context.
|
|
429
|
+
*
|
|
430
|
+
* Deliberately resolved with `Effect.serviceOption` rather than a plain
|
|
431
|
+
* `yield* Embedder`: only entities that declare `vectorIndexes` need one, and
|
|
432
|
+
* widening every entity operation's `R` to include `Embedder` would force the
|
|
433
|
+
* service on consumers who will never call it. A missing embedder surfaces as
|
|
434
|
+
* a tagged {@link EmbeddingError} naming the index, not a type error at the
|
|
435
|
+
* far end of the program.
|
|
436
|
+
*/
|
|
437
|
+
const resolveEmbedder = (indexName) => Effect.flatMap(Effect.serviceOption(Embedder), (maybe) => Option.isSome(maybe)
|
|
438
|
+
? Effect.succeed(maybe.value)
|
|
439
|
+
: Effect.fail(new EmbeddingError({
|
|
440
|
+
entityType,
|
|
441
|
+
index: indexName,
|
|
442
|
+
reason: `No Embedder service is available. Provide one via ` +
|
|
443
|
+
`DynamoClient.make({ embedder }) or Effect.provide(Embedder.layerTest(...)), ` +
|
|
444
|
+
`or supply a pre-computed vector with .withVector("${indexName}", [...]).`,
|
|
445
|
+
})));
|
|
446
|
+
/**
|
|
447
|
+
* Reject `.withVector("typo", ...)` before it silently does nothing.
|
|
448
|
+
*
|
|
449
|
+
* The combinator takes a plain string at runtime, and an unknown logical name
|
|
450
|
+
* would otherwise be dropped by the declared-name lookup — the Embedder would
|
|
451
|
+
* run anyway and the caller's supplied vector would vanish without a trace.
|
|
452
|
+
* Fails fast with the list of declared names instead.
|
|
453
|
+
*/
|
|
454
|
+
const checkWithVectorNames = (withVectors, operation) => {
|
|
455
|
+
if (withVectors === undefined)
|
|
456
|
+
return Effect.void;
|
|
457
|
+
for (const name of Object.keys(withVectors)) {
|
|
458
|
+
if (name in vectorIndexes)
|
|
459
|
+
continue;
|
|
460
|
+
const declared = Object.keys(vectorIndexes).sort().join(", ");
|
|
461
|
+
return Effect.fail(new ValidationError({
|
|
462
|
+
entityType,
|
|
463
|
+
operation: `${operation}.withVector`,
|
|
464
|
+
cause: `Unknown vector index "${name}". Declared vector indexes: ` +
|
|
465
|
+
`${declared.length > 0 ? declared : "(none)"}. ` +
|
|
466
|
+
`\`.withVector()\` takes the LOGICAL name from Entity.make({ vectorIndexes }).`,
|
|
467
|
+
}));
|
|
468
|
+
}
|
|
469
|
+
return Effect.void;
|
|
470
|
+
};
|
|
471
|
+
/** @internal Validate an embedding against the index's declared dimensionality. */
|
|
472
|
+
const checkDimensions = (indexName, definition, vector) => vector.length === definition.dimensions
|
|
473
|
+
? Effect.succeed(vector)
|
|
474
|
+
: Effect.fail(new EmbeddingError({
|
|
475
|
+
entityType,
|
|
476
|
+
index: indexName,
|
|
477
|
+
reason: `Embedding has ${vector.length} dimensions but vector index "${definition.index}" ` +
|
|
478
|
+
`declares ${definition.dimensions}. Dimensions are immutable on a DynamoDB vector index.`,
|
|
479
|
+
}));
|
|
480
|
+
/**
|
|
481
|
+
* Compute the vector + partition attributes for a full (put-style) record.
|
|
482
|
+
*
|
|
483
|
+
* Each index is independent and sparse: an index whose partition composites
|
|
484
|
+
* or source fields are absent from the record contributes nothing, and the
|
|
485
|
+
* item is simply not in that index.
|
|
486
|
+
*
|
|
487
|
+
* `embedFor` restricts which indexes are (re-)embedded — `"all"` on the put
|
|
488
|
+
* path, a gated subset on the update path. The partition value is always
|
|
489
|
+
* recomposed when it can be: it is idempotent, and writing it is what keeps
|
|
490
|
+
* an item in the index.
|
|
491
|
+
*/
|
|
492
|
+
const computeVectorAttributes = (record, withVectors, embedFor) => Effect.gen(function* () {
|
|
493
|
+
const sets = {};
|
|
494
|
+
const removes = [];
|
|
495
|
+
for (const [logicalName, definition] of vectorIndexEntries) {
|
|
496
|
+
const shouldEmbed = embedFor === "all" || embedFor.has(logicalName);
|
|
497
|
+
const partition = KeyComposer.tryComposeVectorPartition(schema, entityType, definition, record);
|
|
498
|
+
if (partition === undefined) {
|
|
499
|
+
// No partition value ⇒ the item cannot be in the index at all. Drop
|
|
500
|
+
// both attributes so a previously-indexed item leaves the index
|
|
501
|
+
// rather than lingering under a stale partition.
|
|
502
|
+
removes.push(definition.partitionField, definition.vectorField);
|
|
503
|
+
continue;
|
|
504
|
+
}
|
|
505
|
+
sets[definition.partitionField] = partition;
|
|
506
|
+
if (!shouldEmbed)
|
|
507
|
+
continue;
|
|
508
|
+
const explicit = withVectors?.[logicalName];
|
|
509
|
+
if (explicit !== undefined) {
|
|
510
|
+
sets[definition.vectorField] = yield* checkDimensions(logicalName, definition, explicit);
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
const text = deriveSourceText(definition, record);
|
|
514
|
+
if (text === undefined) {
|
|
515
|
+
// Every source field is gone. Sparse semantics say the item should
|
|
516
|
+
// leave the index — keeping the previous embedding would make it
|
|
517
|
+
// findable by a description it no longer has.
|
|
518
|
+
removes.push(definition.vectorField, definition.partitionField);
|
|
519
|
+
delete sets[definition.partitionField];
|
|
520
|
+
continue;
|
|
521
|
+
}
|
|
522
|
+
const embedder = yield* resolveEmbedder(logicalName);
|
|
523
|
+
const embedded = yield* embedder.embed(text);
|
|
524
|
+
sets[definition.vectorField] = yield* checkDimensions(logicalName, definition, embedded);
|
|
525
|
+
}
|
|
526
|
+
return { sets, removes };
|
|
527
|
+
});
|
|
528
|
+
/**
|
|
529
|
+
* Which vector indexes does this update require work for?
|
|
530
|
+
*
|
|
531
|
+
* Re-embedding fires when the write touches a `source.fields` member by ANY
|
|
532
|
+
* channel — a `set()` payload entry, an `Entity.remove([...])` clearing it, or
|
|
533
|
+
* a path-based operation whose root segment names it — or when the caller
|
|
534
|
+
* supplied an explicit vector.
|
|
535
|
+
*
|
|
536
|
+
* Enumerating every channel (rather than only the SET payload) is the same
|
|
537
|
+
* discipline the policy-aware GSI composer applies to `removedSet` (§7): a
|
|
538
|
+
* clear is a change, and a writer that clears the source must not leave the
|
|
539
|
+
* old embedding behind. Writers that touch none of it neither pay for an
|
|
540
|
+
* embedding call nor clobber a vector another writer owns.
|
|
541
|
+
*/
|
|
542
|
+
const vectorIndexesNeedingUpdate = (touchedFields, withVectors) => vectorIndexEntries.filter(([logicalName, definition]) => withVectors?.[logicalName] !== undefined ||
|
|
543
|
+
definition.sourceFields.some((field) => touchedFields.has(field)));
|
|
544
|
+
/**
|
|
545
|
+
* Every top-level attribute this update touches, by any channel.
|
|
546
|
+
*
|
|
547
|
+
* `touchesSource` alone only sees the SET payload; `remove([...])`, null
|
|
548
|
+
* clears and path-based operations are just as capable of invalidating an
|
|
549
|
+
* embedding, so they all feed the same set.
|
|
550
|
+
*/
|
|
551
|
+
const collectTouchedFields = (updatePayload, uState) => {
|
|
552
|
+
const touched = new Set(Object.keys(updatePayload));
|
|
553
|
+
for (const attr of uState.remove ?? [])
|
|
554
|
+
touched.add(attr);
|
|
555
|
+
const pathOps = [
|
|
556
|
+
uState.pathSets ?? [],
|
|
557
|
+
uState.pathAdds ?? [],
|
|
558
|
+
uState.pathSubtracts ?? [],
|
|
559
|
+
uState.pathAppends ?? [],
|
|
560
|
+
uState.pathPrepends ?? [],
|
|
561
|
+
uState.pathIfNotExists ?? [],
|
|
562
|
+
uState.pathDeletes ?? [],
|
|
563
|
+
];
|
|
564
|
+
for (const ops of pathOps) {
|
|
565
|
+
for (const op of ops) {
|
|
566
|
+
const root = op.segments[0];
|
|
567
|
+
if (typeof root === "string")
|
|
568
|
+
touched.add(root);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
for (const segments of uState.pathRemoves ?? []) {
|
|
572
|
+
const root = segments[0];
|
|
573
|
+
if (typeof root === "string")
|
|
574
|
+
touched.add(root);
|
|
575
|
+
}
|
|
576
|
+
return touched;
|
|
577
|
+
};
|
|
578
|
+
/**
|
|
579
|
+
* Compute vector-related SET / REMOVE attributes for the standard
|
|
580
|
+
* (UpdateItem) path.
|
|
581
|
+
*
|
|
582
|
+
* When re-embedding is required and the payload does not carry every source
|
|
583
|
+
* field, the current item is read once so the embedding is derived from the
|
|
584
|
+
* complete post-update source text rather than the fragment that happened to
|
|
585
|
+
* be in this payload. Attributes the update clears (via `remove([...])` or a
|
|
586
|
+
* null payload entry) are subtracted from that merged record, so "clear the
|
|
587
|
+
* description" produces the embedding of what actually remains — or drops the
|
|
588
|
+
* item out of the index when nothing does.
|
|
589
|
+
*/
|
|
590
|
+
const computeVectorUpdateAttributes = (decodedKey, marshalledKey, tableName, updatePayload, uState) => Effect.gen(function* () {
|
|
591
|
+
if (!hasVectorIndexes)
|
|
592
|
+
return { sets: {}, removes: [] };
|
|
593
|
+
const withVectors = uState.withVectors;
|
|
594
|
+
const touchedFields = collectTouchedFields(updatePayload, uState);
|
|
595
|
+
const needing = vectorIndexesNeedingUpdate(touchedFields, withVectors);
|
|
596
|
+
const clearedFields = new Set(uState.remove ?? []);
|
|
597
|
+
for (const [attr, value] of Object.entries(updatePayload)) {
|
|
598
|
+
if (value === null || value === undefined)
|
|
599
|
+
clearedFields.add(attr);
|
|
600
|
+
}
|
|
601
|
+
// Base record: key composites plus whatever the payload supplies. Enough
|
|
602
|
+
// to compose partition values (their composites are normally primary-key
|
|
603
|
+
// members) without any extra read.
|
|
604
|
+
let merged = { ...decodedKey, ...updatePayload };
|
|
605
|
+
const needsFullSource = needing.some(([logicalName, definition]) => withVectors?.[logicalName] === undefined &&
|
|
606
|
+
definition.sourceFields.some((field) => !(field in merged)));
|
|
607
|
+
if (needsFullSource) {
|
|
608
|
+
const client = yield* DynamoClient;
|
|
609
|
+
const current = yield* client.getItem({ TableName: tableName, Key: marshalledKey });
|
|
610
|
+
if (current.Item) {
|
|
611
|
+
const currentDomain = fromAttributeMap(current.Item);
|
|
612
|
+
renameFromDynamo(currentDomain);
|
|
613
|
+
merged = { ...currentDomain, ...merged };
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
// Cleared attributes are absent post-update, whichever channel cleared
|
|
617
|
+
// them — the same two-way classification the GSI composer uses (§7).
|
|
618
|
+
for (const attr of clearedFields)
|
|
619
|
+
delete merged[attr];
|
|
620
|
+
const sets = {};
|
|
621
|
+
const removes = [];
|
|
622
|
+
for (const [logicalName, definition] of vectorIndexEntries) {
|
|
623
|
+
const isNeeded = needing.some(([name]) => name === logicalName);
|
|
624
|
+
// Partition value: idempotent and cheap, so recompose whenever the
|
|
625
|
+
// merged record can supply it (same reasoning as the PK-composites-only
|
|
626
|
+
// GSI shape in §7).
|
|
627
|
+
const partition = KeyComposer.tryComposeVectorPartition(schema, entityType, definition, merged);
|
|
628
|
+
if (partition === undefined) {
|
|
629
|
+
// Only drop the partition when THIS writer invalidated it. Otherwise
|
|
630
|
+
// the composite simply wasn't in scope for this call and dropping it
|
|
631
|
+
// would be the multi-writer clobber the §7 gate exists to prevent.
|
|
632
|
+
if (definition.partition.some((attr) => clearedFields.has(attr))) {
|
|
633
|
+
removes.push(definition.partitionField, definition.vectorField);
|
|
634
|
+
}
|
|
635
|
+
continue;
|
|
636
|
+
}
|
|
637
|
+
sets[definition.partitionField] = partition;
|
|
638
|
+
if (!isNeeded)
|
|
639
|
+
continue;
|
|
640
|
+
const explicit = withVectors?.[logicalName];
|
|
641
|
+
if (explicit !== undefined) {
|
|
642
|
+
sets[definition.vectorField] = yield* checkDimensions(logicalName, definition, explicit);
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
const text = deriveSourceText(definition, merged);
|
|
646
|
+
if (text === undefined) {
|
|
647
|
+
removes.push(definition.vectorField, definition.partitionField);
|
|
648
|
+
delete sets[definition.partitionField];
|
|
649
|
+
continue;
|
|
650
|
+
}
|
|
651
|
+
const embedder = yield* resolveEmbedder(logicalName);
|
|
652
|
+
const embedded = yield* embedder.embed(text);
|
|
653
|
+
sets[definition.vectorField] = yield* checkDimensions(logicalName, definition, embedded);
|
|
654
|
+
}
|
|
655
|
+
return { sets, removes };
|
|
656
|
+
});
|
|
403
657
|
/**
|
|
404
658
|
* Build a version snapshot item: same PK, version SK, stripped GSI keys,
|
|
405
659
|
* keeps __edd_e__ for entity type filtering.
|
|
@@ -414,6 +668,10 @@ const makeImpl = (config, precomputedData) => {
|
|
|
414
668
|
for (const field of gsiKeyFields()) {
|
|
415
669
|
delete snapshot[field];
|
|
416
670
|
}
|
|
671
|
+
// Same for vector + partition attributes — a snapshot is never an ANN hit.
|
|
672
|
+
for (const field of vectorKeyFields()) {
|
|
673
|
+
delete snapshot[field];
|
|
674
|
+
}
|
|
417
675
|
// Replace SK with version SK
|
|
418
676
|
snapshot[tableSkField] = DynamoSchema.composeVersionKey(schema, entityType, version);
|
|
419
677
|
// Add optional TTL
|
|
@@ -674,6 +932,7 @@ const makeImpl = (config, precomputedData) => {
|
|
|
674
932
|
const tc = yield* tableTag;
|
|
675
933
|
const tableName = tc.name;
|
|
676
934
|
const ttlAttrName = resolveTtlAttributeName(tc);
|
|
935
|
+
yield* checkWithVectorNames(opts.withVectors, "put");
|
|
677
936
|
// Auto-generated id: fill the configured primary-key field with a
|
|
678
937
|
// fresh UUID BEFORE the encode, but only when the caller omitted it —
|
|
679
938
|
// a caller-supplied value is always respected. The id is sourced from
|
|
@@ -729,6 +988,16 @@ const makeImpl = (config, precomputedData) => {
|
|
|
729
988
|
item.__edd_e__ = entityType;
|
|
730
989
|
// Apply composed keys
|
|
731
990
|
Object.assign(item, keys);
|
|
991
|
+
// Vector search: embed the source text and compose the partition
|
|
992
|
+
// value. Sparse — an index whose source or partition composites are
|
|
993
|
+
// absent contributes nothing. See `DESIGN.md §14`.
|
|
994
|
+
if (hasVectorIndexes) {
|
|
995
|
+
const vectorWrite = yield* computeVectorAttributes(encoded, opts.withVectors, "all");
|
|
996
|
+
Object.assign(item, vectorWrite.sets);
|
|
997
|
+
// `put` writes a whole item, so a REMOVE is just an omission.
|
|
998
|
+
for (const field of vectorWrite.removes)
|
|
999
|
+
delete item[field];
|
|
1000
|
+
}
|
|
732
1001
|
// Rename domain fields to DynamoDB attribute names
|
|
733
1002
|
renameToDynamo(item);
|
|
734
1003
|
// Flatten sparse Map fields into per-entry top-level attributes.
|
|
@@ -886,9 +1155,7 @@ const makeImpl = (config, precomputedData) => {
|
|
|
886
1155
|
const pkField = config.indexes.primary.pk.field;
|
|
887
1156
|
const skField = config.indexes.primary.sk.field;
|
|
888
1157
|
const op = put(input);
|
|
889
|
-
return new EntityPutImpl(op._builder, op._entity, op._input, {
|
|
890
|
-
attributeNotExists: [pkField, skField],
|
|
891
|
-
});
|
|
1158
|
+
return new EntityPutImpl(op._builder, op._entity, op._input, { attributeNotExists: [pkField, skField] }, op._withVectors);
|
|
892
1159
|
};
|
|
893
1160
|
// ---------------------------------------------------------------------------
|
|
894
1161
|
// patch operation — update + attribute_exists condition
|
|
@@ -954,6 +1221,7 @@ const makeImpl = (config, precomputedData) => {
|
|
|
954
1221
|
const ttlAttrName = resolveTtlAttributeName(tc);
|
|
955
1222
|
// Clock-backed time source for the updatedAt timestamp + retain snapshot TTL.
|
|
956
1223
|
const now = yield* DateTime.now;
|
|
1224
|
+
yield* checkWithVectorNames(uState.withVectors, "update");
|
|
957
1225
|
// Decode key
|
|
958
1226
|
const decodedKey = yield* Schema.decodeUnknownEffect(schemas.keySchema)(key).pipe(Effect.mapError((cause) => new ValidationError({
|
|
959
1227
|
entityType,
|
|
@@ -1195,6 +1463,20 @@ const makeImpl = (config, precomputedData) => {
|
|
|
1195
1463
|
});
|
|
1196
1464
|
}
|
|
1197
1465
|
}
|
|
1466
|
+
// Vector search: the retain path already holds the full merged
|
|
1467
|
+
// item, so a put-style recompute is exact — no extra read needed.
|
|
1468
|
+
// Re-embedding is still gated on the payload touching a source
|
|
1469
|
+
// field (`DESIGN.md §14`); untouched indexes keep their stored
|
|
1470
|
+
// vector, which the merge above already carried onto `newItem`.
|
|
1471
|
+
if (hasVectorIndexes) {
|
|
1472
|
+
const embedFor = new Set(vectorIndexesNeedingUpdate(collectTouchedFields(hydratedUpdates, uState), uState.withVectors).map(([logicalName]) => logicalName));
|
|
1473
|
+
// `newItem` is the fully merged post-update item (removals already
|
|
1474
|
+
// applied above), so a put-style recompute is exact.
|
|
1475
|
+
const vectorWrite = yield* computeVectorAttributes(newItem, uState.withVectors, embedFor);
|
|
1476
|
+
Object.assign(newItem, vectorWrite.sets);
|
|
1477
|
+
for (const field of vectorWrite.removes)
|
|
1478
|
+
delete newItem[field];
|
|
1479
|
+
}
|
|
1198
1480
|
// Convert back to DynamoDB attribute names for storage
|
|
1199
1481
|
renameToDynamo(newItem);
|
|
1200
1482
|
// Flatten sparse Map fields into per-entry top-level attributes.
|
|
@@ -1504,6 +1786,28 @@ const makeImpl = (config, precomputedData) => {
|
|
|
1504
1786
|
names[nameKey] = keyField;
|
|
1505
1787
|
removeClauses.push(nameKey);
|
|
1506
1788
|
}
|
|
1789
|
+
// Vector search: partition value is recomposed whenever it can be
|
|
1790
|
+
// (idempotent); the embedding is regenerated only when the payload
|
|
1791
|
+
// touched a `source.fields` member or supplied `.withVector(...)`.
|
|
1792
|
+
// See `DESIGN.md §14 Write path`.
|
|
1793
|
+
if (hasVectorIndexes) {
|
|
1794
|
+
const vectorWrite = yield* computeVectorUpdateAttributes(decodedKey, marshalledKey, tableName, hydratedUpdates, uState);
|
|
1795
|
+
for (const [field, value] of Object.entries(vectorWrite.sets)) {
|
|
1796
|
+
const nameKey = `#u${counter}`;
|
|
1797
|
+
const valKey = `:u${counter}`;
|
|
1798
|
+
names[nameKey] = field;
|
|
1799
|
+
values[valKey] = toAttributeValue(value);
|
|
1800
|
+
setClauses.push(`${nameKey} = ${valKey}`);
|
|
1801
|
+
counter++;
|
|
1802
|
+
}
|
|
1803
|
+
// Clearing every source field takes the item out of the index —
|
|
1804
|
+
// there is no other way to delete a vector index entry.
|
|
1805
|
+
for (const field of new Set(vectorWrite.removes)) {
|
|
1806
|
+
const nameKey = `#r${removeClauses.length}`;
|
|
1807
|
+
names[nameKey] = field;
|
|
1808
|
+
removeClauses.push(nameKey);
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1507
1811
|
// ADD (atomic numeric increment / set addition)
|
|
1508
1812
|
if (uState.add) {
|
|
1509
1813
|
for (const [attr, val] of Object.entries(uState.add)) {
|
|
@@ -1801,6 +2105,17 @@ const makeImpl = (config, precomputedData) => {
|
|
|
1801
2105
|
for (const field of gsiKeyFields()) {
|
|
1802
2106
|
delete deletedItem[field];
|
|
1803
2107
|
}
|
|
2108
|
+
// Vector attributes: stash the embedding under a non-indexed name
|
|
2109
|
+
// before stripping. Sparse semantics drop the tombstone out of the
|
|
2110
|
+
// vector index immediately, and `restore()` un-stashes without
|
|
2111
|
+
// paying for another Embedder call. See `DESIGN.md §14`.
|
|
2112
|
+
for (const [, definition] of vectorIndexEntries) {
|
|
2113
|
+
const stored = deletedItem[definition.vectorField];
|
|
2114
|
+
if (stored !== undefined)
|
|
2115
|
+
deletedItem[definition.stashField] = stored;
|
|
2116
|
+
delete deletedItem[definition.vectorField];
|
|
2117
|
+
delete deletedItem[definition.partitionField];
|
|
2118
|
+
}
|
|
1804
2119
|
// Replace SK with deleted sort key
|
|
1805
2120
|
deletedItem[primary.sk.field] = DynamoSchema.composeDeletedKey(schema, entityType, now);
|
|
1806
2121
|
// Add deletedAt
|
|
@@ -1938,6 +2253,7 @@ const makeImpl = (config, precomputedData) => {
|
|
|
1938
2253
|
const { name: tableName } = yield* tableTag;
|
|
1939
2254
|
// Clock-backed time source for createdAt/updatedAt SET clauses.
|
|
1940
2255
|
const now = yield* DateTime.now;
|
|
2256
|
+
yield* checkWithVectorNames(opts.withVectors, "upsert");
|
|
1941
2257
|
// Encode user input → wire form (see `put` for strategy).
|
|
1942
2258
|
const encodedInput = yield* encodeOrDecodeEncode(schemas.inputSchema, input, entityType, "upsert");
|
|
1943
2259
|
// Hydrate refs: replace ID fields with full entity domain data
|
|
@@ -1950,11 +2266,21 @@ const makeImpl = (config, precomputedData) => {
|
|
|
1950
2266
|
const marshalledKey = toAttributeMap(primaryKey);
|
|
1951
2267
|
// Build UpdateExpression with if_not_exists for immutable fields + createdAt
|
|
1952
2268
|
const setClauses = [];
|
|
2269
|
+
const upsertRemoveClauses = [];
|
|
1953
2270
|
const names = {};
|
|
1954
2271
|
const values = {};
|
|
1955
2272
|
let counter = 0;
|
|
1956
|
-
//
|
|
1957
|
-
|
|
2273
|
+
// The physical key attributes (`pk` / `sk`) live in `Key` and cannot
|
|
2274
|
+
// appear in an UpdateExpression. The COMPOSITE SOURCE fields are
|
|
2275
|
+
// ordinary attributes and must be written like any other model field —
|
|
2276
|
+
// skipping them (as this path used to) left the stored item without
|
|
2277
|
+
// e.g. `productId`, so every read of an upserted item failed to decode
|
|
2278
|
+
// with `Missing key`. Their values are fixed by the key, so a plain
|
|
2279
|
+
// SET is idempotent.
|
|
2280
|
+
const keyAttributeFields = new Set([
|
|
2281
|
+
config.indexes.primary.pk.field,
|
|
2282
|
+
config.indexes.primary.sk.field,
|
|
2283
|
+
]);
|
|
1958
2284
|
// System-colliding fields are written below in the system-field block
|
|
1959
2285
|
// (so their semantics — if_not_exists / always-set — stay consistent).
|
|
1960
2286
|
const systemColliders = new Set();
|
|
@@ -1965,9 +2291,9 @@ const makeImpl = (config, precomputedData) => {
|
|
|
1965
2291
|
if (systemFields.versionCollision && systemFields.version)
|
|
1966
2292
|
systemColliders.add(systemFields.version);
|
|
1967
2293
|
// `item` is already in wire-form via `Schema.encode(inputSchema)`.
|
|
1968
|
-
// All model fields (excluding
|
|
2294
|
+
// All model fields (excluding the physical key attributes themselves)
|
|
1969
2295
|
for (const [attr, val] of Object.entries(item)) {
|
|
1970
|
-
if (
|
|
2296
|
+
if (keyAttributeFields.has(resolveDbName(attr)))
|
|
1971
2297
|
continue;
|
|
1972
2298
|
if (systemColliders.has(attr))
|
|
1973
2299
|
continue;
|
|
@@ -2029,6 +2355,30 @@ const makeImpl = (config, precomputedData) => {
|
|
|
2029
2355
|
setClauses.push(`${nameKey} = ${valKey}`);
|
|
2030
2356
|
counter++;
|
|
2031
2357
|
}
|
|
2358
|
+
// Vector search: upsert writes the full item, so it always re-embeds
|
|
2359
|
+
// (same contract as `put` — see `DESIGN.md §14 Write path`). Emitted
|
|
2360
|
+
// as plain SETs, never `if_not_exists`: an upsert that overwrites the
|
|
2361
|
+
// source fields must overwrite the vector derived from them, or the
|
|
2362
|
+
// item would stay searchable under its previous description forever.
|
|
2363
|
+
if (hasVectorIndexes) {
|
|
2364
|
+
const vectorWrite = yield* computeVectorAttributes(item, opts.withVectors, "all");
|
|
2365
|
+
for (const [field, value] of Object.entries(vectorWrite.sets)) {
|
|
2366
|
+
const nameKey = `#u${counter}`;
|
|
2367
|
+
const valKey = `:u${counter}`;
|
|
2368
|
+
names[nameKey] = field;
|
|
2369
|
+
values[valKey] = toAttributeValue(value);
|
|
2370
|
+
setClauses.push(`${nameKey} = ${valKey}`);
|
|
2371
|
+
counter++;
|
|
2372
|
+
}
|
|
2373
|
+
// An upsert whose input carries no source text takes the item out
|
|
2374
|
+
// of the index, exactly as a `put` of the same input would.
|
|
2375
|
+
for (const field of new Set(vectorWrite.removes)) {
|
|
2376
|
+
const nameKey = `#vr${counter}`;
|
|
2377
|
+
names[nameKey] = field;
|
|
2378
|
+
upsertRemoveClauses.push(nameKey);
|
|
2379
|
+
counter++;
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2032
2382
|
// Add entity type discriminator
|
|
2033
2383
|
{
|
|
2034
2384
|
const nameKey = `#u${counter}`;
|
|
@@ -2074,7 +2424,9 @@ const makeImpl = (config, precomputedData) => {
|
|
|
2074
2424
|
setClauses.push(`${nameKey} = if_not_exists(${nameKey}, ${zeroKey}) + :vinc`);
|
|
2075
2425
|
counter++;
|
|
2076
2426
|
}
|
|
2077
|
-
const updateExpression =
|
|
2427
|
+
const updateExpression = upsertRemoveClauses.length > 0
|
|
2428
|
+
? `SET ${setClauses.join(", ")} REMOVE ${upsertRemoveClauses.join(", ")}`
|
|
2429
|
+
: `SET ${setClauses.join(", ")}`;
|
|
2078
2430
|
// Optional user condition
|
|
2079
2431
|
const condParts = [];
|
|
2080
2432
|
if (opts.condition) {
|
|
@@ -2401,6 +2753,10 @@ const makeImpl = (config, precomputedData) => {
|
|
|
2401
2753
|
for (const field of gsiKeyFields()) {
|
|
2402
2754
|
delete eventItem[field];
|
|
2403
2755
|
}
|
|
2756
|
+
// Only the current item is searchable — event items carry no embedding.
|
|
2757
|
+
for (const field of vectorKeyFields()) {
|
|
2758
|
+
delete eventItem[field];
|
|
2759
|
+
}
|
|
2404
2760
|
// Sparse-map fields are aggregate state, not event state. They live on
|
|
2405
2761
|
// the current item only — strip from event items entirely. (Same
|
|
2406
2762
|
// treatment as enrichment fields outside `appendInput`. By design,
|
|
@@ -2782,6 +3138,20 @@ const makeImpl = (config, precomputedData) => {
|
|
|
2782
3138
|
// Recompose all keys (original SK, GSI keys)
|
|
2783
3139
|
const restoredKeys = composeAllKeys(restoredItem);
|
|
2784
3140
|
Object.assign(restoredItem, restoredKeys);
|
|
3141
|
+
// Un-stash the embedding and recompose the vector partition value —
|
|
3142
|
+
// restoring never costs an Embedder call. See `DESIGN.md §14`.
|
|
3143
|
+
for (const [, definition] of vectorIndexEntries) {
|
|
3144
|
+
const stashed = restoredItem[definition.stashField];
|
|
3145
|
+
if (stashed !== undefined) {
|
|
3146
|
+
restoredItem[definition.vectorField] = stashed;
|
|
3147
|
+
delete restoredItem[definition.stashField];
|
|
3148
|
+
}
|
|
3149
|
+
if (restoredItem[definition.vectorField] === undefined)
|
|
3150
|
+
continue;
|
|
3151
|
+
const partition = KeyComposer.tryComposeVectorPartition(schema, entityType, definition, restoredItem);
|
|
3152
|
+
if (partition !== undefined)
|
|
3153
|
+
restoredItem[definition.partitionField] = partition;
|
|
3154
|
+
}
|
|
2785
3155
|
const marshalledRestoredItem = toAttributeMap(restoredItem);
|
|
2786
3156
|
const transactItems = [];
|
|
2787
3157
|
// Delete the soft-deleted item
|
|
@@ -3027,9 +3397,15 @@ const makeImpl = (config, precomputedData) => {
|
|
|
3027
3397
|
identifier: resolvedIdentifier,
|
|
3028
3398
|
timeSeries: config.timeSeries,
|
|
3029
3399
|
generatedId: config.generatedId,
|
|
3400
|
+
vectorIndexes: config.vectorIndexes,
|
|
3401
|
+
_vectorIndexes: vectorIndexes,
|
|
3030
3402
|
_resolvedRefs: resolvedRefs,
|
|
3031
3403
|
/** @internal Full decode pipeline: rename + schema decode. Used by Batch/Aggregate. */
|
|
3032
3404
|
_decodeRecord: decodeRecord,
|
|
3405
|
+
/** @internal Domain field name → stored attribute name (`storedAs` renames). */
|
|
3406
|
+
_resolveDbName: resolveDbName,
|
|
3407
|
+
/** @internal Entity schema version baked into composed keys. */
|
|
3408
|
+
_entityVersion: entityVersion,
|
|
3033
3409
|
_serializeSparseFields: serializeSparseFields,
|
|
3034
3410
|
_attachPrototype: attachPrototype,
|
|
3035
3411
|
_configure: (injectedSchema, injectedTableTag) => {
|
|
@@ -3140,8 +3516,122 @@ export const bind = (entity) => Effect.gen(function* () {
|
|
|
3140
3516
|
};
|
|
3141
3517
|
return new BoundQueryImpl(q, bqConfig);
|
|
3142
3518
|
};
|
|
3519
|
+
/**
|
|
3520
|
+
* Out-of-band embedding refresh.
|
|
3521
|
+
*
|
|
3522
|
+
* DynamoDB never recomputes a vector, so changing the embedding model (or
|
|
3523
|
+
* the declared `source.fields`) leaves every stored vector stale until
|
|
3524
|
+
* something rewrites it. `reembed` is that something: scan the entity's
|
|
3525
|
+
* items, re-derive the source text, embed, and write the vector +
|
|
3526
|
+
* partition attributes back.
|
|
3527
|
+
*
|
|
3528
|
+
* Version snapshots and soft-delete tombstones carry the same `__edd_e__`
|
|
3529
|
+
* discriminator as live items, so they are filtered out by recomposing the
|
|
3530
|
+
* primary sort key from the decoded record and comparing it with what is
|
|
3531
|
+
* stored — a snapshot's SK never round-trips.
|
|
3532
|
+
*
|
|
3533
|
+
* See `DESIGN.md §14 Write path`.
|
|
3534
|
+
*/
|
|
3535
|
+
const reembed = (options) => {
|
|
3536
|
+
const vectorDefs = Object.entries(entity._vectorIndexes);
|
|
3537
|
+
const boundSchema = entity._schema;
|
|
3538
|
+
const primary = entity.indexes.primary;
|
|
3539
|
+
const boundEntityVersion = entity._entityVersion;
|
|
3540
|
+
const concurrency = options?.concurrency ?? 4;
|
|
3541
|
+
return provide(Effect.gen(function* () {
|
|
3542
|
+
if (vectorDefs.length === 0)
|
|
3543
|
+
return 0;
|
|
3544
|
+
const client = yield* DynamoClient;
|
|
3545
|
+
const { name: tableName } = yield* entity._tableTag;
|
|
3546
|
+
const reembedItem = (marshalled) => Effect.gen(function* () {
|
|
3547
|
+
const raw = fromAttributeMap(marshalled);
|
|
3548
|
+
const storedSk = raw[primary.sk.field];
|
|
3549
|
+
const decoded = (yield* entity._decodeRecord(raw));
|
|
3550
|
+
// Snapshots / tombstones live in the same partition under a
|
|
3551
|
+
// rewritten SK — their SK cannot be recomposed from the record.
|
|
3552
|
+
const liveSk = KeyComposer.composeSk(boundSchema, entity.entityType, boundEntityVersion, primary, decoded);
|
|
3553
|
+
if (storedSk !== liveSk)
|
|
3554
|
+
return false;
|
|
3555
|
+
const attributes = {};
|
|
3556
|
+
for (const [logicalName, definition] of vectorDefs) {
|
|
3557
|
+
const partition = KeyComposer.tryComposeVectorPartition(boundSchema, entity.entityType, definition, decoded);
|
|
3558
|
+
if (partition === undefined)
|
|
3559
|
+
continue;
|
|
3560
|
+
const text = deriveSourceText(definition, decoded);
|
|
3561
|
+
if (text === undefined)
|
|
3562
|
+
continue;
|
|
3563
|
+
const maybeEmbedder = yield* Effect.serviceOption(Embedder);
|
|
3564
|
+
if (Option.isNone(maybeEmbedder)) {
|
|
3565
|
+
return yield* new EmbeddingError({
|
|
3566
|
+
entityType: entity.entityType,
|
|
3567
|
+
index: logicalName,
|
|
3568
|
+
reason: "No Embedder service is available. Provide one via " +
|
|
3569
|
+
"DynamoClient.make({ embedder }) before calling reembed().",
|
|
3570
|
+
});
|
|
3571
|
+
}
|
|
3572
|
+
const vector = yield* maybeEmbedder.value.embed(text);
|
|
3573
|
+
if (vector.length !== definition.dimensions) {
|
|
3574
|
+
return yield* new EmbeddingError({
|
|
3575
|
+
entityType: entity.entityType,
|
|
3576
|
+
index: logicalName,
|
|
3577
|
+
reason: `Embedder produced ${vector.length} dimensions but vector index ` +
|
|
3578
|
+
`"${definition.index}" declares ${definition.dimensions}.`,
|
|
3579
|
+
});
|
|
3580
|
+
}
|
|
3581
|
+
attributes[definition.partitionField] = partition;
|
|
3582
|
+
attributes[definition.vectorField] = vector;
|
|
3583
|
+
}
|
|
3584
|
+
if (Object.keys(attributes).length === 0)
|
|
3585
|
+
return false;
|
|
3586
|
+
const names = { "#pk": primary.pk.field };
|
|
3587
|
+
const values = {};
|
|
3588
|
+
const sets = [];
|
|
3589
|
+
let i = 0;
|
|
3590
|
+
for (const [field, value] of Object.entries(attributes)) {
|
|
3591
|
+
names[`#v${i}`] = field;
|
|
3592
|
+
values[`:v${i}`] = toAttributeValue(value);
|
|
3593
|
+
sets.push(`#v${i} = :v${i}`);
|
|
3594
|
+
i++;
|
|
3595
|
+
}
|
|
3596
|
+
// Scan → embed → update is not atomic: an item can be hard-deleted
|
|
3597
|
+
// between the scan page and this write, and an unconditional
|
|
3598
|
+
// UpdateItem would resurrect it as a key-plus-vector fragment that
|
|
3599
|
+
// decodes to nothing but still answers searches. Guard on the item
|
|
3600
|
+
// still existing and treat the rejection as "skipped".
|
|
3601
|
+
return yield* client
|
|
3602
|
+
.updateItem({
|
|
3603
|
+
TableName: tableName,
|
|
3604
|
+
Key: {
|
|
3605
|
+
[primary.pk.field]: marshalled[primary.pk.field],
|
|
3606
|
+
[primary.sk.field]: marshalled[primary.sk.field],
|
|
3607
|
+
},
|
|
3608
|
+
UpdateExpression: `SET ${sets.join(", ")}`,
|
|
3609
|
+
ConditionExpression: "attribute_exists(#pk)",
|
|
3610
|
+
ExpressionAttributeNames: names,
|
|
3611
|
+
ExpressionAttributeValues: values,
|
|
3612
|
+
})
|
|
3613
|
+
.pipe(Effect.as(true), Effect.catchIf((err) => isAwsConditionalCheckFailed(err.cause), () => Effect.succeed(false)));
|
|
3614
|
+
});
|
|
3615
|
+
let updated = 0;
|
|
3616
|
+
let exclusiveStartKey;
|
|
3617
|
+
do {
|
|
3618
|
+
const page = yield* client.scan({
|
|
3619
|
+
TableName: tableName,
|
|
3620
|
+
FilterExpression: "#et = :et",
|
|
3621
|
+
ExpressionAttributeNames: { "#et": "__edd_e__" },
|
|
3622
|
+
ExpressionAttributeValues: { ":et": { S: entity.entityType } },
|
|
3623
|
+
ExclusiveStartKey: exclusiveStartKey,
|
|
3624
|
+
});
|
|
3625
|
+
const results = yield* Effect.all((page.Items ?? []).map((item) => reembedItem(item)), { concurrency });
|
|
3626
|
+
updated += results.filter((applied) => applied).length;
|
|
3627
|
+
exclusiveStartKey = page.LastEvaluatedKey;
|
|
3628
|
+
} while (exclusiveStartKey !== undefined);
|
|
3629
|
+
return updated;
|
|
3630
|
+
}));
|
|
3631
|
+
};
|
|
3143
3632
|
return {
|
|
3144
3633
|
// CRUD — fluent bound builders (yieldable, no .run() terminal)
|
|
3634
|
+
reembed,
|
|
3145
3635
|
get: (key) => provide(entity.get(key)._run("record")),
|
|
3146
3636
|
put: (input) => makeBoundPut(entity.put(input), boundCrudConfig),
|
|
3147
3637
|
create: (input) => makeBoundPut(entity.create(input), boundCrudConfig),
|