@farukada/aws-langgraph-dynamodb-ts 1.0.0-rc.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +390 -193
- package/dist/{backfill → cjs/backfill}/backfill.d.ts +15 -12
- package/dist/cjs/backfill/backfill.js +177 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/delete-thread.d.ts +1 -1
- package/dist/cjs/checkpointer/actions/delete-thread.js +26 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/get-tuple.d.ts +4 -3
- package/dist/cjs/checkpointer/actions/get-tuple.js +22 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/list.d.ts +7 -4
- package/dist/cjs/checkpointer/actions/list.js +70 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/put-writes.d.ts +8 -6
- package/dist/cjs/checkpointer/actions/put-writes.js +21 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/put.d.ts +21 -15
- package/dist/cjs/checkpointer/actions/put.js +79 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/delta-history.d.ts +38 -26
- package/dist/cjs/checkpointer/internal/delta-history.js +131 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/listing.d.ts +4 -4
- package/dist/cjs/checkpointer/internal/listing.js +104 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/parse.d.ts +51 -15
- package/dist/cjs/checkpointer/internal/parse.js +184 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/pending-writes.d.ts +64 -25
- package/dist/cjs/checkpointer/internal/pending-writes.js +241 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/read.d.ts +15 -10
- package/dist/cjs/checkpointer/internal/read.js +132 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/rows.d.ts +94 -21
- package/dist/cjs/checkpointer/internal/rows.js +356 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/setup.d.ts +13 -20
- package/dist/cjs/checkpointer/internal/setup.js +25 -0
- package/dist/{checkpointer → cjs/checkpointer}/saver.d.ts +72 -36
- package/dist/cjs/checkpointer/saver.js +56 -0
- package/dist/{checkpointer → cjs/checkpointer}/types.d.ts +1 -1
- package/dist/cjs/checkpointer/types.js +2 -0
- package/dist/{factory → cjs/factory}/factory.d.ts +25 -20
- package/dist/cjs/factory/factory.js +137 -0
- package/dist/{factory → cjs/factory}/types.d.ts +12 -12
- package/dist/cjs/factory/types.js +2 -0
- package/dist/{history → cjs/history}/actions/add-messages.d.ts +18 -10
- package/dist/cjs/history/actions/add-messages.js +17 -0
- package/dist/{history → cjs/history}/actions/clear.d.ts +1 -1
- package/dist/cjs/history/actions/clear.js +29 -0
- package/dist/{history → cjs/history}/actions/get-messages.d.ts +11 -9
- package/dist/cjs/history/actions/get-messages.js +58 -0
- package/dist/{history → cjs/history}/actions/list-sessions.d.ts +3 -3
- package/dist/cjs/history/actions/list-sessions.js +67 -0
- package/dist/{history → cjs/history}/actions/reconcile-count.d.ts +13 -9
- package/dist/cjs/history/actions/reconcile-count.js +8 -0
- package/dist/{history → cjs/history}/chat-message-history.d.ts +62 -30
- package/dist/cjs/history/chat-message-history.js +61 -0
- package/dist/{history → cjs/history}/internal/append.d.ts +64 -32
- package/dist/cjs/history/internal/append.js +267 -0
- package/dist/{history → cjs/history}/internal/message-read.d.ts +3 -3
- package/dist/cjs/history/internal/message-read.js +93 -0
- package/dist/{history → cjs/history}/internal/parse.d.ts +2 -2
- package/dist/cjs/history/internal/parse.js +113 -0
- package/dist/{history → cjs/history}/internal/rows.d.ts +11 -10
- package/dist/cjs/history/internal/rows.js +96 -0
- package/dist/{history → cjs/history}/internal/session.d.ts +41 -22
- package/dist/cjs/history/internal/session.js +296 -0
- package/dist/{history → cjs/history}/internal/setup.d.ts +2 -2
- package/dist/cjs/history/internal/setup.js +60 -0
- package/dist/{history → cjs/history}/session-adapter.d.ts +0 -6
- package/dist/cjs/history/session-adapter.js +38 -0
- package/dist/{history → cjs/history}/types.d.ts +18 -15
- package/dist/cjs/history/types.js +2 -0
- package/dist/{index.d.ts → cjs/index.d.ts} +27 -27
- package/dist/cjs/index.js +25 -0
- package/dist/cjs/package.json +1 -0
- package/dist/{shared → cjs/shared}/adapter.d.ts +27 -16
- package/dist/cjs/shared/adapter.js +70 -0
- package/dist/cjs/shared/clock.js +14 -0
- package/dist/{shared → cjs/shared}/codec/codec.d.ts +37 -20
- package/dist/cjs/shared/codec/codec.js +162 -0
- package/dist/{shared → cjs/shared}/codec/compression.d.ts +9 -4
- package/dist/cjs/shared/codec/compression.js +45 -0
- package/dist/cjs/shared/codec/json-serde.js +58 -0
- package/dist/{shared → cjs/shared}/codec/s3/client-types.d.ts +2 -1
- package/dist/cjs/shared/codec/s3/client-types.js +6 -0
- package/dist/{shared → cjs/shared}/codec/s3/client.d.ts +25 -7
- package/dist/cjs/shared/codec/s3/client.js +78 -0
- package/dist/{shared → cjs/shared}/codec/s3/config.d.ts +12 -7
- package/dist/cjs/shared/codec/s3/config.js +92 -0
- package/dist/{shared → cjs/shared}/codec/s3/lifecycle.d.ts +71 -17
- package/dist/cjs/shared/codec/s3/lifecycle.js +223 -0
- package/dist/{shared → cjs/shared}/codec/s3/offloader.d.ts +42 -29
- package/dist/cjs/shared/codec/s3/offloader.js +289 -0
- package/dist/cjs/shared/concurrency.js +31 -0
- package/dist/{shared → cjs/shared}/dynamodb/abort.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/abort.js +16 -0
- package/dist/{shared → cjs/shared}/dynamodb/batch-write.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/batch-write.js +79 -0
- package/dist/cjs/shared/dynamodb/cancellation.js +37 -0
- package/dist/{shared → cjs/shared}/dynamodb/client.d.ts +24 -22
- package/dist/cjs/shared/dynamodb/client.js +40 -0
- package/dist/{shared → cjs/shared}/dynamodb/idempotent-write.d.ts +64 -19
- package/dist/cjs/shared/dynamodb/idempotent-write.js +173 -0
- package/dist/{shared → cjs/shared}/dynamodb/paginate.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/paginate.js +90 -0
- package/dist/{shared → cjs/shared}/dynamodb/partition-delete.d.ts +6 -6
- package/dist/cjs/shared/dynamodb/partition-delete.js +177 -0
- package/dist/{shared → cjs/shared}/dynamodb/recency-index.d.ts +27 -15
- package/dist/cjs/shared/dynamodb/recency-index.js +146 -0
- package/dist/{shared → cjs/shared}/dynamodb/retry.d.ts +37 -6
- package/dist/cjs/shared/dynamodb/retry.js +193 -0
- package/dist/cjs/shared/dynamodb/row-size.d.ts +33 -0
- package/dist/cjs/shared/dynamodb/row-size.js +30 -0
- package/dist/{shared → cjs/shared}/dynamodb/table-schema.d.ts +27 -18
- package/dist/cjs/shared/dynamodb/table-schema.js +52 -0
- package/dist/{shared → cjs/shared}/errors/base-error.d.ts +42 -7
- package/dist/cjs/shared/errors/base-error.js +73 -0
- package/dist/{shared → cjs/shared}/errors/boundary.d.ts +29 -14
- package/dist/cjs/shared/errors/boundary.js +61 -0
- package/dist/cjs/shared/errors/classify.d.ts +206 -0
- package/dist/cjs/shared/errors/classify.js +232 -0
- package/dist/{shared → cjs/shared}/errors/error-code.d.ts +17 -10
- package/dist/cjs/shared/errors/error-code.js +27 -0
- package/dist/{shared → cjs/shared}/errors/errors.d.ts +75 -9
- package/dist/cjs/shared/errors/errors.js +123 -0
- package/dist/cjs/shared/logging/logger.js +30 -0
- package/dist/{shared → cjs/shared}/logging/redaction.d.ts +8 -4
- package/dist/cjs/shared/logging/redaction.js +145 -0
- package/dist/{shared → cjs/shared}/logging/secret-patterns.d.ts +30 -7
- package/dist/cjs/shared/logging/secret-patterns.js +93 -0
- package/dist/cjs/shared/logging/truncate.js +31 -0
- package/dist/{shared → cjs/shared}/options.d.ts +27 -11
- package/dist/cjs/shared/options.js +2 -0
- package/dist/cjs/shared/ulid.js +83 -0
- package/dist/{shared → cjs/shared}/validation/collaborators.d.ts +21 -2
- package/dist/cjs/shared/validation/collaborators.js +79 -0
- package/dist/{shared → cjs/shared}/validation/option-shape.d.ts +7 -5
- package/dist/cjs/shared/validation/option-shape.js +31 -0
- package/dist/{shared → cjs/shared}/validation/options.d.ts +4 -4
- package/dist/cjs/shared/validation/options.js +166 -0
- package/dist/cjs/shared/validation/primitives.js +114 -0
- package/dist/{shared → cjs/shared}/validation/ttl.d.ts +26 -9
- package/dist/cjs/shared/validation/ttl.js +52 -0
- package/dist/{store → cjs/store}/actions/list-namespaces.d.ts +5 -5
- package/dist/cjs/store/actions/list-namespaces.js +92 -0
- package/dist/{store → cjs/store}/actions/put.d.ts +11 -4
- package/dist/cjs/store/actions/put.js +49 -0
- package/dist/{store → cjs/store}/actions/reconcile-vector-index.d.ts +5 -5
- package/dist/cjs/store/actions/reconcile-vector-index.js +13 -0
- package/dist/{store → cjs/store}/actions/search.d.ts +2 -2
- package/dist/cjs/store/actions/search.js +26 -0
- package/dist/cjs/store/internal/batch-plan.d.ts +56 -0
- package/dist/cjs/store/internal/batch-plan.js +32 -0
- package/dist/{store → cjs/store}/internal/filter.d.ts +3 -3
- package/dist/cjs/store/internal/filter.js +64 -0
- package/dist/cjs/store/internal/get-item.d.ts +77 -0
- package/dist/cjs/store/internal/get-item.js +58 -0
- package/dist/{store → cjs/store}/internal/item-write.d.ts +72 -32
- package/dist/cjs/store/internal/item-write.js +181 -0
- package/dist/{store → cjs/store}/internal/parse.d.ts +15 -10
- package/dist/cjs/store/internal/parse.js +180 -0
- package/dist/{store → cjs/store}/internal/rows.d.ts +38 -34
- package/dist/cjs/store/internal/rows.js +166 -0
- package/dist/{store → cjs/store}/internal/semantic-search.d.ts +2 -2
- package/dist/cjs/store/internal/semantic-search.js +211 -0
- package/dist/{store → cjs/store}/internal/setup.d.ts +10 -15
- package/dist/cjs/store/internal/setup.js +103 -0
- package/dist/{store → cjs/store}/internal/table-search.d.ts +5 -5
- package/dist/cjs/store/internal/table-search.js +123 -0
- package/dist/{store → cjs/store}/internal/vector-index.d.ts +26 -15
- package/dist/cjs/store/internal/vector-index.js +269 -0
- package/dist/{store → cjs/store}/store.d.ts +82 -39
- package/dist/cjs/store/store.js +97 -0
- package/dist/{store → cjs/store}/types.d.ts +25 -10
- package/dist/cjs/store/types.js +2 -0
- package/dist/cjs/store/vector-backend.js +4 -0
- package/dist/esm/backfill/backfill.d.ts +171 -0
- package/dist/esm/backfill/backfill.js +170 -0
- package/dist/{checkpointer/actions/delete-thread.js → esm/checkpointer/actions/delete-thread.d.ts} +4 -26
- package/dist/esm/checkpointer/actions/delete-thread.js +23 -0
- package/dist/{checkpointer/actions/get-tuple.js → esm/checkpointer/actions/get-tuple.d.ts} +7 -24
- package/dist/esm/checkpointer/actions/get-tuple.js +19 -0
- package/dist/esm/checkpointer/actions/list.d.ts +55 -0
- package/dist/esm/checkpointer/actions/list.js +67 -0
- package/dist/{checkpointer/actions/put-writes.js → esm/checkpointer/actions/put-writes.d.ts} +10 -37
- package/dist/esm/checkpointer/actions/put-writes.js +18 -0
- package/dist/esm/checkpointer/actions/put.d.ts +95 -0
- package/dist/esm/checkpointer/actions/put.js +75 -0
- package/dist/esm/checkpointer/internal/delta-history.d.ts +124 -0
- package/dist/esm/checkpointer/internal/delta-history.js +126 -0
- package/dist/{checkpointer/internal/listing.js → esm/checkpointer/internal/listing.d.ts} +22 -118
- package/dist/esm/checkpointer/internal/listing.js +95 -0
- package/dist/esm/checkpointer/internal/parse.d.ts +298 -0
- package/dist/esm/checkpointer/internal/parse.js +169 -0
- package/dist/esm/checkpointer/internal/pending-writes.d.ts +314 -0
- package/dist/esm/checkpointer/internal/pending-writes.js +229 -0
- package/dist/esm/checkpointer/internal/read.d.ts +135 -0
- package/dist/esm/checkpointer/internal/read.js +125 -0
- package/dist/esm/checkpointer/internal/rows.d.ts +644 -0
- package/dist/esm/checkpointer/internal/rows.js +323 -0
- package/dist/esm/checkpointer/internal/setup.d.ts +47 -0
- package/dist/esm/checkpointer/internal/setup.js +21 -0
- package/dist/{checkpointer/saver.js → esm/checkpointer/saver.d.ts} +84 -87
- package/dist/esm/checkpointer/saver.js +52 -0
- package/dist/esm/checkpointer/types.d.ts +51 -0
- package/dist/esm/checkpointer/types.js +1 -0
- package/dist/esm/factory/factory.d.ts +154 -0
- package/dist/esm/factory/factory.js +133 -0
- package/dist/esm/factory/types.d.ts +76 -0
- package/dist/esm/factory/types.js +1 -0
- package/dist/{history/actions/add-messages.js → esm/history/actions/add-messages.d.ts} +20 -26
- package/dist/esm/history/actions/add-messages.js +14 -0
- package/dist/{history/actions/clear.js → esm/history/actions/clear.d.ts} +4 -35
- package/dist/esm/history/actions/clear.js +26 -0
- package/dist/esm/history/actions/get-messages.d.ts +60 -0
- package/dist/esm/history/actions/get-messages.js +55 -0
- package/dist/esm/history/actions/list-sessions.d.ts +55 -0
- package/dist/esm/history/actions/list-sessions.js +64 -0
- package/dist/{history/actions/reconcile-count.js → esm/history/actions/reconcile-count.d.ts} +14 -16
- package/dist/esm/history/actions/reconcile-count.js +5 -0
- package/dist/{history/chat-message-history.js → esm/history/chat-message-history.d.ts} +76 -87
- package/dist/esm/history/chat-message-history.js +57 -0
- package/dist/esm/history/internal/append.d.ts +244 -0
- package/dist/esm/history/internal/append.js +259 -0
- package/dist/esm/history/internal/message-read.d.ts +84 -0
- package/dist/esm/history/internal/message-read.js +89 -0
- package/dist/esm/history/internal/parse.d.ts +153 -0
- package/dist/esm/history/internal/parse.js +105 -0
- package/dist/{history/internal/rows.js → esm/history/internal/rows.d.ts} +58 -112
- package/dist/esm/history/internal/rows.js +83 -0
- package/dist/esm/history/internal/session.d.ts +350 -0
- package/dist/esm/history/internal/session.js +285 -0
- package/dist/esm/history/internal/setup.d.ts +64 -0
- package/dist/esm/history/internal/setup.js +56 -0
- package/dist/{history/session-adapter.js → esm/history/session-adapter.d.ts} +25 -39
- package/dist/esm/history/session-adapter.js +34 -0
- package/dist/esm/history/types.d.ts +102 -0
- package/dist/esm/history/types.js +1 -0
- package/dist/esm/index.d.ts +53 -0
- package/dist/esm/index.js +10 -0
- package/dist/esm/shared/adapter.d.ts +146 -0
- package/dist/esm/shared/adapter.js +65 -0
- package/dist/{shared/clock.js → esm/shared/clock.d.ts} +3 -14
- package/dist/esm/shared/clock.js +9 -0
- package/dist/esm/shared/codec/codec.d.ts +336 -0
- package/dist/esm/shared/codec/codec.js +150 -0
- package/dist/esm/shared/codec/compression.d.ts +66 -0
- package/dist/esm/shared/codec/compression.js +40 -0
- package/dist/esm/shared/codec/json-serde.d.ts +80 -0
- package/dist/esm/shared/codec/json-serde.js +54 -0
- package/dist/esm/shared/codec/s3/client-types.d.ts +54 -0
- package/dist/esm/shared/codec/s3/client-types.js +3 -0
- package/dist/esm/shared/codec/s3/client.d.ts +63 -0
- package/dist/esm/shared/codec/s3/client.js +40 -0
- package/dist/{shared/codec/s3/config.js → esm/shared/codec/s3/config.d.ts} +45 -93
- package/dist/esm/shared/codec/s3/config.js +79 -0
- package/dist/esm/shared/codec/s3/lifecycle.d.ts +220 -0
- package/dist/esm/shared/codec/s3/lifecycle.js +214 -0
- package/dist/esm/shared/codec/s3/offloader.d.ts +428 -0
- package/dist/esm/shared/codec/s3/offloader.js +278 -0
- package/dist/{shared/concurrency.js → esm/shared/concurrency.d.ts} +2 -37
- package/dist/esm/shared/concurrency.js +27 -0
- package/dist/{shared/dynamodb/abort.js → esm/shared/dynamodb/abort.d.ts} +4 -16
- package/dist/esm/shared/dynamodb/abort.js +12 -0
- package/dist/esm/shared/dynamodb/batch-write.d.ts +86 -0
- package/dist/esm/shared/dynamodb/batch-write.js +74 -0
- package/dist/{shared/dynamodb/cancellation.js → esm/shared/dynamodb/cancellation.d.ts} +25 -53
- package/dist/esm/shared/dynamodb/cancellation.js +30 -0
- package/dist/{shared/dynamodb/client.js → esm/shared/dynamodb/client.d.ts} +74 -64
- package/dist/esm/shared/dynamodb/client.js +35 -0
- package/dist/{shared/dynamodb/idempotent-write.js → esm/shared/dynamodb/idempotent-write.d.ts} +172 -169
- package/dist/esm/shared/dynamodb/idempotent-write.js +153 -0
- package/dist/esm/shared/dynamodb/paginate.d.ts +111 -0
- package/dist/esm/shared/dynamodb/paginate.js +84 -0
- package/dist/esm/shared/dynamodb/partition-delete.d.ts +204 -0
- package/dist/esm/shared/dynamodb/partition-delete.js +171 -0
- package/dist/esm/shared/dynamodb/recency-index.d.ts +243 -0
- package/dist/esm/shared/dynamodb/recency-index.js +136 -0
- package/dist/esm/shared/dynamodb/retry.d.ts +317 -0
- package/dist/esm/shared/dynamodb/retry.js +180 -0
- package/dist/esm/shared/dynamodb/row-size.d.ts +33 -0
- package/dist/esm/shared/dynamodb/row-size.js +26 -0
- package/dist/{shared/dynamodb/table-schema.js → esm/shared/dynamodb/table-schema.d.ts} +57 -67
- package/dist/esm/shared/dynamodb/table-schema.js +43 -0
- package/dist/esm/shared/errors/base-error.d.ts +231 -0
- package/dist/esm/shared/errors/base-error.js +64 -0
- package/dist/esm/shared/errors/boundary.d.ts +86 -0
- package/dist/esm/shared/errors/boundary.js +54 -0
- package/dist/esm/shared/errors/classify.d.ts +206 -0
- package/dist/esm/shared/errors/classify.js +220 -0
- package/dist/{shared/errors/error-code.js → esm/shared/errors/error-code.d.ts} +39 -43
- package/dist/esm/shared/errors/error-code.js +24 -0
- package/dist/{shared/errors/errors.js → esm/shared/errors/errors.d.ts} +92 -98
- package/dist/esm/shared/errors/errors.js +110 -0
- package/dist/{shared/logging/logger.js → esm/shared/logging/logger.d.ts} +22 -52
- package/dist/esm/shared/logging/logger.js +25 -0
- package/dist/esm/shared/logging/redaction.d.ts +123 -0
- package/dist/esm/shared/logging/redaction.js +140 -0
- package/dist/{shared/logging/secret-patterns.js → esm/shared/logging/secret-patterns.d.ts} +46 -87
- package/dist/esm/shared/logging/secret-patterns.js +83 -0
- package/dist/{shared/logging/truncate.js → esm/shared/logging/truncate.d.ts} +6 -40
- package/dist/esm/shared/logging/truncate.js +25 -0
- package/dist/esm/shared/options.d.ts +100 -0
- package/dist/esm/shared/options.js +1 -0
- package/dist/{shared/ulid.js → esm/shared/ulid.d.ts} +4 -84
- package/dist/esm/shared/ulid.js +77 -0
- package/dist/{shared/validation/collaborators.js → esm/shared/validation/collaborators.d.ts} +37 -65
- package/dist/esm/shared/validation/collaborators.js +70 -0
- package/dist/{shared/validation/option-shape.js → esm/shared/validation/option-shape.d.ts} +14 -36
- package/dist/esm/shared/validation/option-shape.js +24 -0
- package/dist/esm/shared/validation/options.d.ts +145 -0
- package/dist/esm/shared/validation/options.js +156 -0
- package/dist/{shared/validation/primitives.js → esm/shared/validation/primitives.d.ts} +30 -125
- package/dist/esm/shared/validation/primitives.js +100 -0
- package/dist/esm/shared/validation/ttl.d.ts +94 -0
- package/dist/esm/shared/validation/ttl.js +46 -0
- package/dist/esm/store/actions/list-namespaces.d.ts +79 -0
- package/dist/esm/store/actions/list-namespaces.js +86 -0
- package/dist/esm/store/actions/put.d.ts +43 -0
- package/dist/esm/store/actions/put.js +46 -0
- package/dist/{store/actions/reconcile-vector-index.js → esm/store/actions/reconcile-vector-index.d.ts} +8 -16
- package/dist/esm/store/actions/reconcile-vector-index.js +10 -0
- package/dist/esm/store/actions/search.d.ts +38 -0
- package/dist/esm/store/actions/search.js +23 -0
- package/dist/esm/store/internal/batch-plan.d.ts +56 -0
- package/dist/esm/store/internal/batch-plan.js +28 -0
- package/dist/esm/store/internal/filter.d.ts +43 -0
- package/dist/esm/store/internal/filter.js +60 -0
- package/dist/esm/store/internal/get-item.d.ts +77 -0
- package/dist/esm/store/internal/get-item.js +54 -0
- package/dist/esm/store/internal/item-write.d.ts +270 -0
- package/dist/esm/store/internal/item-write.js +173 -0
- package/dist/esm/store/internal/parse.d.ts +230 -0
- package/dist/esm/store/internal/parse.js +169 -0
- package/dist/{store/internal/rows.js → esm/store/internal/rows.d.ts} +111 -199
- package/dist/esm/store/internal/rows.js +147 -0
- package/dist/esm/store/internal/semantic-search.d.ts +168 -0
- package/dist/esm/store/internal/semantic-search.js +200 -0
- package/dist/esm/store/internal/setup.d.ts +96 -0
- package/dist/esm/store/internal/setup.js +98 -0
- package/dist/esm/store/internal/table-search.d.ts +100 -0
- package/dist/esm/store/internal/table-search.js +119 -0
- package/dist/esm/store/internal/vector-index.d.ts +258 -0
- package/dist/esm/store/internal/vector-index.js +256 -0
- package/dist/{store/store.js → esm/store/store.d.ts} +99 -133
- package/dist/esm/store/store.js +93 -0
- package/dist/esm/store/types.d.ts +111 -0
- package/dist/esm/store/types.js +1 -0
- package/dist/esm/store/vector-backend.d.ts +104 -0
- package/dist/esm/store/vector-backend.js +1 -0
- package/package.json +50 -44
- package/dist/backfill/backfill.js +0 -393
- package/dist/checkpointer/actions/list.js +0 -142
- package/dist/checkpointer/actions/put.js +0 -195
- package/dist/checkpointer/internal/delta-history.js +0 -252
- package/dist/checkpointer/internal/parse.js +0 -372
- package/dist/checkpointer/internal/pending-writes.js +0 -588
- package/dist/checkpointer/internal/read.js +0 -264
- package/dist/checkpointer/internal/rows.js +0 -834
- package/dist/checkpointer/internal/setup.js +0 -72
- package/dist/checkpointer/types.js +0 -12
- package/dist/factory/factory.js +0 -266
- package/dist/factory/types.js +0 -10
- package/dist/history/actions/get-messages.js +0 -141
- package/dist/history/actions/list-sessions.js +0 -156
- package/dist/history/internal/append.js +0 -500
- package/dist/history/internal/message-read.js +0 -204
- package/dist/history/internal/parse.js +0 -252
- package/dist/history/internal/session.js +0 -628
- package/dist/history/internal/setup.js +0 -106
- package/dist/history/types.js +0 -12
- package/dist/index.js +0 -51
- package/dist/shared/adapter.js +0 -143
- package/dist/shared/codec/codec.js +0 -441
- package/dist/shared/codec/compression.js +0 -89
- package/dist/shared/codec/json-serde.js +0 -191
- package/dist/shared/codec/s3/client-types.js +0 -26
- package/dist/shared/codec/s3/client.js +0 -97
- package/dist/shared/codec/s3/lifecycle.js +0 -355
- package/dist/shared/codec/s3/offloader.js +0 -658
- package/dist/shared/dynamodb/batch-write.js +0 -173
- package/dist/shared/dynamodb/paginate.js +0 -188
- package/dist/shared/dynamodb/partition-delete.js +0 -340
- package/dist/shared/dynamodb/recency-index.js +0 -377
- package/dist/shared/dynamodb/retry.js +0 -463
- package/dist/shared/errors/base-error.js +0 -174
- package/dist/shared/errors/boundary.js +0 -143
- package/dist/shared/errors/classify.d.ts +0 -97
- package/dist/shared/errors/classify.js +0 -257
- package/dist/shared/logging/redaction.js +0 -316
- package/dist/shared/options.js +0 -11
- package/dist/shared/validation/options.js +0 -328
- package/dist/shared/validation/ttl.js +0 -132
- package/dist/store/actions/list-namespaces.js +0 -197
- package/dist/store/actions/put.js +0 -78
- package/dist/store/actions/search.js +0 -66
- package/dist/store/internal/batch-plan.d.ts +0 -26
- package/dist/store/internal/batch-plan.js +0 -109
- package/dist/store/internal/filter.js +0 -137
- package/dist/store/internal/get-item.d.ts +0 -45
- package/dist/store/internal/get-item.js +0 -115
- package/dist/store/internal/item-write.js +0 -463
- package/dist/store/internal/parse.js +0 -350
- package/dist/store/internal/semantic-search.js +0 -389
- package/dist/store/internal/setup.js +0 -223
- package/dist/store/internal/table-search.js +0 -213
- package/dist/store/internal/vector-index.js +0 -546
- package/dist/store/types.js +0 -15
- package/dist/store/vector-backend.js +0 -17
- /package/dist/{shared → cjs/shared}/clock.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/codec/json-serde.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/concurrency.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/dynamodb/cancellation.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/logging/logger.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/logging/truncate.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/ulid.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/validation/primitives.d.ts +0 -0
- /package/dist/{store → cjs/store}/vector-backend.d.ts +0 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hides which vector index the store talks to.
|
|
3
|
+
*
|
|
4
|
+
* The store sees only `VectorBackend` — upsert, query, delete and an optional
|
|
5
|
+
* listing — with what it promises and requires stated per method, so any
|
|
6
|
+
* index that keeps those terms plugs in without the store changing. The table
|
|
7
|
+
* stays the truth: a backend may lag it and may score by distance, and every
|
|
8
|
+
* match is re-read before a caller sees it.
|
|
9
|
+
*/
|
|
10
|
+
/** A vector-similarity match returned by an external {@link VectorBackend}. */
|
|
11
|
+
export interface VectorMatch {
|
|
12
|
+
namespace: string[];
|
|
13
|
+
key: string;
|
|
14
|
+
/**
|
|
15
|
+
* Relevance, where **higher means a better match** — the same direction as
|
|
16
|
+
* upstream `SearchItem.score`, which this value is forwarded to verbatim.
|
|
17
|
+
*
|
|
18
|
+
* A backend whose native output is a *distance* (S3 Vectors, FAISS L2,
|
|
19
|
+
* pgvector's `<->`) has two options: convert before returning, or declare
|
|
20
|
+
* `vectorScoreDirection: 'distance'` on the store, which negates and
|
|
21
|
+
* re-sorts for you. Forwarding an unconverted distance without declaring it
|
|
22
|
+
* yields results ordered correctly but scored backwards, which silently
|
|
23
|
+
* breaks any caller that thresholds or displays the number; that case is
|
|
24
|
+
* warned about but never reordered.
|
|
25
|
+
*/
|
|
26
|
+
score: number;
|
|
27
|
+
}
|
|
28
|
+
/** A stored vector's location, returned by {@link VectorBackend.listKeys}. */
|
|
29
|
+
export interface VectorRef {
|
|
30
|
+
namespace: string[];
|
|
31
|
+
key: string;
|
|
32
|
+
}
|
|
33
|
+
/** Which direction a {@link VectorBackend} reports its score in. */
|
|
34
|
+
export type VectorScoreDirection = 'relevance' | 'distance';
|
|
35
|
+
/**
|
|
36
|
+
* Pluggable vector index. When provided to the store, embeddings live here and
|
|
37
|
+
* similarity search is delegated to it; DynamoDB still holds the canonical item.
|
|
38
|
+
*
|
|
39
|
+
* This is an interface you implement, so what the store promises and what it
|
|
40
|
+
* requires are both stated per method. Two properties hold throughout: the
|
|
41
|
+
* store writes DynamoDB first and syncs here afterwards, so the backend may lag
|
|
42
|
+
* the canonical item and `reconcileVectorIndex` exists to close that gap; and
|
|
43
|
+
* every result is re-read from DynamoDB before a caller sees it, so a stale
|
|
44
|
+
* entry costs a wasted read, never a wrong answer.
|
|
45
|
+
*/
|
|
46
|
+
export interface VectorBackend {
|
|
47
|
+
/**
|
|
48
|
+
* Store one item's vector.
|
|
49
|
+
*
|
|
50
|
+
* Receives: a validated `namespace` and `key`, and a vector of the width
|
|
51
|
+
* `index.dims` declares. Called once per indexed put, after the item is
|
|
52
|
+
* committed, and again for every item of a `reconcileVectorIndex`.
|
|
53
|
+
*
|
|
54
|
+
* Must: replace any vector already held for that `(namespace, key)` —
|
|
55
|
+
* upserting the same pair repeatedly is normal and must not accumulate
|
|
56
|
+
* entries.
|
|
57
|
+
*
|
|
58
|
+
* May throw: a rejection is logged and swallowed on the put path (the item
|
|
59
|
+
* still stands) and propagates from a reconcile.
|
|
60
|
+
*/
|
|
61
|
+
upsert(namespace: string[], key: string, vector: number[]): Promise<void>;
|
|
62
|
+
/**
|
|
63
|
+
* Return up to `topK` matches under `namespacePrefix`, best first.
|
|
64
|
+
*
|
|
65
|
+
* Receives: the same query vector width as `upsert`, and a `topK` the store
|
|
66
|
+
* raises — up to `maxSearchCandidates` — while its filter leaves the page
|
|
67
|
+
* short, so returning fewer than `topK` is read as "that is all there is".
|
|
68
|
+
*
|
|
69
|
+
* Must: order best-first, and score each match as a relevance, not a
|
|
70
|
+
* distance (see {@link VectorMatch.score}). Matches outside the prefix, and
|
|
71
|
+
* matches whose item has since been deleted, are dropped by the store rather
|
|
72
|
+
* than trusted — over-returning is safe, under-returning silently shortens
|
|
73
|
+
* the page.
|
|
74
|
+
*/
|
|
75
|
+
query(namespacePrefix: string[], queryVector: number[], topK: number): Promise<VectorMatch[]>;
|
|
76
|
+
/**
|
|
77
|
+
* Drop one item's vector.
|
|
78
|
+
*
|
|
79
|
+
* Receives: the item's address. Called when the item is deleted, and also
|
|
80
|
+
* after any put that produced no vector — `index: false`, or a value with no
|
|
81
|
+
* indexable text — because the previous vector would otherwise keep matching
|
|
82
|
+
* an item that no longer has that text.
|
|
83
|
+
*
|
|
84
|
+
* Must: succeed for a key that holds no vector. That is the common case on
|
|
85
|
+
* the no-vector put path, and treating it as an error would warn on every
|
|
86
|
+
* such put.
|
|
87
|
+
*/
|
|
88
|
+
delete(namespace: string[], key: string): Promise<void>;
|
|
89
|
+
/**
|
|
90
|
+
* Optionally enumerate every stored vector under `namespacePrefix`. Enables
|
|
91
|
+
* `reconcileVectorIndex` to prune vectors orphaned by a lost delete. Omit it
|
|
92
|
+
* when the backend cannot enumerate — reconciliation then re-pushes only.
|
|
93
|
+
*
|
|
94
|
+
* Must: not return keys from outside `namespacePrefix`. Under-reporting only
|
|
95
|
+
* leaves an orphan for a later reconcile; over-reporting offers a vector
|
|
96
|
+
* outside the scope for pruning, and the reconcile is what would delete it.
|
|
97
|
+
*/
|
|
98
|
+
listKeys?(namespacePrefix: string[]): Promise<VectorRef[]>;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Every direction `toRelevanceScores` (store/internal/vector-index.ts)
|
|
102
|
+
* recognises, for validating input.
|
|
103
|
+
*/
|
|
104
|
+
export declare const VECTOR_SCORE_DIRECTIONS: readonly VectorScoreDirection[];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const VECTOR_SCORE_DIRECTIONS = ['relevance', 'distance'];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@farukada/aws-langgraph-dynamodb-ts",
|
|
3
|
-
"version": "1.0.0
|
|
3
|
+
"version": "1.0.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "DynamoDB persistence for LangGraph.js: checkpoint saver, long-term memory store with semantic search, and chat message history, with optional S3 offload, compression and TTL",
|
|
6
6
|
"keywords": [
|
|
@@ -26,39 +26,44 @@
|
|
|
26
26
|
"bugs": {
|
|
27
27
|
"url": "https://github.com/FarukAda/aws-langgraph-dynamodb-ts/issues"
|
|
28
28
|
},
|
|
29
|
-
"type": "
|
|
30
|
-
"main": "dist/index.js",
|
|
31
|
-
"types": "dist/index.d.ts",
|
|
29
|
+
"type": "module",
|
|
30
|
+
"main": "./dist/cjs/index.js",
|
|
31
|
+
"types": "./dist/cjs/index.d.ts",
|
|
32
32
|
"exports": {
|
|
33
33
|
".": {
|
|
34
|
-
"
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
"import": {
|
|
35
|
+
"types": "./dist/esm/index.d.ts",
|
|
36
|
+
"default": "./dist/esm/index.js"
|
|
37
|
+
},
|
|
38
|
+
"require": {
|
|
39
|
+
"types": "./dist/cjs/index.d.ts",
|
|
40
|
+
"default": "./dist/cjs/index.js"
|
|
41
|
+
}
|
|
38
42
|
},
|
|
39
43
|
"./package.json": "./package.json"
|
|
40
44
|
},
|
|
41
45
|
"engines": {
|
|
42
|
-
"node": ">=22.0.0"
|
|
43
|
-
"npm": ">=10.0.0"
|
|
46
|
+
"node": ">=22.0.0"
|
|
44
47
|
},
|
|
45
48
|
"scripts": {
|
|
46
|
-
"build": "node scripts/clean.mjs &&
|
|
49
|
+
"build": "node scripts/clean.mjs && npm run build:esm && npm run build:cjs",
|
|
50
|
+
"build:esm": "tsc -p tsconfig.build.json --removeComments --declaration false && tsc -p tsconfig.build.json --emitDeclarationOnly",
|
|
51
|
+
"build:cjs": "tsc -p tsconfig.build.cjs.json --removeComments --declaration false && tsc -p tsconfig.build.cjs.json --emitDeclarationOnly && node scripts/mark-cjs.mjs",
|
|
47
52
|
"typecheck": "tsc -p tsconfig.build.json --noEmit",
|
|
48
53
|
"typecheck:all": "tsc --noEmit",
|
|
49
54
|
"lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\" \"*.config.ts\" \"**/*.mjs\" --no-error-on-unmatched-pattern",
|
|
50
55
|
"lint:fix": "eslint \"src/**/*.ts\" \"test/**/*.ts\" \"*.config.ts\" \"**/*.mjs\" --fix --no-error-on-unmatched-pattern",
|
|
51
56
|
"test": "jest --coverage",
|
|
52
|
-
"test:integration": "jest --config jest.integration.config.
|
|
57
|
+
"test:integration": "jest --config jest.integration.config.mjs",
|
|
53
58
|
"test:integration:up": "docker compose up -d",
|
|
54
59
|
"test:integration:down": "docker compose down",
|
|
55
|
-
"test:conformance": "jest --config jest.conformance.config.
|
|
56
|
-
"test:aws": "jest --config jest.aws.config.
|
|
60
|
+
"test:conformance": "jest --config jest.conformance.config.mjs",
|
|
61
|
+
"test:aws": "jest --config jest.aws.config.mjs",
|
|
57
62
|
"test:static": "jest test/static",
|
|
58
63
|
"test:scripts": "node --test \"test/scripts/**/*.test.mjs\"",
|
|
59
64
|
"test:package-smoke": "node --test test/package-smoke/smoke.test.mjs",
|
|
60
65
|
"test:consumer-types": "node --test test/package-smoke/consumer-types.test.mjs",
|
|
61
|
-
"test:surface": "node scripts/run-with-timeout.mjs 150 -- node --test --test-timeout=120000 test/surface/surface.test.mjs",
|
|
66
|
+
"test:surface": "node scripts/run-with-timeout.mjs 150 -- node --test --test-force-exit --test-timeout=120000 test/surface/surface.test.mjs",
|
|
62
67
|
"test:surface:update": "node scripts/update-surface-baseline.mjs",
|
|
63
68
|
"prepublishOnly": "npm run typecheck && npm run typecheck:all && npm run lint && npm run build && npm test",
|
|
64
69
|
"docs": "typedoc",
|
|
@@ -66,7 +71,6 @@
|
|
|
66
71
|
"check:docs": "node scripts/check-doc-samples.mjs",
|
|
67
72
|
"check:links": "node scripts/check-doc-links.mjs",
|
|
68
73
|
"unused": "knip",
|
|
69
|
-
"depcheck": "depcheck .",
|
|
70
74
|
"peer-floors": "node scripts/peer-floors.mjs",
|
|
71
75
|
"ci:require-green": "node scripts/require-green-ci.mjs",
|
|
72
76
|
"changelog-section": "node scripts/changelog-section.mjs",
|
|
@@ -74,17 +78,21 @@
|
|
|
74
78
|
"cpd": "jscpd ./src --threshold 2 --min-lines 3 --reporters console --silent --format typescript,javascript",
|
|
75
79
|
"cpd:full": "jscpd ./src --threshold 2 --min-lines 3 --reporters console-full --format typescript,javascript",
|
|
76
80
|
"cpd:test": "jscpd ./test --threshold 2 --min-lines 3 --reporters console-full --format typescript,javascript",
|
|
77
|
-
"
|
|
81
|
+
"example:checkpointer": "npm run build && node examples/live-checkpointer.mjs",
|
|
82
|
+
"example:persist": "npm run build && node examples/live-persist.mjs",
|
|
83
|
+
"example:store": "npm run build && node examples/live-store.mjs",
|
|
84
|
+
"example:agent": "npm run build && node examples/live-agent.mjs",
|
|
85
|
+
"pack:check": "node scripts/pack-check.mjs && publint && attw --pack ."
|
|
78
86
|
},
|
|
79
87
|
"dependencies": {
|
|
80
|
-
"@aws-sdk/client-dynamodb": "^3.
|
|
81
|
-
"@aws-sdk/lib-dynamodb": "^3.
|
|
88
|
+
"@aws-sdk/client-dynamodb": "^3.1151.0",
|
|
89
|
+
"@aws-sdk/lib-dynamodb": "^3.1151.0",
|
|
82
90
|
"@aws-sdk/util-dynamodb": "^3.996.9"
|
|
83
91
|
},
|
|
84
92
|
"peerDependencies": {
|
|
85
|
-
"@aws-sdk/client-s3": "^3.
|
|
86
|
-
"@langchain/core": "^1.2.
|
|
87
|
-
"@langchain/langgraph-checkpoint": "^1.
|
|
93
|
+
"@aws-sdk/client-s3": "^3.1151.0",
|
|
94
|
+
"@langchain/core": "^1.2.17",
|
|
95
|
+
"@langchain/langgraph-checkpoint": "^1.2.0"
|
|
88
96
|
},
|
|
89
97
|
"peerDependenciesMeta": {
|
|
90
98
|
"@aws-sdk/client-s3": {
|
|
@@ -93,39 +101,37 @@
|
|
|
93
101
|
},
|
|
94
102
|
"devDependencies": {
|
|
95
103
|
"@arethetypeswrong/cli": "^0.18.5",
|
|
96
|
-
"@aws-sdk/client-s3": "^3.
|
|
104
|
+
"@aws-sdk/client-s3": "^3.1151.0",
|
|
97
105
|
"@eslint/js": "^10.0.1",
|
|
98
|
-
"@langchain/aws": "^1.4.
|
|
99
|
-
"@langchain/langgraph": "^1.
|
|
100
|
-
"@langchain/langgraph-checkpoint-validation": "^1.
|
|
106
|
+
"@langchain/aws": "^1.4.6",
|
|
107
|
+
"@langchain/langgraph": "^1.5.0",
|
|
108
|
+
"@langchain/langgraph-checkpoint-validation": "^1.2.0",
|
|
101
109
|
"@types/jest": "^30.0.0",
|
|
102
|
-
"@types/node": "^26.5
|
|
110
|
+
"@types/node": "^26.6.5",
|
|
103
111
|
"@typescript/native": "npm:typescript@^7.0.2",
|
|
104
112
|
"aws-sdk-client-mock": "^4.1.0",
|
|
105
|
-
"
|
|
106
|
-
"eslint": "^10.10.0",
|
|
113
|
+
"eslint": "^10.12.0",
|
|
107
114
|
"eslint-config-prettier": "^10.1.8",
|
|
108
115
|
"eslint-plugin-no-instanceof": "^1.0.1",
|
|
109
|
-
"eslint-plugin-perfectionist": "^5.
|
|
110
|
-
"eslint-plugin-prettier": "^5.5.
|
|
116
|
+
"eslint-plugin-perfectionist": "^5.12.1",
|
|
117
|
+
"eslint-plugin-prettier": "^5.5.7",
|
|
111
118
|
"eslint-plugin-unused-imports": "^4.4.1",
|
|
112
119
|
"expect-type": "^1.4.0",
|
|
113
|
-
"fast-check": "^4.10.
|
|
114
|
-
"jest": "^30.5.
|
|
115
|
-
"jest-environment-node": "^30.5.
|
|
120
|
+
"fast-check": "^4.10.2",
|
|
121
|
+
"jest": "^30.5.2",
|
|
122
|
+
"jest-environment-node": "^30.5.2",
|
|
116
123
|
"jiti": "^2.7.0",
|
|
117
|
-
"jscpd": "^5.
|
|
118
|
-
"knip": "^6.
|
|
119
|
-
"langchain": "^1.5.
|
|
120
|
-
"prettier": "^3.9.
|
|
121
|
-
"publint": "^0.3.
|
|
122
|
-
"ts-jest": "^29.4.
|
|
123
|
-
"ts-node": "^10.9.2",
|
|
124
|
+
"jscpd": "^5.4.1",
|
|
125
|
+
"knip": "^6.41.0",
|
|
126
|
+
"langchain": "^1.5.16",
|
|
127
|
+
"prettier": "^3.9.10",
|
|
128
|
+
"publint": "^0.3.25",
|
|
129
|
+
"ts-jest": "^29.4.14",
|
|
124
130
|
"typedoc": "^0.28.20",
|
|
125
|
-
"typedoc-plugin-markdown": "^4.
|
|
131
|
+
"typedoc-plugin-markdown": "^4.14.0",
|
|
126
132
|
"typescript": "npm:@typescript/typescript6@^6.0.2",
|
|
127
|
-
"typescript-eslint": "^8.
|
|
128
|
-
"zod": "^4.6.
|
|
133
|
+
"typescript-eslint": "^8.71.2",
|
|
134
|
+
"zod": "^4.6.7"
|
|
129
135
|
},
|
|
130
136
|
"prettier": {
|
|
131
137
|
"semi": true,
|
|
@@ -1,393 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* Hides how rows written before the recency index are given its keys.
|
|
4
|
-
*
|
|
5
|
-
* The index is opt-in (record 8), so a table can hold rows from before it
|
|
6
|
-
* existed. An operator walks the table a page at a time, resumably, and gives
|
|
7
|
-
* each row the keys its adapter would have written; which rows those are is
|
|
8
|
-
* each adapter's own answer, asked here, and a row that gained keys since the
|
|
9
|
-
* scan read it is left alone.
|
|
10
|
-
*/
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.backfillRecencyIndex = backfillRecencyIndex;
|
|
13
|
-
exports.indexTargetOf = indexTargetOf;
|
|
14
|
-
exports.encodeScanCursor = encodeScanCursor;
|
|
15
|
-
exports.decodeScanCursor = decodeScanCursor;
|
|
16
|
-
exports.assertBackfillOptions = assertBackfillOptions;
|
|
17
|
-
const rows_1 = require("../checkpointer/internal/rows");
|
|
18
|
-
const session_1 = require("../history/internal/session");
|
|
19
|
-
const concurrency_1 = require("../shared/concurrency");
|
|
20
|
-
const idempotent_write_1 = require("../shared/dynamodb/idempotent-write");
|
|
21
|
-
const recency_index_1 = require("../shared/dynamodb/recency-index");
|
|
22
|
-
const retry_1 = require("../shared/dynamodb/retry");
|
|
23
|
-
const table_schema_1 = require("../shared/dynamodb/table-schema");
|
|
24
|
-
const boundary_1 = require("../shared/errors/boundary");
|
|
25
|
-
const errors_1 = require("../shared/errors/errors");
|
|
26
|
-
const collaborators_1 = require("../shared/validation/collaborators");
|
|
27
|
-
const option_shape_1 = require("../shared/validation/option-shape");
|
|
28
|
-
const options_1 = require("../shared/validation/options");
|
|
29
|
-
const primitives_1 = require("../shared/validation/primitives");
|
|
30
|
-
const rows_2 = require("../store/internal/rows");
|
|
31
|
-
/**
|
|
32
|
-
* The retry policy every request of one run uses: the caller's `retry`, with
|
|
33
|
-
* the signal that cancels the run. That is the top-level `signal` when one is
|
|
34
|
-
* given and `retry.signal` otherwise — the top-level one is the caller's handle
|
|
35
|
-
* on the whole operation, so it wins when both are set.
|
|
36
|
-
*/
|
|
37
|
-
function runRetry(options) {
|
|
38
|
-
return { ...options.retry, signal: options.signal ?? options.retry?.signal };
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Write one row's index keys; false when this run wrote none for it.
|
|
42
|
-
*
|
|
43
|
-
* False is an ordinary outcome and never a failure. Either the row is not one a
|
|
44
|
-
* listing reaches, so it needs no keys; or the conditional update was refused,
|
|
45
|
-
* which by the terms of that condition means the row already carries keys a
|
|
46
|
-
* live adapter gave it, or the row is gone. In every one of those cases this
|
|
47
|
-
* run has nothing to do for the row, and the row is counted as skipped.
|
|
48
|
-
*
|
|
49
|
-
* Which is why the refusal ends the row rather than the run. Both races are
|
|
50
|
-
* ordinary on a table that is being written to, and that is the only kind of
|
|
51
|
-
* table anyone runs a backfill against; a tool that stopped whenever the table
|
|
52
|
-
* it is migrating is in use would never finish one. Re-running is cheap and
|
|
53
|
-
* safe besides — the scan's own `attribute_not_exists` filter never looks at a
|
|
54
|
-
* row an earlier run indexed.
|
|
55
|
-
*
|
|
56
|
-
* Any other failure is rethrown and ends the run, because nothing about it says
|
|
57
|
-
* this row needed no writing.
|
|
58
|
-
*/
|
|
59
|
-
async function indexRow(options, row, shards) {
|
|
60
|
-
const target = indexTargetOf(row);
|
|
61
|
-
if (target === undefined)
|
|
62
|
-
return false;
|
|
63
|
-
const keys = (0, recency_index_1.indexKeys)(target.tag, target.id, target.at, shards);
|
|
64
|
-
if (options.dryRun)
|
|
65
|
-
return true;
|
|
66
|
-
try {
|
|
67
|
-
await writeIndexKeys(options, row, keys);
|
|
68
|
-
}
|
|
69
|
-
catch (error) {
|
|
70
|
-
if (!(0, idempotent_write_1.isConditionalCheckFailed)(error))
|
|
71
|
-
throw error;
|
|
72
|
-
return false;
|
|
73
|
-
}
|
|
74
|
-
return true;
|
|
75
|
-
}
|
|
76
|
-
/** The conditional `UpdateItem` that gives one row the keys computed for it. */
|
|
77
|
-
async function writeIndexKeys(options, row, keys) {
|
|
78
|
-
await (0, retry_1.withDynamoDBRetry)((request) => options.client.update({
|
|
79
|
-
TableName: options.tableName,
|
|
80
|
-
Key: (0, table_schema_1.rowKeyOf)(row),
|
|
81
|
-
UpdateExpression: 'SET #gpk = :gpk, #gsk = :gsk',
|
|
82
|
-
ExpressionAttributeNames: { '#gpk': 'gsi1pk', '#gsk': 'gsi1sk' },
|
|
83
|
-
ExpressionAttributeValues: { ':gpk': keys.gsi1pk, ':gsk': keys.gsi1sk },
|
|
84
|
-
// Two clauses, and both are load-bearing.
|
|
85
|
-
//
|
|
86
|
-
// `attribute_not_exists(#gpk)` never overwrites keys a row already has:
|
|
87
|
-
// a row a running adapter wrote carries its true timestamp, and
|
|
88
|
-
// replacing it with the pre-index epoch would move a live row to the
|
|
89
|
-
// bottom of every listing.
|
|
90
|
-
//
|
|
91
|
-
// `attribute_exists(PK)` is what makes this an update rather than an
|
|
92
|
-
// upsert, which is what `UpdateItem` is by default. A condition naming
|
|
93
|
-
// only the index attribute is satisfied by a key holding *nothing at
|
|
94
|
-
// all*, so without it, a row deleted between the scan that found it
|
|
95
|
-
// and this update would be re-created as a stub carrying nothing but
|
|
96
|
-
// `PK`, `SK` and the two index keys — and carrying them, it would land
|
|
97
|
-
// in the recency index that the cross-partition listings read. The
|
|
98
|
-
// tool exists to give keys to rows that are already there, so nothing
|
|
99
|
-
// legitimate is refused.
|
|
100
|
-
ConditionExpression: `attribute_exists(${table_schema_1.PARTITION_KEY_ATTRIBUTE}) AND attribute_not_exists(#gpk)`,
|
|
101
|
-
}, request), runRetry(options));
|
|
102
|
-
}
|
|
103
|
-
/** One scan page, and the rows of it that were given keys. */
|
|
104
|
-
async function backfillPage(options, shards, startKey) {
|
|
105
|
-
const result = await (0, retry_1.withDynamoDBRetry)((request) => options.client.scan({
|
|
106
|
-
TableName: options.tableName,
|
|
107
|
-
Limit: options.pageSize ?? 100,
|
|
108
|
-
ExclusiveStartKey: startKey,
|
|
109
|
-
// Rows that already carry keys are not read into memory at all.
|
|
110
|
-
FilterExpression: 'attribute_not_exists(#gpk)',
|
|
111
|
-
ExpressionAttributeNames: { '#gpk': 'gsi1pk' },
|
|
112
|
-
}, request), runRetry(options));
|
|
113
|
-
const rows = (result.Items ?? []);
|
|
114
|
-
const written = await (0, concurrency_1.mapWithConcurrency)(rows, concurrency_1.DEFAULT_READ_CONCURRENCY, (row) => indexRow(options, row, shards));
|
|
115
|
-
return {
|
|
116
|
-
rows: rows.length,
|
|
117
|
-
indexed: written.filter(Boolean).length,
|
|
118
|
-
nextKey: result.LastEvaluatedKey,
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* Give rows written before the recency index their index keys.
|
|
123
|
-
*
|
|
124
|
-
* **Run this before setting `indexName` on any adapter.** A row without the
|
|
125
|
-
* keys is not in the index, so enabling the index first would make every
|
|
126
|
-
* pre-existing session, item and checkpoint silently vanish from the listings
|
|
127
|
-
* that read it — the rows are still there, and every other read still returns
|
|
128
|
-
* them, but a listing would not.
|
|
129
|
-
*
|
|
130
|
-
* Safe to re-run and safe to run while adapters are writing: every write is
|
|
131
|
-
* conditional on the row still being there and having no keys yet, so a row a
|
|
132
|
-
* live adapter has already indexed is left exactly as it is, and a row deleted
|
|
133
|
-
* after the scan found it stays deleted rather than being re-created by an
|
|
134
|
-
* `UpdateItem`, which upserts.
|
|
135
|
-
*
|
|
136
|
-
* `indexShards` must match what the adapters use. A mismatch puts rows on
|
|
137
|
-
* shards no listing queries, which looks exactly like the rows being missing.
|
|
138
|
-
*
|
|
139
|
-
* Accepts: `options` — validated in full before any read: only the keys
|
|
140
|
-
* `BackfillOptions` declares; `tableName`, `indexShards` and the numbers in
|
|
141
|
-
* `retry` by the adapters' rules; `signal` as their methods check it; a
|
|
142
|
-
* `client` providing `scan` and `update`. `options.pageSize` — a positive integer,
|
|
143
|
-
* default 100. `options.cursor` — from a previous run, to resume.
|
|
144
|
-
* `options.maxPages` — how far one run goes, so a large table can be
|
|
145
|
-
* backfilled in bounded slices. `options.indexShards` — must equal the
|
|
146
|
-
* adapters' setting, and has their ceiling. `options.dryRun` — a boolean.
|
|
147
|
-
* `options.signal` — cancels the run; `retry.signal` does so when there is no
|
|
148
|
-
* top-level `signal`, and the top-level one wins when both are given.
|
|
149
|
-
*
|
|
150
|
-
* Returns: how many rows were scanned, how many were given keys and how many
|
|
151
|
-
* were skipped — a row no listing reaches, and a row whose write the condition
|
|
152
|
-
* refused because the row already has keys or is gone — plus a `nextCursor`
|
|
153
|
-
* when the run stopped short of the end. An absent cursor means the table is
|
|
154
|
-
* fully backfilled.
|
|
155
|
-
*
|
|
156
|
-
* Throws: `VALIDATION` naming the offending option, before any DynamoDB
|
|
157
|
-
* call; `RETRY_EXHAUSTED` once a transient failure has used every attempt;
|
|
158
|
-
* `ABORTED` when `signal` fires, or `retry.signal` when no top-level `signal`
|
|
159
|
-
* is given; any other error the scan or the writes throw, wrapped with the
|
|
160
|
-
* code the classifier assigns — this is the function's own error boundary,
|
|
161
|
-
* the same as every adapter's public methods, so a caller's mistake never escapes as a bare
|
|
162
|
-
* exception. A refused write is none of these: it is an outcome for one row,
|
|
163
|
-
* reported in `skipped`.
|
|
164
|
-
*
|
|
165
|
-
* Guarantees: every write is conditional on the row still being there and
|
|
166
|
-
* having no keys yet, so re-running is safe, running against a live table is
|
|
167
|
-
* safe, a row a live adapter has already indexed is left exactly as it is, and
|
|
168
|
-
* a row deleted between the scan and the write is never re-created. Neither
|
|
169
|
-
* refusal stops the run: both mean this run has nothing to do for that row, so
|
|
170
|
-
* the row is counted as skipped and the walk carries on to the rest of the
|
|
171
|
-
* table.
|
|
172
|
-
*/
|
|
173
|
-
async function backfillRecencyIndex(options) {
|
|
174
|
-
return (0, boundary_1.guardPublic)('backfillRecencyIndex', async () => {
|
|
175
|
-
assertBackfillOptions(options);
|
|
176
|
-
const shards = options.indexShards ?? recency_index_1.DEFAULT_INDEX_SHARDS;
|
|
177
|
-
let startKey = options.cursor === undefined ? undefined : decodeScanCursor(options.cursor);
|
|
178
|
-
let scanned = 0;
|
|
179
|
-
let indexed = 0;
|
|
180
|
-
for (let page = 1;; page++) {
|
|
181
|
-
const result = await backfillPage(options, shards, startKey);
|
|
182
|
-
scanned += result.rows;
|
|
183
|
-
indexed += result.indexed;
|
|
184
|
-
startKey = result.nextKey;
|
|
185
|
-
if (startKey === undefined)
|
|
186
|
-
break;
|
|
187
|
-
if (options.maxPages !== undefined && page >= options.maxPages) {
|
|
188
|
-
return {
|
|
189
|
-
scanned,
|
|
190
|
-
indexed,
|
|
191
|
-
skipped: scanned - indexed,
|
|
192
|
-
nextCursor: encodeScanCursor(startKey),
|
|
193
|
-
};
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
return { scanned, indexed, skipped: scanned - indexed };
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Which rows the index covers: each adapter answers for its own, and no row belongs to two.
|
|
201
|
-
*
|
|
202
|
-
* Accepts: `row` — any row a table scan returns, including a foreign one and
|
|
203
|
-
* one whose `PK`/`SK` are not strings.
|
|
204
|
-
*
|
|
205
|
-
* Returns: the index identity, or undefined for a row no listing reaches — a
|
|
206
|
-
* foreign row, a payload or write row, a META row carrying no `checkpointId`.
|
|
207
|
-
*
|
|
208
|
-
* Throws: nothing. A backfill walks the whole table; one unrecognised row must
|
|
209
|
-
* be skipped, not fatal.
|
|
210
|
-
*/
|
|
211
|
-
function indexTargetOf(row) {
|
|
212
|
-
return (0, rows_1.checkpointIndexTarget)(row) ?? (0, rows_2.storeIndexTarget)(row) ?? (0, session_1.sessionIndexTarget)(row);
|
|
213
|
-
}
|
|
214
|
-
/**
|
|
215
|
-
* A scan position, as an opaque string.
|
|
216
|
-
*
|
|
217
|
-
* Accepts: `key` — a `LastEvaluatedKey` from the scan being resumed.
|
|
218
|
-
*
|
|
219
|
-
* Returns: it base64url-encoded. Opaque to the caller: its shape is not a
|
|
220
|
-
* promise, which is what leaves the encoding free to change.
|
|
221
|
-
*
|
|
222
|
-
* Throws: nothing.
|
|
223
|
-
*/
|
|
224
|
-
function encodeScanCursor(key) {
|
|
225
|
-
return Buffer.from(JSON.stringify(key), 'utf8').toString('base64url');
|
|
226
|
-
}
|
|
227
|
-
/** Whether `value` is exactly the base table's primary key: `PK` and `SK`, both strings, nothing else. */
|
|
228
|
-
function isTableKeyShape(value) {
|
|
229
|
-
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
230
|
-
return false;
|
|
231
|
-
const keys = Object.keys(value);
|
|
232
|
-
return keys.length === 2 && typeof value.PK === 'string' && typeof value.SK === 'string';
|
|
233
|
-
}
|
|
234
|
-
/**
|
|
235
|
-
* The scan position a cursor encodes.
|
|
236
|
-
*
|
|
237
|
-
* Accepts: `cursor` — as a previous page returned it.
|
|
238
|
-
*
|
|
239
|
-
* Returns: the `ExclusiveStartKey` to resume from — always exactly `{ PK,
|
|
240
|
-
* SK }`, both strings, since a plain table `Scan` (no `IndexName`) never
|
|
241
|
-
* returns a `LastEvaluatedKey` shaped any other way.
|
|
242
|
-
*
|
|
243
|
-
* Throws: `VALIDATION` naming `cursor` for anything this tool did not
|
|
244
|
-
* issue — text that is not base64url, that does not decode to JSON, or that
|
|
245
|
-
* decodes to anything but `{ PK: string, SK: string }`: an array, an object
|
|
246
|
-
* missing either key, carrying an extra one, or carrying a non-string value
|
|
247
|
-
* for either. A cursor is fed straight back to DynamoDB as
|
|
248
|
-
* `ExclusiveStartKey`, so a value of the wrong shape is refused here rather
|
|
249
|
-
* than surfacing as a raw `ValidationException` from the service.
|
|
250
|
-
*/
|
|
251
|
-
function decodeScanCursor(cursor) {
|
|
252
|
-
try {
|
|
253
|
-
const decoded = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
|
|
254
|
-
if (!isTableKeyShape(decoded))
|
|
255
|
-
throw new Error('not a scan position');
|
|
256
|
-
return decoded;
|
|
257
|
-
}
|
|
258
|
-
catch {
|
|
259
|
-
throw (0, errors_1.validationError)('cursor is not one this tool issued', 'cursor');
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
/**
|
|
263
|
-
* The keys of {@link BackfillOptions}, exhaustive in both directions —
|
|
264
|
-
* `allKeysOf<T>` fails to compile if this list omits or invents one.
|
|
265
|
-
*/
|
|
266
|
-
const BACKFILL_KEYS = (0, option_shape_1.allKeysOf)({
|
|
267
|
-
client: 'client',
|
|
268
|
-
tableName: 'tableName',
|
|
269
|
-
indexShards: 'indexShards',
|
|
270
|
-
pageSize: 'pageSize',
|
|
271
|
-
maxPages: 'maxPages',
|
|
272
|
-
cursor: 'cursor',
|
|
273
|
-
dryRun: 'dryRun',
|
|
274
|
-
retry: 'retry',
|
|
275
|
-
signal: 'signal',
|
|
276
|
-
});
|
|
277
|
-
/** The `DynamoDBDocument` methods the backfill calls on an injected `client`. */
|
|
278
|
-
const BACKFILL_CLIENT_MEMBERS = ['scan', 'update'];
|
|
279
|
-
/**
|
|
280
|
-
* The keys of {@link RetryOptions}, exhaustive in both directions. Backfill
|
|
281
|
-
* accepts the full retry surface, not the adapters' narrower `RetryPolicy`
|
|
282
|
-
* `assertRetryPolicy` checks — `onRetry` is backfill's only way to observe
|
|
283
|
-
* retries, since it takes no `logger`.
|
|
284
|
-
*
|
|
285
|
-
* `deadlineAt` is the one exclusion: it is an internal per-call bound a write
|
|
286
|
-
* path sets on itself, not something an application names, so it stays an
|
|
287
|
-
* unknown key here and is refused like any other. Excluding it by `Omit`
|
|
288
|
-
* rather than by leaving it out keeps the list exhaustive, so a genuinely new
|
|
289
|
-
* option still fails to compile until it is decided on here.
|
|
290
|
-
*/
|
|
291
|
-
const BACKFILL_RETRY_KEYS = (0, option_shape_1.allKeysOf)({
|
|
292
|
-
maxAttempts: 'maxAttempts',
|
|
293
|
-
baseDelayMs: 'baseDelayMs',
|
|
294
|
-
maxDelayMs: 'maxDelayMs',
|
|
295
|
-
retryableErrors: 'retryableErrors',
|
|
296
|
-
isRetryable: 'isRetryable',
|
|
297
|
-
onRetry: 'onRetry',
|
|
298
|
-
signal: 'signal',
|
|
299
|
-
rng: 'rng',
|
|
300
|
-
});
|
|
301
|
-
/**
|
|
302
|
-
* Validate a `backfillRecencyIndex` retry policy against the full
|
|
303
|
-
* {@link RetryOptions} surface.
|
|
304
|
-
*
|
|
305
|
-
* Accepts: `retry` — must be an object naming only {@link BACKFILL_RETRY_KEYS}.
|
|
306
|
-
* `maxAttempts`/`baseDelayMs`/`maxDelayMs` share the adapters' own bounds
|
|
307
|
-
* ({@link assertRetryBounds}), so backfill and the adapters it feeds cannot
|
|
308
|
-
* drift apart on what a legal value is. `retryableErrors`, when given, must be
|
|
309
|
-
* an array of strings. `isRetryable`, `onRetry` and `rng`, when given, must
|
|
310
|
-
* each be a function. `signal` is checked the same way the top-level
|
|
311
|
-
* `options.signal` is. Either one cancels the run; when both are given the
|
|
312
|
-
* top-level one wins, since the caller's own signal is meant to cancel the
|
|
313
|
-
* whole operation.
|
|
314
|
-
*
|
|
315
|
-
* Returns: nothing: the value is kept under its declared type, and this
|
|
316
|
-
* checks it.
|
|
317
|
-
*
|
|
318
|
-
* Throws: `VALIDATION` naming `retry`, `retry.<numeric key>`,
|
|
319
|
-
* `retry.retryableErrors`, `retry.isRetryable`, `retry.onRetry`, `retry.rng`
|
|
320
|
-
* or `retry.signal`.
|
|
321
|
-
*/
|
|
322
|
-
function assertBackfillRetryOptions(retry) {
|
|
323
|
-
(0, option_shape_1.assertShape)(retry, BACKFILL_RETRY_KEYS, 'retry');
|
|
324
|
-
(0, options_1.assertRetryBounds)(retry);
|
|
325
|
-
if (retry.retryableErrors !== undefined) {
|
|
326
|
-
(0, primitives_1.assertStringArray)(retry.retryableErrors, 'retry.retryableErrors');
|
|
327
|
-
}
|
|
328
|
-
if (retry.isRetryable !== undefined && typeof retry.isRetryable !== 'function') {
|
|
329
|
-
throw (0, errors_1.validationError)('retry.isRetryable must be a function', 'retry.isRetryable');
|
|
330
|
-
}
|
|
331
|
-
if (retry.onRetry !== undefined && typeof retry.onRetry !== 'function') {
|
|
332
|
-
throw (0, errors_1.validationError)('retry.onRetry must be a function', 'retry.onRetry');
|
|
333
|
-
}
|
|
334
|
-
if (retry.rng !== undefined && typeof retry.rng !== 'function') {
|
|
335
|
-
throw (0, errors_1.validationError)('retry.rng must be a function', 'retry.rng');
|
|
336
|
-
}
|
|
337
|
-
(0, collaborators_1.assertSignalLike)(retry.signal, 'retry.signal');
|
|
338
|
-
}
|
|
339
|
-
/**
|
|
340
|
-
* Reject a non-integer positive bound, `undefined` left to its own default.
|
|
341
|
-
*
|
|
342
|
-
* Accepts: `value` — `options.pageSize`, `options.maxPages` or
|
|
343
|
-
* `options.indexShards`, as the caller gave it. `field` — what the error
|
|
344
|
-
* names. `max` — an upper bound, when one applies.
|
|
345
|
-
*
|
|
346
|
-
* Returns: nothing: the value is kept under its declared type, and this
|
|
347
|
-
* checks it.
|
|
348
|
-
*
|
|
349
|
-
* Throws: `VALIDATION` naming `field` — including for `null`, which is a
|
|
350
|
-
* caller's explicit (wrong) value, not "unset", so it is refused rather than
|
|
351
|
-
* silently falling through to the default the way `??` alone would.
|
|
352
|
-
*/
|
|
353
|
-
function assertPositiveBound(value, field, max) {
|
|
354
|
-
if (value === undefined)
|
|
355
|
-
return;
|
|
356
|
-
(0, primitives_1.assertInteger)(value, field, max === undefined ? { min: 1 } : { min: 1, max });
|
|
357
|
-
}
|
|
358
|
-
/**
|
|
359
|
-
* Validate every option `backfillRecencyIndex` reads, before any of them is
|
|
360
|
-
* read for real.
|
|
361
|
-
*
|
|
362
|
-
* Accepts: `options` — must be an object naming only the nine keys
|
|
363
|
-
* {@link BackfillOptions} declares. `tableName` and `client` are required, the
|
|
364
|
-
* rest optional; each, when given, follows the same rule an adapter's own
|
|
365
|
-
* option of the same name does. `tableName` reuses the adapters' own
|
|
366
|
-
* `tableName` validator outright; `indexShards` is held to the adapters'
|
|
367
|
-
* shard cap, `MAX_INDEX_SHARDS`; and `retry` shares its numeric bounds with
|
|
368
|
-
* the adapters' own `retry` validator while accepting the wider surface
|
|
369
|
-
* backfill's `RetryOptions` needs, so a mismatch between backfill and the
|
|
370
|
-
* adapters it feeds cannot drift in on what a bound means.
|
|
371
|
-
*
|
|
372
|
-
* Returns: nothing: the value is kept under its declared type, and this
|
|
373
|
-
* checks it.
|
|
374
|
-
*
|
|
375
|
-
* Throws: `VALIDATION` naming `options.<key>` for an unknown key;
|
|
376
|
-
* `tableName`; `client` or `client.<member>`; `indexShards`, `pageSize` or
|
|
377
|
-
* `maxPages` for a non-positive-integer bound (`indexShards` is additionally
|
|
378
|
-
* capped); `dryRun` for a non-boolean; `retry`/`retry.<key>`; `signal`.
|
|
379
|
-
*/
|
|
380
|
-
function assertBackfillOptions(options) {
|
|
381
|
-
(0, option_shape_1.assertShape)(options, BACKFILL_KEYS, 'options');
|
|
382
|
-
(0, options_1.assertTableName)(options.tableName);
|
|
383
|
-
(0, collaborators_1.assertMembers)(options.client, BACKFILL_CLIENT_MEMBERS, 'client');
|
|
384
|
-
assertPositiveBound(options.indexShards, 'indexShards', recency_index_1.MAX_INDEX_SHARDS);
|
|
385
|
-
assertPositiveBound(options.pageSize, 'pageSize');
|
|
386
|
-
assertPositiveBound(options.maxPages, 'maxPages');
|
|
387
|
-
if (options.dryRun !== undefined && typeof options.dryRun !== 'boolean') {
|
|
388
|
-
throw (0, errors_1.validationError)('dryRun must be a boolean', 'dryRun');
|
|
389
|
-
}
|
|
390
|
-
if (options.retry !== undefined)
|
|
391
|
-
assertBackfillRetryOptions(options.retry);
|
|
392
|
-
(0, collaborators_1.assertSignalLike)(options.signal);
|
|
393
|
-
}
|