@farukada/aws-langgraph-dynamodb-ts 0.9.0 → 1.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1720 -154
- package/dist/backfill/backfill.d.ts +168 -0
- package/dist/backfill/backfill.js +393 -0
- package/dist/checkpointer/actions/delete-thread.d.ts +47 -6
- package/dist/checkpointer/actions/delete-thread.js +58 -21
- package/dist/checkpointer/actions/get-tuple.d.ts +29 -4
- package/dist/checkpointer/actions/get-tuple.js +44 -10
- package/dist/checkpointer/actions/list.d.ts +46 -4
- package/dist/checkpointer/actions/list.js +121 -66
- package/dist/checkpointer/actions/put-writes.d.ts +41 -9
- package/dist/checkpointer/actions/put-writes.js +62 -77
- package/dist/checkpointer/actions/put.d.ts +83 -4
- package/dist/checkpointer/actions/put.js +177 -25
- package/dist/checkpointer/internal/delta-history.d.ts +112 -0
- package/dist/checkpointer/internal/delta-history.js +252 -0
- package/dist/checkpointer/internal/listing.d.ts +149 -0
- package/dist/checkpointer/internal/listing.js +245 -0
- package/dist/checkpointer/internal/parse.d.ts +262 -0
- package/dist/checkpointer/internal/parse.js +372 -0
- package/dist/checkpointer/internal/pending-writes.d.ts +275 -0
- package/dist/checkpointer/internal/pending-writes.js +588 -0
- package/dist/checkpointer/internal/read.d.ts +130 -0
- package/dist/checkpointer/internal/read.js +264 -0
- package/dist/checkpointer/internal/rows.d.ts +571 -0
- package/dist/checkpointer/internal/rows.js +834 -0
- package/dist/checkpointer/internal/setup.d.ts +42 -19
- package/dist/checkpointer/internal/setup.js +65 -29
- package/dist/checkpointer/saver.d.ts +256 -16
- package/dist/checkpointer/saver.js +275 -29
- package/dist/checkpointer/types.d.ts +39 -39
- package/dist/checkpointer/types.js +10 -1
- package/dist/factory/factory.d.ts +134 -28
- package/dist/factory/factory.js +240 -21
- package/dist/factory/types.d.ts +76 -0
- package/dist/factory/types.js +10 -0
- package/dist/history/actions/add-messages.d.ts +31 -4
- package/dist/history/actions/add-messages.js +38 -58
- package/dist/history/actions/clear.d.ts +49 -6
- package/dist/history/actions/clear.js +66 -14
- package/dist/history/actions/get-messages.d.ts +54 -6
- package/dist/history/actions/get-messages.js +126 -43
- package/dist/history/actions/list-sessions.d.ts +52 -10
- package/dist/history/actions/list-sessions.js +139 -40
- package/dist/history/actions/reconcile-count.d.ts +42 -10
- package/dist/history/actions/reconcile-count.js +45 -45
- package/dist/history/chat-message-history.d.ts +220 -33
- package/dist/history/chat-message-history.js +240 -43
- package/dist/history/internal/append.d.ts +212 -0
- package/dist/history/internal/append.js +500 -0
- package/dist/history/internal/message-read.d.ts +84 -0
- package/dist/history/internal/message-read.js +204 -0
- package/dist/history/internal/parse.d.ts +153 -0
- package/dist/history/internal/parse.js +252 -0
- package/dist/history/internal/rows.d.ts +195 -0
- package/dist/history/internal/rows.js +250 -0
- package/dist/history/internal/session.d.ts +331 -0
- package/dist/history/internal/session.js +628 -0
- package/dist/history/internal/setup.d.ts +52 -17
- package/dist/history/internal/setup.js +92 -21
- package/dist/history/session-adapter.d.ts +102 -7
- package/dist/history/session-adapter.js +103 -9
- package/dist/history/types.d.ts +80 -29
- package/dist/history/types.js +10 -1
- package/dist/index.d.ts +42 -11
- package/dist/index.js +33 -12
- package/dist/shared/adapter.d.ts +135 -0
- package/dist/shared/adapter.js +143 -0
- package/dist/shared/clock.d.ts +51 -2
- package/dist/shared/clock.js +57 -2
- package/dist/shared/codec/codec.d.ts +288 -13
- package/dist/shared/codec/codec.js +416 -19
- package/dist/shared/codec/compression.d.ts +43 -7
- package/dist/shared/codec/compression.js +53 -13
- package/dist/shared/codec/json-serde.d.ts +76 -4
- package/dist/shared/codec/json-serde.js +181 -8
- package/dist/shared/codec/s3/client-types.d.ts +53 -0
- package/dist/shared/codec/s3/client-types.js +26 -0
- package/dist/shared/codec/s3/client.d.ts +43 -10
- package/dist/shared/codec/s3/client.js +82 -9
- package/dist/shared/codec/s3/config.d.ts +242 -11
- package/dist/shared/codec/s3/config.js +293 -11
- package/dist/shared/codec/s3/lifecycle.d.ts +164 -6
- package/dist/shared/codec/s3/lifecycle.js +335 -27
- package/dist/shared/codec/s3/offloader.d.ts +393 -18
- package/dist/shared/codec/s3/offloader.js +595 -37
- package/dist/shared/concurrency.d.ts +43 -0
- package/dist/shared/concurrency.js +78 -0
- package/dist/shared/dynamodb/abort.d.ts +47 -0
- package/dist/shared/dynamodb/abort.js +59 -0
- package/dist/shared/dynamodb/batch-write.d.ts +77 -14
- package/dist/shared/dynamodb/batch-write.js +146 -27
- package/dist/shared/dynamodb/cancellation.d.ts +121 -4
- package/dist/shared/dynamodb/cancellation.js +147 -3
- package/dist/shared/dynamodb/client.d.ts +162 -8
- package/dist/shared/dynamodb/client.js +153 -5
- package/dist/shared/dynamodb/idempotent-write.d.ts +551 -0
- package/dist/shared/dynamodb/idempotent-write.js +593 -0
- package/dist/shared/dynamodb/paginate.d.ts +105 -9
- package/dist/shared/dynamodb/paginate.js +175 -7
- package/dist/shared/dynamodb/partition-delete.d.ts +185 -14
- package/dist/shared/dynamodb/partition-delete.js +314 -44
- package/dist/shared/dynamodb/recency-index.d.ts +231 -0
- package/dist/shared/dynamodb/recency-index.js +377 -0
- package/dist/shared/dynamodb/retry.d.ts +276 -8
- package/dist/shared/dynamodb/retry.js +433 -23
- package/dist/shared/dynamodb/table-schema.d.ts +190 -0
- package/dist/shared/dynamodb/table-schema.js +209 -0
- package/dist/shared/errors/base-error.d.ts +184 -10
- package/dist/shared/errors/base-error.js +160 -14
- package/dist/shared/errors/boundary.d.ts +71 -0
- package/dist/shared/errors/boundary.js +143 -0
- package/dist/shared/errors/classify.d.ts +97 -0
- package/dist/shared/errors/classify.js +257 -0
- package/dist/shared/errors/error-code.d.ts +77 -2
- package/dist/shared/errors/error-code.js +83 -1
- package/dist/shared/errors/errors.d.ts +158 -59
- package/dist/shared/errors/errors.js +219 -92
- package/dist/shared/logging/logger.d.ts +69 -3
- package/dist/shared/logging/logger.js +97 -3
- package/dist/shared/logging/redaction.d.ts +92 -8
- package/dist/shared/logging/redaction.js +273 -17
- package/dist/shared/logging/secret-patterns.d.ts +149 -19
- package/dist/shared/logging/secret-patterns.js +188 -27
- package/dist/shared/logging/truncate.d.ts +197 -0
- package/dist/shared/logging/truncate.js +231 -0
- package/dist/shared/options.d.ts +59 -7
- package/dist/shared/options.js +9 -1
- package/dist/shared/ulid.d.ts +77 -7
- package/dist/shared/ulid.js +103 -8
- package/dist/shared/validation/collaborators.d.ts +141 -0
- package/dist/shared/validation/collaborators.js +188 -0
- package/dist/shared/validation/option-shape.d.ts +89 -0
- package/dist/shared/validation/option-shape.js +113 -0
- package/dist/shared/validation/options.d.ts +145 -0
- package/dist/shared/validation/options.js +328 -0
- package/dist/shared/validation/primitives.d.ts +288 -21
- package/dist/shared/validation/primitives.js +353 -50
- package/dist/shared/validation/ttl.d.ts +66 -10
- package/dist/shared/validation/ttl.js +113 -15
- package/dist/store/actions/list-namespaces.d.ts +76 -6
- package/dist/store/actions/list-namespaces.js +166 -24
- package/dist/store/actions/put.d.ts +33 -8
- package/dist/store/actions/put.js +53 -60
- package/dist/store/actions/reconcile-vector-index.d.ts +31 -10
- package/dist/store/actions/reconcile-vector-index.js +34 -15
- package/dist/store/actions/search.d.ts +34 -6
- package/dist/store/actions/search.js +56 -51
- package/dist/store/internal/batch-plan.d.ts +26 -0
- package/dist/store/internal/batch-plan.js +109 -0
- package/dist/store/internal/filter.d.ts +36 -3
- package/dist/store/internal/filter.js +66 -15
- package/dist/store/internal/get-item.d.ts +45 -0
- package/dist/store/internal/get-item.js +115 -0
- package/dist/store/internal/item-write.d.ts +230 -0
- package/dist/store/internal/item-write.js +463 -0
- package/dist/store/internal/parse.d.ts +225 -0
- package/dist/store/internal/parse.js +350 -0
- package/dist/store/internal/rows.d.ts +355 -0
- package/dist/store/internal/rows.js +447 -0
- package/dist/store/internal/semantic-search.d.ts +161 -6
- package/dist/store/internal/semantic-search.js +360 -18
- package/dist/store/internal/setup.d.ts +77 -20
- package/dist/store/internal/setup.js +178 -47
- package/dist/store/internal/table-search.d.ts +100 -0
- package/dist/store/internal/table-search.js +213 -0
- package/dist/store/internal/vector-index.d.ts +247 -0
- package/dist/store/internal/vector-index.js +546 -0
- package/dist/store/store.d.ts +270 -17
- package/dist/store/store.js +329 -38
- package/dist/store/types.d.ts +76 -26
- package/dist/store/types.js +13 -1
- package/dist/store/vector-backend.d.ts +64 -4
- package/dist/store/vector-backend.js +15 -1
- package/package.json +58 -36
- package/dist/checkpointer/actions/delete-thread.d.ts.map +0 -1
- package/dist/checkpointer/actions/delete-thread.js.map +0 -1
- package/dist/checkpointer/actions/get-tuple.d.ts.map +0 -1
- package/dist/checkpointer/actions/get-tuple.js.map +0 -1
- package/dist/checkpointer/actions/list.d.ts.map +0 -1
- package/dist/checkpointer/actions/list.js.map +0 -1
- package/dist/checkpointer/actions/put-writes.d.ts.map +0 -1
- package/dist/checkpointer/actions/put-writes.js.map +0 -1
- package/dist/checkpointer/actions/put.d.ts.map +0 -1
- package/dist/checkpointer/actions/put.js.map +0 -1
- package/dist/checkpointer/internal/assemble.d.ts +0 -10
- package/dist/checkpointer/internal/assemble.d.ts.map +0 -1
- package/dist/checkpointer/internal/assemble.js +0 -37
- package/dist/checkpointer/internal/assemble.js.map +0 -1
- package/dist/checkpointer/internal/configurable.d.ts +0 -13
- package/dist/checkpointer/internal/configurable.d.ts.map +0 -1
- package/dist/checkpointer/internal/configurable.js +0 -23
- package/dist/checkpointer/internal/configurable.js.map +0 -1
- package/dist/checkpointer/internal/fetch.d.ts +0 -10
- package/dist/checkpointer/internal/fetch.d.ts.map +0 -1
- package/dist/checkpointer/internal/fetch.js +0 -46
- package/dist/checkpointer/internal/fetch.js.map +0 -1
- package/dist/checkpointer/internal/filter-match.d.ts +0 -12
- package/dist/checkpointer/internal/filter-match.d.ts.map +0 -1
- package/dist/checkpointer/internal/filter-match.js +0 -14
- package/dist/checkpointer/internal/filter-match.js.map +0 -1
- package/dist/checkpointer/internal/item-reader.d.ts +0 -55
- package/dist/checkpointer/internal/item-reader.d.ts.map +0 -1
- package/dist/checkpointer/internal/item-reader.js +0 -88
- package/dist/checkpointer/internal/item-reader.js.map +0 -1
- package/dist/checkpointer/internal/item-writer.d.ts +0 -26
- package/dist/checkpointer/internal/item-writer.d.ts.map +0 -1
- package/dist/checkpointer/internal/item-writer.js +0 -92
- package/dist/checkpointer/internal/item-writer.js.map +0 -1
- package/dist/checkpointer/internal/keys.d.ts +0 -31
- package/dist/checkpointer/internal/keys.d.ts.map +0 -1
- package/dist/checkpointer/internal/keys.js +0 -87
- package/dist/checkpointer/internal/keys.js.map +0 -1
- package/dist/checkpointer/internal/query.d.ts +0 -20
- package/dist/checkpointer/internal/query.d.ts.map +0 -1
- package/dist/checkpointer/internal/query.js +0 -36
- package/dist/checkpointer/internal/query.js.map +0 -1
- package/dist/checkpointer/internal/setup.d.ts.map +0 -1
- package/dist/checkpointer/internal/setup.js.map +0 -1
- package/dist/checkpointer/internal/special-write-cas.d.ts +0 -30
- package/dist/checkpointer/internal/special-write-cas.d.ts.map +0 -1
- package/dist/checkpointer/internal/special-write-cas.js +0 -104
- package/dist/checkpointer/internal/special-write-cas.js.map +0 -1
- package/dist/checkpointer/internal/special-write-cleanup.d.ts +0 -24
- package/dist/checkpointer/internal/special-write-cleanup.d.ts.map +0 -1
- package/dist/checkpointer/internal/special-write-cleanup.js +0 -47
- package/dist/checkpointer/internal/special-write-cleanup.js.map +0 -1
- package/dist/checkpointer/internal/special-write-verify.d.ts +0 -54
- package/dist/checkpointer/internal/special-write-verify.d.ts.map +0 -1
- package/dist/checkpointer/internal/special-write-verify.js +0 -65
- package/dist/checkpointer/internal/special-write-verify.js.map +0 -1
- package/dist/checkpointer/internal/validation.d.ts +0 -13
- package/dist/checkpointer/internal/validation.d.ts.map +0 -1
- package/dist/checkpointer/internal/validation.js +0 -30
- package/dist/checkpointer/internal/validation.js.map +0 -1
- package/dist/checkpointer/internal/write-guard.d.ts +0 -13
- package/dist/checkpointer/internal/write-guard.d.ts.map +0 -1
- package/dist/checkpointer/internal/write-guard.js +0 -39
- package/dist/checkpointer/internal/write-guard.js.map +0 -1
- package/dist/checkpointer/internal/write-index.d.ts +0 -37
- package/dist/checkpointer/internal/write-index.d.ts.map +0 -1
- package/dist/checkpointer/internal/write-index.js +0 -42
- package/dist/checkpointer/internal/write-index.js.map +0 -1
- package/dist/checkpointer/saver.d.ts.map +0 -1
- package/dist/checkpointer/saver.js.map +0 -1
- package/dist/checkpointer/types.d.ts.map +0 -1
- package/dist/checkpointer/types.js.map +0 -1
- package/dist/factory/factory.d.ts.map +0 -1
- package/dist/factory/factory.js.map +0 -1
- package/dist/history/actions/add-messages.d.ts.map +0 -1
- package/dist/history/actions/add-messages.js.map +0 -1
- package/dist/history/actions/clear.d.ts.map +0 -1
- package/dist/history/actions/clear.js.map +0 -1
- package/dist/history/actions/get-messages.d.ts.map +0 -1
- package/dist/history/actions/get-messages.js.map +0 -1
- package/dist/history/actions/list-sessions.d.ts.map +0 -1
- package/dist/history/actions/list-sessions.js.map +0 -1
- package/dist/history/actions/reconcile-count.d.ts.map +0 -1
- package/dist/history/actions/reconcile-count.js.map +0 -1
- package/dist/history/chat-message-history.d.ts.map +0 -1
- package/dist/history/chat-message-history.js.map +0 -1
- package/dist/history/internal/append-saga.d.ts +0 -20
- package/dist/history/internal/append-saga.d.ts.map +0 -1
- package/dist/history/internal/append-saga.js +0 -35
- package/dist/history/internal/append-saga.js.map +0 -1
- package/dist/history/internal/compensation.d.ts +0 -21
- package/dist/history/internal/compensation.d.ts.map +0 -1
- package/dist/history/internal/compensation.js +0 -84
- package/dist/history/internal/compensation.js.map +0 -1
- package/dist/history/internal/item-mapper.d.ts +0 -12
- package/dist/history/internal/item-mapper.d.ts.map +0 -1
- package/dist/history/internal/item-mapper.js +0 -33
- package/dist/history/internal/item-mapper.js.map +0 -1
- package/dist/history/internal/keys.d.ts +0 -17
- package/dist/history/internal/keys.d.ts.map +0 -1
- package/dist/history/internal/keys.js +0 -49
- package/dist/history/internal/keys.js.map +0 -1
- package/dist/history/internal/message-chunker.d.ts +0 -14
- package/dist/history/internal/message-chunker.d.ts.map +0 -1
- package/dist/history/internal/message-chunker.js +0 -68
- package/dist/history/internal/message-chunker.js.map +0 -1
- package/dist/history/internal/message-transaction.d.ts +0 -26
- package/dist/history/internal/message-transaction.d.ts.map +0 -1
- package/dist/history/internal/message-transaction.js +0 -60
- package/dist/history/internal/message-transaction.js.map +0 -1
- package/dist/history/internal/query.d.ts +0 -10
- package/dist/history/internal/query.d.ts.map +0 -1
- package/dist/history/internal/query.js +0 -31
- package/dist/history/internal/query.js.map +0 -1
- package/dist/history/internal/session-count.d.ts +0 -41
- package/dist/history/internal/session-count.d.ts.map +0 -1
- package/dist/history/internal/session-count.js +0 -109
- package/dist/history/internal/session-count.js.map +0 -1
- package/dist/history/internal/session-title.d.ts +0 -20
- package/dist/history/internal/session-title.d.ts.map +0 -1
- package/dist/history/internal/session-title.js +0 -44
- package/dist/history/internal/session-title.js.map +0 -1
- package/dist/history/internal/session-update.d.ts +0 -28
- package/dist/history/internal/session-update.d.ts.map +0 -1
- package/dist/history/internal/session-update.js +0 -70
- package/dist/history/internal/session-update.js.map +0 -1
- package/dist/history/internal/setup.d.ts.map +0 -1
- package/dist/history/internal/setup.js.map +0 -1
- package/dist/history/internal/title-generator.d.ts +0 -13
- package/dist/history/internal/title-generator.d.ts.map +0 -1
- package/dist/history/internal/title-generator.js +0 -25
- package/dist/history/internal/title-generator.js.map +0 -1
- package/dist/history/internal/ttl-anchor.d.ts +0 -25
- package/dist/history/internal/ttl-anchor.d.ts.map +0 -1
- package/dist/history/internal/ttl-anchor.js +0 -38
- package/dist/history/internal/ttl-anchor.js.map +0 -1
- package/dist/history/internal/validation.d.ts +0 -9
- package/dist/history/internal/validation.d.ts.map +0 -1
- package/dist/history/internal/validation.js +0 -16
- package/dist/history/internal/validation.js.map +0 -1
- package/dist/history/session-adapter.d.ts.map +0 -1
- package/dist/history/session-adapter.js.map +0 -1
- package/dist/history/types.d.ts.map +0 -1
- package/dist/history/types.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/shared/clock.d.ts.map +0 -1
- package/dist/shared/clock.js.map +0 -1
- package/dist/shared/codec/codec.d.ts.map +0 -1
- package/dist/shared/codec/codec.js.map +0 -1
- package/dist/shared/codec/compression.d.ts.map +0 -1
- package/dist/shared/codec/compression.js.map +0 -1
- package/dist/shared/codec/descriptor-keys.d.ts +0 -4
- package/dist/shared/codec/descriptor-keys.d.ts.map +0 -1
- package/dist/shared/codec/descriptor-keys.js +0 -14
- package/dist/shared/codec/descriptor-keys.js.map +0 -1
- package/dist/shared/codec/json-serde.d.ts.map +0 -1
- package/dist/shared/codec/json-serde.js.map +0 -1
- package/dist/shared/codec/s3/client.d.ts.map +0 -1
- package/dist/shared/codec/s3/client.js.map +0 -1
- package/dist/shared/codec/s3/config.d.ts.map +0 -1
- package/dist/shared/codec/s3/config.js.map +0 -1
- package/dist/shared/codec/s3/delete.d.ts +0 -8
- package/dist/shared/codec/s3/delete.d.ts.map +0 -1
- package/dist/shared/codec/s3/delete.js +0 -29
- package/dist/shared/codec/s3/delete.js.map +0 -1
- package/dist/shared/codec/s3/lifecycle.d.ts.map +0 -1
- package/dist/shared/codec/s3/lifecycle.js.map +0 -1
- package/dist/shared/codec/s3/offloader.d.ts.map +0 -1
- package/dist/shared/codec/s3/offloader.js.map +0 -1
- package/dist/shared/codec/s3/orphans.d.ts +0 -18
- package/dist/shared/codec/s3/orphans.d.ts.map +0 -1
- package/dist/shared/codec/s3/orphans.js +0 -58
- package/dist/shared/codec/s3/orphans.js.map +0 -1
- package/dist/shared/codec/s3/read-write.d.ts +0 -14
- package/dist/shared/codec/s3/read-write.d.ts.map +0 -1
- package/dist/shared/codec/s3/read-write.js +0 -43
- package/dist/shared/codec/s3/read-write.js.map +0 -1
- package/dist/shared/codec/s3/retry.d.ts +0 -5
- package/dist/shared/codec/s3/retry.d.ts.map +0 -1
- package/dist/shared/codec/s3/retry.js +0 -25
- package/dist/shared/codec/s3/retry.js.map +0 -1
- package/dist/shared/constants.d.ts +0 -64
- package/dist/shared/constants.d.ts.map +0 -1
- package/dist/shared/constants.js +0 -67
- package/dist/shared/constants.js.map +0 -1
- package/dist/shared/dynamodb/backoff.d.ts +0 -15
- package/dist/shared/dynamodb/backoff.d.ts.map +0 -1
- package/dist/shared/dynamodb/backoff.js +0 -48
- package/dist/shared/dynamodb/backoff.js.map +0 -1
- package/dist/shared/dynamodb/batch-write.d.ts.map +0 -1
- package/dist/shared/dynamodb/batch-write.js.map +0 -1
- package/dist/shared/dynamodb/cancellation.d.ts.map +0 -1
- package/dist/shared/dynamodb/cancellation.js.map +0 -1
- package/dist/shared/dynamodb/client.d.ts.map +0 -1
- package/dist/shared/dynamodb/client.js.map +0 -1
- package/dist/shared/dynamodb/conditional-put.d.ts +0 -51
- package/dist/shared/dynamodb/conditional-put.d.ts.map +0 -1
- package/dist/shared/dynamodb/conditional-put.js +0 -59
- package/dist/shared/dynamodb/conditional-put.js.map +0 -1
- package/dist/shared/dynamodb/drain-unprocessed.d.ts +0 -19
- package/dist/shared/dynamodb/drain-unprocessed.d.ts.map +0 -1
- package/dist/shared/dynamodb/drain-unprocessed.js +0 -44
- package/dist/shared/dynamodb/drain-unprocessed.js.map +0 -1
- package/dist/shared/dynamodb/paginate-core.d.ts +0 -22
- package/dist/shared/dynamodb/paginate-core.d.ts.map +0 -1
- package/dist/shared/dynamodb/paginate-core.js +0 -52
- package/dist/shared/dynamodb/paginate-core.js.map +0 -1
- package/dist/shared/dynamodb/paginate.d.ts.map +0 -1
- package/dist/shared/dynamodb/paginate.js.map +0 -1
- package/dist/shared/dynamodb/partition-delete.d.ts.map +0 -1
- package/dist/shared/dynamodb/partition-delete.js.map +0 -1
- package/dist/shared/dynamodb/retry-classifier.d.ts +0 -9
- package/dist/shared/dynamodb/retry-classifier.d.ts.map +0 -1
- package/dist/shared/dynamodb/retry-classifier.js +0 -87
- package/dist/shared/dynamodb/retry-classifier.js.map +0 -1
- package/dist/shared/dynamodb/retry.d.ts.map +0 -1
- package/dist/shared/dynamodb/retry.js.map +0 -1
- package/dist/shared/dynamodb/scan.d.ts +0 -15
- package/dist/shared/dynamodb/scan.d.ts.map +0 -1
- package/dist/shared/dynamodb/scan.js +0 -20
- package/dist/shared/dynamodb/scan.js.map +0 -1
- package/dist/shared/dynamodb/types.d.ts +0 -24
- package/dist/shared/dynamodb/types.d.ts.map +0 -1
- package/dist/shared/dynamodb/types.js +0 -3
- package/dist/shared/dynamodb/types.js.map +0 -1
- package/dist/shared/errors/base-error.d.ts.map +0 -1
- package/dist/shared/errors/base-error.js.map +0 -1
- package/dist/shared/errors/error-code.d.ts.map +0 -1
- package/dist/shared/errors/error-code.js.map +0 -1
- package/dist/shared/errors/errors.d.ts.map +0 -1
- package/dist/shared/errors/errors.js.map +0 -1
- package/dist/shared/errors/wrap-error.d.ts +0 -16
- package/dist/shared/errors/wrap-error.d.ts.map +0 -1
- package/dist/shared/errors/wrap-error.js +0 -30
- package/dist/shared/errors/wrap-error.js.map +0 -1
- package/dist/shared/logging/logger.d.ts.map +0 -1
- package/dist/shared/logging/logger.js.map +0 -1
- package/dist/shared/logging/redaction-walk.d.ts +0 -23
- package/dist/shared/logging/redaction-walk.d.ts.map +0 -1
- package/dist/shared/logging/redaction-walk.js +0 -92
- package/dist/shared/logging/redaction-walk.js.map +0 -1
- package/dist/shared/logging/redaction.d.ts.map +0 -1
- package/dist/shared/logging/redaction.js.map +0 -1
- package/dist/shared/logging/secret-patterns.d.ts.map +0 -1
- package/dist/shared/logging/secret-patterns.js.map +0 -1
- package/dist/shared/options.d.ts.map +0 -1
- package/dist/shared/options.js.map +0 -1
- package/dist/shared/ulid.d.ts.map +0 -1
- package/dist/shared/ulid.js.map +0 -1
- package/dist/shared/validation/primitives.d.ts.map +0 -1
- package/dist/shared/validation/primitives.js.map +0 -1
- package/dist/shared/validation/ttl.d.ts.map +0 -1
- package/dist/shared/validation/ttl.js.map +0 -1
- package/dist/store/actions/get.d.ts +0 -5
- package/dist/store/actions/get.d.ts.map +0 -1
- package/dist/store/actions/get.js +0 -35
- package/dist/store/actions/get.js.map +0 -1
- package/dist/store/actions/list-namespaces.d.ts.map +0 -1
- package/dist/store/actions/list-namespaces.js.map +0 -1
- package/dist/store/actions/put.d.ts.map +0 -1
- package/dist/store/actions/put.js.map +0 -1
- package/dist/store/actions/reconcile-vector-index.d.ts.map +0 -1
- package/dist/store/actions/reconcile-vector-index.js.map +0 -1
- package/dist/store/actions/search.d.ts.map +0 -1
- package/dist/store/actions/search.js.map +0 -1
- package/dist/store/internal/backend-search.d.ts +0 -5
- package/dist/store/internal/backend-search.d.ts.map +0 -1
- package/dist/store/internal/backend-search.js +0 -68
- package/dist/store/internal/backend-search.js.map +0 -1
- package/dist/store/internal/filter.d.ts.map +0 -1
- package/dist/store/internal/filter.js.map +0 -1
- package/dist/store/internal/index-reconcile.d.ts +0 -22
- package/dist/store/internal/index-reconcile.d.ts.map +0 -1
- package/dist/store/internal/index-reconcile.js +0 -105
- package/dist/store/internal/index-reconcile.js.map +0 -1
- package/dist/store/internal/index-sync.d.ts +0 -11
- package/dist/store/internal/index-sync.d.ts.map +0 -1
- package/dist/store/internal/index-sync.js +0 -26
- package/dist/store/internal/index-sync.js.map +0 -1
- package/dist/store/internal/item-mapper.d.ts +0 -25
- package/dist/store/internal/item-mapper.d.ts.map +0 -1
- package/dist/store/internal/item-mapper.js +0 -53
- package/dist/store/internal/item-mapper.js.map +0 -1
- package/dist/store/internal/keys.d.ts +0 -18
- package/dist/store/internal/keys.d.ts.map +0 -1
- package/dist/store/internal/keys.js +0 -42
- package/dist/store/internal/keys.js.map +0 -1
- package/dist/store/internal/namespace-match.d.ts +0 -12
- package/dist/store/internal/namespace-match.d.ts.map +0 -1
- package/dist/store/internal/namespace-match.js +0 -41
- package/dist/store/internal/namespace-match.js.map +0 -1
- package/dist/store/internal/overwrite-swap.d.ts +0 -33
- package/dist/store/internal/overwrite-swap.d.ts.map +0 -1
- package/dist/store/internal/overwrite-swap.js +0 -62
- package/dist/store/internal/overwrite-swap.js.map +0 -1
- package/dist/store/internal/persist.d.ts +0 -27
- package/dist/store/internal/persist.d.ts.map +0 -1
- package/dist/store/internal/persist.js +0 -59
- package/dist/store/internal/persist.js.map +0 -1
- package/dist/store/internal/query.d.ts +0 -6
- package/dist/store/internal/query.d.ts.map +0 -1
- package/dist/store/internal/query.js +0 -32
- package/dist/store/internal/query.js.map +0 -1
- package/dist/store/internal/ranker.d.ts +0 -13
- package/dist/store/internal/ranker.d.ts.map +0 -1
- package/dist/store/internal/ranker.js +0 -31
- package/dist/store/internal/ranker.js.map +0 -1
- package/dist/store/internal/read-existing.d.ts +0 -19
- package/dist/store/internal/read-existing.d.ts.map +0 -1
- package/dist/store/internal/read-existing.js +0 -29
- package/dist/store/internal/read-existing.js.map +0 -1
- package/dist/store/internal/score-direction.d.ts +0 -32
- package/dist/store/internal/score-direction.d.ts.map +0 -1
- package/dist/store/internal/score-direction.js +0 -39
- package/dist/store/internal/score-direction.js.map +0 -1
- package/dist/store/internal/search-filter.d.ts +0 -4
- package/dist/store/internal/search-filter.d.ts.map +0 -1
- package/dist/store/internal/search-filter.js +0 -11
- package/dist/store/internal/search-filter.js.map +0 -1
- package/dist/store/internal/semantic-search.d.ts.map +0 -1
- package/dist/store/internal/semantic-search.js.map +0 -1
- package/dist/store/internal/setup.d.ts.map +0 -1
- package/dist/store/internal/setup.js.map +0 -1
- package/dist/store/internal/validation.d.ts +0 -13
- package/dist/store/internal/validation.d.ts.map +0 -1
- package/dist/store/internal/validation.js +0 -35
- package/dist/store/internal/validation.js.map +0 -1
- package/dist/store/internal/write-verify.d.ts +0 -37
- package/dist/store/internal/write-verify.d.ts.map +0 -1
- package/dist/store/internal/write-verify.js +0 -68
- package/dist/store/internal/write-verify.js.map +0 -1
- package/dist/store/store.d.ts.map +0 -1
- package/dist/store/store.js.map +0 -1
- package/dist/store/types.d.ts.map +0 -1
- package/dist/store/types.js.map +0 -1
- package/dist/store/vector-backend.d.ts.map +0 -1
- package/dist/store/vector-backend.js.map +0 -1
|
@@ -1,15 +1,111 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Hides that a Query or a Scan is many pages.
|
|
3
|
+
*
|
|
4
|
+
* A read receives rows one at a time and never sees a page boundary. How many
|
|
5
|
+
* rows and pages it may collect before it refuses with `RESULT_TRUNCATED`
|
|
6
|
+
* rather than truncating silently, how a read that stopped exactly at its row
|
|
7
|
+
* cap probes whether anything remained, and where cancellation is checked
|
|
8
|
+
* between pages are decided here, the same way for a Query and a Scan.
|
|
9
|
+
*/
|
|
10
|
+
import type { QueryCommandInput, ScanCommandInput } from '@aws-sdk/lib-dynamodb';
|
|
11
|
+
import type { DynamoDBDocumentLike, AttributeMap } from './client';
|
|
12
|
+
import { type RetryOptions } from './retry';
|
|
13
|
+
/** Hard cap on query-pagination loop iterations (runaway-loop guard). */
|
|
14
|
+
export declare const MAX_LOOP_ITERATIONS = 1000;
|
|
15
|
+
/** Hard cap on rows collected into memory across a paginated query. */
|
|
16
|
+
export declare const MAX_TOTAL_ROWS_IN_MEMORY = 10000;
|
|
17
|
+
/**
|
|
18
|
+
* Raw rows a single `listCheckpoints` call may pull before it warns. The read
|
|
19
|
+
* itself is deliberately unbounded: capping it would count raw rows rather
|
|
20
|
+
* than filter-matched ones, so a caller asking for a handful of rare matches
|
|
21
|
+
* over a large thread would get a hard error instead of the true answer. The
|
|
22
|
+
* warning gives the operational signal without that wrong error.
|
|
23
|
+
*
|
|
24
|
+
* Its own literal, deliberately: this is the point at which a scan is worth
|
|
25
|
+
* telling an operator about, which is independent of
|
|
26
|
+
* {@link MAX_TOTAL_ROWS_IN_MEMORY}'s hard collection cap. Aliasing the two —
|
|
27
|
+
* so one reused the other's literal — would let retuning the memory cap
|
|
28
|
+
* silently move the warning threshold too, for constants that answer
|
|
29
|
+
* unrelated questions, and would make knip report the pair as a duplicate
|
|
30
|
+
* export.
|
|
31
|
+
*/
|
|
32
|
+
export declare const LIST_SCAN_WARN_THRESHOLD = 10000;
|
|
4
33
|
/** Options for {@link paginateQuery}. */
|
|
5
34
|
export interface PaginateOptions extends PaginateCoreOptions {
|
|
6
|
-
client:
|
|
35
|
+
client: DynamoDBDocumentLike;
|
|
7
36
|
params: QueryCommandInput;
|
|
8
37
|
}
|
|
9
38
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
39
|
+
* Every row a Query returns, across all its pages.
|
|
40
|
+
*
|
|
41
|
+
* Accepts: `params` — the Query input; `ExclusiveStartKey` is set per page and
|
|
42
|
+
* anything the caller put there is replaced. `retry` and `signal` are applied
|
|
43
|
+
* to each page read, `maxItems` / `maxIterations` to the walk (see
|
|
44
|
+
* {@link paginatePages}).
|
|
45
|
+
*
|
|
46
|
+
* Returns: an async generator over the rows, following `LastEvaluatedKey`
|
|
47
|
+
* until it is absent. A page carrying no `Items` is an empty page, not the end.
|
|
48
|
+
*
|
|
49
|
+
* Throws: whatever the page read throws, plus the caps and abort behaviour of
|
|
50
|
+
* {@link paginatePages}.
|
|
51
|
+
*/
|
|
52
|
+
export declare function paginateQuery(options: PaginateOptions): AsyncGenerator<AttributeMap>;
|
|
53
|
+
/** One page of results plus the key to resume from (undefined when exhausted). */
|
|
54
|
+
export interface PageResult {
|
|
55
|
+
items: AttributeMap[];
|
|
56
|
+
lastKey: AttributeMap | undefined;
|
|
57
|
+
}
|
|
58
|
+
/** Options shared by the paginators built on {@link paginatePages}. */
|
|
59
|
+
export interface PaginateCoreOptions {
|
|
60
|
+
/** The adapter's retry options, applied to every page read. */
|
|
61
|
+
retry?: RetryOptions;
|
|
62
|
+
signal?: AbortSignal;
|
|
63
|
+
maxItems?: number;
|
|
64
|
+
maxIterations?: number;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Drive a paged DynamoDB read to completion, yielding each item.
|
|
68
|
+
*
|
|
69
|
+
* Accepts: `fetchPage` — performs one page read from a start key.
|
|
70
|
+
* `options.maxItems` — how many items may be yielded, default
|
|
71
|
+
* {@link MAX_TOTAL_ROWS_IN_MEMORY}. `options.maxIterations` — how many page
|
|
72
|
+
* reads may be issued, default {@link MAX_LOOP_ITERATIONS}; the probe below is
|
|
73
|
+
* charged against it too. Both accept `Infinity` to read to true completion
|
|
74
|
+
* (deletes do), and both must otherwise be at least 1 — an accepted cap of 0
|
|
75
|
+
* would yield one item before noticing. `options.signal` — checked before every
|
|
76
|
+
* fetch. `options.retry` is the page reader's own concern.
|
|
77
|
+
*
|
|
78
|
+
* Returns: an async generator over the items, continuing past empty pages.
|
|
79
|
+
*
|
|
80
|
+
* Throws: `VALIDATION` naming `maxItems` or `maxIterations` for a cap below
|
|
81
|
+
* 1; `ABORTED` when the signal is already aborted at a page boundary; and
|
|
82
|
+
* `RESULT_TRUNCATED` when a cap is reached while data actually
|
|
83
|
+
* remains — a partial result is never returned silently.
|
|
84
|
+
*
|
|
85
|
+
* Guarantees: reaching `maxItems` on the last item of a page is not by itself a
|
|
86
|
+
* truncation. DynamoDB returns a `LastEvaluatedKey` whenever it stopped
|
|
87
|
+
* *evaluating* at the 1 MB boundary, whether or not a later item passes the
|
|
88
|
+
* filter, so the remaining keys are followed until one carries an item
|
|
89
|
+
* (truncated) or they run out (the result was complete).
|
|
90
|
+
*/
|
|
91
|
+
export declare function paginatePages(fetchPage: (startKey: AttributeMap | undefined) => Promise<PageResult>, options?: PaginateCoreOptions): AsyncGenerator<AttributeMap>;
|
|
92
|
+
/** Options for {@link paginateScan}. */
|
|
93
|
+
export interface ScanOptions extends PaginateCoreOptions {
|
|
94
|
+
client: DynamoDBDocumentLike;
|
|
95
|
+
params: ScanCommandInput;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Every item a Scan returns, across all its pages.
|
|
99
|
+
*
|
|
100
|
+
* Accepts: as {@link paginateQuery}; only the request differs.
|
|
101
|
+
*
|
|
102
|
+
* Returns: as {@link paginateQuery} — an async generator over the items, which
|
|
103
|
+
* a consumer may abandon early to stop reading.
|
|
104
|
+
*
|
|
105
|
+
* Throws: as {@link paginateQuery}.
|
|
106
|
+
*
|
|
107
|
+
* A `Scan` reads every row of the table before filtering, so the four reads
|
|
108
|
+
* allowed to call this are fixed and guarded; `test/static/guards/scan-sites.ts`
|
|
109
|
+
* lists them and states the rule they follow.
|
|
13
110
|
*/
|
|
14
|
-
export declare function
|
|
15
|
-
//# sourceMappingURL=paginate.d.ts.map
|
|
111
|
+
export declare function paginateScan(options: ScanOptions): AsyncGenerator<AttributeMap>;
|
|
@@ -1,20 +1,188 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Hides that a Query or a Scan is many pages.
|
|
4
|
+
*
|
|
5
|
+
* A read receives rows one at a time and never sees a page boundary. How many
|
|
6
|
+
* rows and pages it may collect before it refuses with `RESULT_TRUNCATED`
|
|
7
|
+
* rather than truncating silently, how a read that stopped exactly at its row
|
|
8
|
+
* cap probes whether anything remained, and where cancellation is checked
|
|
9
|
+
* between pages are decided here, the same way for a Query and a Scan.
|
|
10
|
+
*/
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.LIST_SCAN_WARN_THRESHOLD = exports.MAX_TOTAL_ROWS_IN_MEMORY = exports.MAX_LOOP_ITERATIONS = void 0;
|
|
3
13
|
exports.paginateQuery = paginateQuery;
|
|
4
|
-
|
|
14
|
+
exports.paginatePages = paginatePages;
|
|
15
|
+
exports.paginateScan = paginateScan;
|
|
16
|
+
const errors_1 = require("../errors/errors");
|
|
17
|
+
const abort_1 = require("./abort");
|
|
5
18
|
const retry_1 = require("./retry");
|
|
19
|
+
/** Hard cap on query-pagination loop iterations (runaway-loop guard). */
|
|
20
|
+
exports.MAX_LOOP_ITERATIONS = 1000;
|
|
21
|
+
/** Hard cap on rows collected into memory across a paginated query. */
|
|
22
|
+
exports.MAX_TOTAL_ROWS_IN_MEMORY = 10000;
|
|
23
|
+
/**
|
|
24
|
+
* Raw rows a single `listCheckpoints` call may pull before it warns. The read
|
|
25
|
+
* itself is deliberately unbounded: capping it would count raw rows rather
|
|
26
|
+
* than filter-matched ones, so a caller asking for a handful of rare matches
|
|
27
|
+
* over a large thread would get a hard error instead of the true answer. The
|
|
28
|
+
* warning gives the operational signal without that wrong error.
|
|
29
|
+
*
|
|
30
|
+
* Its own literal, deliberately: this is the point at which a scan is worth
|
|
31
|
+
* telling an operator about, which is independent of
|
|
32
|
+
* {@link MAX_TOTAL_ROWS_IN_MEMORY}'s hard collection cap. Aliasing the two —
|
|
33
|
+
* so one reused the other's literal — would let retuning the memory cap
|
|
34
|
+
* silently move the warning threshold too, for constants that answer
|
|
35
|
+
* unrelated questions, and would make knip report the pair as a duplicate
|
|
36
|
+
* export.
|
|
37
|
+
*/
|
|
38
|
+
exports.LIST_SCAN_WARN_THRESHOLD = 10000;
|
|
6
39
|
/**
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
40
|
+
* Every row a Query returns, across all its pages.
|
|
41
|
+
*
|
|
42
|
+
* Accepts: `params` — the Query input; `ExclusiveStartKey` is set per page and
|
|
43
|
+
* anything the caller put there is replaced. `retry` and `signal` are applied
|
|
44
|
+
* to each page read, `maxItems` / `maxIterations` to the walk (see
|
|
45
|
+
* {@link paginatePages}).
|
|
46
|
+
*
|
|
47
|
+
* Returns: an async generator over the rows, following `LastEvaluatedKey`
|
|
48
|
+
* until it is absent. A page carrying no `Items` is an empty page, not the end.
|
|
49
|
+
*
|
|
50
|
+
* Throws: whatever the page read throws, plus the caps and abort behaviour of
|
|
51
|
+
* {@link paginatePages}.
|
|
10
52
|
*/
|
|
11
53
|
function paginateQuery(options) {
|
|
12
|
-
return
|
|
13
|
-
const page = await (0, retry_1.withDynamoDBRetry)(() => options.client.query({ ...options.params, ExclusiveStartKey: startKey }), { signal: options.signal });
|
|
54
|
+
return paginatePages(async (startKey) => {
|
|
55
|
+
const page = await (0, retry_1.withDynamoDBRetry)((request) => options.client.query({ ...options.params, ExclusiveStartKey: startKey }, request), { ...options.retry, signal: options.signal });
|
|
56
|
+
return {
|
|
57
|
+
items: page.Items ?? [],
|
|
58
|
+
lastKey: page.LastEvaluatedKey,
|
|
59
|
+
};
|
|
60
|
+
}, options);
|
|
61
|
+
}
|
|
62
|
+
/** Read the next page, honouring the abort signal and the iteration cap first. */
|
|
63
|
+
async function readPage(reader, startKey) {
|
|
64
|
+
if (reader.iterations >= reader.maxIterations) {
|
|
65
|
+
throw (0, errors_1.resultTruncatedError)('maxIterations', reader.maxIterations);
|
|
66
|
+
}
|
|
67
|
+
if (reader.signal?.aborted)
|
|
68
|
+
throw (0, abort_1.abortErrorFrom)(reader.signal);
|
|
69
|
+
reader.iterations += 1;
|
|
70
|
+
return reader.fetchPage(startKey);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Yield one page's rows, counting toward the shared `state.yielded` budget.
|
|
74
|
+
* Reaching the cap with unyielded rows still on the page is a truncation;
|
|
75
|
+
* reaching it on the last row is reported as `capped` for the caller to settle.
|
|
76
|
+
*
|
|
77
|
+
* Synchronous and private: nothing here awaits, and `paginatePages` delegates
|
|
78
|
+
* to it with `yield*` from inside its own `async function*`, so the rows it
|
|
79
|
+
* yields still reach callers through the same async-generator protocol.
|
|
80
|
+
*/
|
|
81
|
+
function* yieldPageRows(page, state, maxItems) {
|
|
82
|
+
for (let index = 0; index < page.items.length; index++) {
|
|
83
|
+
yield page.items[index];
|
|
84
|
+
state.yielded += 1;
|
|
85
|
+
if (state.yielded >= maxItems) {
|
|
86
|
+
if (index < page.items.length - 1)
|
|
87
|
+
throw (0, errors_1.resultTruncatedError)('maxItems', maxItems);
|
|
88
|
+
return 'capped';
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return 'exhausted';
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Whether anything remains past `startKey`. DynamoDB returns a
|
|
95
|
+
* `LastEvaluatedKey` whenever it stopped *evaluating* at the 1 MB boundary,
|
|
96
|
+
* whether or not a later item passes the filter, so a trailing key alone proves
|
|
97
|
+
* nothing: the keys are followed until a page carries an item (truncated) or
|
|
98
|
+
* they run out (the result was complete). The probe is charged against the same
|
|
99
|
+
* iteration budget and honours the same signal as the read itself.
|
|
100
|
+
*/
|
|
101
|
+
async function dataRemains(reader, startKey) {
|
|
102
|
+
let key = startKey;
|
|
103
|
+
while (key !== undefined) {
|
|
104
|
+
const page = await readPage(reader, key);
|
|
105
|
+
if (page.items.length > 0)
|
|
106
|
+
return true;
|
|
107
|
+
key = page.lastKey;
|
|
108
|
+
}
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
/** Reject a cap that admits nothing; `Infinity` is the way to ask for no cap. */
|
|
112
|
+
function assertPositiveCap(value, field) {
|
|
113
|
+
if (!(value >= 1)) {
|
|
114
|
+
throw (0, errors_1.validationError)(`${field} must be at least 1 (pass Infinity to read to completion)`, field);
|
|
115
|
+
}
|
|
116
|
+
return value;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Drive a paged DynamoDB read to completion, yielding each item.
|
|
120
|
+
*
|
|
121
|
+
* Accepts: `fetchPage` — performs one page read from a start key.
|
|
122
|
+
* `options.maxItems` — how many items may be yielded, default
|
|
123
|
+
* {@link MAX_TOTAL_ROWS_IN_MEMORY}. `options.maxIterations` — how many page
|
|
124
|
+
* reads may be issued, default {@link MAX_LOOP_ITERATIONS}; the probe below is
|
|
125
|
+
* charged against it too. Both accept `Infinity` to read to true completion
|
|
126
|
+
* (deletes do), and both must otherwise be at least 1 — an accepted cap of 0
|
|
127
|
+
* would yield one item before noticing. `options.signal` — checked before every
|
|
128
|
+
* fetch. `options.retry` is the page reader's own concern.
|
|
129
|
+
*
|
|
130
|
+
* Returns: an async generator over the items, continuing past empty pages.
|
|
131
|
+
*
|
|
132
|
+
* Throws: `VALIDATION` naming `maxItems` or `maxIterations` for a cap below
|
|
133
|
+
* 1; `ABORTED` when the signal is already aborted at a page boundary; and
|
|
134
|
+
* `RESULT_TRUNCATED` when a cap is reached while data actually
|
|
135
|
+
* remains — a partial result is never returned silently.
|
|
136
|
+
*
|
|
137
|
+
* Guarantees: reaching `maxItems` on the last item of a page is not by itself a
|
|
138
|
+
* truncation. DynamoDB returns a `LastEvaluatedKey` whenever it stopped
|
|
139
|
+
* *evaluating* at the 1 MB boundary, whether or not a later item passes the
|
|
140
|
+
* filter, so the remaining keys are followed until one carries an item
|
|
141
|
+
* (truncated) or they run out (the result was complete).
|
|
142
|
+
*/
|
|
143
|
+
async function* paginatePages(fetchPage, options = {}) {
|
|
144
|
+
const maxItems = assertPositiveCap(options.maxItems ?? exports.MAX_TOTAL_ROWS_IN_MEMORY, 'maxItems');
|
|
145
|
+
const reader = {
|
|
146
|
+
fetchPage,
|
|
147
|
+
maxIterations: assertPositiveCap(options.maxIterations ?? exports.MAX_LOOP_ITERATIONS, 'maxIterations'),
|
|
148
|
+
iterations: 0,
|
|
149
|
+
signal: options.signal,
|
|
150
|
+
};
|
|
151
|
+
const state = { yielded: 0 };
|
|
152
|
+
let startKey;
|
|
153
|
+
for (;;) {
|
|
154
|
+
const page = await readPage(reader, startKey);
|
|
155
|
+
const outcome = yield* yieldPageRows(page, state, maxItems);
|
|
156
|
+
if (outcome === 'capped') {
|
|
157
|
+
if (await dataRemains(reader, page.lastKey))
|
|
158
|
+
throw (0, errors_1.resultTruncatedError)('maxItems', maxItems);
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
startKey = page.lastKey;
|
|
162
|
+
if (startKey === undefined)
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Every item a Scan returns, across all its pages.
|
|
168
|
+
*
|
|
169
|
+
* Accepts: as {@link paginateQuery}; only the request differs.
|
|
170
|
+
*
|
|
171
|
+
* Returns: as {@link paginateQuery} — an async generator over the items, which
|
|
172
|
+
* a consumer may abandon early to stop reading.
|
|
173
|
+
*
|
|
174
|
+
* Throws: as {@link paginateQuery}.
|
|
175
|
+
*
|
|
176
|
+
* A `Scan` reads every row of the table before filtering, so the four reads
|
|
177
|
+
* allowed to call this are fixed and guarded; `test/static/guards/scan-sites.ts`
|
|
178
|
+
* lists them and states the rule they follow.
|
|
179
|
+
*/
|
|
180
|
+
function paginateScan(options) {
|
|
181
|
+
return paginatePages(async (startKey) => {
|
|
182
|
+
const page = await (0, retry_1.withDynamoDBRetry)((request) => options.client.scan({ ...options.params, ExclusiveStartKey: startKey }, request), { ...options.retry, signal: options.signal });
|
|
14
183
|
return {
|
|
15
184
|
items: page.Items ?? [],
|
|
16
185
|
lastKey: page.LastEvaluatedKey,
|
|
17
186
|
};
|
|
18
187
|
}, options);
|
|
19
188
|
}
|
|
20
|
-
//# sourceMappingURL=paginate.js.map
|
|
@@ -1,14 +1,70 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Hides deleting exactly the rows of a partition that a read observed.
|
|
3
|
+
*
|
|
4
|
+
* A partition on a shared table can hold another adapter's rows, and a row can
|
|
5
|
+
* be rewritten between the read and the delete. A foreign row is left in place,
|
|
6
|
+
* each delete is pinned to the write id the read saw so a rewritten row
|
|
7
|
+
* survives, the rows of a unit are skipped once one of them is refused, and an
|
|
8
|
+
* object is released only once the row naming it is confirmed gone.
|
|
9
|
+
*/
|
|
10
|
+
import type { QueryCommandInput } from '@aws-sdk/lib-dynamodb';
|
|
11
|
+
import { type PayloadDescriptor, type DescriptorRef } from '../codec/codec';
|
|
12
|
+
import { type S3Offloader } from '../codec/s3/offloader';
|
|
4
13
|
import type { Logger } from '../logging/logger';
|
|
5
|
-
import type {
|
|
14
|
+
import type { DynamoDBDocumentLike, AttributeMap } from './client';
|
|
15
|
+
import { type RevisionGuard } from './idempotent-write';
|
|
16
|
+
import { type RetryOptions } from './retry';
|
|
17
|
+
/**
|
|
18
|
+
* Conditional row deletes a partition-wide delete keeps in flight. Pinning a
|
|
19
|
+
* row on the write that produced it costs one request per row where a batch
|
|
20
|
+
* carried twenty-five, so issuing them one at a time would have paid a round
|
|
21
|
+
* trip per row; eight at once gives most of that back while keeping a
|
|
22
|
+
* partition of any size from opening a socket per row. A fixed value rather
|
|
23
|
+
* than a caller option: this is a maintenance path, and it has no other knob.
|
|
24
|
+
*
|
|
25
|
+
* Its own literal at the same value as `DEFAULT_READ_CONCURRENCY`
|
|
26
|
+
* (`src/shared/concurrency.ts`), not an alias of it — aliasing two limits
|
|
27
|
+
* would move one whenever the other is retuned (see
|
|
28
|
+
* `LIST_SCAN_WARN_THRESHOLD` (`src/shared/dynamodb/paginate.ts`)).
|
|
29
|
+
*/
|
|
30
|
+
export declare const DELETE_CONCURRENCY = 8;
|
|
31
|
+
/**
|
|
32
|
+
* One offloaded payload a row references, named by the attribute holding it.
|
|
33
|
+
* The name is not decoration: the row is pinned through a document path over
|
|
34
|
+
* that attribute, and a refused row's objects must stay where they are.
|
|
35
|
+
*/
|
|
36
|
+
export interface NamedDescriptor {
|
|
37
|
+
attribute: string;
|
|
38
|
+
descriptor: PayloadDescriptor;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* One row attribute read as a named descriptor, which is the only way this
|
|
42
|
+
* library fills a {@link NamedDescriptor}.
|
|
43
|
+
*
|
|
44
|
+
* Accepts: `row` — a row the partition read returned, whose attributes this
|
|
45
|
+
* library did not necessarily write. `attribute` — the name the payload would
|
|
46
|
+
* be held under.
|
|
47
|
+
*
|
|
48
|
+
* Returns: the named descriptor, or nothing when the row carries no usable one
|
|
49
|
+
* there. An absent attribute and one holding `null` are the same answer,
|
|
50
|
+
* because neither names a payload: the row contributes no id to pin on and no
|
|
51
|
+
* object to release. Narrowing here rather than at each caller is what keeps
|
|
52
|
+
* `descriptor` the non-null thing the type claims — a `null` cast into the
|
|
53
|
+
* array by a caller reached `pinFor`, which reads a write id off it.
|
|
54
|
+
*
|
|
55
|
+
* Throws: nothing.
|
|
56
|
+
*/
|
|
57
|
+
export declare function namedDescriptor(row: AttributeMap, attribute: string): NamedDescriptor | undefined;
|
|
6
58
|
/** Collaborators and per-adapter policy for one partition-wide delete. */
|
|
7
59
|
export interface PartitionDeleteOptions {
|
|
8
|
-
client:
|
|
60
|
+
client: DynamoDBDocumentLike;
|
|
9
61
|
tableName: string;
|
|
10
62
|
params: QueryCommandInput;
|
|
11
63
|
logger: Logger;
|
|
64
|
+
/** The adapter's retry options for the page reads and the row deletes. */
|
|
65
|
+
retry?: RetryOptions;
|
|
66
|
+
/** Aborting it stops the read between pages and rejects with an `ABORTED` error. */
|
|
67
|
+
signal?: AbortSignal;
|
|
12
68
|
offloader?: S3Offloader;
|
|
13
69
|
/** Label for log lines and S3-cleanup diagnostics, e.g. `deleteThread`. */
|
|
14
70
|
operation: string;
|
|
@@ -18,16 +74,131 @@ export interface PartitionDeleteOptions {
|
|
|
18
74
|
* holding a foreign row would have that row deleted too.
|
|
19
75
|
*/
|
|
20
76
|
ownsSortKey: (sortKey: string) => boolean;
|
|
21
|
-
/**
|
|
22
|
-
|
|
77
|
+
/**
|
|
78
|
+
* The offloaded payload descriptors a row references, each named by its
|
|
79
|
+
* attribute, and each read off the row with {@link namedDescriptor} so that
|
|
80
|
+
* an attribute holding `null` yields no entry rather than an unusable one.
|
|
81
|
+
*/
|
|
82
|
+
descriptorsOf: (row: AttributeMap) => NamedDescriptor[];
|
|
83
|
+
/**
|
|
84
|
+
* Top-level attribute carrying a row's per-write id, for the row kinds that
|
|
85
|
+
* have one. Preferred over a descriptor's own id where it exists: it needs no
|
|
86
|
+
* document path, and it is the same attribute the writer already pins on.
|
|
87
|
+
*/
|
|
88
|
+
idAttribute?: string;
|
|
89
|
+
/**
|
|
90
|
+
* The logical unit a row belongs to. A unit's rows are written together but
|
|
91
|
+
* deleted one by one, so a refusal on an earlier one has to suppress the
|
|
92
|
+
* rest; an adapter whose rows form no unit supplies nothing.
|
|
93
|
+
*/
|
|
94
|
+
unitOf?: (row: AttributeMap) => string;
|
|
95
|
+
/**
|
|
96
|
+
* The row kind bounding a flush. Rows arrive kind by kind, so flushing when
|
|
97
|
+
* the kind changes is what settles a refusal before the rows it must suppress
|
|
98
|
+
* are issued.
|
|
99
|
+
*/
|
|
100
|
+
kindOf?: (row: AttributeMap) => string;
|
|
101
|
+
/** The partition's own leading S3 key parts; objects outside their path are never deleted. */
|
|
102
|
+
scope: readonly string[];
|
|
23
103
|
}
|
|
24
104
|
/**
|
|
25
|
-
* Delete
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
105
|
+
* Delete exactly the rows of one partition that this adapter's read observed.
|
|
106
|
+
*
|
|
107
|
+
* Accepts: `params` — the partition query, which carries no sort-key
|
|
108
|
+
* condition. `ownsSortKey` — decides per row; a row it rejects is left in
|
|
109
|
+
* place and reported at `warn`, which is what keeps a shared table's other
|
|
110
|
+
* adapters intact. `descriptorsOf` — the offloaded payloads a row references,
|
|
111
|
+
* each named by the attribute holding it. `idAttribute`, `unitOf` and `kindOf`
|
|
112
|
+
* — the per-write id, the unit and the kind boundary, for an adapter whose
|
|
113
|
+
* rows have them. `scope` — the partition's own leading S3 key parts; an object
|
|
114
|
+
* outside their path is never deleted. `signal` — stops the read between pages.
|
|
115
|
+
*
|
|
116
|
+
* Returns: how many rows were deleted, not counting the ones left in place.
|
|
117
|
+
*
|
|
118
|
+
* Throws: `ABORTED` when the signal fires, whether between pages or during
|
|
119
|
+
* a row's delete, unwrapped and with no further row issued — a cancel is not a
|
|
120
|
+
* delete that half-landed. Otherwise `BATCH_WRITE_INCOMPLETE` when
|
|
121
|
+
* a row's delete fails, carrying what did succeed across every earlier flush.
|
|
122
|
+
* S3 cleanup never throws, whatever it finds.
|
|
123
|
+
*
|
|
124
|
+
* Guarantees: every delete is pinned on the per-write id the read observed, so
|
|
125
|
+
* a row rewritten after that read is left in place and reported rather than
|
|
126
|
+
* erased with the object it names. A row observed carrying no id is deleted
|
|
127
|
+
* unconditionally, as it was before the pin existed. The read is deliberately
|
|
128
|
+
* uncapped (`maxItems` and `maxIterations` are `Infinity`), so a partition of
|
|
129
|
+
* any size is deleted to completion rather than truncated at the in-memory page
|
|
130
|
+
* caps — memory stays bounded because rows are flushed in batches of
|
|
131
|
+
* {@link BATCH_WRITE_MAX} and never accumulated. The carry-forward depends on
|
|
132
|
+
* the scan being **ascending**, which is the default the partition queries rely
|
|
133
|
+
* on: a kind's rows are settled before the next kind's are issued, so a refusal
|
|
134
|
+
* suppresses the rest of its unit. A failure part-way keeps the rows already
|
|
135
|
+
* deleted; this is a single pass over a quiescent partition, not a transaction.
|
|
31
136
|
*/
|
|
32
137
|
export declare function deletePartitionRows(options: PartitionDeleteOptions): Promise<number>;
|
|
33
|
-
|
|
138
|
+
/** One row a pass has read and means to delete. */
|
|
139
|
+
export interface PendingDelete {
|
|
140
|
+
key: AttributeMap;
|
|
141
|
+
/** The pin the read's observation supports; absent for a row that carried no id. */
|
|
142
|
+
guard?: RevisionGuard;
|
|
143
|
+
/** The objects this row names, released only if the row is confirmed gone. */
|
|
144
|
+
descriptors: DescriptorRef[];
|
|
145
|
+
/** The logical unit the row belongs to, when the caller names one. */
|
|
146
|
+
unit?: string;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* What one flush needs, narrowed from the caller's options rather than taking
|
|
150
|
+
* the options interface itself, so the seam between the pass and its flush
|
|
151
|
+
* stays as small as what crosses it.
|
|
152
|
+
*/
|
|
153
|
+
export interface FlushDeps {
|
|
154
|
+
client: DynamoDBDocumentLike;
|
|
155
|
+
tableName: string;
|
|
156
|
+
logger: Logger;
|
|
157
|
+
/** Names the pass in every log line, e.g. `deleteThread`. */
|
|
158
|
+
operation: string;
|
|
159
|
+
/** The partition's own leading S3 key parts; an object outside them is never deleted. */
|
|
160
|
+
scope: readonly string[];
|
|
161
|
+
/** The adapter's retry options, which already carry the caller's abort signal. */
|
|
162
|
+
retry?: RetryOptions;
|
|
163
|
+
offloader?: S3Offloader;
|
|
164
|
+
}
|
|
165
|
+
/** What one flush settled, folded into the pass's running totals by its caller. */
|
|
166
|
+
export interface FlushTally {
|
|
167
|
+
deleted: number;
|
|
168
|
+
refused: number;
|
|
169
|
+
/** The units of the refused rows, so a later row of the same unit can be skipped. */
|
|
170
|
+
refusedUnits: string[];
|
|
171
|
+
failures: Error[];
|
|
172
|
+
/** The descriptors of the rows confirmed gone, and only those. */
|
|
173
|
+
released: DescriptorRef[];
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Delete a buffer of rows, each pinned on what the read observed of it, then
|
|
177
|
+
* release the objects of the rows that are confirmed gone.
|
|
178
|
+
*
|
|
179
|
+
* Accepts: `deps` — the client, the table, and the logging, retry and offload
|
|
180
|
+
* collaborators of the pass this flush belongs to. `rows` — the buffer,
|
|
181
|
+
* each row carrying its key, its pin and the objects it names.
|
|
182
|
+
*
|
|
183
|
+
* Returns: the tally — rows deleted, rows refused and the units they belonged
|
|
184
|
+
* to, the failures, and the descriptors released.
|
|
185
|
+
*
|
|
186
|
+
* Throws: nothing. Every failure stops the flush from starting further rows and
|
|
187
|
+
* is handed back in `failures` for the caller to end the pass with — the
|
|
188
|
+
* delete's own rejection, and equally a throw from decoding a rejection's
|
|
189
|
+
* attached row or from the caller's logger, neither of which is this pass's to
|
|
190
|
+
* absorb. A refusal never stops anything. S3 cleanup never throws either
|
|
191
|
+
* ({@link cleanUpS3Orphans}).
|
|
192
|
+
*
|
|
193
|
+
* Guarantees: an empty `failures` means every row of the buffer was settled.
|
|
194
|
+
* The flush cannot both lose a failure and hand back a clean tally, which is
|
|
195
|
+
* what let a pass log a deleted thread over a partition it had mostly left
|
|
196
|
+
* alone. At most {@link DELETE_CONCURRENCY} requests are in flight. There
|
|
197
|
+
* is one request per row - that is the price of a condition, which a batch
|
|
198
|
+
* write silently ignores - but they cost a bounded number of sequential rounds
|
|
199
|
+
* rather than one per row, and a partition of any size cannot open a socket
|
|
200
|
+
* per row. A row the pin
|
|
201
|
+
* turned away is left exactly as the racing writer left it, and nothing it
|
|
202
|
+
* names is released — a live row still names those objects.
|
|
203
|
+
*/
|
|
204
|
+
export declare function flushPendingDeletes(deps: FlushDeps, rows: readonly PendingDelete[]): Promise<FlushTally>;
|