@farukada/aws-langgraph-dynamodb-ts 1.0.0-rc.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +390 -193
- package/dist/{backfill → cjs/backfill}/backfill.d.ts +15 -12
- package/dist/cjs/backfill/backfill.js +177 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/delete-thread.d.ts +1 -1
- package/dist/cjs/checkpointer/actions/delete-thread.js +26 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/get-tuple.d.ts +4 -3
- package/dist/cjs/checkpointer/actions/get-tuple.js +22 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/list.d.ts +7 -4
- package/dist/cjs/checkpointer/actions/list.js +70 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/put-writes.d.ts +8 -6
- package/dist/cjs/checkpointer/actions/put-writes.js +21 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/put.d.ts +21 -15
- package/dist/cjs/checkpointer/actions/put.js +79 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/delta-history.d.ts +38 -26
- package/dist/cjs/checkpointer/internal/delta-history.js +131 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/listing.d.ts +4 -4
- package/dist/cjs/checkpointer/internal/listing.js +104 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/parse.d.ts +51 -15
- package/dist/cjs/checkpointer/internal/parse.js +184 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/pending-writes.d.ts +64 -25
- package/dist/cjs/checkpointer/internal/pending-writes.js +241 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/read.d.ts +15 -10
- package/dist/cjs/checkpointer/internal/read.js +132 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/rows.d.ts +94 -21
- package/dist/cjs/checkpointer/internal/rows.js +356 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/setup.d.ts +13 -20
- package/dist/cjs/checkpointer/internal/setup.js +25 -0
- package/dist/{checkpointer → cjs/checkpointer}/saver.d.ts +72 -36
- package/dist/cjs/checkpointer/saver.js +56 -0
- package/dist/{checkpointer → cjs/checkpointer}/types.d.ts +1 -1
- package/dist/cjs/checkpointer/types.js +2 -0
- package/dist/{factory → cjs/factory}/factory.d.ts +25 -20
- package/dist/cjs/factory/factory.js +137 -0
- package/dist/{factory → cjs/factory}/types.d.ts +12 -12
- package/dist/cjs/factory/types.js +2 -0
- package/dist/{history → cjs/history}/actions/add-messages.d.ts +18 -10
- package/dist/cjs/history/actions/add-messages.js +17 -0
- package/dist/{history → cjs/history}/actions/clear.d.ts +1 -1
- package/dist/cjs/history/actions/clear.js +29 -0
- package/dist/{history → cjs/history}/actions/get-messages.d.ts +11 -9
- package/dist/cjs/history/actions/get-messages.js +58 -0
- package/dist/{history → cjs/history}/actions/list-sessions.d.ts +3 -3
- package/dist/cjs/history/actions/list-sessions.js +67 -0
- package/dist/{history → cjs/history}/actions/reconcile-count.d.ts +13 -9
- package/dist/cjs/history/actions/reconcile-count.js +8 -0
- package/dist/{history → cjs/history}/chat-message-history.d.ts +62 -30
- package/dist/cjs/history/chat-message-history.js +61 -0
- package/dist/{history → cjs/history}/internal/append.d.ts +64 -32
- package/dist/cjs/history/internal/append.js +267 -0
- package/dist/{history → cjs/history}/internal/message-read.d.ts +3 -3
- package/dist/cjs/history/internal/message-read.js +93 -0
- package/dist/{history → cjs/history}/internal/parse.d.ts +2 -2
- package/dist/cjs/history/internal/parse.js +113 -0
- package/dist/{history → cjs/history}/internal/rows.d.ts +11 -10
- package/dist/cjs/history/internal/rows.js +96 -0
- package/dist/{history → cjs/history}/internal/session.d.ts +41 -22
- package/dist/cjs/history/internal/session.js +296 -0
- package/dist/{history → cjs/history}/internal/setup.d.ts +2 -2
- package/dist/cjs/history/internal/setup.js +60 -0
- package/dist/{history → cjs/history}/session-adapter.d.ts +0 -6
- package/dist/cjs/history/session-adapter.js +38 -0
- package/dist/{history → cjs/history}/types.d.ts +18 -15
- package/dist/cjs/history/types.js +2 -0
- package/dist/{index.d.ts → cjs/index.d.ts} +27 -27
- package/dist/cjs/index.js +25 -0
- package/dist/cjs/package.json +1 -0
- package/dist/{shared → cjs/shared}/adapter.d.ts +27 -16
- package/dist/cjs/shared/adapter.js +70 -0
- package/dist/cjs/shared/clock.js +14 -0
- package/dist/{shared → cjs/shared}/codec/codec.d.ts +37 -20
- package/dist/cjs/shared/codec/codec.js +162 -0
- package/dist/{shared → cjs/shared}/codec/compression.d.ts +9 -4
- package/dist/cjs/shared/codec/compression.js +45 -0
- package/dist/cjs/shared/codec/json-serde.js +58 -0
- package/dist/{shared → cjs/shared}/codec/s3/client-types.d.ts +2 -1
- package/dist/cjs/shared/codec/s3/client-types.js +6 -0
- package/dist/{shared → cjs/shared}/codec/s3/client.d.ts +25 -7
- package/dist/cjs/shared/codec/s3/client.js +78 -0
- package/dist/{shared → cjs/shared}/codec/s3/config.d.ts +12 -7
- package/dist/cjs/shared/codec/s3/config.js +92 -0
- package/dist/{shared → cjs/shared}/codec/s3/lifecycle.d.ts +71 -17
- package/dist/cjs/shared/codec/s3/lifecycle.js +223 -0
- package/dist/{shared → cjs/shared}/codec/s3/offloader.d.ts +42 -29
- package/dist/cjs/shared/codec/s3/offloader.js +289 -0
- package/dist/cjs/shared/concurrency.js +31 -0
- package/dist/{shared → cjs/shared}/dynamodb/abort.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/abort.js +16 -0
- package/dist/{shared → cjs/shared}/dynamodb/batch-write.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/batch-write.js +79 -0
- package/dist/cjs/shared/dynamodb/cancellation.js +37 -0
- package/dist/{shared → cjs/shared}/dynamodb/client.d.ts +24 -22
- package/dist/cjs/shared/dynamodb/client.js +40 -0
- package/dist/{shared → cjs/shared}/dynamodb/idempotent-write.d.ts +64 -19
- package/dist/cjs/shared/dynamodb/idempotent-write.js +173 -0
- package/dist/{shared → cjs/shared}/dynamodb/paginate.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/paginate.js +90 -0
- package/dist/{shared → cjs/shared}/dynamodb/partition-delete.d.ts +6 -6
- package/dist/cjs/shared/dynamodb/partition-delete.js +177 -0
- package/dist/{shared → cjs/shared}/dynamodb/recency-index.d.ts +27 -15
- package/dist/cjs/shared/dynamodb/recency-index.js +146 -0
- package/dist/{shared → cjs/shared}/dynamodb/retry.d.ts +37 -6
- package/dist/cjs/shared/dynamodb/retry.js +193 -0
- package/dist/cjs/shared/dynamodb/row-size.d.ts +33 -0
- package/dist/cjs/shared/dynamodb/row-size.js +30 -0
- package/dist/{shared → cjs/shared}/dynamodb/table-schema.d.ts +27 -18
- package/dist/cjs/shared/dynamodb/table-schema.js +52 -0
- package/dist/{shared → cjs/shared}/errors/base-error.d.ts +42 -7
- package/dist/cjs/shared/errors/base-error.js +73 -0
- package/dist/{shared → cjs/shared}/errors/boundary.d.ts +29 -14
- package/dist/cjs/shared/errors/boundary.js +61 -0
- package/dist/cjs/shared/errors/classify.d.ts +206 -0
- package/dist/cjs/shared/errors/classify.js +232 -0
- package/dist/{shared → cjs/shared}/errors/error-code.d.ts +17 -10
- package/dist/cjs/shared/errors/error-code.js +27 -0
- package/dist/{shared → cjs/shared}/errors/errors.d.ts +75 -9
- package/dist/cjs/shared/errors/errors.js +123 -0
- package/dist/cjs/shared/logging/logger.js +30 -0
- package/dist/{shared → cjs/shared}/logging/redaction.d.ts +8 -4
- package/dist/cjs/shared/logging/redaction.js +145 -0
- package/dist/{shared → cjs/shared}/logging/secret-patterns.d.ts +30 -7
- package/dist/cjs/shared/logging/secret-patterns.js +93 -0
- package/dist/cjs/shared/logging/truncate.js +31 -0
- package/dist/{shared → cjs/shared}/options.d.ts +27 -11
- package/dist/cjs/shared/options.js +2 -0
- package/dist/cjs/shared/ulid.js +83 -0
- package/dist/{shared → cjs/shared}/validation/collaborators.d.ts +21 -2
- package/dist/cjs/shared/validation/collaborators.js +79 -0
- package/dist/{shared → cjs/shared}/validation/option-shape.d.ts +7 -5
- package/dist/cjs/shared/validation/option-shape.js +31 -0
- package/dist/{shared → cjs/shared}/validation/options.d.ts +4 -4
- package/dist/cjs/shared/validation/options.js +166 -0
- package/dist/cjs/shared/validation/primitives.js +114 -0
- package/dist/{shared → cjs/shared}/validation/ttl.d.ts +26 -9
- package/dist/cjs/shared/validation/ttl.js +52 -0
- package/dist/{store → cjs/store}/actions/list-namespaces.d.ts +5 -5
- package/dist/cjs/store/actions/list-namespaces.js +92 -0
- package/dist/{store → cjs/store}/actions/put.d.ts +11 -4
- package/dist/cjs/store/actions/put.js +49 -0
- package/dist/{store → cjs/store}/actions/reconcile-vector-index.d.ts +5 -5
- package/dist/cjs/store/actions/reconcile-vector-index.js +13 -0
- package/dist/{store → cjs/store}/actions/search.d.ts +2 -2
- package/dist/cjs/store/actions/search.js +26 -0
- package/dist/cjs/store/internal/batch-plan.d.ts +56 -0
- package/dist/cjs/store/internal/batch-plan.js +32 -0
- package/dist/{store → cjs/store}/internal/filter.d.ts +3 -3
- package/dist/cjs/store/internal/filter.js +64 -0
- package/dist/cjs/store/internal/get-item.d.ts +77 -0
- package/dist/cjs/store/internal/get-item.js +58 -0
- package/dist/{store → cjs/store}/internal/item-write.d.ts +72 -32
- package/dist/cjs/store/internal/item-write.js +181 -0
- package/dist/{store → cjs/store}/internal/parse.d.ts +15 -10
- package/dist/cjs/store/internal/parse.js +180 -0
- package/dist/{store → cjs/store}/internal/rows.d.ts +38 -34
- package/dist/cjs/store/internal/rows.js +166 -0
- package/dist/{store → cjs/store}/internal/semantic-search.d.ts +2 -2
- package/dist/cjs/store/internal/semantic-search.js +211 -0
- package/dist/{store → cjs/store}/internal/setup.d.ts +10 -15
- package/dist/cjs/store/internal/setup.js +103 -0
- package/dist/{store → cjs/store}/internal/table-search.d.ts +5 -5
- package/dist/cjs/store/internal/table-search.js +123 -0
- package/dist/{store → cjs/store}/internal/vector-index.d.ts +26 -15
- package/dist/cjs/store/internal/vector-index.js +269 -0
- package/dist/{store → cjs/store}/store.d.ts +82 -39
- package/dist/cjs/store/store.js +97 -0
- package/dist/{store → cjs/store}/types.d.ts +25 -10
- package/dist/cjs/store/types.js +2 -0
- package/dist/cjs/store/vector-backend.js +4 -0
- package/dist/esm/backfill/backfill.d.ts +171 -0
- package/dist/esm/backfill/backfill.js +170 -0
- package/dist/{checkpointer/actions/delete-thread.js → esm/checkpointer/actions/delete-thread.d.ts} +4 -26
- package/dist/esm/checkpointer/actions/delete-thread.js +23 -0
- package/dist/{checkpointer/actions/get-tuple.js → esm/checkpointer/actions/get-tuple.d.ts} +7 -24
- package/dist/esm/checkpointer/actions/get-tuple.js +19 -0
- package/dist/esm/checkpointer/actions/list.d.ts +55 -0
- package/dist/esm/checkpointer/actions/list.js +67 -0
- package/dist/{checkpointer/actions/put-writes.js → esm/checkpointer/actions/put-writes.d.ts} +10 -37
- package/dist/esm/checkpointer/actions/put-writes.js +18 -0
- package/dist/esm/checkpointer/actions/put.d.ts +95 -0
- package/dist/esm/checkpointer/actions/put.js +75 -0
- package/dist/esm/checkpointer/internal/delta-history.d.ts +124 -0
- package/dist/esm/checkpointer/internal/delta-history.js +126 -0
- package/dist/{checkpointer/internal/listing.js → esm/checkpointer/internal/listing.d.ts} +22 -118
- package/dist/esm/checkpointer/internal/listing.js +95 -0
- package/dist/esm/checkpointer/internal/parse.d.ts +298 -0
- package/dist/esm/checkpointer/internal/parse.js +169 -0
- package/dist/esm/checkpointer/internal/pending-writes.d.ts +314 -0
- package/dist/esm/checkpointer/internal/pending-writes.js +229 -0
- package/dist/esm/checkpointer/internal/read.d.ts +135 -0
- package/dist/esm/checkpointer/internal/read.js +125 -0
- package/dist/esm/checkpointer/internal/rows.d.ts +644 -0
- package/dist/esm/checkpointer/internal/rows.js +323 -0
- package/dist/esm/checkpointer/internal/setup.d.ts +47 -0
- package/dist/esm/checkpointer/internal/setup.js +21 -0
- package/dist/{checkpointer/saver.js → esm/checkpointer/saver.d.ts} +84 -87
- package/dist/esm/checkpointer/saver.js +52 -0
- package/dist/esm/checkpointer/types.d.ts +51 -0
- package/dist/esm/checkpointer/types.js +1 -0
- package/dist/esm/factory/factory.d.ts +154 -0
- package/dist/esm/factory/factory.js +133 -0
- package/dist/esm/factory/types.d.ts +76 -0
- package/dist/esm/factory/types.js +1 -0
- package/dist/{history/actions/add-messages.js → esm/history/actions/add-messages.d.ts} +20 -26
- package/dist/esm/history/actions/add-messages.js +14 -0
- package/dist/{history/actions/clear.js → esm/history/actions/clear.d.ts} +4 -35
- package/dist/esm/history/actions/clear.js +26 -0
- package/dist/esm/history/actions/get-messages.d.ts +60 -0
- package/dist/esm/history/actions/get-messages.js +55 -0
- package/dist/esm/history/actions/list-sessions.d.ts +55 -0
- package/dist/esm/history/actions/list-sessions.js +64 -0
- package/dist/{history/actions/reconcile-count.js → esm/history/actions/reconcile-count.d.ts} +14 -16
- package/dist/esm/history/actions/reconcile-count.js +5 -0
- package/dist/{history/chat-message-history.js → esm/history/chat-message-history.d.ts} +76 -87
- package/dist/esm/history/chat-message-history.js +57 -0
- package/dist/esm/history/internal/append.d.ts +244 -0
- package/dist/esm/history/internal/append.js +259 -0
- package/dist/esm/history/internal/message-read.d.ts +84 -0
- package/dist/esm/history/internal/message-read.js +89 -0
- package/dist/esm/history/internal/parse.d.ts +153 -0
- package/dist/esm/history/internal/parse.js +105 -0
- package/dist/{history/internal/rows.js → esm/history/internal/rows.d.ts} +58 -112
- package/dist/esm/history/internal/rows.js +83 -0
- package/dist/esm/history/internal/session.d.ts +350 -0
- package/dist/esm/history/internal/session.js +285 -0
- package/dist/esm/history/internal/setup.d.ts +64 -0
- package/dist/esm/history/internal/setup.js +56 -0
- package/dist/{history/session-adapter.js → esm/history/session-adapter.d.ts} +25 -39
- package/dist/esm/history/session-adapter.js +34 -0
- package/dist/esm/history/types.d.ts +102 -0
- package/dist/esm/history/types.js +1 -0
- package/dist/esm/index.d.ts +53 -0
- package/dist/esm/index.js +10 -0
- package/dist/esm/shared/adapter.d.ts +146 -0
- package/dist/esm/shared/adapter.js +65 -0
- package/dist/{shared/clock.js → esm/shared/clock.d.ts} +3 -14
- package/dist/esm/shared/clock.js +9 -0
- package/dist/esm/shared/codec/codec.d.ts +336 -0
- package/dist/esm/shared/codec/codec.js +150 -0
- package/dist/esm/shared/codec/compression.d.ts +66 -0
- package/dist/esm/shared/codec/compression.js +40 -0
- package/dist/esm/shared/codec/json-serde.d.ts +80 -0
- package/dist/esm/shared/codec/json-serde.js +54 -0
- package/dist/esm/shared/codec/s3/client-types.d.ts +54 -0
- package/dist/esm/shared/codec/s3/client-types.js +3 -0
- package/dist/esm/shared/codec/s3/client.d.ts +63 -0
- package/dist/esm/shared/codec/s3/client.js +40 -0
- package/dist/{shared/codec/s3/config.js → esm/shared/codec/s3/config.d.ts} +45 -93
- package/dist/esm/shared/codec/s3/config.js +79 -0
- package/dist/esm/shared/codec/s3/lifecycle.d.ts +220 -0
- package/dist/esm/shared/codec/s3/lifecycle.js +214 -0
- package/dist/esm/shared/codec/s3/offloader.d.ts +428 -0
- package/dist/esm/shared/codec/s3/offloader.js +278 -0
- package/dist/{shared/concurrency.js → esm/shared/concurrency.d.ts} +2 -37
- package/dist/esm/shared/concurrency.js +27 -0
- package/dist/{shared/dynamodb/abort.js → esm/shared/dynamodb/abort.d.ts} +4 -16
- package/dist/esm/shared/dynamodb/abort.js +12 -0
- package/dist/esm/shared/dynamodb/batch-write.d.ts +86 -0
- package/dist/esm/shared/dynamodb/batch-write.js +74 -0
- package/dist/{shared/dynamodb/cancellation.js → esm/shared/dynamodb/cancellation.d.ts} +25 -53
- package/dist/esm/shared/dynamodb/cancellation.js +30 -0
- package/dist/{shared/dynamodb/client.js → esm/shared/dynamodb/client.d.ts} +74 -64
- package/dist/esm/shared/dynamodb/client.js +35 -0
- package/dist/{shared/dynamodb/idempotent-write.js → esm/shared/dynamodb/idempotent-write.d.ts} +172 -169
- package/dist/esm/shared/dynamodb/idempotent-write.js +153 -0
- package/dist/esm/shared/dynamodb/paginate.d.ts +111 -0
- package/dist/esm/shared/dynamodb/paginate.js +84 -0
- package/dist/esm/shared/dynamodb/partition-delete.d.ts +204 -0
- package/dist/esm/shared/dynamodb/partition-delete.js +171 -0
- package/dist/esm/shared/dynamodb/recency-index.d.ts +243 -0
- package/dist/esm/shared/dynamodb/recency-index.js +136 -0
- package/dist/esm/shared/dynamodb/retry.d.ts +317 -0
- package/dist/esm/shared/dynamodb/retry.js +180 -0
- package/dist/esm/shared/dynamodb/row-size.d.ts +33 -0
- package/dist/esm/shared/dynamodb/row-size.js +26 -0
- package/dist/{shared/dynamodb/table-schema.js → esm/shared/dynamodb/table-schema.d.ts} +57 -67
- package/dist/esm/shared/dynamodb/table-schema.js +43 -0
- package/dist/esm/shared/errors/base-error.d.ts +231 -0
- package/dist/esm/shared/errors/base-error.js +64 -0
- package/dist/esm/shared/errors/boundary.d.ts +86 -0
- package/dist/esm/shared/errors/boundary.js +54 -0
- package/dist/esm/shared/errors/classify.d.ts +206 -0
- package/dist/esm/shared/errors/classify.js +220 -0
- package/dist/{shared/errors/error-code.js → esm/shared/errors/error-code.d.ts} +39 -43
- package/dist/esm/shared/errors/error-code.js +24 -0
- package/dist/{shared/errors/errors.js → esm/shared/errors/errors.d.ts} +92 -98
- package/dist/esm/shared/errors/errors.js +110 -0
- package/dist/{shared/logging/logger.js → esm/shared/logging/logger.d.ts} +22 -52
- package/dist/esm/shared/logging/logger.js +25 -0
- package/dist/esm/shared/logging/redaction.d.ts +123 -0
- package/dist/esm/shared/logging/redaction.js +140 -0
- package/dist/{shared/logging/secret-patterns.js → esm/shared/logging/secret-patterns.d.ts} +46 -87
- package/dist/esm/shared/logging/secret-patterns.js +83 -0
- package/dist/{shared/logging/truncate.js → esm/shared/logging/truncate.d.ts} +6 -40
- package/dist/esm/shared/logging/truncate.js +25 -0
- package/dist/esm/shared/options.d.ts +100 -0
- package/dist/esm/shared/options.js +1 -0
- package/dist/{shared/ulid.js → esm/shared/ulid.d.ts} +4 -84
- package/dist/esm/shared/ulid.js +77 -0
- package/dist/{shared/validation/collaborators.js → esm/shared/validation/collaborators.d.ts} +37 -65
- package/dist/esm/shared/validation/collaborators.js +70 -0
- package/dist/{shared/validation/option-shape.js → esm/shared/validation/option-shape.d.ts} +14 -36
- package/dist/esm/shared/validation/option-shape.js +24 -0
- package/dist/esm/shared/validation/options.d.ts +145 -0
- package/dist/esm/shared/validation/options.js +156 -0
- package/dist/{shared/validation/primitives.js → esm/shared/validation/primitives.d.ts} +30 -125
- package/dist/esm/shared/validation/primitives.js +100 -0
- package/dist/esm/shared/validation/ttl.d.ts +94 -0
- package/dist/esm/shared/validation/ttl.js +46 -0
- package/dist/esm/store/actions/list-namespaces.d.ts +79 -0
- package/dist/esm/store/actions/list-namespaces.js +86 -0
- package/dist/esm/store/actions/put.d.ts +43 -0
- package/dist/esm/store/actions/put.js +46 -0
- package/dist/{store/actions/reconcile-vector-index.js → esm/store/actions/reconcile-vector-index.d.ts} +8 -16
- package/dist/esm/store/actions/reconcile-vector-index.js +10 -0
- package/dist/esm/store/actions/search.d.ts +38 -0
- package/dist/esm/store/actions/search.js +23 -0
- package/dist/esm/store/internal/batch-plan.d.ts +56 -0
- package/dist/esm/store/internal/batch-plan.js +28 -0
- package/dist/esm/store/internal/filter.d.ts +43 -0
- package/dist/esm/store/internal/filter.js +60 -0
- package/dist/esm/store/internal/get-item.d.ts +77 -0
- package/dist/esm/store/internal/get-item.js +54 -0
- package/dist/esm/store/internal/item-write.d.ts +270 -0
- package/dist/esm/store/internal/item-write.js +173 -0
- package/dist/esm/store/internal/parse.d.ts +230 -0
- package/dist/esm/store/internal/parse.js +169 -0
- package/dist/{store/internal/rows.js → esm/store/internal/rows.d.ts} +111 -199
- package/dist/esm/store/internal/rows.js +147 -0
- package/dist/esm/store/internal/semantic-search.d.ts +168 -0
- package/dist/esm/store/internal/semantic-search.js +200 -0
- package/dist/esm/store/internal/setup.d.ts +96 -0
- package/dist/esm/store/internal/setup.js +98 -0
- package/dist/esm/store/internal/table-search.d.ts +100 -0
- package/dist/esm/store/internal/table-search.js +119 -0
- package/dist/esm/store/internal/vector-index.d.ts +258 -0
- package/dist/esm/store/internal/vector-index.js +256 -0
- package/dist/{store/store.js → esm/store/store.d.ts} +99 -133
- package/dist/esm/store/store.js +93 -0
- package/dist/esm/store/types.d.ts +111 -0
- package/dist/esm/store/types.js +1 -0
- package/dist/esm/store/vector-backend.d.ts +104 -0
- package/dist/esm/store/vector-backend.js +1 -0
- package/package.json +50 -44
- package/dist/backfill/backfill.js +0 -393
- package/dist/checkpointer/actions/list.js +0 -142
- package/dist/checkpointer/actions/put.js +0 -195
- package/dist/checkpointer/internal/delta-history.js +0 -252
- package/dist/checkpointer/internal/parse.js +0 -372
- package/dist/checkpointer/internal/pending-writes.js +0 -588
- package/dist/checkpointer/internal/read.js +0 -264
- package/dist/checkpointer/internal/rows.js +0 -834
- package/dist/checkpointer/internal/setup.js +0 -72
- package/dist/checkpointer/types.js +0 -12
- package/dist/factory/factory.js +0 -266
- package/dist/factory/types.js +0 -10
- package/dist/history/actions/get-messages.js +0 -141
- package/dist/history/actions/list-sessions.js +0 -156
- package/dist/history/internal/append.js +0 -500
- package/dist/history/internal/message-read.js +0 -204
- package/dist/history/internal/parse.js +0 -252
- package/dist/history/internal/session.js +0 -628
- package/dist/history/internal/setup.js +0 -106
- package/dist/history/types.js +0 -12
- package/dist/index.js +0 -51
- package/dist/shared/adapter.js +0 -143
- package/dist/shared/codec/codec.js +0 -441
- package/dist/shared/codec/compression.js +0 -89
- package/dist/shared/codec/json-serde.js +0 -191
- package/dist/shared/codec/s3/client-types.js +0 -26
- package/dist/shared/codec/s3/client.js +0 -97
- package/dist/shared/codec/s3/lifecycle.js +0 -355
- package/dist/shared/codec/s3/offloader.js +0 -658
- package/dist/shared/dynamodb/batch-write.js +0 -173
- package/dist/shared/dynamodb/paginate.js +0 -188
- package/dist/shared/dynamodb/partition-delete.js +0 -340
- package/dist/shared/dynamodb/recency-index.js +0 -377
- package/dist/shared/dynamodb/retry.js +0 -463
- package/dist/shared/errors/base-error.js +0 -174
- package/dist/shared/errors/boundary.js +0 -143
- package/dist/shared/errors/classify.d.ts +0 -97
- package/dist/shared/errors/classify.js +0 -257
- package/dist/shared/logging/redaction.js +0 -316
- package/dist/shared/options.js +0 -11
- package/dist/shared/validation/options.js +0 -328
- package/dist/shared/validation/ttl.js +0 -132
- package/dist/store/actions/list-namespaces.js +0 -197
- package/dist/store/actions/put.js +0 -78
- package/dist/store/actions/search.js +0 -66
- package/dist/store/internal/batch-plan.d.ts +0 -26
- package/dist/store/internal/batch-plan.js +0 -109
- package/dist/store/internal/filter.js +0 -137
- package/dist/store/internal/get-item.d.ts +0 -45
- package/dist/store/internal/get-item.js +0 -115
- package/dist/store/internal/item-write.js +0 -463
- package/dist/store/internal/parse.js +0 -350
- package/dist/store/internal/semantic-search.js +0 -389
- package/dist/store/internal/setup.js +0 -223
- package/dist/store/internal/table-search.js +0 -213
- package/dist/store/internal/vector-index.js +0 -546
- package/dist/store/types.js +0 -15
- package/dist/store/vector-backend.js +0 -17
- /package/dist/{shared → cjs/shared}/clock.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/codec/json-serde.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/concurrency.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/dynamodb/cancellation.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/logging/logger.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/logging/truncate.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/ulid.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/validation/primitives.d.ts +0 -0
- /package/dist/{store → cjs/store}/vector-backend.d.ts +0 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
import { isAbortError } from '../../dynamodb/abort.js';
|
|
2
|
+
import { fullJitter, INITIAL_BACKOFF_DELAY_MS, isTransientS3Error, nextBackoffDelay, sleep, withRetry, } from '../../dynamodb/retry.js';
|
|
3
|
+
import { copyForCaller, DynamoDBLangGraphError, failureLabel } from '../../errors/base-error.js';
|
|
4
|
+
import { awsDiagnostics, classifiableCause, classifyAwsError } from '../../errors/classify.js';
|
|
5
|
+
import { ErrorCode } from '../../errors/error-code.js';
|
|
6
|
+
import { validationError } from '../../errors/errors.js';
|
|
7
|
+
import { absorbLoggerFailure, SILENT_LOGGER } from '../../logging/logger.js';
|
|
8
|
+
import { redactedMessage } from '../../logging/secret-patterns.js';
|
|
9
|
+
import { truncateForLog } from '../../logging/truncate.js';
|
|
10
|
+
import { createDefaultS3Client, loadS3Sdk, withS3ClientDefaults } from './client.js';
|
|
11
|
+
import { assertKeyInScope, buildS3Key, DEFAULT_S3_KEY_PREFIX, encodeKeyPart, isKeyInScope, } from './config.js';
|
|
12
|
+
import { ensureLifecycleRule } from './lifecycle.js';
|
|
13
|
+
export const DEFAULT_S3_THRESHOLD_BYTES = 350 * 1024;
|
|
14
|
+
export const DEFAULT_S3_SSE = 'AES256';
|
|
15
|
+
const S3_MAX_ATTEMPTS = 3;
|
|
16
|
+
export const DEFAULT_MAX_S3_DOWNLOAD_BYTES = 50 * 1024 * 1024;
|
|
17
|
+
export const S3_DELETE_BATCH_MAX = 1000;
|
|
18
|
+
export class S3Offloader {
|
|
19
|
+
clientPromise;
|
|
20
|
+
resolvedClient;
|
|
21
|
+
destroyed = false;
|
|
22
|
+
bucketName;
|
|
23
|
+
keyPrefix;
|
|
24
|
+
thresholdBytes;
|
|
25
|
+
sse;
|
|
26
|
+
sseKmsKeyId;
|
|
27
|
+
maxDownloadBytes;
|
|
28
|
+
config;
|
|
29
|
+
logger;
|
|
30
|
+
constructor(config, logger = SILENT_LOGGER) {
|
|
31
|
+
this.config = config;
|
|
32
|
+
this.logger = logger;
|
|
33
|
+
this.bucketName = config.bucketName;
|
|
34
|
+
this.keyPrefix = config.keyPrefix ?? DEFAULT_S3_KEY_PREFIX;
|
|
35
|
+
this.thresholdBytes = config.thresholdBytes ?? DEFAULT_S3_THRESHOLD_BYTES;
|
|
36
|
+
this.sse = config.serverSideEncryption ?? DEFAULT_S3_SSE;
|
|
37
|
+
this.sseKmsKeyId = config.sseKmsKeyId;
|
|
38
|
+
this.maxDownloadBytes = config.maxDownloadBytes ?? DEFAULT_MAX_S3_DOWNLOAD_BYTES;
|
|
39
|
+
void loadS3Sdk().catch(() => undefined);
|
|
40
|
+
}
|
|
41
|
+
getClient() {
|
|
42
|
+
if (!this.clientPromise) {
|
|
43
|
+
const cfg = this.config.clientConfig ?? {};
|
|
44
|
+
this.clientPromise = (this.config.createS3Client
|
|
45
|
+
? Promise.resolve(this.config.createS3Client(withS3ClientDefaults(cfg)))
|
|
46
|
+
: createDefaultS3Client(cfg)).then((client) => {
|
|
47
|
+
this.resolvedClient = client;
|
|
48
|
+
if (this.destroyed)
|
|
49
|
+
client.destroy();
|
|
50
|
+
return client;
|
|
51
|
+
}, (error) => {
|
|
52
|
+
this.clientPromise = undefined;
|
|
53
|
+
throw error;
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
return this.clientPromise.catch((error) => {
|
|
57
|
+
throw copyForCaller(error);
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
shouldOffload(data) {
|
|
61
|
+
return data.length >= this.thresholdBytes;
|
|
62
|
+
}
|
|
63
|
+
buildKey(parts, objectId) {
|
|
64
|
+
return buildS3Key(this.keyPrefix, parts, objectId);
|
|
65
|
+
}
|
|
66
|
+
getKeyPrefix() {
|
|
67
|
+
return this.keyPrefix;
|
|
68
|
+
}
|
|
69
|
+
ownsKey(key, scope) {
|
|
70
|
+
return isKeyInScope(key, this.keyPrefix, scope);
|
|
71
|
+
}
|
|
72
|
+
assertOwnedKey(key, scope) {
|
|
73
|
+
assertKeyInScope(key, this.keyPrefix, scope);
|
|
74
|
+
}
|
|
75
|
+
retryReporter(operation) {
|
|
76
|
+
return ({ attempt, delayMs, error }) => absorbLoggerFailure(() => this.logger.debug('retrying after a transient error', {
|
|
77
|
+
attempt,
|
|
78
|
+
delayMs,
|
|
79
|
+
operation,
|
|
80
|
+
error: truncateForLog(failureLabel(error)),
|
|
81
|
+
}));
|
|
82
|
+
}
|
|
83
|
+
async upload(key, data, row, signal) {
|
|
84
|
+
if (data.length > this.maxDownloadBytes) {
|
|
85
|
+
throw validationError(`payload of ${data.length} bytes exceeds s3.maxDownloadBytes (${this.maxDownloadBytes}), ` +
|
|
86
|
+
'so no reader configured like this adapter could download it; raise ' +
|
|
87
|
+
's3.maxDownloadBytes or store a smaller value', 'payload');
|
|
88
|
+
}
|
|
89
|
+
await uploadObject(await this.getClient(), {
|
|
90
|
+
bucket: this.bucketName,
|
|
91
|
+
key,
|
|
92
|
+
data,
|
|
93
|
+
serverSideEncryption: this.sse,
|
|
94
|
+
sseKmsKeyId: this.sseKmsKeyId,
|
|
95
|
+
metadata: backlinkMetadata(row),
|
|
96
|
+
signal,
|
|
97
|
+
onRetry: this.retryReporter('upload'),
|
|
98
|
+
});
|
|
99
|
+
return key;
|
|
100
|
+
}
|
|
101
|
+
async download(key, signal) {
|
|
102
|
+
return downloadObject(await this.getClient(), { bucket: this.bucketName, key, maxBytes: this.maxDownloadBytes }, signal, this.retryReporter('download'));
|
|
103
|
+
}
|
|
104
|
+
async deleteBatch(keys) {
|
|
105
|
+
return deleteObjects(await this.getClient(), this.bucketName, keys);
|
|
106
|
+
}
|
|
107
|
+
async ensureLifecycleRule(ttlDays, logger) {
|
|
108
|
+
return ensureLifecycleRule(await this.getClient(), { bucket: this.bucketName, prefix: this.keyPrefix, days: ttlDays }, logger);
|
|
109
|
+
}
|
|
110
|
+
destroy() {
|
|
111
|
+
this.destroyed = true;
|
|
112
|
+
this.resolvedClient?.destroy();
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function rethrowIfCancelled(error) {
|
|
116
|
+
if (isAbortError(error))
|
|
117
|
+
throw error;
|
|
118
|
+
}
|
|
119
|
+
function alreadyStored(error) {
|
|
120
|
+
return classifyAwsError(error) === ErrorCode.CONDITION_CONFLICT;
|
|
121
|
+
}
|
|
122
|
+
export async function uploadObject(client, params) {
|
|
123
|
+
const { PutObjectCommand } = await loadS3Sdk();
|
|
124
|
+
try {
|
|
125
|
+
await withRetry((request) => client.send(new PutObjectCommand({
|
|
126
|
+
Bucket: params.bucket,
|
|
127
|
+
Key: params.key,
|
|
128
|
+
Body: params.data,
|
|
129
|
+
ContentType: 'application/octet-stream',
|
|
130
|
+
IfNoneMatch: '*',
|
|
131
|
+
ServerSideEncryption: params.serverSideEncryption,
|
|
132
|
+
...(params.sseKmsKeyId ? { SSEKMSKeyId: params.sseKmsKeyId } : {}),
|
|
133
|
+
...(params.metadata ? { Metadata: params.metadata } : {}),
|
|
134
|
+
}), request), {
|
|
135
|
+
maxAttempts: S3_MAX_ATTEMPTS,
|
|
136
|
+
isRetryable: isTransientS3Error,
|
|
137
|
+
signal: params.signal,
|
|
138
|
+
onRetry: params.onRetry,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
if (alreadyStored(error))
|
|
143
|
+
return;
|
|
144
|
+
rethrowIfCancelled(error);
|
|
145
|
+
throw new DynamoDBLangGraphError(redactedMessage(error), ErrorCode.S3_OFFLOAD_FAILED, {
|
|
146
|
+
operation: 'upload',
|
|
147
|
+
key: params.key,
|
|
148
|
+
...awsDiagnostics(classifiableCause(error)),
|
|
149
|
+
}, error);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
export async function downloadObject(client, object, signal, onRetry) {
|
|
153
|
+
const { GetObjectCommand } = await loadS3Sdk();
|
|
154
|
+
const { bucket, key, maxBytes } = object;
|
|
155
|
+
try {
|
|
156
|
+
return await withRetry(async (request) => {
|
|
157
|
+
const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }), request);
|
|
158
|
+
if (typeof response.ContentLength === 'number' && response.ContentLength > maxBytes) {
|
|
159
|
+
response.Body?.destroy?.();
|
|
160
|
+
throw oversizedObjectError(key, response.ContentLength, maxBytes);
|
|
161
|
+
}
|
|
162
|
+
if (!response.Body) {
|
|
163
|
+
throw new Error(`S3 object body is empty for key: ${truncateForLog(key)}`);
|
|
164
|
+
}
|
|
165
|
+
return readBodyBounded(response.Body, key, maxBytes);
|
|
166
|
+
}, { maxAttempts: S3_MAX_ATTEMPTS, isRetryable: isTransientS3Error, signal, onRetry });
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
rethrowIfCancelled(error);
|
|
170
|
+
throw new DynamoDBLangGraphError(redactedMessage(error), ErrorCode.S3_OFFLOAD_FAILED, { operation: 'download', key, ...awsDiagnostics(classifiableCause(error)) }, error);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
export function oversizedObjectError(key, bytes, maxBytes) {
|
|
174
|
+
return new DynamoDBLangGraphError(`S3 object ${truncateForLog(key)} exceeds the ${maxBytes}-byte maxDownloadBytes cap ` +
|
|
175
|
+
`(${bytes} bytes declared or read)`, ErrorCode.S3_OFFLOAD_FAILED, { operation: 'download', key });
|
|
176
|
+
}
|
|
177
|
+
function concat(chunks, total) {
|
|
178
|
+
const out = new Uint8Array(total);
|
|
179
|
+
let offset = 0;
|
|
180
|
+
for (const chunk of chunks) {
|
|
181
|
+
out.set(chunk, offset);
|
|
182
|
+
offset += chunk.length;
|
|
183
|
+
}
|
|
184
|
+
return out;
|
|
185
|
+
}
|
|
186
|
+
export async function readBodyBounded(body, key, maxBytes) {
|
|
187
|
+
const streaming = body;
|
|
188
|
+
const iterate = streaming[Symbol.asyncIterator];
|
|
189
|
+
if (typeof iterate !== 'function') {
|
|
190
|
+
const whole = new Uint8Array(await body.transformToByteArray());
|
|
191
|
+
if (whole.length > maxBytes)
|
|
192
|
+
throw oversizedObjectError(key, whole.length, maxBytes);
|
|
193
|
+
return whole;
|
|
194
|
+
}
|
|
195
|
+
const chunks = [];
|
|
196
|
+
let total = 0;
|
|
197
|
+
for await (const chunk of { [Symbol.asyncIterator]: iterate.bind(streaming) }) {
|
|
198
|
+
total += chunk.length;
|
|
199
|
+
if (total > maxBytes) {
|
|
200
|
+
streaming.destroy?.();
|
|
201
|
+
throw oversizedObjectError(key, total, maxBytes);
|
|
202
|
+
}
|
|
203
|
+
chunks.push(chunk);
|
|
204
|
+
}
|
|
205
|
+
return concat(chunks, total);
|
|
206
|
+
}
|
|
207
|
+
export async function deleteObjects(client, bucket, keys) {
|
|
208
|
+
if (keys.length === 0)
|
|
209
|
+
return [];
|
|
210
|
+
const { DeleteObjectsCommand } = await loadS3Sdk();
|
|
211
|
+
const failed = [];
|
|
212
|
+
for (let offset = 0; offset < keys.length; offset += S3_DELETE_BATCH_MAX) {
|
|
213
|
+
const chunk = keys.slice(offset, offset + S3_DELETE_BATCH_MAX);
|
|
214
|
+
const response = await client.send(new DeleteObjectsCommand({
|
|
215
|
+
Bucket: bucket,
|
|
216
|
+
Delete: { Objects: chunk.map((key) => ({ Key: key })), Quiet: true },
|
|
217
|
+
}));
|
|
218
|
+
for (const error of response.Errors ?? []) {
|
|
219
|
+
if (error.Key)
|
|
220
|
+
failed.push(error.Key);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return failed;
|
|
224
|
+
}
|
|
225
|
+
const PK_FIELD = 'dynamodb-pk-b64';
|
|
226
|
+
const SK_FIELD = 'dynamodb-sk-b64';
|
|
227
|
+
export function backlinkMetadata(row) {
|
|
228
|
+
return { [PK_FIELD]: encodeKeyPart(row.pk), [SK_FIELD]: encodeKeyPart(row.sk) };
|
|
229
|
+
}
|
|
230
|
+
async function backoffSleep(delayMs, release) {
|
|
231
|
+
try {
|
|
232
|
+
await sleep(fullJitter(delayMs, release.rng), release.signal);
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
return true;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
function ownedOnly(offloader, keys, release, scope) {
|
|
240
|
+
return keys.filter((key) => {
|
|
241
|
+
if (offloader.ownsKey(key, scope))
|
|
242
|
+
return true;
|
|
243
|
+
absorbLoggerFailure(() => release.logger.warn(`${release.operation}: refusing to delete an S3 object outside this row's scope`, {
|
|
244
|
+
key: truncateForLog(key),
|
|
245
|
+
}));
|
|
246
|
+
return false;
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
function selectOrphans(offloader, release) {
|
|
250
|
+
const present = release.keys.filter((key) => typeof key === 'string' && key.length > 0);
|
|
251
|
+
return release.scope ? ownedOnly(offloader, present, release, release.scope) : present;
|
|
252
|
+
}
|
|
253
|
+
export async function cleanUpS3Orphans(offloader, release) {
|
|
254
|
+
const orphans = selectOrphans(offloader, release);
|
|
255
|
+
if (orphans.length === 0)
|
|
256
|
+
return;
|
|
257
|
+
const maxAttempts = release.maxAttempts ?? S3_MAX_ATTEMPTS;
|
|
258
|
+
let delay = INITIAL_BACKOFF_DELAY_MS;
|
|
259
|
+
let lastError = new Error('S3 orphan cleanup did not run');
|
|
260
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
261
|
+
try {
|
|
262
|
+
const failed = await offloader.deleteBatch(orphans);
|
|
263
|
+
if (failed.length === 0)
|
|
264
|
+
return;
|
|
265
|
+
absorbLoggerFailure(() => release.logger.warn(`Some orphaned S3 objects could not be deleted after ${release.operation}; a lifecycle rule from ensureS3LifecycleRule() would sweep them, otherwise clean up manually`, { failedCount: failed.length }));
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
catch (error) {
|
|
269
|
+
lastError = error;
|
|
270
|
+
if (attempt >= maxAttempts || !isTransientS3Error(lastError))
|
|
271
|
+
break;
|
|
272
|
+
if (await backoffSleep(delay, release))
|
|
273
|
+
break;
|
|
274
|
+
delay = nextBackoffDelay(delay);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
absorbLoggerFailure(() => release.logger.warn(`Failed to clean up orphaned S3 objects after ${release.operation}; a lifecycle rule from ensureS3LifecycleRule() would sweep them, otherwise clean up manually`, { reason: truncateForLog(failureLabel(lastError)) }));
|
|
278
|
+
}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/**
|
|
3
2
|
* Hides how many calls run at once, and which failure a fan-out reports.
|
|
4
3
|
*
|
|
@@ -8,9 +7,6 @@
|
|
|
8
7
|
* work are decided here, as is the default of eight in flight, so tuning it
|
|
9
8
|
* touches no reader.
|
|
10
9
|
*/
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.DEFAULT_READ_CONCURRENCY = void 0;
|
|
13
|
-
exports.mapWithConcurrency = mapWithConcurrency;
|
|
14
10
|
/**
|
|
15
11
|
* Offloaded payloads decoded at once by one read (`getTuple` pending writes,
|
|
16
12
|
* `search` candidates, `getMessages`). Each offloaded row costs one S3 GET, so
|
|
@@ -18,7 +14,7 @@ exports.mapWithConcurrency = mapWithConcurrency;
|
|
|
18
14
|
* keeps the win without bursting a bucket. Also the recency-index shards one
|
|
19
15
|
* listing queries at once when the adapter names no `readConcurrency`.
|
|
20
16
|
*/
|
|
21
|
-
|
|
17
|
+
export declare const DEFAULT_READ_CONCURRENCY = 8;
|
|
22
18
|
/**
|
|
23
19
|
* Map `items` through `fn` with at most `limit` calls in flight.
|
|
24
20
|
*
|
|
@@ -44,35 +40,4 @@ exports.DEFAULT_READ_CONCURRENCY = 8;
|
|
|
44
40
|
* at all: tested by value, it would be swallowed, and its slot in the results
|
|
45
41
|
* would stay a hole.
|
|
46
42
|
*/
|
|
47
|
-
|
|
48
|
-
const results = [];
|
|
49
|
-
let next = 0;
|
|
50
|
-
let failed = false;
|
|
51
|
-
let failure;
|
|
52
|
-
const worker = async () => {
|
|
53
|
-
while (!failed && next < items.length) {
|
|
54
|
-
const index = next;
|
|
55
|
-
next += 1;
|
|
56
|
-
try {
|
|
57
|
-
results[index] = await fn(items[index], index);
|
|
58
|
-
}
|
|
59
|
-
catch (error) {
|
|
60
|
-
if (!failed) {
|
|
61
|
-
failed = true;
|
|
62
|
-
failure = error;
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
};
|
|
67
|
-
const workers = Math.min(limit >= 1 ? Math.floor(limit) : 1, Math.max(items.length, 1));
|
|
68
|
-
await Promise.all(Array.from({ length: workers }, worker));
|
|
69
|
-
// `failure` is stored `Error | undefined` only because that is as far as a
|
|
70
|
-
// `catch` binding's value can be named without `unknown`, which is banned
|
|
71
|
-
// in src; the JSDoc above states the real contract — whatever `fn` rejected
|
|
72
|
-
// with, unchanged, and that can be `undefined` itself. The assertion below
|
|
73
|
-
// changes nothing at runtime; it only lets `only-throw-error` see the type
|
|
74
|
-
// this throw already had before that rule existed.
|
|
75
|
-
if (failed)
|
|
76
|
-
throw failure;
|
|
77
|
-
return results;
|
|
78
|
-
}
|
|
43
|
+
export declare function mapWithConcurrency<T, R>(items: readonly T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export const DEFAULT_READ_CONCURRENCY = 8;
|
|
2
|
+
export async function mapWithConcurrency(items, limit, fn) {
|
|
3
|
+
const results = [];
|
|
4
|
+
let next = 0;
|
|
5
|
+
let failed = false;
|
|
6
|
+
let failure;
|
|
7
|
+
const worker = async () => {
|
|
8
|
+
while (!failed && next < items.length) {
|
|
9
|
+
const index = next;
|
|
10
|
+
next += 1;
|
|
11
|
+
try {
|
|
12
|
+
results[index] = await fn(items[index], index);
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
if (!failed) {
|
|
16
|
+
failed = true;
|
|
17
|
+
failure = error;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
const workers = Math.min(limit >= 1 ? Math.floor(limit) : 1, Math.max(items.length, 1));
|
|
23
|
+
await Promise.all(Array.from({ length: workers }, worker));
|
|
24
|
+
if (failed)
|
|
25
|
+
throw failure;
|
|
26
|
+
return results;
|
|
27
|
+
}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/**
|
|
3
2
|
* Hides what an abort looks like.
|
|
4
3
|
*
|
|
@@ -8,12 +7,8 @@
|
|
|
8
7
|
* twice, and one test on the library's brand and that code is what tells a
|
|
9
8
|
* cancellation from a failure, so no call site inspects a reason's shape.
|
|
10
9
|
*/
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
exports.abortErrorFrom = abortErrorFrom;
|
|
14
|
-
const base_error_1 = require("../errors/base-error");
|
|
15
|
-
const error_code_1 = require("../errors/error-code");
|
|
16
|
-
const errors_1 = require("../errors/errors");
|
|
10
|
+
import { DynamoDBLangGraphError } from '../errors/base-error.js';
|
|
11
|
+
import { ErrorCode } from '../errors/error-code.js';
|
|
17
12
|
/**
|
|
18
13
|
* Whether `error` is a cancellation rather than a failure.
|
|
19
14
|
*
|
|
@@ -31,9 +26,7 @@ const errors_1 = require("../errors/errors");
|
|
|
31
26
|
* Throws: **nothing**, for any value. A value that cannot carry a property is
|
|
32
27
|
* not a cancellation, which is the answer an uncoded `Error` gets too.
|
|
33
28
|
*/
|
|
34
|
-
function isAbortError(error)
|
|
35
|
-
return error !== undefined && (0, base_error_1.hasErrorCode)(error, error_code_1.ErrorCode.ABORTED);
|
|
36
|
-
}
|
|
29
|
+
export declare function isAbortError(error: Error | undefined): error is DynamoDBLangGraphError<ErrorCode.ABORTED>;
|
|
37
30
|
/**
|
|
38
31
|
* This library's error for an aborted `signal`.
|
|
39
32
|
*
|
|
@@ -51,9 +44,4 @@ function isAbortError(error) {
|
|
|
51
44
|
* Guarantees: `code === 'ABORTED'` holds however the signal was aborted, so a
|
|
52
45
|
* caller branches on the code rather than on the reason's shape.
|
|
53
46
|
*/
|
|
54
|
-
function abortErrorFrom(signal)
|
|
55
|
-
const reason = signal.reason;
|
|
56
|
-
if (isAbortError(reason))
|
|
57
|
-
return reason;
|
|
58
|
-
return (0, errors_1.abortError)('Operation aborted', reason === undefined ? undefined : (0, base_error_1.toError)(reason));
|
|
59
|
-
}
|
|
47
|
+
export declare function abortErrorFrom(signal: AbortSignal): DynamoDBLangGraphError<ErrorCode.ABORTED>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { hasErrorCode, toError } from '../errors/base-error.js';
|
|
2
|
+
import { ErrorCode } from '../errors/error-code.js';
|
|
3
|
+
import { abortError } from '../errors/errors.js';
|
|
4
|
+
export function isAbortError(error) {
|
|
5
|
+
return error !== undefined && hasErrorCode(error, ErrorCode.ABORTED);
|
|
6
|
+
}
|
|
7
|
+
export function abortErrorFrom(signal) {
|
|
8
|
+
const reason = signal.reason;
|
|
9
|
+
if (isAbortError(reason))
|
|
10
|
+
return reason;
|
|
11
|
+
return abortError('Operation aborted', reason === undefined ? undefined : toError(reason));
|
|
12
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hides BatchWriteItem.
|
|
3
|
+
*
|
|
4
|
+
* A call carries at most twenty-five requests, and the service may hand some
|
|
5
|
+
* back unprocessed; they are re-sent with backoff until a bound, and a batch
|
|
6
|
+
* that still cannot finish is reported with how much of it did.
|
|
7
|
+
*/
|
|
8
|
+
import type { DynamoDBDocumentLike, WriteRequest } from './client.js';
|
|
9
|
+
import { type RetryOptions } from './retry.js';
|
|
10
|
+
/**
|
|
11
|
+
* DynamoDB BatchWriteItem maximum requests per call.
|
|
12
|
+
*
|
|
13
|
+
* @see https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
|
|
14
|
+
*/
|
|
15
|
+
export declare const BATCH_WRITE_MAX = 25;
|
|
16
|
+
/** Maximum retries draining UnprocessedItems / UnprocessedKeys. */
|
|
17
|
+
export declare const MAX_UNPROCESSED_RETRIES = 10;
|
|
18
|
+
/**
|
|
19
|
+
* Write an arbitrary number of requests, chunked into batches of 25 (the
|
|
20
|
+
* BatchWriteItem limit). Every chunk is attempted regardless of an earlier
|
|
21
|
+
* chunk's failure — order-independent writes (deletes/puts) should never
|
|
22
|
+
* lose "later" chunks just because an earlier one failed. If any chunk fails,
|
|
23
|
+
* throws `BATCH_WRITE_INCOMPLETE` reporting exactly how many
|
|
24
|
+
* chunks succeeded vs. failed, and exactly how many individual writes
|
|
25
|
+
* persisted, once every chunk has been attempted.
|
|
26
|
+
*
|
|
27
|
+
* Accepts: `requests` — any number, in any order, of writes that do not depend
|
|
28
|
+
* on each other; none is a no-op. `options` — the drain's retry budget and
|
|
29
|
+
* signal.
|
|
30
|
+
*
|
|
31
|
+
* Returns: nothing, and only when every request persisted.
|
|
32
|
+
*
|
|
33
|
+
* Throws: `ABORTED` the moment a chunk reports one, unwrapped and with no
|
|
34
|
+
* further chunk attempted — a caller who cancelled did not encounter a fault,
|
|
35
|
+
* and spending the remaining requests on a cancelled call is the opposite of
|
|
36
|
+
* what the cancel asked for. Otherwise `BATCH_WRITE_INCOMPLETE`,
|
|
37
|
+
* once every chunk has been attempted, reporting how many chunks succeeded and
|
|
38
|
+
* how many individual writes persisted. Its one caller — the rollback in
|
|
39
|
+
* history/internal/append.ts — type-asserts a caught error straight to
|
|
40
|
+
* its code's details (not `instanceof`, banned repo-wide) instead of
|
|
41
|
+
* narrowing it, on the narrower guarantee that it passes no signal, so the
|
|
42
|
+
* abort path cannot arise there; a call site that does pass one must narrow
|
|
43
|
+
* instead.
|
|
44
|
+
*
|
|
45
|
+
* Guarantees: every chunk is attempted regardless of an earlier chunk's
|
|
46
|
+
* failure — these writes are order-independent, so losing the later ones to an
|
|
47
|
+
* earlier failure would delete less than the caller asked and report no more
|
|
48
|
+
* for it. A cancel is the one exception, because it is not a failure. The
|
|
49
|
+
* count the error carries is exact, which is what lets a compensating caller
|
|
50
|
+
* revert precisely what landed, and it never counts a chunk whose error
|
|
51
|
+
* carried no count of its own.
|
|
52
|
+
*/
|
|
53
|
+
export declare function batchWriteAll(client: DynamoDBDocumentLike, tableName: string, requests: WriteRequest[], options?: DrainOptions): Promise<void>;
|
|
54
|
+
/** Backoff/abort options shared by the drain helpers. */
|
|
55
|
+
export interface DrainOptions {
|
|
56
|
+
/** The adapter's retry options, applied to every BatchWriteItem round. */
|
|
57
|
+
retry?: RetryOptions;
|
|
58
|
+
signal?: AbortSignal;
|
|
59
|
+
rng?: () => number;
|
|
60
|
+
maxRetries?: number;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Write `requests` with `BatchWriteItem`, re-submitting what DynamoDB returns
|
|
64
|
+
* as `UnprocessedItems` until the batch drains.
|
|
65
|
+
*
|
|
66
|
+
* Accepts: `requests` — any length, including empty, which issues no request.
|
|
67
|
+
* `options.maxRetries` — re-submission rounds, default
|
|
68
|
+
* {@link MAX_UNPROCESSED_RETRIES}. `options.retry` — the adapter's policy, whose
|
|
69
|
+
* `baseDelayMs` and `maxDelayMs` set the backoff between rounds, so one policy
|
|
70
|
+
* governs every wait this package performs. `options.signal` — aborts a wait.
|
|
71
|
+
*
|
|
72
|
+
* Returns: nothing; success means every request persisted.
|
|
73
|
+
*
|
|
74
|
+
* Throws: `BATCH_WRITE_INCOMPLETE` when the batch does not drain
|
|
75
|
+
* within the rounds allowed, or when a round's write call fails outright — its
|
|
76
|
+
* `succeededCount` is every earlier round's confirmed persists and the
|
|
77
|
+
* triggering error is the `cause`. An `ABORTED` error passes through unchanged: a
|
|
78
|
+
* caller who cancelled did not get an incomplete batch write, and every
|
|
79
|
+
* cancellable method documents `ABORTED`.
|
|
80
|
+
*
|
|
81
|
+
* Guarantees: **every** error this function throws is one of those two — a
|
|
82
|
+
* `BATCH_WRITE_INCOMPLETE` error carrying an accurate `succeededCount`, or an
|
|
83
|
+
* `ABORTED` error. {@link batchWriteAll} adds up those counts across chunks and
|
|
84
|
+
* depends on it.
|
|
85
|
+
*/
|
|
86
|
+
export declare function drainUnprocessedWrites(client: DynamoDBDocumentLike, tableName: string, requests: WriteRequest[], options?: DrainOptions): Promise<void>;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { hasErrorCode } from '../errors/base-error.js';
|
|
2
|
+
import { ErrorCode } from '../errors/error-code.js';
|
|
3
|
+
import { batchWriteAllIncompleteError, batchWriteIncompleteError } from '../errors/errors.js';
|
|
4
|
+
import { isAbortError } from './abort.js';
|
|
5
|
+
import { backoffCapMs, fullJitter, INITIAL_BACKOFF_DELAY_MS, nextBackoffDelay, sleep, withDynamoDBRetry, } from './retry.js';
|
|
6
|
+
export const BATCH_WRITE_MAX = 25;
|
|
7
|
+
export const MAX_UNPROCESSED_RETRIES = 10;
|
|
8
|
+
export async function batchWriteAll(client, tableName, requests, options = {}) {
|
|
9
|
+
const totalChunks = Math.ceil(requests.length / BATCH_WRITE_MAX);
|
|
10
|
+
let succeededChunks = 0;
|
|
11
|
+
let succeededCount = 0;
|
|
12
|
+
const failedChunks = [];
|
|
13
|
+
for (let offset = 0; offset < requests.length; offset += BATCH_WRITE_MAX) {
|
|
14
|
+
const chunk = requests.slice(offset, offset + BATCH_WRITE_MAX);
|
|
15
|
+
try {
|
|
16
|
+
await drainUnprocessedWrites(client, tableName, chunk, options);
|
|
17
|
+
succeededChunks += 1;
|
|
18
|
+
succeededCount += chunk.length;
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
const failure = error;
|
|
22
|
+
if (!hasErrorCode(failure, ErrorCode.BATCH_WRITE_INCOMPLETE))
|
|
23
|
+
throw failure;
|
|
24
|
+
failedChunks.push(failure);
|
|
25
|
+
succeededCount += failure.details.succeededCount;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
if (failedChunks.length > 0) {
|
|
29
|
+
throw batchWriteAllIncompleteError({
|
|
30
|
+
succeeded: succeededChunks,
|
|
31
|
+
total: totalChunks,
|
|
32
|
+
failures: failedChunks,
|
|
33
|
+
succeededCount,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function drainBackoff(retry) {
|
|
38
|
+
const base = retry?.baseDelayMs ?? INITIAL_BACKOFF_DELAY_MS;
|
|
39
|
+
const max = backoffCapMs(base, retry?.maxDelayMs);
|
|
40
|
+
return { base: Math.min(base, max), max };
|
|
41
|
+
}
|
|
42
|
+
function drainFailure(error, succeededCount, pending, retries) {
|
|
43
|
+
if (isAbortError(error))
|
|
44
|
+
return error;
|
|
45
|
+
return batchWriteIncompleteError(succeededCount, pending, retries, error);
|
|
46
|
+
}
|
|
47
|
+
export async function drainUnprocessedWrites(client, tableName, requests, options = {}) {
|
|
48
|
+
if (requests.length === 0)
|
|
49
|
+
return;
|
|
50
|
+
const maxRetries = options.maxRetries ?? MAX_UNPROCESSED_RETRIES;
|
|
51
|
+
const initialCount = requests.length;
|
|
52
|
+
const { base: baseDelay, max: maxDelay } = drainBackoff(options.retry);
|
|
53
|
+
let pending = requests;
|
|
54
|
+
let delay = baseDelay;
|
|
55
|
+
let retries = 0;
|
|
56
|
+
while (pending.length > 0) {
|
|
57
|
+
try {
|
|
58
|
+
const result = await withDynamoDBRetry((request) => client.batchWrite({ RequestItems: { [tableName]: pending } }, request), { ...options.retry, signal: options.signal });
|
|
59
|
+
const leftover = result.UnprocessedItems?.[tableName] ?? [];
|
|
60
|
+
if (leftover.length === 0)
|
|
61
|
+
return;
|
|
62
|
+
pending = leftover;
|
|
63
|
+
retries += 1;
|
|
64
|
+
if (retries > maxRetries)
|
|
65
|
+
break;
|
|
66
|
+
await sleep(fullJitter(delay, options.rng), options.signal);
|
|
67
|
+
delay = nextBackoffDelay(delay, maxDelay);
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
throw drainFailure(error, initialCount - pending.length, pending, retries);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
throw batchWriteIncompleteError(initialCount - pending.length, pending, maxRetries);
|
|
74
|
+
}
|