@farukada/aws-langgraph-dynamodb-ts 1.0.0-rc.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +390 -193
- package/dist/{backfill → cjs/backfill}/backfill.d.ts +15 -12
- package/dist/cjs/backfill/backfill.js +177 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/delete-thread.d.ts +1 -1
- package/dist/cjs/checkpointer/actions/delete-thread.js +26 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/get-tuple.d.ts +4 -3
- package/dist/cjs/checkpointer/actions/get-tuple.js +22 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/list.d.ts +7 -4
- package/dist/cjs/checkpointer/actions/list.js +70 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/put-writes.d.ts +8 -6
- package/dist/cjs/checkpointer/actions/put-writes.js +21 -0
- package/dist/{checkpointer → cjs/checkpointer}/actions/put.d.ts +21 -15
- package/dist/cjs/checkpointer/actions/put.js +79 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/delta-history.d.ts +38 -26
- package/dist/cjs/checkpointer/internal/delta-history.js +131 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/listing.d.ts +4 -4
- package/dist/cjs/checkpointer/internal/listing.js +104 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/parse.d.ts +51 -15
- package/dist/cjs/checkpointer/internal/parse.js +184 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/pending-writes.d.ts +64 -25
- package/dist/cjs/checkpointer/internal/pending-writes.js +241 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/read.d.ts +15 -10
- package/dist/cjs/checkpointer/internal/read.js +132 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/rows.d.ts +94 -21
- package/dist/cjs/checkpointer/internal/rows.js +356 -0
- package/dist/{checkpointer → cjs/checkpointer}/internal/setup.d.ts +13 -20
- package/dist/cjs/checkpointer/internal/setup.js +25 -0
- package/dist/{checkpointer → cjs/checkpointer}/saver.d.ts +72 -36
- package/dist/cjs/checkpointer/saver.js +56 -0
- package/dist/{checkpointer → cjs/checkpointer}/types.d.ts +1 -1
- package/dist/cjs/checkpointer/types.js +2 -0
- package/dist/{factory → cjs/factory}/factory.d.ts +25 -20
- package/dist/cjs/factory/factory.js +137 -0
- package/dist/{factory → cjs/factory}/types.d.ts +12 -12
- package/dist/cjs/factory/types.js +2 -0
- package/dist/{history → cjs/history}/actions/add-messages.d.ts +18 -10
- package/dist/cjs/history/actions/add-messages.js +17 -0
- package/dist/{history → cjs/history}/actions/clear.d.ts +1 -1
- package/dist/cjs/history/actions/clear.js +29 -0
- package/dist/{history → cjs/history}/actions/get-messages.d.ts +11 -9
- package/dist/cjs/history/actions/get-messages.js +58 -0
- package/dist/{history → cjs/history}/actions/list-sessions.d.ts +3 -3
- package/dist/cjs/history/actions/list-sessions.js +67 -0
- package/dist/{history → cjs/history}/actions/reconcile-count.d.ts +13 -9
- package/dist/cjs/history/actions/reconcile-count.js +8 -0
- package/dist/{history → cjs/history}/chat-message-history.d.ts +62 -30
- package/dist/cjs/history/chat-message-history.js +61 -0
- package/dist/{history → cjs/history}/internal/append.d.ts +64 -32
- package/dist/cjs/history/internal/append.js +267 -0
- package/dist/{history → cjs/history}/internal/message-read.d.ts +3 -3
- package/dist/cjs/history/internal/message-read.js +93 -0
- package/dist/{history → cjs/history}/internal/parse.d.ts +2 -2
- package/dist/cjs/history/internal/parse.js +113 -0
- package/dist/{history → cjs/history}/internal/rows.d.ts +11 -10
- package/dist/cjs/history/internal/rows.js +96 -0
- package/dist/{history → cjs/history}/internal/session.d.ts +41 -22
- package/dist/cjs/history/internal/session.js +296 -0
- package/dist/{history → cjs/history}/internal/setup.d.ts +2 -2
- package/dist/cjs/history/internal/setup.js +60 -0
- package/dist/{history → cjs/history}/session-adapter.d.ts +0 -6
- package/dist/cjs/history/session-adapter.js +38 -0
- package/dist/{history → cjs/history}/types.d.ts +18 -15
- package/dist/cjs/history/types.js +2 -0
- package/dist/{index.d.ts → cjs/index.d.ts} +27 -27
- package/dist/cjs/index.js +25 -0
- package/dist/cjs/package.json +1 -0
- package/dist/{shared → cjs/shared}/adapter.d.ts +27 -16
- package/dist/cjs/shared/adapter.js +70 -0
- package/dist/cjs/shared/clock.js +14 -0
- package/dist/{shared → cjs/shared}/codec/codec.d.ts +37 -20
- package/dist/cjs/shared/codec/codec.js +162 -0
- package/dist/{shared → cjs/shared}/codec/compression.d.ts +9 -4
- package/dist/cjs/shared/codec/compression.js +45 -0
- package/dist/cjs/shared/codec/json-serde.js +58 -0
- package/dist/{shared → cjs/shared}/codec/s3/client-types.d.ts +2 -1
- package/dist/cjs/shared/codec/s3/client-types.js +6 -0
- package/dist/{shared → cjs/shared}/codec/s3/client.d.ts +25 -7
- package/dist/cjs/shared/codec/s3/client.js +78 -0
- package/dist/{shared → cjs/shared}/codec/s3/config.d.ts +12 -7
- package/dist/cjs/shared/codec/s3/config.js +92 -0
- package/dist/{shared → cjs/shared}/codec/s3/lifecycle.d.ts +71 -17
- package/dist/cjs/shared/codec/s3/lifecycle.js +223 -0
- package/dist/{shared → cjs/shared}/codec/s3/offloader.d.ts +42 -29
- package/dist/cjs/shared/codec/s3/offloader.js +289 -0
- package/dist/cjs/shared/concurrency.js +31 -0
- package/dist/{shared → cjs/shared}/dynamodb/abort.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/abort.js +16 -0
- package/dist/{shared → cjs/shared}/dynamodb/batch-write.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/batch-write.js +79 -0
- package/dist/cjs/shared/dynamodb/cancellation.js +37 -0
- package/dist/{shared → cjs/shared}/dynamodb/client.d.ts +24 -22
- package/dist/cjs/shared/dynamodb/client.js +40 -0
- package/dist/{shared → cjs/shared}/dynamodb/idempotent-write.d.ts +64 -19
- package/dist/cjs/shared/dynamodb/idempotent-write.js +173 -0
- package/dist/{shared → cjs/shared}/dynamodb/paginate.d.ts +2 -2
- package/dist/cjs/shared/dynamodb/paginate.js +90 -0
- package/dist/{shared → cjs/shared}/dynamodb/partition-delete.d.ts +6 -6
- package/dist/cjs/shared/dynamodb/partition-delete.js +177 -0
- package/dist/{shared → cjs/shared}/dynamodb/recency-index.d.ts +27 -15
- package/dist/cjs/shared/dynamodb/recency-index.js +146 -0
- package/dist/{shared → cjs/shared}/dynamodb/retry.d.ts +37 -6
- package/dist/cjs/shared/dynamodb/retry.js +193 -0
- package/dist/cjs/shared/dynamodb/row-size.d.ts +33 -0
- package/dist/cjs/shared/dynamodb/row-size.js +30 -0
- package/dist/{shared → cjs/shared}/dynamodb/table-schema.d.ts +27 -18
- package/dist/cjs/shared/dynamodb/table-schema.js +52 -0
- package/dist/{shared → cjs/shared}/errors/base-error.d.ts +42 -7
- package/dist/cjs/shared/errors/base-error.js +73 -0
- package/dist/{shared → cjs/shared}/errors/boundary.d.ts +29 -14
- package/dist/cjs/shared/errors/boundary.js +61 -0
- package/dist/cjs/shared/errors/classify.d.ts +206 -0
- package/dist/cjs/shared/errors/classify.js +232 -0
- package/dist/{shared → cjs/shared}/errors/error-code.d.ts +17 -10
- package/dist/cjs/shared/errors/error-code.js +27 -0
- package/dist/{shared → cjs/shared}/errors/errors.d.ts +75 -9
- package/dist/cjs/shared/errors/errors.js +123 -0
- package/dist/cjs/shared/logging/logger.js +30 -0
- package/dist/{shared → cjs/shared}/logging/redaction.d.ts +8 -4
- package/dist/cjs/shared/logging/redaction.js +145 -0
- package/dist/{shared → cjs/shared}/logging/secret-patterns.d.ts +30 -7
- package/dist/cjs/shared/logging/secret-patterns.js +93 -0
- package/dist/cjs/shared/logging/truncate.js +31 -0
- package/dist/{shared → cjs/shared}/options.d.ts +27 -11
- package/dist/cjs/shared/options.js +2 -0
- package/dist/cjs/shared/ulid.js +83 -0
- package/dist/{shared → cjs/shared}/validation/collaborators.d.ts +21 -2
- package/dist/cjs/shared/validation/collaborators.js +79 -0
- package/dist/{shared → cjs/shared}/validation/option-shape.d.ts +7 -5
- package/dist/cjs/shared/validation/option-shape.js +31 -0
- package/dist/{shared → cjs/shared}/validation/options.d.ts +4 -4
- package/dist/cjs/shared/validation/options.js +166 -0
- package/dist/cjs/shared/validation/primitives.js +114 -0
- package/dist/{shared → cjs/shared}/validation/ttl.d.ts +26 -9
- package/dist/cjs/shared/validation/ttl.js +52 -0
- package/dist/{store → cjs/store}/actions/list-namespaces.d.ts +5 -5
- package/dist/cjs/store/actions/list-namespaces.js +92 -0
- package/dist/{store → cjs/store}/actions/put.d.ts +11 -4
- package/dist/cjs/store/actions/put.js +49 -0
- package/dist/{store → cjs/store}/actions/reconcile-vector-index.d.ts +5 -5
- package/dist/cjs/store/actions/reconcile-vector-index.js +13 -0
- package/dist/{store → cjs/store}/actions/search.d.ts +2 -2
- package/dist/cjs/store/actions/search.js +26 -0
- package/dist/cjs/store/internal/batch-plan.d.ts +56 -0
- package/dist/cjs/store/internal/batch-plan.js +32 -0
- package/dist/{store → cjs/store}/internal/filter.d.ts +3 -3
- package/dist/cjs/store/internal/filter.js +64 -0
- package/dist/cjs/store/internal/get-item.d.ts +77 -0
- package/dist/cjs/store/internal/get-item.js +58 -0
- package/dist/{store → cjs/store}/internal/item-write.d.ts +72 -32
- package/dist/cjs/store/internal/item-write.js +181 -0
- package/dist/{store → cjs/store}/internal/parse.d.ts +15 -10
- package/dist/cjs/store/internal/parse.js +180 -0
- package/dist/{store → cjs/store}/internal/rows.d.ts +38 -34
- package/dist/cjs/store/internal/rows.js +166 -0
- package/dist/{store → cjs/store}/internal/semantic-search.d.ts +2 -2
- package/dist/cjs/store/internal/semantic-search.js +211 -0
- package/dist/{store → cjs/store}/internal/setup.d.ts +10 -15
- package/dist/cjs/store/internal/setup.js +103 -0
- package/dist/{store → cjs/store}/internal/table-search.d.ts +5 -5
- package/dist/cjs/store/internal/table-search.js +123 -0
- package/dist/{store → cjs/store}/internal/vector-index.d.ts +26 -15
- package/dist/cjs/store/internal/vector-index.js +269 -0
- package/dist/{store → cjs/store}/store.d.ts +82 -39
- package/dist/cjs/store/store.js +97 -0
- package/dist/{store → cjs/store}/types.d.ts +25 -10
- package/dist/cjs/store/types.js +2 -0
- package/dist/cjs/store/vector-backend.js +4 -0
- package/dist/esm/backfill/backfill.d.ts +171 -0
- package/dist/esm/backfill/backfill.js +170 -0
- package/dist/{checkpointer/actions/delete-thread.js → esm/checkpointer/actions/delete-thread.d.ts} +4 -26
- package/dist/esm/checkpointer/actions/delete-thread.js +23 -0
- package/dist/{checkpointer/actions/get-tuple.js → esm/checkpointer/actions/get-tuple.d.ts} +7 -24
- package/dist/esm/checkpointer/actions/get-tuple.js +19 -0
- package/dist/esm/checkpointer/actions/list.d.ts +55 -0
- package/dist/esm/checkpointer/actions/list.js +67 -0
- package/dist/{checkpointer/actions/put-writes.js → esm/checkpointer/actions/put-writes.d.ts} +10 -37
- package/dist/esm/checkpointer/actions/put-writes.js +18 -0
- package/dist/esm/checkpointer/actions/put.d.ts +95 -0
- package/dist/esm/checkpointer/actions/put.js +75 -0
- package/dist/esm/checkpointer/internal/delta-history.d.ts +124 -0
- package/dist/esm/checkpointer/internal/delta-history.js +126 -0
- package/dist/{checkpointer/internal/listing.js → esm/checkpointer/internal/listing.d.ts} +22 -118
- package/dist/esm/checkpointer/internal/listing.js +95 -0
- package/dist/esm/checkpointer/internal/parse.d.ts +298 -0
- package/dist/esm/checkpointer/internal/parse.js +169 -0
- package/dist/esm/checkpointer/internal/pending-writes.d.ts +314 -0
- package/dist/esm/checkpointer/internal/pending-writes.js +229 -0
- package/dist/esm/checkpointer/internal/read.d.ts +135 -0
- package/dist/esm/checkpointer/internal/read.js +125 -0
- package/dist/esm/checkpointer/internal/rows.d.ts +644 -0
- package/dist/esm/checkpointer/internal/rows.js +323 -0
- package/dist/esm/checkpointer/internal/setup.d.ts +47 -0
- package/dist/esm/checkpointer/internal/setup.js +21 -0
- package/dist/{checkpointer/saver.js → esm/checkpointer/saver.d.ts} +84 -87
- package/dist/esm/checkpointer/saver.js +52 -0
- package/dist/esm/checkpointer/types.d.ts +51 -0
- package/dist/esm/checkpointer/types.js +1 -0
- package/dist/esm/factory/factory.d.ts +154 -0
- package/dist/esm/factory/factory.js +133 -0
- package/dist/esm/factory/types.d.ts +76 -0
- package/dist/esm/factory/types.js +1 -0
- package/dist/{history/actions/add-messages.js → esm/history/actions/add-messages.d.ts} +20 -26
- package/dist/esm/history/actions/add-messages.js +14 -0
- package/dist/{history/actions/clear.js → esm/history/actions/clear.d.ts} +4 -35
- package/dist/esm/history/actions/clear.js +26 -0
- package/dist/esm/history/actions/get-messages.d.ts +60 -0
- package/dist/esm/history/actions/get-messages.js +55 -0
- package/dist/esm/history/actions/list-sessions.d.ts +55 -0
- package/dist/esm/history/actions/list-sessions.js +64 -0
- package/dist/{history/actions/reconcile-count.js → esm/history/actions/reconcile-count.d.ts} +14 -16
- package/dist/esm/history/actions/reconcile-count.js +5 -0
- package/dist/{history/chat-message-history.js → esm/history/chat-message-history.d.ts} +76 -87
- package/dist/esm/history/chat-message-history.js +57 -0
- package/dist/esm/history/internal/append.d.ts +244 -0
- package/dist/esm/history/internal/append.js +259 -0
- package/dist/esm/history/internal/message-read.d.ts +84 -0
- package/dist/esm/history/internal/message-read.js +89 -0
- package/dist/esm/history/internal/parse.d.ts +153 -0
- package/dist/esm/history/internal/parse.js +105 -0
- package/dist/{history/internal/rows.js → esm/history/internal/rows.d.ts} +58 -112
- package/dist/esm/history/internal/rows.js +83 -0
- package/dist/esm/history/internal/session.d.ts +350 -0
- package/dist/esm/history/internal/session.js +285 -0
- package/dist/esm/history/internal/setup.d.ts +64 -0
- package/dist/esm/history/internal/setup.js +56 -0
- package/dist/{history/session-adapter.js → esm/history/session-adapter.d.ts} +25 -39
- package/dist/esm/history/session-adapter.js +34 -0
- package/dist/esm/history/types.d.ts +102 -0
- package/dist/esm/history/types.js +1 -0
- package/dist/esm/index.d.ts +53 -0
- package/dist/esm/index.js +10 -0
- package/dist/esm/shared/adapter.d.ts +146 -0
- package/dist/esm/shared/adapter.js +65 -0
- package/dist/{shared/clock.js → esm/shared/clock.d.ts} +3 -14
- package/dist/esm/shared/clock.js +9 -0
- package/dist/esm/shared/codec/codec.d.ts +336 -0
- package/dist/esm/shared/codec/codec.js +150 -0
- package/dist/esm/shared/codec/compression.d.ts +66 -0
- package/dist/esm/shared/codec/compression.js +40 -0
- package/dist/esm/shared/codec/json-serde.d.ts +80 -0
- package/dist/esm/shared/codec/json-serde.js +54 -0
- package/dist/esm/shared/codec/s3/client-types.d.ts +54 -0
- package/dist/esm/shared/codec/s3/client-types.js +3 -0
- package/dist/esm/shared/codec/s3/client.d.ts +63 -0
- package/dist/esm/shared/codec/s3/client.js +40 -0
- package/dist/{shared/codec/s3/config.js → esm/shared/codec/s3/config.d.ts} +45 -93
- package/dist/esm/shared/codec/s3/config.js +79 -0
- package/dist/esm/shared/codec/s3/lifecycle.d.ts +220 -0
- package/dist/esm/shared/codec/s3/lifecycle.js +214 -0
- package/dist/esm/shared/codec/s3/offloader.d.ts +428 -0
- package/dist/esm/shared/codec/s3/offloader.js +278 -0
- package/dist/{shared/concurrency.js → esm/shared/concurrency.d.ts} +2 -37
- package/dist/esm/shared/concurrency.js +27 -0
- package/dist/{shared/dynamodb/abort.js → esm/shared/dynamodb/abort.d.ts} +4 -16
- package/dist/esm/shared/dynamodb/abort.js +12 -0
- package/dist/esm/shared/dynamodb/batch-write.d.ts +86 -0
- package/dist/esm/shared/dynamodb/batch-write.js +74 -0
- package/dist/{shared/dynamodb/cancellation.js → esm/shared/dynamodb/cancellation.d.ts} +25 -53
- package/dist/esm/shared/dynamodb/cancellation.js +30 -0
- package/dist/{shared/dynamodb/client.js → esm/shared/dynamodb/client.d.ts} +74 -64
- package/dist/esm/shared/dynamodb/client.js +35 -0
- package/dist/{shared/dynamodb/idempotent-write.js → esm/shared/dynamodb/idempotent-write.d.ts} +172 -169
- package/dist/esm/shared/dynamodb/idempotent-write.js +153 -0
- package/dist/esm/shared/dynamodb/paginate.d.ts +111 -0
- package/dist/esm/shared/dynamodb/paginate.js +84 -0
- package/dist/esm/shared/dynamodb/partition-delete.d.ts +204 -0
- package/dist/esm/shared/dynamodb/partition-delete.js +171 -0
- package/dist/esm/shared/dynamodb/recency-index.d.ts +243 -0
- package/dist/esm/shared/dynamodb/recency-index.js +136 -0
- package/dist/esm/shared/dynamodb/retry.d.ts +317 -0
- package/dist/esm/shared/dynamodb/retry.js +180 -0
- package/dist/esm/shared/dynamodb/row-size.d.ts +33 -0
- package/dist/esm/shared/dynamodb/row-size.js +26 -0
- package/dist/{shared/dynamodb/table-schema.js → esm/shared/dynamodb/table-schema.d.ts} +57 -67
- package/dist/esm/shared/dynamodb/table-schema.js +43 -0
- package/dist/esm/shared/errors/base-error.d.ts +231 -0
- package/dist/esm/shared/errors/base-error.js +64 -0
- package/dist/esm/shared/errors/boundary.d.ts +86 -0
- package/dist/esm/shared/errors/boundary.js +54 -0
- package/dist/esm/shared/errors/classify.d.ts +206 -0
- package/dist/esm/shared/errors/classify.js +220 -0
- package/dist/{shared/errors/error-code.js → esm/shared/errors/error-code.d.ts} +39 -43
- package/dist/esm/shared/errors/error-code.js +24 -0
- package/dist/{shared/errors/errors.js → esm/shared/errors/errors.d.ts} +92 -98
- package/dist/esm/shared/errors/errors.js +110 -0
- package/dist/{shared/logging/logger.js → esm/shared/logging/logger.d.ts} +22 -52
- package/dist/esm/shared/logging/logger.js +25 -0
- package/dist/esm/shared/logging/redaction.d.ts +123 -0
- package/dist/esm/shared/logging/redaction.js +140 -0
- package/dist/{shared/logging/secret-patterns.js → esm/shared/logging/secret-patterns.d.ts} +46 -87
- package/dist/esm/shared/logging/secret-patterns.js +83 -0
- package/dist/{shared/logging/truncate.js → esm/shared/logging/truncate.d.ts} +6 -40
- package/dist/esm/shared/logging/truncate.js +25 -0
- package/dist/esm/shared/options.d.ts +100 -0
- package/dist/esm/shared/options.js +1 -0
- package/dist/{shared/ulid.js → esm/shared/ulid.d.ts} +4 -84
- package/dist/esm/shared/ulid.js +77 -0
- package/dist/{shared/validation/collaborators.js → esm/shared/validation/collaborators.d.ts} +37 -65
- package/dist/esm/shared/validation/collaborators.js +70 -0
- package/dist/{shared/validation/option-shape.js → esm/shared/validation/option-shape.d.ts} +14 -36
- package/dist/esm/shared/validation/option-shape.js +24 -0
- package/dist/esm/shared/validation/options.d.ts +145 -0
- package/dist/esm/shared/validation/options.js +156 -0
- package/dist/{shared/validation/primitives.js → esm/shared/validation/primitives.d.ts} +30 -125
- package/dist/esm/shared/validation/primitives.js +100 -0
- package/dist/esm/shared/validation/ttl.d.ts +94 -0
- package/dist/esm/shared/validation/ttl.js +46 -0
- package/dist/esm/store/actions/list-namespaces.d.ts +79 -0
- package/dist/esm/store/actions/list-namespaces.js +86 -0
- package/dist/esm/store/actions/put.d.ts +43 -0
- package/dist/esm/store/actions/put.js +46 -0
- package/dist/{store/actions/reconcile-vector-index.js → esm/store/actions/reconcile-vector-index.d.ts} +8 -16
- package/dist/esm/store/actions/reconcile-vector-index.js +10 -0
- package/dist/esm/store/actions/search.d.ts +38 -0
- package/dist/esm/store/actions/search.js +23 -0
- package/dist/esm/store/internal/batch-plan.d.ts +56 -0
- package/dist/esm/store/internal/batch-plan.js +28 -0
- package/dist/esm/store/internal/filter.d.ts +43 -0
- package/dist/esm/store/internal/filter.js +60 -0
- package/dist/esm/store/internal/get-item.d.ts +77 -0
- package/dist/esm/store/internal/get-item.js +54 -0
- package/dist/esm/store/internal/item-write.d.ts +270 -0
- package/dist/esm/store/internal/item-write.js +173 -0
- package/dist/esm/store/internal/parse.d.ts +230 -0
- package/dist/esm/store/internal/parse.js +169 -0
- package/dist/{store/internal/rows.js → esm/store/internal/rows.d.ts} +111 -199
- package/dist/esm/store/internal/rows.js +147 -0
- package/dist/esm/store/internal/semantic-search.d.ts +168 -0
- package/dist/esm/store/internal/semantic-search.js +200 -0
- package/dist/esm/store/internal/setup.d.ts +96 -0
- package/dist/esm/store/internal/setup.js +98 -0
- package/dist/esm/store/internal/table-search.d.ts +100 -0
- package/dist/esm/store/internal/table-search.js +119 -0
- package/dist/esm/store/internal/vector-index.d.ts +258 -0
- package/dist/esm/store/internal/vector-index.js +256 -0
- package/dist/{store/store.js → esm/store/store.d.ts} +99 -133
- package/dist/esm/store/store.js +93 -0
- package/dist/esm/store/types.d.ts +111 -0
- package/dist/esm/store/types.js +1 -0
- package/dist/esm/store/vector-backend.d.ts +104 -0
- package/dist/esm/store/vector-backend.js +1 -0
- package/package.json +50 -44
- package/dist/backfill/backfill.js +0 -393
- package/dist/checkpointer/actions/list.js +0 -142
- package/dist/checkpointer/actions/put.js +0 -195
- package/dist/checkpointer/internal/delta-history.js +0 -252
- package/dist/checkpointer/internal/parse.js +0 -372
- package/dist/checkpointer/internal/pending-writes.js +0 -588
- package/dist/checkpointer/internal/read.js +0 -264
- package/dist/checkpointer/internal/rows.js +0 -834
- package/dist/checkpointer/internal/setup.js +0 -72
- package/dist/checkpointer/types.js +0 -12
- package/dist/factory/factory.js +0 -266
- package/dist/factory/types.js +0 -10
- package/dist/history/actions/get-messages.js +0 -141
- package/dist/history/actions/list-sessions.js +0 -156
- package/dist/history/internal/append.js +0 -500
- package/dist/history/internal/message-read.js +0 -204
- package/dist/history/internal/parse.js +0 -252
- package/dist/history/internal/session.js +0 -628
- package/dist/history/internal/setup.js +0 -106
- package/dist/history/types.js +0 -12
- package/dist/index.js +0 -51
- package/dist/shared/adapter.js +0 -143
- package/dist/shared/codec/codec.js +0 -441
- package/dist/shared/codec/compression.js +0 -89
- package/dist/shared/codec/json-serde.js +0 -191
- package/dist/shared/codec/s3/client-types.js +0 -26
- package/dist/shared/codec/s3/client.js +0 -97
- package/dist/shared/codec/s3/lifecycle.js +0 -355
- package/dist/shared/codec/s3/offloader.js +0 -658
- package/dist/shared/dynamodb/batch-write.js +0 -173
- package/dist/shared/dynamodb/paginate.js +0 -188
- package/dist/shared/dynamodb/partition-delete.js +0 -340
- package/dist/shared/dynamodb/recency-index.js +0 -377
- package/dist/shared/dynamodb/retry.js +0 -463
- package/dist/shared/errors/base-error.js +0 -174
- package/dist/shared/errors/boundary.js +0 -143
- package/dist/shared/errors/classify.d.ts +0 -97
- package/dist/shared/errors/classify.js +0 -257
- package/dist/shared/logging/redaction.js +0 -316
- package/dist/shared/options.js +0 -11
- package/dist/shared/validation/options.js +0 -328
- package/dist/shared/validation/ttl.js +0 -132
- package/dist/store/actions/list-namespaces.js +0 -197
- package/dist/store/actions/put.js +0 -78
- package/dist/store/actions/search.js +0 -66
- package/dist/store/internal/batch-plan.d.ts +0 -26
- package/dist/store/internal/batch-plan.js +0 -109
- package/dist/store/internal/filter.js +0 -137
- package/dist/store/internal/get-item.d.ts +0 -45
- package/dist/store/internal/get-item.js +0 -115
- package/dist/store/internal/item-write.js +0 -463
- package/dist/store/internal/parse.js +0 -350
- package/dist/store/internal/semantic-search.js +0 -389
- package/dist/store/internal/setup.js +0 -223
- package/dist/store/internal/table-search.js +0 -213
- package/dist/store/internal/vector-index.js +0 -546
- package/dist/store/types.js +0 -15
- package/dist/store/vector-backend.js +0 -17
- /package/dist/{shared → cjs/shared}/clock.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/codec/json-serde.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/concurrency.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/dynamodb/cancellation.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/logging/logger.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/logging/truncate.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/ulid.d.ts +0 -0
- /package/dist/{shared → cjs/shared}/validation/primitives.d.ts +0 -0
- /package/dist/{store → cjs/store}/vector-backend.d.ts +0 -0
|
@@ -2,13 +2,16 @@
|
|
|
2
2
|
* Hides the bucket lifecycle configuration this package maintains.
|
|
3
3
|
*
|
|
4
4
|
* Two rules per prefix — one expiring objects a day after the ttl plus the
|
|
5
|
-
* sweep margin and keeping a released payload's noncurrent version for a
|
|
6
|
-
* window, one reclaiming expired delete markers — are written
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* sweep margin and keeping a released payload's noncurrent version for a
|
|
6
|
+
* grace window, one reclaiming expired delete markers — are written beside
|
|
7
|
+
* whatever else the read that built the write showed, rewritten only when
|
|
8
|
+
* they differ. A lifecycle GET carries no version, so a later read can show
|
|
9
|
+
* less than an earlier one; a rule that read does not show — this package's
|
|
10
|
+
* own included — is dropped by the write built from it. This is followed by
|
|
11
|
+
* a report of whether the bucket keeps versions at all.
|
|
9
12
|
*/
|
|
10
13
|
import type { LifecycleRule, S3Client } from '@aws-sdk/client-s3';
|
|
11
|
-
import type { Logger } from '../../logging/logger';
|
|
14
|
+
import type { Logger } from '../../logging/logger.js';
|
|
12
15
|
/**
|
|
13
16
|
* Days a released payload's noncurrent version survives behind its delete
|
|
14
17
|
* marker before S3 reclaims it. One day is the smallest the lifecycle API
|
|
@@ -27,6 +30,12 @@ export interface LifecycleTarget {
|
|
|
27
30
|
readonly prefix: string;
|
|
28
31
|
readonly days: number;
|
|
29
32
|
}
|
|
33
|
+
/** Rounds {@link ensureLifecycleRule} polls before it reports a writer it cannot outlast. */
|
|
34
|
+
export declare const LIFECYCLE_SETTLE_WRITES = 5;
|
|
35
|
+
/** How {@link ensureLifecycleRule} waits between re-reads; a test passes one that takes no time. */
|
|
36
|
+
export interface LifecyclePace {
|
|
37
|
+
wait?: (delayMs: number) => Promise<void>;
|
|
38
|
+
}
|
|
30
39
|
/**
|
|
31
40
|
* Ensure the two rules scoped to `target.prefix` exist on `target.bucket`: a
|
|
32
41
|
* `target.days`-day expiration for current versions, and the reclaim that
|
|
@@ -38,24 +47,69 @@ export interface LifecycleTarget {
|
|
|
38
47
|
* versions are governed by the release grace instead (see {@link ttlRule}),
|
|
39
48
|
* which is a floor measured against every rule already governing these keys.
|
|
40
49
|
* Both are inert on an unversioned bucket, which keeps no noncurrent version
|
|
41
|
-
* and leaves no marker.
|
|
50
|
+
* and leaves no marker. `pace.wait` — how to wait between re-reads; the
|
|
51
|
+
* default sleeps.
|
|
42
52
|
*
|
|
43
53
|
* Returns: nothing. Idempotent: when both rules already say this, no write is
|
|
44
|
-
* issued.
|
|
45
|
-
*
|
|
46
|
-
*
|
|
54
|
+
* issued. Otherwise it writes them, waits, and re-reads. A bucket's
|
|
55
|
+
* configuration is served eventually consistently rather than
|
|
56
|
+
* read-your-writes
|
|
57
|
+
* (https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html, "Amazon
|
|
58
|
+
* S3 data consistency model": "Bucket configurations have an eventual
|
|
59
|
+
* consistency model"), and S3 documents specifically that a lifecycle
|
|
60
|
+
* configuration can take a few minutes to fully propagate
|
|
61
|
+
* (https://docs.aws.amazon.com/AmazonS3/latest/userguide/how-to-set-lifecycle-configuration-intro.html,
|
|
62
|
+
* "Lifecycle configuration propagation delay"). So a re-read that still shows
|
|
63
|
+
* the same rules — by id and the fields this comparison examines — as the
|
|
64
|
+
* read behind this call's last write is treated as that lag, not as loss, and
|
|
65
|
+
* prompts another wait rather than another write; a re-read that instead
|
|
66
|
+
* shows some *other* configuration, still without this call's rules, is a
|
|
67
|
+
* competing writer, and the rules are written again, merged with whatever
|
|
68
|
+
* that read now holds. The waits (1, 2, 4, 8 s) and the five-round limit are
|
|
69
|
+
* this library's own polling policy — a choice, not a bound AWS documents.
|
|
70
|
+
* `CONTENTION` is reserved for a writer this call never once found the bucket
|
|
71
|
+
* at rest for: it fires only when every one of the five rounds needs a
|
|
72
|
+
* write, meaning no re-read this call took ever agreed with the one before
|
|
73
|
+
* it. A last round that writes without reaching that count, or that finds
|
|
74
|
+
* only lag, ends the same way instead — a `warn` and a normal return: the
|
|
75
|
+
* rules were written, only their visibility could not be confirmed here, and
|
|
76
|
+
* a later call (or the same one, run again after a few minutes) can still
|
|
77
|
+
* find them settled. The bucket's versioning state is reported on every exit
|
|
78
|
+
* the loop itself chooses — settled, lag, a late rival, or `CONTENTION` —
|
|
79
|
+
* because the containment a released payload depends on is missing or
|
|
80
|
+
* present regardless of how this call ended. A read or write failure, or a
|
|
81
|
+
* `VALIDATION` `planRules` raises from a re-read, propagates directly instead
|
|
82
|
+
* and skips it, the same as it would skip any other cleanup this never got
|
|
83
|
+
* to.
|
|
47
84
|
*
|
|
48
85
|
* Throws: `VALIDATION` naming `s3.keyPrefix` for an unscoped prefix, or when
|
|
49
86
|
* either rule id this prefix produces is already held by a different prefix
|
|
50
|
-
* (see {@link assertNoIdCollision});
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
87
|
+
* (see {@link assertNoIdCollision}); `CONTENTION` when every one of the five
|
|
88
|
+
* rounds this call polls needs a write — a competing writer replacing the
|
|
89
|
+
* configuration on every single re-read, never once leaving it long enough
|
|
90
|
+
* for lag alone to explain what this call sees; otherwise whatever the SDK
|
|
91
|
+
* rejects with. A bucket with no lifecycle configuration at all is not an
|
|
92
|
+
* error — S3 reports `NoSuchLifecycleConfiguration` and this starts from an
|
|
93
|
+
* empty rule set.
|
|
94
|
+
*
|
|
95
|
+
* Guarantees: a write carries forward whatever its own read showed — every
|
|
96
|
+
* rule this package did not add or replace, and the bucket-level
|
|
97
|
+
* `TransitionDefaultMinimumObjectSize` — because a Put replaces the whole
|
|
98
|
+
* configuration and dropping either would silently reset the bucket. That
|
|
99
|
+
* guarantee is only as good as the read, though: a lifecycle GET carries no
|
|
100
|
+
* version, so a later read can show less than an earlier one, and a rule —
|
|
101
|
+
* this package's own included — that a write's own read does not show is
|
|
102
|
+
* dropped by that write, whether or not it is genuinely still on the bucket.
|
|
103
|
+
* Building each write from a read taken right before it, rather than from a
|
|
104
|
+
* plan an earlier wait left stale, narrows this without closing it: a rule a
|
|
105
|
+
* competing writer added is carried forward when this call's own next
|
|
106
|
+
* re-read happens to show it, and lost when a still staler one does not.
|
|
107
|
+
* None of this reaches a *different* call's own first read either: run
|
|
108
|
+
* provisioning calls one at a time, and when more than one adapter or
|
|
109
|
+
* process shares a bucket, re-run each after a few minutes once every one of
|
|
110
|
+
* them has run.
|
|
57
111
|
*/
|
|
58
|
-
export declare function ensureLifecycleRule(client: S3Client, target: LifecycleTarget, logger: Logger): Promise<void>;
|
|
112
|
+
export declare function ensureLifecycleRule(client: S3Client, target: LifecycleTarget, logger: Logger, pace?: LifecyclePace): Promise<void>;
|
|
59
113
|
/** The rule to build: its id, its prefix, and the days after which current versions expire. */
|
|
60
114
|
interface RuleTarget {
|
|
61
115
|
readonly id: string;
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LIFECYCLE_SETTLE_WRITES = exports.S3_RELEASE_GRACE_DAYS = void 0;
|
|
4
|
+
exports.ensureLifecycleRule = ensureLifecycleRule;
|
|
5
|
+
exports.ttlRule = ttlRule;
|
|
6
|
+
exports.markerRule = markerRule;
|
|
7
|
+
exports.assertNoIdCollision = assertNoIdCollision;
|
|
8
|
+
exports.alreadyCorrect = alreadyCorrect;
|
|
9
|
+
exports.reportBucketVersioning = reportBucketVersioning;
|
|
10
|
+
const retry_js_1 = require("../../dynamodb/retry.js");
|
|
11
|
+
const base_error_js_1 = require("../../errors/base-error.js");
|
|
12
|
+
const classify_js_1 = require("../../errors/classify.js");
|
|
13
|
+
const error_code_js_1 = require("../../errors/error-code.js");
|
|
14
|
+
const errors_js_1 = require("../../errors/errors.js");
|
|
15
|
+
const truncate_js_1 = require("../../logging/truncate.js");
|
|
16
|
+
const client_js_1 = require("./client.js");
|
|
17
|
+
const config_js_1 = require("./config.js");
|
|
18
|
+
exports.S3_RELEASE_GRACE_DAYS = 1;
|
|
19
|
+
async function readState(client, bucket) {
|
|
20
|
+
const { GetBucketLifecycleConfigurationCommand } = await (0, client_js_1.loadS3Sdk)();
|
|
21
|
+
try {
|
|
22
|
+
const existing = await client.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket }));
|
|
23
|
+
return {
|
|
24
|
+
rules: existing.Rules ?? [],
|
|
25
|
+
transitionDefaultMinimumObjectSize: existing.TransitionDefaultMinimumObjectSize,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if ((0, classify_js_1.isMissingLifecycleConfiguration)(error))
|
|
30
|
+
return { rules: [] };
|
|
31
|
+
throw error;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function upsert(rules, rule) {
|
|
35
|
+
return rules.some((held) => held.ID === rule.ID)
|
|
36
|
+
? rules.map((held) => (held.ID === rule.ID ? rule : held))
|
|
37
|
+
: [...rules, rule];
|
|
38
|
+
}
|
|
39
|
+
async function putRules(client, bucket, state, rules) {
|
|
40
|
+
const { PutBucketLifecycleConfigurationCommand } = await (0, client_js_1.loadS3Sdk)();
|
|
41
|
+
await client.send(new PutBucketLifecycleConfigurationCommand({
|
|
42
|
+
Bucket: bucket,
|
|
43
|
+
LifecycleConfiguration: { Rules: rules },
|
|
44
|
+
...(state.transitionDefaultMinimumObjectSize === undefined
|
|
45
|
+
? {}
|
|
46
|
+
: { TransitionDefaultMinimumObjectSize: state.transitionDefaultMinimumObjectSize }),
|
|
47
|
+
}));
|
|
48
|
+
}
|
|
49
|
+
exports.LIFECYCLE_SETTLE_WRITES = 5;
|
|
50
|
+
const LIFECYCLE_SETTLE_BASE_MS = 1_000;
|
|
51
|
+
function planRules(state, target, ids) {
|
|
52
|
+
const heldTtl = state.rules.find((rule) => rule.ID === ids.ttl);
|
|
53
|
+
const heldMarker = state.rules.find((rule) => rule.ID === ids.marker);
|
|
54
|
+
assertNoIdCollision(heldTtl, target.prefix, ids.ttl);
|
|
55
|
+
assertNoIdCollision(heldMarker, target.prefix, ids.marker);
|
|
56
|
+
const expiry = ttlRule({ id: ids.ttl, prefix: target.prefix, days: target.days }, state.rules, heldTtl);
|
|
57
|
+
const reclaim = markerRule(ids.marker, target.prefix, heldMarker);
|
|
58
|
+
return {
|
|
59
|
+
settled: alreadyCorrect(heldTtl, expiry) && alreadyCorrect(heldMarker, reclaim),
|
|
60
|
+
rules: upsert(upsert(state.rules, expiry), reclaim),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
function delayForAttempt(attempt) {
|
|
64
|
+
return LIFECYCLE_SETTLE_BASE_MS * 2 ** (attempt - 1);
|
|
65
|
+
}
|
|
66
|
+
function ruleFingerprint(rule) {
|
|
67
|
+
return JSON.stringify([
|
|
68
|
+
rule.ID,
|
|
69
|
+
rule.Status,
|
|
70
|
+
scopeOf(rule),
|
|
71
|
+
rule.Expiration?.Days,
|
|
72
|
+
rule.Expiration?.ExpiredObjectDeleteMarker,
|
|
73
|
+
rule.Expiration?.Date,
|
|
74
|
+
rule.NoncurrentVersionExpiration?.NoncurrentDays,
|
|
75
|
+
]);
|
|
76
|
+
}
|
|
77
|
+
function sameRuleSet(a, b) {
|
|
78
|
+
const fingerprint = (rules) => rules.map(ruleFingerprint).sort().join('\n');
|
|
79
|
+
return fingerprint(a) === fingerprint(b);
|
|
80
|
+
}
|
|
81
|
+
function lifecycleContention(target) {
|
|
82
|
+
return new base_error_js_1.DynamoDBLangGraphError(`the lifecycle configuration of bucket ${(0, truncate_js_1.truncateForLog)(target.bucket)} kept losing the ` +
|
|
83
|
+
`rules for ${(0, truncate_js_1.truncateForLog)(target.prefix)} to a different configuration: every one of ` +
|
|
84
|
+
`the ${exports.LIFECYCLE_SETTLE_WRITES} rounds this call polls needed a write, another writer ` +
|
|
85
|
+
'replacing it on every re-read; run ensureS3LifecycleRule again once that writer has ' +
|
|
86
|
+
'finished', error_code_js_1.ErrorCode.CONTENTION, {});
|
|
87
|
+
}
|
|
88
|
+
function warnLifecyclePending(target, logger) {
|
|
89
|
+
logger.warn('ensureS3LifecycleRule: wrote the lifecycle rules but a re-read did not show them within the ' +
|
|
90
|
+
'polling window; S3 documents that propagation can take a few minutes, so this is most ' +
|
|
91
|
+
'likely lag rather than a lost write — call ensureS3LifecycleRule() again later to confirm', { bucket: (0, truncate_js_1.truncateForLog)(target.bucket), prefix: (0, truncate_js_1.truncateForLog)(target.prefix) });
|
|
92
|
+
}
|
|
93
|
+
async function ensureLifecycleRule(client, target, logger, pace = {}) {
|
|
94
|
+
(0, config_js_1.assertScopedKeyPrefix)(target.prefix);
|
|
95
|
+
const ids = {
|
|
96
|
+
ttl: (0, config_js_1.buildLifecycleRuleId)(target.prefix),
|
|
97
|
+
marker: (0, config_js_1.buildMarkerRuleId)(target.prefix),
|
|
98
|
+
};
|
|
99
|
+
const wait = pace.wait ?? ((delayMs) => (0, retry_js_1.sleep)(delayMs));
|
|
100
|
+
let before = [];
|
|
101
|
+
let writes = 0;
|
|
102
|
+
let contended = false;
|
|
103
|
+
for (let attempt = 0;; attempt += 1) {
|
|
104
|
+
if (attempt > 0)
|
|
105
|
+
await wait(delayForAttempt(attempt));
|
|
106
|
+
const state = await readState(client, target.bucket);
|
|
107
|
+
const plan = planRules(state, target, ids);
|
|
108
|
+
if (plan.settled)
|
|
109
|
+
break;
|
|
110
|
+
const lastAttempt = attempt === exports.LIFECYCLE_SETTLE_WRITES - 1;
|
|
111
|
+
if (attempt > 0 && sameRuleSet(state.rules, before)) {
|
|
112
|
+
if (lastAttempt) {
|
|
113
|
+
warnLifecyclePending(target, logger);
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
logger.debug('ensureS3LifecycleRule: a re-read still shows what was there before this write; ' +
|
|
117
|
+
'waiting for it to propagate', { attempt, waitedMs: delayForAttempt(attempt) });
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (attempt > 0) {
|
|
121
|
+
logger.debug('ensureS3LifecycleRule: a re-read shows a different configuration without these ' +
|
|
122
|
+
'rules; writing them again', { attempt, waitedMs: delayForAttempt(attempt) });
|
|
123
|
+
}
|
|
124
|
+
await putRules(client, target.bucket, state, plan.rules);
|
|
125
|
+
writes += 1;
|
|
126
|
+
before = state.rules;
|
|
127
|
+
if (lastAttempt) {
|
|
128
|
+
if (writes === exports.LIFECYCLE_SETTLE_WRITES)
|
|
129
|
+
contended = true;
|
|
130
|
+
else
|
|
131
|
+
warnLifecyclePending(target, logger);
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
await reportBucketVersioning(client, target.bucket, logger);
|
|
136
|
+
if (contended)
|
|
137
|
+
throw lifecycleContention(target);
|
|
138
|
+
}
|
|
139
|
+
function scopeOf(rule) {
|
|
140
|
+
return rule.Filter?.Prefix ?? rule.Filter?.And?.Prefix ?? rule.Prefix;
|
|
141
|
+
}
|
|
142
|
+
function governs(rule, prefix) {
|
|
143
|
+
const scope = scopeOf(rule);
|
|
144
|
+
return rule.Status === 'Enabled' && (scope === undefined || prefix.startsWith(scope));
|
|
145
|
+
}
|
|
146
|
+
function noncurrentDays(rules, prefix, existing) {
|
|
147
|
+
const held = rules
|
|
148
|
+
.filter((rule) => governs(rule, prefix))
|
|
149
|
+
.map((rule) => rule.NoncurrentVersionExpiration?.NoncurrentDays ?? 0);
|
|
150
|
+
return Math.max(...held, existing?.NoncurrentVersionExpiration?.NoncurrentDays ?? 0, exports.S3_RELEASE_GRACE_DAYS);
|
|
151
|
+
}
|
|
152
|
+
function carried(existing) {
|
|
153
|
+
const kept = { ...existing };
|
|
154
|
+
delete kept.Prefix;
|
|
155
|
+
return kept;
|
|
156
|
+
}
|
|
157
|
+
function ttlRule(rule, rules, existing) {
|
|
158
|
+
return {
|
|
159
|
+
...carried(existing),
|
|
160
|
+
ID: rule.id,
|
|
161
|
+
Filter: { Prefix: rule.prefix },
|
|
162
|
+
Status: 'Enabled',
|
|
163
|
+
Expiration: { Days: rule.days },
|
|
164
|
+
NoncurrentVersionExpiration: {
|
|
165
|
+
...existing?.NoncurrentVersionExpiration,
|
|
166
|
+
NoncurrentDays: noncurrentDays(rules, rule.prefix, existing),
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
function markerRule(id, prefix, existing) {
|
|
171
|
+
return {
|
|
172
|
+
...carried(existing),
|
|
173
|
+
ID: id,
|
|
174
|
+
Filter: { Prefix: prefix },
|
|
175
|
+
Status: 'Enabled',
|
|
176
|
+
Expiration: { ExpiredObjectDeleteMarker: true },
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
function assertNoIdCollision(rule, prefix, id) {
|
|
180
|
+
if (rule === undefined)
|
|
181
|
+
return;
|
|
182
|
+
const found = scopeOf(rule);
|
|
183
|
+
if (found === undefined || found === prefix)
|
|
184
|
+
return;
|
|
185
|
+
throw (0, errors_js_1.validationError)(`the S3 lifecycle rule id "${(0, truncate_js_1.truncateForLog)(id)}" is already used by the prefix ` +
|
|
186
|
+
`"${(0, truncate_js_1.truncateForLog)(found)}"; an id is the key ` +
|
|
187
|
+
'prefix with every non-alphanumeric character replaced by "-", and the marker rule appends ' +
|
|
188
|
+
'"-markers" to that, so "a/b/" takes the id of "a-b/" and "app/" takes the marker id of ' +
|
|
189
|
+
'"app-markers/" — choose an s3.keyPrefix that produces neither id of any other prefix on ' +
|
|
190
|
+
'this bucket', 's3.keyPrefix');
|
|
191
|
+
}
|
|
192
|
+
function managedShape(rule) {
|
|
193
|
+
return JSON.stringify([
|
|
194
|
+
rule?.Status,
|
|
195
|
+
rule?.Filter?.Prefix,
|
|
196
|
+
rule?.Expiration?.Days,
|
|
197
|
+
rule?.Expiration?.ExpiredObjectDeleteMarker,
|
|
198
|
+
rule?.NoncurrentVersionExpiration?.NoncurrentDays,
|
|
199
|
+
]);
|
|
200
|
+
}
|
|
201
|
+
function alreadyCorrect(existing, desired) {
|
|
202
|
+
return managedShape(existing) === managedShape(desired);
|
|
203
|
+
}
|
|
204
|
+
async function reportBucketVersioning(client, bucket, logger) {
|
|
205
|
+
const { GetBucketVersioningCommand } = await (0, client_js_1.loadS3Sdk)();
|
|
206
|
+
const named = (0, truncate_js_1.truncateForLog)(bucket);
|
|
207
|
+
let status;
|
|
208
|
+
try {
|
|
209
|
+
status = (await client.send(new GetBucketVersioningCommand({ Bucket: bucket }))).Status;
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
const reason = (0, truncate_js_1.truncateForLog)(error?.name ?? 'unknown');
|
|
213
|
+
logger.warn('ensureS3LifecycleRule: could not read the offload bucket versioning state, so whether a released payload is recoverable is unknown; the lifecycle rules were written, and the role needs s3:GetBucketVersioning', { bucket: named, reason });
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
if (status === 'Enabled')
|
|
217
|
+
return;
|
|
218
|
+
if (status === undefined) {
|
|
219
|
+
logger.warn('ensureS3LifecycleRule: versioning is off on the offload bucket, so releasing a payload deletes it outright with no recovery window; enable bucket versioning to gain one', { bucket: named });
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
logger.warn('ensureS3LifecycleRule: versioning is suspended on the offload bucket, so releasing a payload deletes it outright; re-enable versioning, and note that the payloads released while it was suspended are already gone', { bucket: named });
|
|
223
|
+
}
|
|
@@ -9,9 +9,10 @@
|
|
|
9
9
|
* never deletes outside the row's own path, and logs rather than throws.
|
|
10
10
|
*/
|
|
11
11
|
import type { S3Client } from '@aws-sdk/client-s3';
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
12
|
+
import { type RetryAttemptInfo } from '../../dynamodb/retry.js';
|
|
13
|
+
import { DynamoDBLangGraphError } from '../../errors/base-error.js';
|
|
14
|
+
import { type Logger } from '../../logging/logger.js';
|
|
15
|
+
import { S3OffloadConfig } from './config.js';
|
|
15
16
|
/** Default payload size that triggers S3 offload (350 KB; 50 KB under the DDB 400 KB cap). */
|
|
16
17
|
export declare const DEFAULT_S3_THRESHOLD_BYTES: number;
|
|
17
18
|
/** Default S3 server-side encryption algorithm. */
|
|
@@ -41,9 +42,11 @@ export declare class S3Offloader {
|
|
|
41
42
|
private readonly sseKmsKeyId?;
|
|
42
43
|
private readonly maxDownloadBytes;
|
|
43
44
|
private readonly config;
|
|
45
|
+
private readonly logger;
|
|
44
46
|
/**
|
|
45
47
|
* Accepts: `config` — already validated by the adapter that builds this, with
|
|
46
|
-
* its key prefix resolved to the adapter's own path.
|
|
48
|
+
* its key prefix resolved to the adapter's own path. `logger` — the
|
|
49
|
+
* adapter's, which each S3 transfer retry is reported to at `debug`.
|
|
47
50
|
*
|
|
48
51
|
* Returns: an offloader whose S3 client is built lazily, on the first
|
|
49
52
|
* operation that needs one.
|
|
@@ -52,7 +55,7 @@ export declare class S3Offloader {
|
|
|
52
55
|
* import is warmed so the failure surfaces, typed, on the first S3 operation
|
|
53
56
|
* rather than on the first oversize payload days later.
|
|
54
57
|
*/
|
|
55
|
-
constructor(config: S3OffloadConfig);
|
|
58
|
+
constructor(config: S3OffloadConfig, logger?: Logger);
|
|
56
59
|
private getClient;
|
|
57
60
|
/**
|
|
58
61
|
* Whether `data` is large enough to warrant S3 offload.
|
|
@@ -117,6 +120,12 @@ export declare class S3Offloader {
|
|
|
117
120
|
* object belonging to another row.
|
|
118
121
|
*/
|
|
119
122
|
assertOwnedKey(key: string, scope: readonly string[]): void;
|
|
123
|
+
/**
|
|
124
|
+
* The line one S3 transfer retry writes: the message a DynamoDB retry writes,
|
|
125
|
+
* so a count of retries by error name covers both, plus the transfer it was.
|
|
126
|
+
* The error is quoted by name only, bounded like every other quoted name.
|
|
127
|
+
*/
|
|
128
|
+
private retryReporter;
|
|
120
129
|
/**
|
|
121
130
|
* Upload `data` under `key`, writing only while the key is free.
|
|
122
131
|
*
|
|
@@ -129,8 +138,10 @@ export declare class S3Offloader {
|
|
|
129
138
|
* attempt of this upload did (see `uploadObject`); the caller's obligation is
|
|
130
139
|
* the same either way.
|
|
131
140
|
*
|
|
132
|
-
* Throws: `
|
|
133
|
-
*
|
|
141
|
+
* Throws: `VALIDATION` naming `payload` for bytes over `maxDownloadBytes`,
|
|
142
|
+
* before any request; `S3_OFFLOAD_FAILED` carrying the key; `ABORTED` when
|
|
143
|
+
* the signal fires, which is the caller's own stop rather than a failed
|
|
144
|
+
* offload.
|
|
134
145
|
*/
|
|
135
146
|
upload(key: string, data: Uint8Array, row: BacklinkRow, signal?: AbortSignal): Promise<string>;
|
|
136
147
|
/**
|
|
@@ -170,11 +181,17 @@ export declare class S3Offloader {
|
|
|
170
181
|
* reported rather than enforced.
|
|
171
182
|
*
|
|
172
183
|
* Returns: nothing. Rules that are already correct are left alone, so this
|
|
173
|
-
* is safe to call on every deploy.
|
|
184
|
+
* is safe to call on every deploy. When it writes the rules but cannot
|
|
185
|
+
* confirm within its polling window that a re-read shows them — S3
|
|
186
|
+
* documents that a lifecycle configuration can take minutes to propagate —
|
|
187
|
+
* it logs a `warn` and returns rather than throwing: the rules were
|
|
188
|
+
* written, and a later call can confirm them.
|
|
174
189
|
*
|
|
175
190
|
* Throws: `VALIDATION` naming `s3.keyPrefix` when a rule id this prefix
|
|
176
191
|
* would take is already held by a different prefix; whatever reading or
|
|
177
|
-
* writing the bucket's lifecycle configuration throws
|
|
192
|
+
* writing the bucket's lifecycle configuration throws; `CONTENTION` when
|
|
193
|
+
* every one of the five rounds this call polls needs a write — a competing
|
|
194
|
+
* writer replacing the configuration on every single re-read.
|
|
178
195
|
*/
|
|
179
196
|
ensureLifecycleRule(ttlDays: number, logger: Logger): Promise<void>;
|
|
180
197
|
/**
|
|
@@ -202,6 +219,8 @@ export interface UploadParams {
|
|
|
202
219
|
metadata?: Record<string, string>;
|
|
203
220
|
/** The caller's cancellation; absent, the upload cannot be interrupted. */
|
|
204
221
|
signal?: AbortSignal;
|
|
222
|
+
/** Called before each retry's backoff, as `withRetry` calls it. */
|
|
223
|
+
onRetry?: (info: RetryAttemptInfo) => void;
|
|
205
224
|
}
|
|
206
225
|
/**
|
|
207
226
|
* Upload `data` to S3 unless an object already exists under `key`, wrapping
|
|
@@ -229,7 +248,8 @@ export interface UploadParams {
|
|
|
229
248
|
* carrying the key and the underlying error, after three attempts on a
|
|
230
249
|
* transient failure. Its message quotes the SDK's, with credential shapes
|
|
231
250
|
* redacted — a signing failure names the key it signed with, and this message
|
|
232
|
-
* reaches `err.message` on a public error.
|
|
251
|
+
* reaches `err.message` on a public error. Its context carries the S3
|
|
252
|
+
* failure's `awsErrorName`, `httpStatusCode` and `requestId` when it was AWS's.
|
|
233
253
|
*
|
|
234
254
|
* Guarantees: a cancelled upload leaves at most the object it was writing, at
|
|
235
255
|
* a key ending in this write's own object id, which no row names because the
|
|
@@ -250,19 +270,23 @@ export interface StoredObject {
|
|
|
250
270
|
*
|
|
251
271
|
* Accepts: `object.maxBytes` — the largest object this call will buffer.
|
|
252
272
|
* `signal` — cancels the request, and with it a body already streaming.
|
|
273
|
+
* `onRetry` — called before each retry's backoff.
|
|
253
274
|
*
|
|
254
275
|
* Returns: the object's bytes.
|
|
255
276
|
*
|
|
256
277
|
* Throws: `ABORTED` when the signal fires, unwrapped; `S3_OFFLOAD_FAILED`
|
|
257
278
|
* naming the key — for an object over `maxBytes`, for a response with no body,
|
|
258
279
|
* and for any SDK failure that survives the retries. Its message quotes the
|
|
259
|
-
* underlying one with credential shapes redacted; the
|
|
260
|
-
* `cause
|
|
261
|
-
*
|
|
280
|
+
* underlying one with credential shapes redacted; the failure is kept as
|
|
281
|
+
* `cause` — the SDK error itself for a failure that was not retried, the
|
|
282
|
+
* `RETRY_EXHAUSTED` error wrapping it once the retry budget is spent — so
|
|
283
|
+
* `NoSuchKey` stays distinguishable ({@link isMissingObjectError}). Its context carries the S3 failure's
|
|
284
|
+
* `awsErrorName`, `httpStatusCode` and `requestId` when it was AWS's.
|
|
262
285
|
*
|
|
263
286
|
* Guarantees: an object over the cap is refused from its declared
|
|
264
|
-
* `ContentLength` before the body is touched, and
|
|
265
|
-
*
|
|
287
|
+
* `ContentLength` before the body is touched, and that body is destroyed
|
|
288
|
+
* rather than left open, and while streaming when the length is absent, so a
|
|
289
|
+
* replaced or hostile object cannot exhaust memory.
|
|
266
290
|
* Transient failures are retried by this package alone — the client is built
|
|
267
291
|
* with `maxAttempts: 1`.
|
|
268
292
|
*
|
|
@@ -274,7 +298,7 @@ export interface StoredObject {
|
|
|
274
298
|
* leaves nothing behind at all — no local file, no partial object, and the
|
|
275
299
|
* chunks already buffered are dropped with the call.
|
|
276
300
|
*/
|
|
277
|
-
export declare function downloadObject(client: S3Client, object: StoredObject, signal?: AbortSignal): Promise<Uint8Array>;
|
|
301
|
+
export declare function downloadObject(client: S3Client, object: StoredObject, signal?: AbortSignal, onRetry?: (info: RetryAttemptInfo) => void): Promise<Uint8Array>;
|
|
278
302
|
/** The part of an S3 `GetObject` body this module relies on. */
|
|
279
303
|
export interface S3Body {
|
|
280
304
|
transformToByteArray(): Promise<Uint8Array>;
|
|
@@ -360,17 +384,6 @@ export interface BacklinkRow {
|
|
|
360
384
|
* Throws: nothing.
|
|
361
385
|
*/
|
|
362
386
|
export declare function backlinkMetadata(row: BacklinkRow): Record<string, string>;
|
|
363
|
-
/**
|
|
364
|
-
* The total UTF-8 bytes `metadata` costs against S3's 2 KB user-metadata cap.
|
|
365
|
-
*
|
|
366
|
-
* Accepts: any metadata map. Names and values both count, which is how S3
|
|
367
|
-
* measures it.
|
|
368
|
-
*
|
|
369
|
-
* Returns: the byte total.
|
|
370
|
-
*
|
|
371
|
-
* Throws: nothing.
|
|
372
|
-
*/
|
|
373
|
-
export declare function metadataBytes(metadata: Record<string, string>): number;
|
|
374
387
|
/** What one release of orphaned objects needs. */
|
|
375
388
|
export interface OrphanRelease {
|
|
376
389
|
/** The keys to delete; an absent or empty one is skipped. */
|
|
@@ -397,8 +410,8 @@ export interface OrphanRelease {
|
|
|
397
410
|
*
|
|
398
411
|
* Returns: nothing.
|
|
399
412
|
*
|
|
400
|
-
* Throws: **nothing**, ever
|
|
401
|
-
*
|
|
413
|
+
* Throws: **nothing**, ever: a cleanup failure must not mask the write
|
|
414
|
+
* failure that caused it. Every
|
|
402
415
|
* outcome that is not a clean delete is logged at `warn`, carrying counts and
|
|
403
416
|
* the failing error's *name* — never its message, which can hold a credential
|
|
404
417
|
* fragment. A `logger` that throws is one more thing this absorbs (see
|