@farukada/aws-langgraph-dynamodb-ts 1.0.0-rc.1 → 1.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1621 -288
- package/dist/backfill/backfill.d.ts +168 -0
- package/dist/backfill/backfill.js +393 -0
- package/dist/checkpointer/actions/delete-thread.d.ts +44 -4
- package/dist/checkpointer/actions/delete-thread.js +56 -22
- package/dist/checkpointer/actions/get-tuple.d.ts +29 -5
- package/dist/checkpointer/actions/get-tuple.js +41 -13
- package/dist/checkpointer/actions/list.d.ts +42 -10
- package/dist/checkpointer/actions/list.js +86 -57
- package/dist/checkpointer/actions/put-writes.d.ts +41 -9
- package/dist/checkpointer/actions/put-writes.js +62 -48
- package/dist/checkpointer/actions/put.d.ts +82 -10
- package/dist/checkpointer/actions/put.js +166 -44
- package/dist/checkpointer/internal/delta-history.d.ts +112 -0
- package/dist/checkpointer/internal/delta-history.js +252 -0
- package/dist/checkpointer/internal/listing.d.ts +149 -0
- package/dist/checkpointer/internal/listing.js +245 -0
- package/dist/checkpointer/internal/parse.d.ts +262 -0
- package/dist/checkpointer/internal/parse.js +372 -0
- package/dist/checkpointer/internal/pending-writes.d.ts +275 -0
- package/dist/checkpointer/internal/pending-writes.js +588 -0
- package/dist/checkpointer/internal/read.d.ts +130 -0
- package/dist/checkpointer/internal/read.js +264 -0
- package/dist/checkpointer/internal/rows.d.ts +571 -0
- package/dist/checkpointer/internal/rows.js +834 -0
- package/dist/checkpointer/internal/setup.d.ts +42 -21
- package/dist/checkpointer/internal/setup.js +65 -31
- package/dist/checkpointer/saver.d.ts +242 -42
- package/dist/checkpointer/saver.js +257 -53
- package/dist/checkpointer/types.d.ts +37 -44
- package/dist/checkpointer/types.js +10 -0
- package/dist/factory/factory.d.ts +120 -11
- package/dist/factory/factory.js +195 -16
- package/dist/factory/types.d.ts +23 -4
- package/dist/factory/types.js +8 -0
- package/dist/history/actions/add-messages.d.ts +30 -2
- package/dist/history/actions/add-messages.js +37 -57
- package/dist/history/actions/clear.d.ts +46 -4
- package/dist/history/actions/clear.js +64 -15
- package/dist/history/actions/get-messages.d.ts +41 -0
- package/dist/history/actions/get-messages.js +94 -32
- package/dist/history/actions/list-sessions.d.ts +52 -10
- package/dist/history/actions/list-sessions.js +141 -38
- package/dist/history/actions/reconcile-count.d.ts +41 -8
- package/dist/history/actions/reconcile-count.js +44 -57
- package/dist/history/chat-message-history.d.ts +200 -38
- package/dist/history/chat-message-history.js +218 -51
- package/dist/history/internal/append.d.ts +212 -0
- package/dist/history/internal/append.js +500 -0
- package/dist/history/internal/message-read.d.ts +84 -0
- package/dist/history/internal/message-read.js +204 -0
- package/dist/history/internal/parse.d.ts +153 -0
- package/dist/history/internal/parse.js +252 -0
- package/dist/history/internal/rows.d.ts +195 -0
- package/dist/history/internal/rows.js +250 -0
- package/dist/history/internal/session.d.ts +331 -0
- package/dist/history/internal/session.js +628 -0
- package/dist/history/internal/setup.d.ts +52 -19
- package/dist/history/internal/setup.js +90 -27
- package/dist/history/session-adapter.d.ts +95 -6
- package/dist/history/session-adapter.js +99 -8
- package/dist/history/types.d.ts +61 -31
- package/dist/history/types.js +10 -0
- package/dist/index.d.ts +38 -14
- package/dist/index.js +31 -12
- package/dist/shared/adapter.d.ts +135 -0
- package/dist/shared/adapter.js +143 -0
- package/dist/shared/clock.d.ts +48 -4
- package/dist/shared/clock.js +51 -4
- package/dist/shared/codec/codec.d.ts +275 -9
- package/dist/shared/codec/codec.js +394 -19
- package/dist/shared/codec/compression.d.ts +43 -6
- package/dist/shared/codec/compression.js +52 -11
- package/dist/shared/codec/json-serde.d.ts +76 -3
- package/dist/shared/codec/json-serde.js +181 -7
- package/dist/shared/codec/s3/client-types.d.ts +20 -1
- package/dist/shared/codec/s3/client-types.js +20 -1
- package/dist/shared/codec/s3/client.d.ts +35 -8
- package/dist/shared/codec/s3/client.js +61 -10
- package/dist/shared/codec/s3/config.d.ts +226 -22
- package/dist/shared/codec/s3/config.js +274 -24
- package/dist/shared/codec/s3/lifecycle.d.ts +163 -11
- package/dist/shared/codec/s3/lifecycle.js +328 -36
- package/dist/shared/codec/s3/offloader.d.ts +390 -19
- package/dist/shared/codec/s3/offloader.js +590 -50
- package/dist/shared/concurrency.d.ts +40 -3
- package/dist/shared/concurrency.js +55 -7
- package/dist/shared/dynamodb/abort.d.ts +45 -7
- package/dist/shared/dynamodb/abort.js +46 -14
- package/dist/shared/dynamodb/batch-write.d.ts +77 -13
- package/dist/shared/dynamodb/batch-write.js +146 -26
- package/dist/shared/dynamodb/cancellation.d.ts +121 -3
- package/dist/shared/dynamodb/cancellation.js +147 -2
- package/dist/shared/dynamodb/client.d.ts +159 -12
- package/dist/shared/dynamodb/client.js +135 -10
- package/dist/shared/dynamodb/idempotent-write.d.ts +551 -0
- package/dist/shared/dynamodb/idempotent-write.js +593 -0
- package/dist/shared/dynamodb/paginate.d.ts +105 -8
- package/dist/shared/dynamodb/paginate.js +175 -6
- package/dist/shared/dynamodb/partition-delete.d.ts +181 -16
- package/dist/shared/dynamodb/partition-delete.js +312 -43
- package/dist/shared/dynamodb/recency-index.d.ts +231 -0
- package/dist/shared/dynamodb/recency-index.js +377 -0
- package/dist/shared/dynamodb/retry.d.ts +265 -10
- package/dist/shared/dynamodb/retry.js +426 -20
- package/dist/shared/dynamodb/table-schema.d.ts +190 -0
- package/dist/shared/dynamodb/table-schema.js +209 -0
- package/dist/shared/errors/base-error.d.ts +174 -8
- package/dist/shared/errors/base-error.js +154 -7
- package/dist/shared/errors/boundary.d.ts +63 -10
- package/dist/shared/errors/boundary.js +110 -13
- package/dist/shared/errors/classify.d.ts +97 -0
- package/dist/shared/errors/classify.js +257 -0
- package/dist/shared/errors/error-code.d.ts +76 -1
- package/dist/shared/errors/error-code.js +83 -1
- package/dist/shared/errors/errors.d.ts +157 -58
- package/dist/shared/errors/errors.js +218 -93
- package/dist/shared/logging/logger.d.ts +63 -1
- package/dist/shared/logging/logger.js +97 -2
- package/dist/shared/logging/redaction.d.ts +86 -6
- package/dist/shared/logging/redaction.js +265 -24
- package/dist/shared/logging/secret-patterns.d.ts +136 -24
- package/dist/shared/logging/secret-patterns.js +165 -30
- package/dist/shared/logging/truncate.d.ts +197 -0
- package/dist/shared/logging/truncate.js +231 -0
- package/dist/shared/options.d.ts +52 -5
- package/dist/shared/options.js +9 -0
- package/dist/shared/ulid.d.ts +71 -14
- package/dist/shared/ulid.js +77 -14
- package/dist/shared/validation/collaborators.d.ts +141 -0
- package/dist/shared/validation/collaborators.js +188 -0
- package/dist/shared/validation/option-shape.d.ts +89 -0
- package/dist/shared/validation/option-shape.js +113 -0
- package/dist/shared/validation/options.d.ts +142 -5
- package/dist/shared/validation/options.js +284 -46
- package/dist/shared/validation/primitives.d.ts +287 -24
- package/dist/shared/validation/primitives.js +347 -42
- package/dist/shared/validation/ttl.d.ts +61 -12
- package/dist/shared/validation/ttl.js +102 -22
- package/dist/store/actions/list-namespaces.d.ts +76 -5
- package/dist/store/actions/list-namespaces.js +160 -22
- package/dist/store/actions/put.d.ts +33 -7
- package/dist/store/actions/put.js +53 -71
- package/dist/store/actions/reconcile-vector-index.d.ts +28 -8
- package/dist/store/actions/reconcile-vector-index.js +33 -13
- package/dist/store/actions/search.d.ts +34 -5
- package/dist/store/actions/search.js +52 -20
- package/dist/store/internal/batch-plan.d.ts +24 -9
- package/dist/store/internal/batch-plan.js +91 -34
- package/dist/store/internal/filter.d.ts +36 -2
- package/dist/store/internal/filter.js +66 -14
- package/dist/store/internal/get-item.d.ts +45 -0
- package/dist/store/internal/get-item.js +115 -0
- package/dist/store/internal/item-write.d.ts +230 -0
- package/dist/store/internal/item-write.js +463 -0
- package/dist/store/internal/parse.d.ts +225 -0
- package/dist/store/internal/parse.js +350 -0
- package/dist/store/internal/rows.d.ts +355 -0
- package/dist/store/internal/rows.js +447 -0
- package/dist/store/internal/semantic-search.d.ts +146 -12
- package/dist/store/internal/semantic-search.js +317 -15
- package/dist/store/internal/setup.d.ts +86 -21
- package/dist/store/internal/setup.js +209 -26
- package/dist/store/internal/table-search.d.ts +100 -0
- package/dist/store/internal/table-search.js +213 -0
- package/dist/store/internal/vector-index.d.ts +247 -0
- package/dist/store/internal/vector-index.js +546 -0
- package/dist/store/store.d.ts +258 -33
- package/dist/store/store.js +309 -53
- package/dist/store/types.d.ts +74 -28
- package/dist/store/types.js +13 -0
- package/dist/store/vector-backend.d.ts +64 -3
- package/dist/store/vector-backend.js +15 -0
- package/package.json +41 -25
- package/dist/checkpointer/internal/assemble.d.ts +0 -17
- package/dist/checkpointer/internal/assemble.js +0 -38
- package/dist/checkpointer/internal/checkpoint-write-verify.d.ts +0 -24
- package/dist/checkpointer/internal/checkpoint-write-verify.js +0 -67
- package/dist/checkpointer/internal/configurable.d.ts +0 -12
- package/dist/checkpointer/internal/configurable.js +0 -24
- package/dist/checkpointer/internal/fetch.d.ts +0 -19
- package/dist/checkpointer/internal/fetch.js +0 -88
- package/dist/checkpointer/internal/filter-match.d.ts +0 -13
- package/dist/checkpointer/internal/filter-match.js +0 -15
- package/dist/checkpointer/internal/item-reader.d.ts +0 -66
- package/dist/checkpointer/internal/item-reader.js +0 -113
- package/dist/checkpointer/internal/item-writer.d.ts +0 -35
- package/dist/checkpointer/internal/item-writer.js +0 -107
- package/dist/checkpointer/internal/keys.d.ts +0 -41
- package/dist/checkpointer/internal/keys.js +0 -117
- package/dist/checkpointer/internal/list-scope.d.ts +0 -61
- package/dist/checkpointer/internal/list-scope.js +0 -108
- package/dist/checkpointer/internal/pending-sends.d.ts +0 -13
- package/dist/checkpointer/internal/pending-sends.js +0 -29
- package/dist/checkpointer/internal/query.d.ts +0 -21
- package/dist/checkpointer/internal/query.js +0 -42
- package/dist/checkpointer/internal/regular-write.d.ts +0 -28
- package/dist/checkpointer/internal/regular-write.js +0 -66
- package/dist/checkpointer/internal/special-write-cas.d.ts +0 -29
- package/dist/checkpointer/internal/special-write-cas.js +0 -104
- package/dist/checkpointer/internal/special-write-cleanup.d.ts +0 -23
- package/dist/checkpointer/internal/special-write-cleanup.js +0 -50
- package/dist/checkpointer/internal/special-write-verify.d.ts +0 -53
- package/dist/checkpointer/internal/special-write-verify.js +0 -80
- package/dist/checkpointer/internal/stored-channels.d.ts +0 -31
- package/dist/checkpointer/internal/stored-channels.js +0 -70
- package/dist/checkpointer/internal/validation.d.ts +0 -18
- package/dist/checkpointer/internal/validation.js +0 -40
- package/dist/checkpointer/internal/write-guard.d.ts +0 -20
- package/dist/checkpointer/internal/write-guard.js +0 -48
- package/dist/checkpointer/internal/write-index.d.ts +0 -36
- package/dist/checkpointer/internal/write-index.js +0 -41
- package/dist/history/internal/append-saga.d.ts +0 -25
- package/dist/history/internal/append-saga.js +0 -79
- package/dist/history/internal/compensation.d.ts +0 -25
- package/dist/history/internal/compensation.js +0 -92
- package/dist/history/internal/item-mapper.d.ts +0 -9
- package/dist/history/internal/item-mapper.js +0 -27
- package/dist/history/internal/keys.d.ts +0 -16
- package/dist/history/internal/keys.js +0 -48
- package/dist/history/internal/message-chunker.d.ts +0 -13
- package/dist/history/internal/message-chunker.js +0 -67
- package/dist/history/internal/message-transaction.d.ts +0 -25
- package/dist/history/internal/message-transaction.js +0 -61
- package/dist/history/internal/message-window.d.ts +0 -17
- package/dist/history/internal/message-window.js +0 -49
- package/dist/history/internal/query.d.ts +0 -23
- package/dist/history/internal/query.js +0 -39
- package/dist/history/internal/session-count.d.ts +0 -47
- package/dist/history/internal/session-count.js +0 -115
- package/dist/history/internal/session-title.d.ts +0 -19
- package/dist/history/internal/session-title.js +0 -43
- package/dist/history/internal/session-update.d.ts +0 -27
- package/dist/history/internal/session-update.js +0 -69
- package/dist/history/internal/title-generator.d.ts +0 -12
- package/dist/history/internal/title-generator.js +0 -41
- package/dist/history/internal/ttl-anchor.d.ts +0 -24
- package/dist/history/internal/ttl-anchor.js +0 -38
- package/dist/history/internal/validation.d.ts +0 -25
- package/dist/history/internal/validation.js +0 -53
- package/dist/shared/codec/descriptor-keys.d.ts +0 -12
- package/dist/shared/codec/descriptor-keys.js +0 -14
- package/dist/shared/codec/encode.d.ts +0 -13
- package/dist/shared/codec/encode.js +0 -43
- package/dist/shared/codec/payload-loss.d.ts +0 -12
- package/dist/shared/codec/payload-loss.js +0 -38
- package/dist/shared/codec/s3/adapter-config.d.ts +0 -13
- package/dist/shared/codec/s3/adapter-config.js +0 -22
- package/dist/shared/codec/s3/bounded-body.d.ts +0 -14
- package/dist/shared/codec/s3/bounded-body.js +0 -47
- package/dist/shared/codec/s3/delete.d.ts +0 -7
- package/dist/shared/codec/s3/delete.js +0 -28
- package/dist/shared/codec/s3/key-scope.d.ts +0 -23
- package/dist/shared/codec/s3/key-scope.js +0 -44
- package/dist/shared/codec/s3/orphans.d.ts +0 -23
- package/dist/shared/codec/s3/orphans.js +0 -71
- package/dist/shared/codec/s3/read-write.d.ts +0 -18
- package/dist/shared/codec/s3/read-write.js +0 -51
- package/dist/shared/codec/s3/retry.d.ts +0 -5
- package/dist/shared/codec/s3/retry.js +0 -22
- package/dist/shared/constants.d.ts +0 -112
- package/dist/shared/constants.js +0 -115
- package/dist/shared/dynamodb/backoff.d.ts +0 -15
- package/dist/shared/dynamodb/backoff.js +0 -48
- package/dist/shared/dynamodb/conditional-put.d.ts +0 -67
- package/dist/shared/dynamodb/conditional-put.js +0 -76
- package/dist/shared/dynamodb/drain-unprocessed.d.ts +0 -21
- package/dist/shared/dynamodb/drain-unprocessed.js +0 -43
- package/dist/shared/dynamodb/expiry.d.ts +0 -16
- package/dist/shared/dynamodb/expiry.js +0 -29
- package/dist/shared/dynamodb/paginate-core.d.ts +0 -24
- package/dist/shared/dynamodb/paginate-core.js +0 -82
- package/dist/shared/dynamodb/retry-classifier.d.ts +0 -17
- package/dist/shared/dynamodb/retry-classifier.js +0 -118
- package/dist/shared/dynamodb/retry-policy.d.ts +0 -26
- package/dist/shared/dynamodb/retry-policy.js +0 -22
- package/dist/shared/dynamodb/scan.d.ts +0 -14
- package/dist/shared/dynamodb/scan.js +0 -19
- package/dist/shared/dynamodb/types.d.ts +0 -23
- package/dist/shared/dynamodb/types.js +0 -2
- package/dist/shared/errors/upstream-error.d.ts +0 -16
- package/dist/shared/errors/upstream-error.js +0 -27
- package/dist/shared/errors/wrap-error.d.ts +0 -7
- package/dist/shared/errors/wrap-error.js +0 -15
- package/dist/shared/logging/redaction-walk.d.ts +0 -23
- package/dist/shared/logging/redaction-walk.js +0 -112
- package/dist/store/actions/get.d.ts +0 -13
- package/dist/store/actions/get.js +0 -73
- package/dist/store/internal/backend-search.d.ts +0 -4
- package/dist/store/internal/backend-search.js +0 -74
- package/dist/store/internal/candidates.d.ts +0 -23
- package/dist/store/internal/candidates.js +0 -94
- package/dist/store/internal/index-reconcile.d.ts +0 -21
- package/dist/store/internal/index-reconcile.js +0 -118
- package/dist/store/internal/index-sync.d.ts +0 -10
- package/dist/store/internal/index-sync.js +0 -25
- package/dist/store/internal/item-mapper.d.ts +0 -27
- package/dist/store/internal/item-mapper.js +0 -61
- package/dist/store/internal/keys.d.ts +0 -17
- package/dist/store/internal/keys.js +0 -41
- package/dist/store/internal/namespace-match.d.ts +0 -11
- package/dist/store/internal/namespace-match.js +0 -40
- package/dist/store/internal/option-validation.d.ts +0 -13
- package/dist/store/internal/option-validation.js +0 -77
- package/dist/store/internal/overwrite-swap.d.ts +0 -32
- package/dist/store/internal/overwrite-swap.js +0 -68
- package/dist/store/internal/persist.d.ts +0 -28
- package/dist/store/internal/persist.js +0 -64
- package/dist/store/internal/query.d.ts +0 -11
- package/dist/store/internal/query.js +0 -48
- package/dist/store/internal/ranker.d.ts +0 -18
- package/dist/store/internal/ranker.js +0 -39
- package/dist/store/internal/read-existing.d.ts +0 -23
- package/dist/store/internal/read-existing.js +0 -41
- package/dist/store/internal/score-direction.d.ts +0 -31
- package/dist/store/internal/score-direction.js +0 -38
- package/dist/store/internal/search-filter.d.ts +0 -3
- package/dist/store/internal/search-filter.js +0 -10
- package/dist/store/internal/text-path.d.ts +0 -17
- package/dist/store/internal/text-path.js +0 -147
- package/dist/store/internal/validation.d.ts +0 -18
- package/dist/store/internal/validation.js +0 -51
- package/dist/store/internal/write-verify.d.ts +0 -42
- package/dist/store/internal/write-verify.js +0 -71
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.validateSessionId = validateSessionId;
|
|
4
|
-
exports.validateStorableMessages = validateStorableMessages;
|
|
5
|
-
exports.validateMessageWindow = validateMessageWindow;
|
|
6
|
-
const messages_1 = require("@langchain/core/messages");
|
|
7
|
-
const constants_1 = require("../../shared/constants");
|
|
8
|
-
const errors_1 = require("../../shared/errors/errors");
|
|
9
|
-
const primitives_1 = require("../../shared/validation/primitives");
|
|
10
|
-
const keys_1 = require("./keys");
|
|
11
|
-
/**
|
|
12
|
-
* Validate a session id is a non-blank, separator- and control-char-free string
|
|
13
|
-
* of at most 1024 bytes. Applied on every entry point, not just the write path: a bad value
|
|
14
|
-
* used to reach DynamoDB and surface as a raw AWS SDK exception on reads,
|
|
15
|
-
* while the identical value threw this library's typed `ValidationError` on a
|
|
16
|
-
* write.
|
|
17
|
-
*/
|
|
18
|
-
function validateSessionId(sessionId) {
|
|
19
|
-
(0, primitives_1.validateIdentifier)(sessionId, keys_1.SORT_KEY_SEPARATOR, 'sessionId', constants_1.MAX_PARTITION_ID_BYTES);
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Reject a message this adapter could never read back. The check *is* the
|
|
23
|
-
* read side's own rebuild (`mapStoredMessagesToChatMessages`), so write and
|
|
24
|
-
* read agree by construction: a `RemoveMessage`, or a tool, function or
|
|
25
|
-
* generic message missing its required field, fails here with a typed error
|
|
26
|
-
* instead of being persisted and then skipped or thrown by `getMessages`.
|
|
27
|
-
*/
|
|
28
|
-
function validateStorableMessages(stored) {
|
|
29
|
-
stored.forEach((message, index) => {
|
|
30
|
-
try {
|
|
31
|
-
(0, messages_1.mapStoredMessagesToChatMessages)([message]);
|
|
32
|
-
}
|
|
33
|
-
catch (error) {
|
|
34
|
-
throw new errors_1.ValidationError(`messages[${index}] of type "${message.type}" cannot be stored: ${error.message}`, 'messages');
|
|
35
|
-
}
|
|
36
|
-
});
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Validate a `getMessages` window: `limit`, when given, a positive integer;
|
|
40
|
-
* `before`, when given, a valid `Date`. Checked before any DynamoDB call so a
|
|
41
|
-
* bad value fails with a typed error rather than a raw `ValidationException`
|
|
42
|
-
* (or, for an invalid Date, a NaN-derived sort key that matches nothing).
|
|
43
|
-
*/
|
|
44
|
-
function validateMessageWindow(window) {
|
|
45
|
-
if (window.limit !== undefined && (!Number.isInteger(window.limit) || window.limit < 1)) {
|
|
46
|
-
throw new errors_1.ValidationError('limit must be a positive integer', 'limit');
|
|
47
|
-
}
|
|
48
|
-
if (window.before !== undefined) {
|
|
49
|
-
const time = typeof window.before.getTime === 'function' ? window.before.getTime() : Number.NaN;
|
|
50
|
-
if (!Number.isFinite(time))
|
|
51
|
-
throw new errors_1.ValidationError('before must be a valid Date', 'before');
|
|
52
|
-
}
|
|
53
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import { PayloadLocation } from './codec';
|
|
2
|
-
/**
|
|
3
|
-
* The part of a descriptor cleanup needs: where the payload lives and, for S3,
|
|
4
|
-
* its key. A full `PayloadDescriptor` satisfies it, and so does the projection
|
|
5
|
-
* a pre-write read returns without the inline bytes.
|
|
6
|
-
*/
|
|
7
|
-
export interface DescriptorRef {
|
|
8
|
-
location: PayloadLocation;
|
|
9
|
-
s3Key?: string;
|
|
10
|
-
}
|
|
11
|
-
/** Collect the S3 keys of any offloaded descriptors, for orphan cleanup. */
|
|
12
|
-
export declare function collectS3Keys(descriptors: readonly DescriptorRef[]): string[];
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.collectS3Keys = collectS3Keys;
|
|
4
|
-
const codec_1 = require("./codec");
|
|
5
|
-
/** Collect the S3 keys of any offloaded descriptors, for orphan cleanup. */
|
|
6
|
-
function collectS3Keys(descriptors) {
|
|
7
|
-
const keys = [];
|
|
8
|
-
for (const descriptor of descriptors) {
|
|
9
|
-
if (descriptor.location === codec_1.PayloadLocation.S3 && descriptor.s3Key !== undefined) {
|
|
10
|
-
keys.push(descriptor.s3Key);
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
return keys;
|
|
14
|
-
}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { type CodecDeps, type PayloadDescriptor } from './codec';
|
|
2
|
-
/** Options controlling where an offloaded payload's S3 key is built from. */
|
|
3
|
-
export interface EncodeOptions {
|
|
4
|
-
keyParts: readonly string[];
|
|
5
|
-
}
|
|
6
|
-
/**
|
|
7
|
-
* Encode `value`: serialize via serde, compress (if configured), then offload
|
|
8
|
-
* to S3 when the compressed bytes exceed the offloader's threshold. Returns a
|
|
9
|
-
* descriptor recording how to read it back. Without an offloader, bytes that
|
|
10
|
-
* cannot fit a DynamoDB item are rejected here (see
|
|
11
|
-
* {@link assertInlinePayloadFits}).
|
|
12
|
-
*/
|
|
13
|
-
export declare function encodePayload<T>(value: T, deps: CodecDeps, options: EncodeOptions): Promise<PayloadDescriptor>;
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.encodePayload = encodePayload;
|
|
4
|
-
const constants_1 = require("../constants");
|
|
5
|
-
const errors_1 = require("../errors/errors");
|
|
6
|
-
const codec_1 = require("./codec");
|
|
7
|
-
const compression_1 = require("./compression");
|
|
8
|
-
/**
|
|
9
|
-
* Reject bytes that cannot be stored inline. Without an offloader the only
|
|
10
|
-
* alternative is a raw `ValidationException` from DynamoDB after the network
|
|
11
|
-
* round trip, which names neither the cause nor the remedy.
|
|
12
|
-
*/
|
|
13
|
-
function assertInlinePayloadFits(bytes, deps) {
|
|
14
|
-
if (bytes.length <= constants_1.MAX_INLINE_PAYLOAD_BYTES)
|
|
15
|
-
return;
|
|
16
|
-
const hint = deps.compression?.enabled
|
|
17
|
-
? ''
|
|
18
|
-
: ', or enable compression if the data compresses well';
|
|
19
|
-
throw new errors_1.ValidationError(`payload of ${bytes.length} bytes exceeds the ${constants_1.MAX_INLINE_PAYLOAD_BYTES}-byte inline limit ` +
|
|
20
|
-
`(DynamoDB items are capped at 400 KB); configure s3 offloading${hint}`, 'payload');
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Encode `value`: serialize via serde, compress (if configured), then offload
|
|
24
|
-
* to S3 when the compressed bytes exceed the offloader's threshold. Returns a
|
|
25
|
-
* descriptor recording how to read it back. Without an offloader, bytes that
|
|
26
|
-
* cannot fit a DynamoDB item are rejected here (see
|
|
27
|
-
* {@link assertInlinePayloadFits}).
|
|
28
|
-
*/
|
|
29
|
-
async function encodePayload(value, deps, options) {
|
|
30
|
-
const [serdeType, raw] = await deps.serde.dumpsTyped(value);
|
|
31
|
-
const { bytes, compressed } = deps.compression
|
|
32
|
-
? await (0, compression_1.compress)(raw, deps.compression)
|
|
33
|
-
: { bytes: raw, compressed: false };
|
|
34
|
-
const base = { schemaVersion: codec_1.DESCRIPTOR_SCHEMA_VERSION, serdeType, compressed };
|
|
35
|
-
if (deps.offloader && deps.offloader.shouldOffload(bytes)) {
|
|
36
|
-
const s3Key = deps.offloader.buildKey(options.keyParts);
|
|
37
|
-
await deps.offloader.upload(s3Key, bytes);
|
|
38
|
-
return { ...base, location: codec_1.PayloadLocation.S3, s3Key };
|
|
39
|
-
}
|
|
40
|
-
if (!deps.offloader)
|
|
41
|
-
assertInlinePayloadFits(bytes, deps);
|
|
42
|
-
return { ...base, location: codec_1.PayloadLocation.INLINE, bytes };
|
|
43
|
-
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* True when an offloaded object no longer exists — a lifecycle sweep removed
|
|
3
|
-
* it, or a competing overwrite deleted it between a row read and the download.
|
|
4
|
-
*/
|
|
5
|
-
export declare function isMissingObjectError(error: Error): boolean;
|
|
6
|
-
/**
|
|
7
|
-
* True when a payload can never be read again — its object is gone, it trips
|
|
8
|
-
* the decompression guard, or its key lies outside the row's own path — as
|
|
9
|
-
* opposed to a failure that may succeed on retry or after a configuration fix
|
|
10
|
-
* (throttling, network, permissions).
|
|
11
|
-
*/
|
|
12
|
-
export declare function isPermanentPayloadLoss(error: Error): boolean;
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.isMissingObjectError = isMissingObjectError;
|
|
4
|
-
exports.isPermanentPayloadLoss = isPermanentPayloadLoss;
|
|
5
|
-
const error_code_1 = require("../errors/error-code");
|
|
6
|
-
/**
|
|
7
|
-
* True when an offloaded object no longer exists — a lifecycle sweep removed
|
|
8
|
-
* it, or a competing overwrite deleted it between a row read and the download.
|
|
9
|
-
*/
|
|
10
|
-
function isMissingObjectError(error) {
|
|
11
|
-
const coded = error;
|
|
12
|
-
return coded.code === error_code_1.ErrorCode.S3_OFFLOAD_FAILED && coded.cause?.name === 'NoSuchKey';
|
|
13
|
-
}
|
|
14
|
-
/** Validation fields that condemn the row itself rather than the caller's input. */
|
|
15
|
-
const ROW_REJECTION_FIELDS = ['s3Key', 'descriptor'];
|
|
16
|
-
/**
|
|
17
|
-
* True when a row's descriptor can never be read by this adapter: its key lies
|
|
18
|
-
* outside the S3 path its own identifiers allow (see `assertKeyInScope`), or
|
|
19
|
-
* its shape is one this version does not understand.
|
|
20
|
-
*/
|
|
21
|
-
function isRowRejection(error) {
|
|
22
|
-
const coded = error;
|
|
23
|
-
return (coded.code === error_code_1.ErrorCode.VALIDATION &&
|
|
24
|
-
coded.context?.field !== undefined &&
|
|
25
|
-
ROW_REJECTION_FIELDS.includes(coded.context.field));
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* True when a payload can never be read again — its object is gone, it trips
|
|
29
|
-
* the decompression guard, or its key lies outside the row's own path — as
|
|
30
|
-
* opposed to a failure that may succeed on retry or after a configuration fix
|
|
31
|
-
* (throttling, network, permissions).
|
|
32
|
-
*/
|
|
33
|
-
function isPermanentPayloadLoss(error) {
|
|
34
|
-
const coded = error;
|
|
35
|
-
return (coded.code === error_code_1.ErrorCode.COMPRESSION_LIMIT ||
|
|
36
|
-
isMissingObjectError(error) ||
|
|
37
|
-
isRowRejection(error));
|
|
38
|
-
}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { DynamoDBClientConfig } from '@aws-sdk/client-dynamodb';
|
|
2
|
-
import { type S3OffloadConfig } from './config';
|
|
3
|
-
/** The adapters that share one bucket, each under its own default key prefix. */
|
|
4
|
-
export type AdapterName = 'checkpointer' | 'store' | 'history';
|
|
5
|
-
/**
|
|
6
|
-
* Resolve an adapter's `s3` option into the offloader's configuration: the
|
|
7
|
-
* adapter-scoped default key prefix when none is given, and the DynamoDB
|
|
8
|
-
* client's region when the S3 client config names none. The S3 SDK does not
|
|
9
|
-
* follow region redirects by default, so a bucket reachable only through the
|
|
10
|
-
* region the DynamoDB side was configured with used to fail with an opaque
|
|
11
|
-
* `PermanentRedirect` on the first offload.
|
|
12
|
-
*/
|
|
13
|
-
export declare function offloaderConfigFor(s3: S3OffloadConfig, adapter: AdapterName, clientConfig?: DynamoDBClientConfig): S3OffloadConfig;
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.offloaderConfigFor = offloaderConfigFor;
|
|
4
|
-
const constants_1 = require("../../constants");
|
|
5
|
-
const client_types_1 = require("./client-types");
|
|
6
|
-
const config_1 = require("./config");
|
|
7
|
-
/**
|
|
8
|
-
* Resolve an adapter's `s3` option into the offloader's configuration: the
|
|
9
|
-
* adapter-scoped default key prefix when none is given, and the DynamoDB
|
|
10
|
-
* client's region when the S3 client config names none. The S3 SDK does not
|
|
11
|
-
* follow region redirects by default, so a bucket reachable only through the
|
|
12
|
-
* region the DynamoDB side was configured with used to fail with an opaque
|
|
13
|
-
* `PermanentRedirect` on the first offload.
|
|
14
|
-
*/
|
|
15
|
-
function offloaderConfigFor(s3, adapter, clientConfig) {
|
|
16
|
-
const region = (0, client_types_1.s3ClientOptions)(s3.clientConfig).region ?? clientConfig?.region;
|
|
17
|
-
return {
|
|
18
|
-
...s3,
|
|
19
|
-
keyPrefix: s3.keyPrefix ?? (0, config_1.defaultAdapterKeyPrefix)(constants_1.DEFAULT_S3_KEY_PREFIX, adapter),
|
|
20
|
-
...(region === undefined ? {} : { clientConfig: { ...s3.clientConfig, region } }),
|
|
21
|
-
};
|
|
22
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { DynamoDBLangGraphError } from '../../errors/base-error';
|
|
2
|
-
/** The part of an S3 `GetObject` body this module relies on. */
|
|
3
|
-
export interface S3Body {
|
|
4
|
-
transformToByteArray(): Promise<Uint8Array>;
|
|
5
|
-
}
|
|
6
|
-
/** The typed error for an object over the download cap. */
|
|
7
|
-
export declare function oversizedObjectError(key: string, bytes: number, maxBytes: number): DynamoDBLangGraphError;
|
|
8
|
-
/**
|
|
9
|
-
* Buffer an S3 body without ever holding more than `maxBytes` of it. A
|
|
10
|
-
* streaming body is consumed chunk by chunk and abandoned the moment the
|
|
11
|
-
* running total passes the cap; a body that only offers
|
|
12
|
-
* `transformToByteArray()` is read whole and then checked.
|
|
13
|
-
*/
|
|
14
|
-
export declare function readBodyBounded(body: S3Body, key: string, maxBytes: number): Promise<Uint8Array>;
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.oversizedObjectError = oversizedObjectError;
|
|
4
|
-
exports.readBodyBounded = readBodyBounded;
|
|
5
|
-
const base_error_1 = require("../../errors/base-error");
|
|
6
|
-
const error_code_1 = require("../../errors/error-code");
|
|
7
|
-
/** The typed error for an object over the download cap. */
|
|
8
|
-
function oversizedObjectError(key, bytes, maxBytes) {
|
|
9
|
-
return new base_error_1.DynamoDBLangGraphError(`S3 object ${key} exceeds the ${maxBytes}-byte maxDownloadBytes cap ` +
|
|
10
|
-
`(${bytes} bytes declared or read)`, error_code_1.ErrorCode.S3_OFFLOAD_FAILED, { operation: 'download', key });
|
|
11
|
-
}
|
|
12
|
-
function concat(chunks, total) {
|
|
13
|
-
const out = new Uint8Array(total);
|
|
14
|
-
let offset = 0;
|
|
15
|
-
for (const chunk of chunks) {
|
|
16
|
-
out.set(chunk, offset);
|
|
17
|
-
offset += chunk.length;
|
|
18
|
-
}
|
|
19
|
-
return out;
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Buffer an S3 body without ever holding more than `maxBytes` of it. A
|
|
23
|
-
* streaming body is consumed chunk by chunk and abandoned the moment the
|
|
24
|
-
* running total passes the cap; a body that only offers
|
|
25
|
-
* `transformToByteArray()` is read whole and then checked.
|
|
26
|
-
*/
|
|
27
|
-
async function readBodyBounded(body, key, maxBytes) {
|
|
28
|
-
const streaming = body;
|
|
29
|
-
const iterate = streaming[Symbol.asyncIterator];
|
|
30
|
-
if (typeof iterate !== 'function') {
|
|
31
|
-
const whole = new Uint8Array(await body.transformToByteArray());
|
|
32
|
-
if (whole.length > maxBytes)
|
|
33
|
-
throw oversizedObjectError(key, whole.length, maxBytes);
|
|
34
|
-
return whole;
|
|
35
|
-
}
|
|
36
|
-
const chunks = [];
|
|
37
|
-
let total = 0;
|
|
38
|
-
for await (const chunk of { [Symbol.asyncIterator]: iterate.bind(streaming) }) {
|
|
39
|
-
total += chunk.length;
|
|
40
|
-
if (total > maxBytes) {
|
|
41
|
-
streaming.destroy?.();
|
|
42
|
-
throw oversizedObjectError(key, total, maxBytes);
|
|
43
|
-
}
|
|
44
|
-
chunks.push(chunk);
|
|
45
|
-
}
|
|
46
|
-
return concat(chunks, total);
|
|
47
|
-
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { S3Client } from '@aws-sdk/client-s3';
|
|
2
|
-
/**
|
|
3
|
-
* Delete `keys` from `bucket` in chunks of 1000 (the DeleteObjects limit).
|
|
4
|
-
* Returns the keys S3 reported as failed — never throws on per-key errors so
|
|
5
|
-
* callers (orphan cleanup) can decide how loudly to react.
|
|
6
|
-
*/
|
|
7
|
-
export declare function deleteObjects(client: S3Client, bucket: string, keys: string[]): Promise<string[]>;
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.deleteObjects = deleteObjects;
|
|
4
|
-
const constants_1 = require("../../constants");
|
|
5
|
-
const client_1 = require("./client");
|
|
6
|
-
/**
|
|
7
|
-
* Delete `keys` from `bucket` in chunks of 1000 (the DeleteObjects limit).
|
|
8
|
-
* Returns the keys S3 reported as failed — never throws on per-key errors so
|
|
9
|
-
* callers (orphan cleanup) can decide how loudly to react.
|
|
10
|
-
*/
|
|
11
|
-
async function deleteObjects(client, bucket, keys) {
|
|
12
|
-
if (keys.length === 0)
|
|
13
|
-
return [];
|
|
14
|
-
const { DeleteObjectsCommand } = await (0, client_1.loadS3Sdk)();
|
|
15
|
-
const failed = [];
|
|
16
|
-
for (let offset = 0; offset < keys.length; offset += constants_1.S3_DELETE_BATCH_MAX) {
|
|
17
|
-
const chunk = keys.slice(offset, offset + constants_1.S3_DELETE_BATCH_MAX);
|
|
18
|
-
const response = await client.send(new DeleteObjectsCommand({
|
|
19
|
-
Bucket: bucket,
|
|
20
|
-
Delete: { Objects: chunk.map((key) => ({ Key: key })), Quiet: true },
|
|
21
|
-
}));
|
|
22
|
-
for (const error of response.Errors ?? []) {
|
|
23
|
-
if (error.Key)
|
|
24
|
-
failed.push(error.Key);
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
return failed;
|
|
28
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
/** base64url-encode one key part; the output alphabet never contains `/` or `.`. */
|
|
2
|
-
export declare function encodeKeyPart(part: string): string;
|
|
3
|
-
/**
|
|
4
|
-
* The path every key `buildS3Key(prefix, [...parts, ...more])` shares: the
|
|
5
|
-
* prefix plus the encoded `parts` joined by `/`, with neither `.bin` nor a
|
|
6
|
-
* trailing `/`.
|
|
7
|
-
*/
|
|
8
|
-
export declare function s3KeyScope(prefix: string, parts: readonly string[]): string;
|
|
9
|
-
/**
|
|
10
|
-
* True when `key` was produced by `buildS3Key` from exactly `parts`
|
|
11
|
-
* (`<scope>.bin`) or from `parts` plus further parts (`<scope>/…`). An empty
|
|
12
|
-
* `parts` degrades to a prefix-only check. Because parts are base64url-encoded
|
|
13
|
-
* and joined by `/`, an identifier sharing a leading substring with another
|
|
14
|
-
* (`t` and `t1`) never matches its scope.
|
|
15
|
-
*/
|
|
16
|
-
export declare function isKeyInScope(key: string, prefix: string, parts: readonly string[]): boolean;
|
|
17
|
-
/**
|
|
18
|
-
* Refuse a row-sourced `s3Key` that lies outside the path the row's own
|
|
19
|
-
* identifiers produce. A row is trusted for its shape, not for the object it
|
|
20
|
-
* points at: a writer able to place one row in a partition must not be able
|
|
21
|
-
* to make this library download or delete another tenant's object.
|
|
22
|
-
*/
|
|
23
|
-
export declare function assertKeyInScope(key: string, prefix: string, parts: readonly string[]): void;
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.encodeKeyPart = encodeKeyPart;
|
|
4
|
-
exports.s3KeyScope = s3KeyScope;
|
|
5
|
-
exports.isKeyInScope = isKeyInScope;
|
|
6
|
-
exports.assertKeyInScope = assertKeyInScope;
|
|
7
|
-
const errors_1 = require("../../errors/errors");
|
|
8
|
-
/** base64url-encode one key part; the output alphabet never contains `/` or `.`. */
|
|
9
|
-
function encodeKeyPart(part) {
|
|
10
|
-
return Buffer.from(part, 'utf8').toString('base64url');
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* The path every key `buildS3Key(prefix, [...parts, ...more])` shares: the
|
|
14
|
-
* prefix plus the encoded `parts` joined by `/`, with neither `.bin` nor a
|
|
15
|
-
* trailing `/`.
|
|
16
|
-
*/
|
|
17
|
-
function s3KeyScope(prefix, parts) {
|
|
18
|
-
return `${prefix}${parts.map(encodeKeyPart).join('/')}`;
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* True when `key` was produced by `buildS3Key` from exactly `parts`
|
|
22
|
-
* (`<scope>.bin`) or from `parts` plus further parts (`<scope>/…`). An empty
|
|
23
|
-
* `parts` degrades to a prefix-only check. Because parts are base64url-encoded
|
|
24
|
-
* and joined by `/`, an identifier sharing a leading substring with another
|
|
25
|
-
* (`t` and `t1`) never matches its scope.
|
|
26
|
-
*/
|
|
27
|
-
function isKeyInScope(key, prefix, parts) {
|
|
28
|
-
const scope = s3KeyScope(prefix, parts);
|
|
29
|
-
if (parts.length === 0)
|
|
30
|
-
return key.startsWith(scope);
|
|
31
|
-
return key === `${scope}.bin` || key.startsWith(`${scope}/`);
|
|
32
|
-
}
|
|
33
|
-
/**
|
|
34
|
-
* Refuse a row-sourced `s3Key` that lies outside the path the row's own
|
|
35
|
-
* identifiers produce. A row is trusted for its shape, not for the object it
|
|
36
|
-
* points at: a writer able to place one row in a partition must not be able
|
|
37
|
-
* to make this library download or delete another tenant's object.
|
|
38
|
-
*/
|
|
39
|
-
function assertKeyInScope(key, prefix, parts) {
|
|
40
|
-
if (isKeyInScope(key, prefix, parts))
|
|
41
|
-
return;
|
|
42
|
-
throw new errors_1.ValidationError(`s3Key "${key}" lies outside the S3 path this row may reference ` +
|
|
43
|
-
`("${s3KeyScope(prefix, parts)}"); refusing to touch an object the row does not own`, 's3Key');
|
|
44
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import type { Logger } from '../../logging/logger';
|
|
2
|
-
import type { S3Offloader } from './offloader';
|
|
3
|
-
/** Options for {@link cleanUpS3Orphans}. */
|
|
4
|
-
export interface OrphanCleanupOptions {
|
|
5
|
-
rng?: () => number;
|
|
6
|
-
maxAttempts?: number;
|
|
7
|
-
signal?: AbortSignal;
|
|
8
|
-
/**
|
|
9
|
-
* The row's own leading key parts when `keys` were read back from a row.
|
|
10
|
-
* A key outside the path they produce is never deleted, only reported; own
|
|
11
|
-
* uploads (built by this call) pass no scope.
|
|
12
|
-
*/
|
|
13
|
-
scope?: readonly string[];
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* Best-effort delete of S3 objects orphaned by a failed DynamoDB write. Retries
|
|
17
|
-
* transient errors with full-jitter backoff; on persistent failure or when S3
|
|
18
|
-
* reports keys it could not delete, it logs at `warn` and never throws — the
|
|
19
|
-
* sole non-throwing path in the library. There is no automatic backstop: an
|
|
20
|
-
* S3 lifecycle rule sweeps the leftovers only if one was provisioned via
|
|
21
|
-
* `ensureS3LifecycleRule()`, which is opt-in.
|
|
22
|
-
*/
|
|
23
|
-
export declare function cleanUpS3Orphans(offloader: S3Offloader, keys: ReadonlyArray<string | undefined>, context: string, logger: Logger, options?: OrphanCleanupOptions): Promise<void>;
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.cleanUpS3Orphans = cleanUpS3Orphans;
|
|
4
|
-
const backoff_1 = require("../../dynamodb/backoff");
|
|
5
|
-
const retry_1 = require("./retry");
|
|
6
|
-
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
7
|
-
const BASE_DELAY_MS = 100;
|
|
8
|
-
/**
|
|
9
|
-
* Wait out one backoff window, cancellable via `options.signal`. Resolves `true`
|
|
10
|
-
* when the signal aborts the wait (so the caller stops retrying) and `false`
|
|
11
|
-
* otherwise — never rejects, preserving the best-effort, non-throwing contract.
|
|
12
|
-
*/
|
|
13
|
-
async function backoffSleep(delayMs, options) {
|
|
14
|
-
try {
|
|
15
|
-
await (0, backoff_1.sleep)((0, backoff_1.fullJitter)(delayMs, options.rng), options.signal);
|
|
16
|
-
return false;
|
|
17
|
-
}
|
|
18
|
-
catch {
|
|
19
|
-
return true;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
/** Drop every key outside the row's scope, reporting each one. */
|
|
23
|
-
function ownedOnly(offloader, keys, scope, context, logger) {
|
|
24
|
-
return keys.filter((key) => {
|
|
25
|
-
if (offloader.ownsKey(key, scope))
|
|
26
|
-
return true;
|
|
27
|
-
logger.warn(`${context}: refusing to delete an S3 object outside this row's scope`, { key });
|
|
28
|
-
return false;
|
|
29
|
-
});
|
|
30
|
-
}
|
|
31
|
-
/** The non-empty keys, restricted to the row's scope when one is given. */
|
|
32
|
-
function selectOrphans(offloader, keys, context, logger, options) {
|
|
33
|
-
const present = keys.filter((key) => typeof key === 'string' && key.length > 0);
|
|
34
|
-
return options.scope ? ownedOnly(offloader, present, options.scope, context, logger) : present;
|
|
35
|
-
}
|
|
36
|
-
/**
|
|
37
|
-
* Best-effort delete of S3 objects orphaned by a failed DynamoDB write. Retries
|
|
38
|
-
* transient errors with full-jitter backoff; on persistent failure or when S3
|
|
39
|
-
* reports keys it could not delete, it logs at `warn` and never throws — the
|
|
40
|
-
* sole non-throwing path in the library. There is no automatic backstop: an
|
|
41
|
-
* S3 lifecycle rule sweeps the leftovers only if one was provisioned via
|
|
42
|
-
* `ensureS3LifecycleRule()`, which is opt-in.
|
|
43
|
-
*/
|
|
44
|
-
async function cleanUpS3Orphans(offloader, keys, context, logger, options = {}) {
|
|
45
|
-
const orphans = selectOrphans(offloader, keys, context, logger, options);
|
|
46
|
-
if (orphans.length === 0)
|
|
47
|
-
return;
|
|
48
|
-
const maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
49
|
-
let delay = BASE_DELAY_MS;
|
|
50
|
-
let lastError = new Error('S3 orphan cleanup did not run');
|
|
51
|
-
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
52
|
-
try {
|
|
53
|
-
const failed = await offloader.deleteBatch(orphans);
|
|
54
|
-
if (failed.length === 0)
|
|
55
|
-
return;
|
|
56
|
-
logger.warn(`Some orphaned S3 objects could not be deleted after ${context}; a lifecycle rule from ensureS3LifecycleRule() would sweep them, otherwise clean up manually`, { failedCount: failed.length });
|
|
57
|
-
return;
|
|
58
|
-
}
|
|
59
|
-
catch (error) {
|
|
60
|
-
lastError = error;
|
|
61
|
-
if (attempt >= maxAttempts || !(0, retry_1.isTransientS3Error)(lastError))
|
|
62
|
-
break;
|
|
63
|
-
if (await backoffSleep(delay, options))
|
|
64
|
-
break;
|
|
65
|
-
delay = (0, backoff_1.nextBackoffDelay)(delay);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
logger.warn(`Failed to clean up orphaned S3 objects after ${context}; a lifecycle rule from ensureS3LifecycleRule() would sweep them, otherwise clean up manually`, {
|
|
69
|
-
message: lastError.message,
|
|
70
|
-
});
|
|
71
|
-
}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import type { S3Client } from '@aws-sdk/client-s3';
|
|
2
|
-
/** Parameters for {@link uploadObject}. */
|
|
3
|
-
export interface UploadParams {
|
|
4
|
-
bucket: string;
|
|
5
|
-
key: string;
|
|
6
|
-
data: Uint8Array;
|
|
7
|
-
serverSideEncryption?: string;
|
|
8
|
-
sseKmsKeyId?: string;
|
|
9
|
-
}
|
|
10
|
-
/** Upload `data` to S3, wrapping failures as `S3_OFFLOAD_FAILED`. */
|
|
11
|
-
export declare function uploadObject(client: S3Client, params: UploadParams): Promise<void>;
|
|
12
|
-
/**
|
|
13
|
-
* Download an object's bytes, wrapping failures/empty bodies as
|
|
14
|
-
* `S3_OFFLOAD_FAILED`. An object over `maxBytes` is refused from its declared
|
|
15
|
-
* `ContentLength` before the body is touched, or while streaming when the
|
|
16
|
-
* length is unknown, so a replaced or hostile object can never exhaust memory.
|
|
17
|
-
*/
|
|
18
|
-
export declare function downloadObject(client: S3Client, bucket: string, key: string, maxBytes: number): Promise<Uint8Array>;
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.uploadObject = uploadObject;
|
|
4
|
-
exports.downloadObject = downloadObject;
|
|
5
|
-
const retry_1 = require("../../dynamodb/retry");
|
|
6
|
-
const base_error_1 = require("../../errors/base-error");
|
|
7
|
-
const error_code_1 = require("../../errors/error-code");
|
|
8
|
-
const bounded_body_1 = require("./bounded-body");
|
|
9
|
-
const client_1 = require("./client");
|
|
10
|
-
const retry_2 = require("./retry");
|
|
11
|
-
/** Upload `data` to S3, wrapping failures as `S3_OFFLOAD_FAILED`. */
|
|
12
|
-
async function uploadObject(client, params) {
|
|
13
|
-
const { PutObjectCommand } = await (0, client_1.loadS3Sdk)();
|
|
14
|
-
try {
|
|
15
|
-
await (0, retry_1.withRetry)(() => client.send(new PutObjectCommand({
|
|
16
|
-
Bucket: params.bucket,
|
|
17
|
-
Key: params.key,
|
|
18
|
-
Body: params.data,
|
|
19
|
-
ContentType: 'application/octet-stream',
|
|
20
|
-
ServerSideEncryption: params.serverSideEncryption,
|
|
21
|
-
...(params.sseKmsKeyId ? { SSEKMSKeyId: params.sseKmsKeyId } : {}),
|
|
22
|
-
})), { maxAttempts: 3, isRetryable: retry_2.isTransientS3Error });
|
|
23
|
-
}
|
|
24
|
-
catch (error) {
|
|
25
|
-
throw new base_error_1.DynamoDBLangGraphError(error.message, error_code_1.ErrorCode.S3_OFFLOAD_FAILED, { operation: 'upload', key: params.key }, error);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Download an object's bytes, wrapping failures/empty bodies as
|
|
30
|
-
* `S3_OFFLOAD_FAILED`. An object over `maxBytes` is refused from its declared
|
|
31
|
-
* `ContentLength` before the body is touched, or while streaming when the
|
|
32
|
-
* length is unknown, so a replaced or hostile object can never exhaust memory.
|
|
33
|
-
*/
|
|
34
|
-
async function downloadObject(client, bucket, key, maxBytes) {
|
|
35
|
-
const { GetObjectCommand } = await (0, client_1.loadS3Sdk)();
|
|
36
|
-
try {
|
|
37
|
-
return await (0, retry_1.withRetry)(async () => {
|
|
38
|
-
const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
|
|
39
|
-
if (typeof response.ContentLength === 'number' && response.ContentLength > maxBytes) {
|
|
40
|
-
throw (0, bounded_body_1.oversizedObjectError)(key, response.ContentLength, maxBytes);
|
|
41
|
-
}
|
|
42
|
-
if (!response.Body) {
|
|
43
|
-
throw new Error(`S3 object body is empty for key: ${key}`);
|
|
44
|
-
}
|
|
45
|
-
return (0, bounded_body_1.readBodyBounded)(response.Body, key, maxBytes);
|
|
46
|
-
}, { maxAttempts: 3, isRetryable: retry_2.isTransientS3Error });
|
|
47
|
-
}
|
|
48
|
-
catch (error) {
|
|
49
|
-
throw new base_error_1.DynamoDBLangGraphError(error.message, error_code_1.ErrorCode.S3_OFFLOAD_FAILED, { operation: 'download', key }, error);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.isTransientS3Error = isTransientS3Error;
|
|
4
|
-
const retry_classifier_1 = require("../../dynamodb/retry-classifier");
|
|
5
|
-
/**
|
|
6
|
-
* Transient S3 signals: everything the DynamoDB classifier already treats as
|
|
7
|
-
* transient (the SDK transport `TimeoutError`, socket errors, `RequestTimeout`,
|
|
8
|
-
* `ServiceUnavailable`, …) plus the two names only S3 uses. HTTP 429/5xx and
|
|
9
|
-
* the `$retryable` trait are recognised by the shared classifier itself.
|
|
10
|
-
*/
|
|
11
|
-
const RETRYABLE_S3_SIGNALS = [
|
|
12
|
-
...retry_classifier_1.DEFAULT_RETRYABLE_ERRORS,
|
|
13
|
-
'SlowDown',
|
|
14
|
-
'InternalError',
|
|
15
|
-
];
|
|
16
|
-
/**
|
|
17
|
-
* True when `error` looks like a transient S3 failure worth retrying — the one
|
|
18
|
-
* classifier for uploads, downloads and orphan cleanup.
|
|
19
|
-
*/
|
|
20
|
-
function isTransientS3Error(error) {
|
|
21
|
-
return (0, retry_classifier_1.isRetryableError)(error, RETRYABLE_S3_SIGNALS);
|
|
22
|
-
}
|