@farukada/aws-langgraph-dynamodb-ts 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -5
- package/dist/checkpointer/actions/delete-thread.d.ts.map +1 -1
- package/dist/checkpointer/actions/delete-thread.js +17 -9
- package/dist/checkpointer/actions/delete-thread.js.map +1 -1
- package/dist/checkpointer/actions/get-tuple.d.ts.map +1 -1
- package/dist/checkpointer/actions/get-tuple.js +70 -32
- package/dist/checkpointer/actions/get-tuple.js.map +1 -1
- package/dist/checkpointer/actions/put-writes.d.ts.map +1 -1
- package/dist/checkpointer/actions/put-writes.js +36 -19
- package/dist/checkpointer/actions/put-writes.js.map +1 -1
- package/dist/checkpointer/actions/put.d.ts.map +1 -1
- package/dist/checkpointer/actions/put.js +75 -18
- package/dist/checkpointer/actions/put.js.map +1 -1
- package/dist/checkpointer/actions/writer.d.ts +6 -4
- package/dist/checkpointer/actions/writer.d.ts.map +1 -1
- package/dist/checkpointer/actions/writer.js +12 -12
- package/dist/checkpointer/actions/writer.js.map +1 -1
- package/dist/checkpointer/index.d.ts +11 -4
- package/dist/checkpointer/index.d.ts.map +1 -1
- package/dist/checkpointer/index.js +97 -72
- package/dist/checkpointer/index.js.map +1 -1
- package/dist/checkpointer/types/index.d.ts +8 -0
- package/dist/checkpointer/types/index.d.ts.map +1 -1
- package/dist/checkpointer/utils/deserialization.d.ts.map +1 -1
- package/dist/checkpointer/utils/deserialization.js +28 -4
- package/dist/checkpointer/utils/deserialization.js.map +1 -1
- package/dist/checkpointer/utils/validation.d.ts +8 -8
- package/dist/checkpointer/utils/validation.d.ts.map +1 -1
- package/dist/checkpointer/utils/validation.js +45 -1
- package/dist/checkpointer/utils/validation.js.map +1 -1
- package/dist/factory.d.ts +14 -0
- package/dist/factory.d.ts.map +1 -1
- package/dist/factory.js +39 -20
- package/dist/factory.js.map +1 -1
- package/dist/history/actions/add-message.d.ts +5 -9
- package/dist/history/actions/add-message.d.ts.map +1 -1
- package/dist/history/actions/add-message.js +14 -42
- package/dist/history/actions/add-message.js.map +1 -1
- package/dist/history/actions/add-messages.d.ts +6 -10
- package/dist/history/actions/add-messages.d.ts.map +1 -1
- package/dist/history/actions/add-messages.js +33 -38
- package/dist/history/actions/add-messages.js.map +1 -1
- package/dist/history/actions/clear.d.ts.map +1 -1
- package/dist/history/actions/clear.js +3 -3
- package/dist/history/actions/clear.js.map +1 -1
- package/dist/history/actions/get-messages.d.ts.map +1 -1
- package/dist/history/actions/get-messages.js +2 -2
- package/dist/history/actions/get-messages.js.map +1 -1
- package/dist/history/actions/list-sessions.d.ts.map +1 -1
- package/dist/history/actions/list-sessions.js +6 -3
- package/dist/history/actions/list-sessions.js.map +1 -1
- package/dist/history/index.d.ts +41 -5
- package/dist/history/index.d.ts.map +1 -1
- package/dist/history/index.js +52 -18
- package/dist/history/index.js.map +1 -1
- package/dist/history/session-adapter.d.ts +39 -0
- package/dist/history/session-adapter.d.ts.map +1 -0
- package/dist/history/session-adapter.js +74 -0
- package/dist/history/session-adapter.js.map +1 -0
- package/dist/history/types/index.d.ts +32 -1
- package/dist/history/types/index.d.ts.map +1 -1
- package/dist/history/utils/index.d.ts +1 -0
- package/dist/history/utils/index.d.ts.map +1 -1
- package/dist/history/utils/index.js +1 -0
- package/dist/history/utils/index.js.map +1 -1
- package/dist/history/utils/optimistic-retry.d.ts +31 -0
- package/dist/history/utils/optimistic-retry.d.ts.map +1 -0
- package/dist/history/utils/optimistic-retry.js +74 -0
- package/dist/history/utils/optimistic-retry.js.map +1 -0
- package/dist/history/utils/title-generator.d.ts.map +1 -1
- package/dist/history/utils/title-generator.js +8 -3
- package/dist/history/utils/title-generator.js.map +1 -1
- package/dist/history/utils/update-expression-builder.d.ts +18 -21
- package/dist/history/utils/update-expression-builder.d.ts.map +1 -1
- package/dist/history/utils/update-expression-builder.js +31 -46
- package/dist/history/utils/update-expression-builder.js.map +1 -1
- package/dist/history/utils/validation.d.ts.map +1 -1
- package/dist/history/utils/validation.js +23 -3
- package/dist/history/utils/validation.js.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -1
- package/dist/index.js.map +1 -1
- package/dist/shared/utils/batch-write.d.ts +44 -6
- package/dist/shared/utils/batch-write.d.ts.map +1 -1
- package/dist/shared/utils/batch-write.js +76 -28
- package/dist/shared/utils/batch-write.js.map +1 -1
- package/dist/shared/utils/client-factory.d.ts +25 -0
- package/dist/shared/utils/client-factory.d.ts.map +1 -0
- package/dist/shared/utils/client-factory.js +23 -0
- package/dist/shared/utils/client-factory.js.map +1 -0
- package/dist/shared/utils/compressor.d.ts +21 -3
- package/dist/shared/utils/compressor.d.ts.map +1 -1
- package/dist/shared/utils/compressor.js +75 -10
- package/dist/shared/utils/compressor.js.map +1 -1
- package/dist/shared/utils/constants.d.ts +1 -1
- package/dist/shared/utils/constants.d.ts.map +1 -1
- package/dist/shared/utils/constants.js +5 -1
- package/dist/shared/utils/constants.js.map +1 -1
- package/dist/shared/utils/index.d.ts +3 -0
- package/dist/shared/utils/index.d.ts.map +1 -1
- package/dist/shared/utils/index.js +3 -0
- package/dist/shared/utils/index.js.map +1 -1
- package/dist/shared/utils/logger.d.ts +27 -0
- package/dist/shared/utils/logger.d.ts.map +1 -1
- package/dist/shared/utils/logger.js +86 -0
- package/dist/shared/utils/logger.js.map +1 -1
- package/dist/shared/utils/retry.d.ts +7 -0
- package/dist/shared/utils/retry.d.ts.map +1 -1
- package/dist/shared/utils/retry.js +75 -17
- package/dist/shared/utils/retry.js.map +1 -1
- package/dist/shared/utils/s3-offloader.d.ts +10 -3
- package/dist/shared/utils/s3-offloader.d.ts.map +1 -1
- package/dist/shared/utils/s3-offloader.js +19 -9
- package/dist/shared/utils/s3-offloader.js.map +1 -1
- package/dist/shared/utils/s3-orphans.d.ts +16 -0
- package/dist/shared/utils/s3-orphans.d.ts.map +1 -0
- package/dist/shared/utils/s3-orphans.js +79 -0
- package/dist/shared/utils/s3-orphans.js.map +1 -0
- package/dist/shared/utils/sleep.d.ts +48 -0
- package/dist/shared/utils/sleep.d.ts.map +1 -0
- package/dist/shared/utils/sleep.js +82 -0
- package/dist/shared/utils/sleep.js.map +1 -0
- package/dist/shared/utils/ttl.d.ts +6 -2
- package/dist/shared/utils/ttl.d.ts.map +1 -1
- package/dist/shared/utils/ttl.js +17 -2
- package/dist/shared/utils/ttl.js.map +1 -1
- package/dist/store/actions/get-operation.d.ts.map +1 -1
- package/dist/store/actions/get-operation.js +2 -2
- package/dist/store/actions/get-operation.js.map +1 -1
- package/dist/store/actions/list-namespaces-operation.d.ts.map +1 -1
- package/dist/store/actions/list-namespaces-operation.js +44 -29
- package/dist/store/actions/list-namespaces-operation.js.map +1 -1
- package/dist/store/actions/put-operation.d.ts.map +1 -1
- package/dist/store/actions/put-operation.js +19 -5
- package/dist/store/actions/put-operation.js.map +1 -1
- package/dist/store/actions/search-operation.d.ts.map +1 -1
- package/dist/store/actions/search-operation.js +54 -21
- package/dist/store/actions/search-operation.js.map +1 -1
- package/dist/store/index.d.ts +1 -0
- package/dist/store/index.d.ts.map +1 -1
- package/dist/store/index.js +24 -20
- package/dist/store/index.js.map +1 -1
- package/dist/store/types/index.d.ts +33 -1
- package/dist/store/types/index.d.ts.map +1 -1
- package/dist/store/utils/filter.d.ts.map +1 -1
- package/dist/store/utils/filter.js +63 -1
- package/dist/store/utils/filter.js.map +1 -1
- package/dist/store/utils/validation.d.ts +6 -2
- package/dist/store/utils/validation.d.ts.map +1 -1
- package/dist/store/utils/validation.js +21 -5
- package/dist/store/utils/validation.js.map +1 -1
- package/package.json +33 -22
|
@@ -25,6 +25,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
25
25
|
exports.setGlobalLogger = setGlobalLogger;
|
|
26
26
|
exports.getLogger = getLogger;
|
|
27
27
|
exports.resetLogger = resetLogger;
|
|
28
|
+
exports.redactSecrets = redactSecrets;
|
|
29
|
+
exports.redactLogger = redactLogger;
|
|
28
30
|
/**
|
|
29
31
|
* Default logger using console
|
|
30
32
|
*/
|
|
@@ -59,4 +61,88 @@ function getLogger() {
|
|
|
59
61
|
function resetLogger() {
|
|
60
62
|
globalLogger = defaultLogger;
|
|
61
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Default field names treated as secrets. Case-insensitive substring match — so
|
|
66
|
+
* `AccessKeyId`, `aws_access_key_id`, `X-Amz-Security-Token`, `authorization`
|
|
67
|
+
* all get caught. Extend via {@link redactLogger}'s `extraKeys` option if your
|
|
68
|
+
* workload introduces more sensitive fields (e.g. `api_key`, `bearer`).
|
|
69
|
+
*/
|
|
70
|
+
const DEFAULT_SECRET_KEY_PATTERNS = [
|
|
71
|
+
'accesskey',
|
|
72
|
+
'secretkey',
|
|
73
|
+
'secret',
|
|
74
|
+
'sessiontoken',
|
|
75
|
+
'securitytoken',
|
|
76
|
+
'authorization',
|
|
77
|
+
'password',
|
|
78
|
+
'credential',
|
|
79
|
+
'apikey',
|
|
80
|
+
'bearer',
|
|
81
|
+
'token',
|
|
82
|
+
'privatekey',
|
|
83
|
+
];
|
|
84
|
+
const REDACTED_PLACEHOLDER = '[REDACTED]';
|
|
85
|
+
function isSecretKey(key, patterns) {
|
|
86
|
+
const lc = key.toLowerCase();
|
|
87
|
+
return patterns.some((p) => lc.includes(p));
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Recursively clone an object, replacing values at any secret-looking key with
|
|
91
|
+
* `[REDACTED]`. Cycles are broken with a WeakSet. Leaves primitives and non-
|
|
92
|
+
* enumerable values untouched. Does not mutate the input.
|
|
93
|
+
*
|
|
94
|
+
* @param value - Arbitrary value to redact
|
|
95
|
+
* @param patterns - Lower-cased substrings to match against each key
|
|
96
|
+
*/
|
|
97
|
+
function redactSecrets(value, patterns = DEFAULT_SECRET_KEY_PATTERNS) {
|
|
98
|
+
const seen = new WeakSet();
|
|
99
|
+
const walk = (v) => {
|
|
100
|
+
if (v === null || typeof v !== 'object')
|
|
101
|
+
return v;
|
|
102
|
+
if (seen.has(v))
|
|
103
|
+
return '[Circular]';
|
|
104
|
+
seen.add(v);
|
|
105
|
+
if (Array.isArray(v)) {
|
|
106
|
+
return v.map((el) => walk(el));
|
|
107
|
+
}
|
|
108
|
+
// Preserve Error shapes so stack traces survive redaction.
|
|
109
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
110
|
+
if (v instanceof Error) {
|
|
111
|
+
return v;
|
|
112
|
+
}
|
|
113
|
+
const out = {};
|
|
114
|
+
for (const [k, val] of Object.entries(v)) {
|
|
115
|
+
out[k] = isSecretKey(k, patterns) ? REDACTED_PLACEHOLDER : walk(val);
|
|
116
|
+
}
|
|
117
|
+
return out;
|
|
118
|
+
};
|
|
119
|
+
return walk(value);
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Wrap an existing logger with automatic secret redaction applied to the
|
|
123
|
+
* variadic args (the message string itself is passed through unchanged — don't
|
|
124
|
+
* interpolate secrets into messages). Strings in args are left as-is; only
|
|
125
|
+
* object properties whose keys match a secret pattern are replaced.
|
|
126
|
+
*
|
|
127
|
+
* @example
|
|
128
|
+
* ```ts
|
|
129
|
+
* import { setGlobalLogger, redactLogger, getLogger } from '@farukada/aws-langgraph-dynamodb-ts';
|
|
130
|
+
* setGlobalLogger(redactLogger(getLogger()));
|
|
131
|
+
* ```
|
|
132
|
+
*
|
|
133
|
+
* @param inner - Logger to wrap
|
|
134
|
+
* @param options.extraKeys - Additional lower-cased substrings to treat as secret keys
|
|
135
|
+
*/
|
|
136
|
+
function redactLogger(inner, options = {}) {
|
|
137
|
+
const patterns = options.extraKeys
|
|
138
|
+
? [...DEFAULT_SECRET_KEY_PATTERNS, ...options.extraKeys.map((k) => k.toLowerCase())]
|
|
139
|
+
: DEFAULT_SECRET_KEY_PATTERNS;
|
|
140
|
+
const wrap = (args) => args.map((a) => redactSecrets(a, patterns));
|
|
141
|
+
return {
|
|
142
|
+
info: (msg, ...args) => inner.info(msg, ...wrap(args)),
|
|
143
|
+
warn: (msg, ...args) => inner.warn(msg, ...wrap(args)),
|
|
144
|
+
error: (msg, ...args) => inner.error(msg, ...wrap(args)),
|
|
145
|
+
debug: (msg, ...args) => inner.debug(msg, ...wrap(args)),
|
|
146
|
+
};
|
|
147
|
+
}
|
|
62
148
|
//# sourceMappingURL=logger.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../../src/shared/utils/logger.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;AAsCH,0CAEC;AAOD,8BAEC;AAKD,kCAEC;
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../../src/shared/utils/logger.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;AAsCH,0CAEC;AAOD,8BAEC;AAKD,kCAEC;AAsCD,sCA6BC;AAiBD,oCAcC;AA9ID;;GAEG;AACH,4GAA4G;AAC5G,MAAM,aAAa,GAAW;IAC5B,IAAI,EAAE,CAAC,OAAe,EAAE,GAAG,IAAe,EAAE,EAAE,CAC5C,OAAO,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC;IAE1D,IAAI,EAAE,CAAC,OAAe,EAAE,GAAG,IAAe,EAAE,EAAE,CAC5C,OAAO,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC;IAE1D,KAAK,EAAE,CAAC,OAAe,EAAE,GAAG,IAAe,EAAE,EAAE,CAC7C,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC;IAE3D,KAAK,EAAE,CAAC,OAAe,EAAE,GAAG,IAAe,EAAE,EAAE,CAC7C,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC;CAC5D,CAAC;AACF,8BAA8B;AAE9B,IAAI,YAAY,GAAW,aAAa,CAAC;AAEzC;;;;GAIG;AACH,SAAgB,eAAe,CAAC,MAAc;IAC5C,YAAY,GAAG,MAAM,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,SAAgB,SAAS;IACvB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW;IACzB,YAAY,GAAG,aAAa,CAAC;AAC/B,CAAC;AAED;;;;;GAKG;AACH,MAAM,2BAA2B,GAAsB;IACrD,WAAW;IACX,WAAW;IACX,QAAQ;IACR,cAAc;IACd,eAAe;IACf,eAAe;IACf,UAAU;IACV,YAAY;IACZ,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,YAAY;CACb,CAAC;AAEF,MAAM,oBAAoB,GAAG,YAAY,CAAC;AAE1C,SAAS,WAAW,CAAC,GAAW,EAAE,QAA2B;IAC3D,MAAM,EAAE,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAC7B,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,aAAa,CAC3B,KAAc,EACd,WAA8B,2BAA2B;IAEzD,MAAM,IAAI,GAAG,IAAI,OAAO,EAAU,CAAC;IAEnC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAW,EAAE;QACnC,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,OAAO,CAAC,CAAC;QAClD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAW,CAAC;YAAE,OAAO,YAAY,CAAC;QAC/C,IAAI,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;QAEtB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACrB,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACjC,CAAC;QAED,2DAA2D;QAC3D,uDAAuD;QACvD,IAAI,CAAC,YAAY,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAA4B,CAAC,EAAE,CAAC;YACpE,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;AACrB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAgB,YAAY,CAC1B,KAAa,EACb,UAA6C,EAAE;IAE/C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS;QAChC,CAAC,CAAC,CAAC,GAAG,2BAA2B,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACpF,CAAC,CAAC,2BAA2B,CAAC;IAChC,MAAM,IAAI,GAAG,CAAC,IAAe,EAAa,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;IACzF,OAAO;QACL,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACtD,KAAK,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,KAAK,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;KACzD,CAAC;AACJ,CAAC"}
|
|
@@ -7,6 +7,13 @@ export interface RetryOptions {
|
|
|
7
7
|
baseDelayMs?: number;
|
|
8
8
|
maxDelayMs?: number;
|
|
9
9
|
retryableErrors?: string[];
|
|
10
|
+
/**
|
|
11
|
+
* Optional abort signal. When aborted, any in-flight sleep between retries
|
|
12
|
+
* resolves immediately and `withRetry` rejects with the signal's abort reason.
|
|
13
|
+
* The signal is checked before every attempt, so an abort mid-backoff short-
|
|
14
|
+
* circuits the remaining retries rather than consuming the full schedule.
|
|
15
|
+
*/
|
|
16
|
+
signal?: AbortSignal;
|
|
10
17
|
}
|
|
11
18
|
/**
|
|
12
19
|
* Retry a function with exponential backoff
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/retry.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/retry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;;;;OAKG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AA0FD;;GAEG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,CAAC,CAAC,CAkE/F;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,CAAC,EACvC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,SAAS,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAChC,OAAO,CAAC,CAAC,CAAC,CAOZ"}
|
|
@@ -6,49 +6,105 @@
|
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
7
|
exports.withRetry = withRetry;
|
|
8
8
|
exports.withDynamoDBRetry = withDynamoDBRetry;
|
|
9
|
+
const sleep_1 = require("./sleep");
|
|
10
|
+
// TransactionCanceledException is NOT retried here: its CancellationReasons include
|
|
11
|
+
// permanent failures (ConditionalCheckFailed, ValidationError, ItemCollectionSizeLimitExceeded)
|
|
12
|
+
// that must not be retried blindly. Callers that need to retry on specific sub-reasons
|
|
13
|
+
// (e.g. ThrottlingError) must inspect CancellationReasons themselves.
|
|
14
|
+
//
|
|
15
|
+
// TransactionConflictException (the *transient* sibling — one row racing with another
|
|
16
|
+
// concurrent transaction) IS retried. It's the recommended retry path for writes under
|
|
17
|
+
// contention and is distinct from TransactionCanceledException.
|
|
9
18
|
const DEFAULT_OPTIONS = {
|
|
10
19
|
maxAttempts: 5,
|
|
11
20
|
baseDelayMs: 100,
|
|
12
21
|
maxDelayMs: 5000,
|
|
13
22
|
retryableErrors: [
|
|
23
|
+
// DynamoDB throttling / capacity
|
|
14
24
|
'ProvisionedThroughputExceededException',
|
|
15
25
|
'ThrottlingException',
|
|
16
26
|
'RequestLimitExceeded',
|
|
27
|
+
// Service-side transient
|
|
17
28
|
'InternalServerError',
|
|
18
29
|
'ServiceUnavailable',
|
|
19
|
-
|
|
30
|
+
// Transaction contention (transient — distinct from TransactionCanceledException)
|
|
31
|
+
'TransactionConflictException',
|
|
32
|
+
// Node-level network transients. Matched against both the error `code` and
|
|
33
|
+
// `name` fields so we catch plain socket errors as well as SDK-wrapped ones.
|
|
34
|
+
'ECONNRESET',
|
|
35
|
+
'ECONNREFUSED',
|
|
36
|
+
'ETIMEDOUT',
|
|
37
|
+
'EPIPE',
|
|
38
|
+
'EAI_AGAIN',
|
|
39
|
+
'NetworkingError',
|
|
40
|
+
'TimeoutError',
|
|
20
41
|
],
|
|
21
42
|
};
|
|
22
43
|
/**
|
|
23
|
-
*
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
* Calculate delay with exponential backoff and jitter
|
|
44
|
+
* Calculate delay with full jitter (AWS recommended) and a max-delay cap.
|
|
45
|
+
*
|
|
46
|
+
* `delay = random(0, min(cap, base * 2^(attempt-1)))` — spreads retries
|
|
47
|
+
* uniformly across the backoff window so concurrent clients don't synchronize
|
|
48
|
+
* into a thundering herd.
|
|
49
|
+
* See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
|
|
30
50
|
*/
|
|
31
51
|
function calculateDelay(attempt, baseDelayMs, maxDelayMs) {
|
|
32
52
|
const exponentialDelay = baseDelayMs * Math.pow(2, attempt - 1);
|
|
33
|
-
const
|
|
34
|
-
return Math.
|
|
53
|
+
const cap = Math.min(exponentialDelay, maxDelayMs);
|
|
54
|
+
return Math.random() * cap;
|
|
35
55
|
}
|
|
36
56
|
/**
|
|
37
|
-
* Check if an error is retryable
|
|
57
|
+
* Check if an error is retryable.
|
|
58
|
+
*
|
|
59
|
+
* Matches against the error's `name` and `code` fields (and, when available,
|
|
60
|
+
* a nested `$metadata`/`cause`) so we catch AWS SDK exceptions, Node socket
|
|
61
|
+
* errors (`ECONNRESET`, `ETIMEDOUT`, …), and wrapped `cause` chains.
|
|
38
62
|
*/
|
|
39
63
|
function isRetryableError(error, retryableErrors) {
|
|
40
64
|
if (!error)
|
|
41
65
|
return false;
|
|
42
|
-
const
|
|
43
|
-
const
|
|
44
|
-
|
|
66
|
+
const seen = new WeakSet();
|
|
67
|
+
const signals = [];
|
|
68
|
+
// Bound recursion depth — WeakSet breaks cycles but does not bound unique chain
|
|
69
|
+
// length. An attacker (or a buggy SDK middleware) could craft a 10k-deep
|
|
70
|
+
// `cause` chain and stack-overflow the process. 32 is generous: realistic AWS
|
|
71
|
+
// SDK wrapping nests 3-5 deep.
|
|
72
|
+
const MAX_CAUSE_DEPTH = 32;
|
|
73
|
+
const collect = (err, depth) => {
|
|
74
|
+
if (depth > MAX_CAUSE_DEPTH)
|
|
75
|
+
return;
|
|
76
|
+
if (!err || typeof err !== 'object')
|
|
77
|
+
return;
|
|
78
|
+
if (seen.has(err))
|
|
79
|
+
return;
|
|
80
|
+
seen.add(err);
|
|
81
|
+
const o = err;
|
|
82
|
+
if (typeof o.name === 'string')
|
|
83
|
+
signals.push(o.name);
|
|
84
|
+
if (typeof o.code === 'string')
|
|
85
|
+
signals.push(o.code);
|
|
86
|
+
if (typeof o.errno === 'string')
|
|
87
|
+
signals.push(o.errno);
|
|
88
|
+
if (typeof o.syscall === 'string')
|
|
89
|
+
signals.push(o.syscall);
|
|
90
|
+
collect(o.cause, depth + 1);
|
|
91
|
+
};
|
|
92
|
+
collect(error, 0);
|
|
93
|
+
if (signals.length === 0)
|
|
94
|
+
return false;
|
|
95
|
+
return retryableErrors.some((retryable) => signals.some((sig) => sig.includes(retryable)));
|
|
45
96
|
}
|
|
46
97
|
/**
|
|
47
98
|
* Retry a function with exponential backoff
|
|
48
99
|
*/
|
|
49
100
|
async function withRetry(fn, options = {}) {
|
|
50
|
-
const
|
|
101
|
+
const { signal, ...rest } = options;
|
|
102
|
+
const opts = { ...DEFAULT_OPTIONS, ...rest };
|
|
51
103
|
let lastError;
|
|
104
|
+
// Surface pre-aborted signals before consuming an attempt.
|
|
105
|
+
if (signal?.aborted) {
|
|
106
|
+
throw signal.reason ?? new Error('Aborted');
|
|
107
|
+
}
|
|
52
108
|
for (let attempt = 1; attempt <= opts.maxAttempts; attempt++) {
|
|
53
109
|
try {
|
|
54
110
|
return await fn();
|
|
@@ -75,9 +131,11 @@ async function withRetry(fn, options = {}) {
|
|
|
75
131
|
}
|
|
76
132
|
throw wrappedError;
|
|
77
133
|
}
|
|
78
|
-
//
|
|
134
|
+
// Abort mid-backoff short-circuits the remaining retries — sleep() rejects
|
|
135
|
+
// with the signal's reason, which we re-throw so the caller sees the abort
|
|
136
|
+
// rather than the last retry error.
|
|
79
137
|
const delay = calculateDelay(attempt, opts.baseDelayMs, opts.maxDelayMs);
|
|
80
|
-
await sleep(delay);
|
|
138
|
+
await (0, sleep_1.sleep)(delay, signal);
|
|
81
139
|
}
|
|
82
140
|
}
|
|
83
141
|
// Ensure we always throw an Error object
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../../src/shared/utils/retry.ts"],"names":[],"mappings":";AAAA;;;GAGG;;
|
|
1
|
+
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../../src/shared/utils/retry.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AA6GH,8BAkEC;AAKD,8CAUC;AA5LD,mCAAgC;AAgBhC,oFAAoF;AACpF,gGAAgG;AAChG,uFAAuF;AACvF,sEAAsE;AACtE,EAAE;AACF,sFAAsF;AACtF,uFAAuF;AACvF,gEAAgE;AAChE,MAAM,eAAe,GAA2C;IAC9D,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,GAAG;IAChB,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE;QACf,iCAAiC;QACjC,wCAAwC;QACxC,qBAAqB;QACrB,sBAAsB;QACtB,yBAAyB;QACzB,qBAAqB;QACrB,oBAAoB;QACpB,kFAAkF;QAClF,8BAA8B;QAC9B,2EAA2E;QAC3E,6EAA6E;QAC7E,YAAY;QACZ,cAAc;QACd,WAAW;QACX,OAAO;QACP,WAAW;QACX,iBAAiB;QACjB,cAAc;KACf;CACF,CAAC;AAEF;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,OAAe,EAAE,WAAmB,EAAE,UAAkB;IAC9E,MAAM,gBAAgB,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;IAChE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;IACnD,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;AAC7B,CAAC;AAED;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,KAAc,EAAE,eAAyB;IACjE,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IAEzB,MAAM,IAAI,GAAG,IAAI,OAAO,EAAU,CAAC;IACnC,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,gFAAgF;IAChF,yEAAyE;IACzE,8EAA8E;IAC9E,+BAA+B;IAC/B,MAAM,eAAe,GAAG,EAAE,CAAC;IAE3B,MAAM,OAAO,GAAG,CAAC,GAAY,EAAE,KAAa,EAAQ,EAAE;QACpD,IAAI,KAAK,GAAG,eAAe;YAAE,OAAO;QACpC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,OAAO;QAC5C,IAAI,IAAI,CAAC,GAAG,CAAC,GAAa,CAAC;YAAE,OAAO;QACpC,IAAI,CAAC,GAAG,CAAC,GAAa,CAAC,CAAC;QAExB,MAAM,CAAC,GAAG,GAA8B,CAAC;QACzC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACvD,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3D,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9B,CAAC,CAAC;IAEF,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAElB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAC7F,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,SAAS,CAAI,EAAoB,EAAE,UAAwB,EAAE;IACjF,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACpC,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,IAAI,EAAE,CAAC;IAC7C,IAAI,SAAkB,CAAC;IAEvB,2DAA2D;IAC3D,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QAC7D,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAElB,0CAA0C;YAC1C,IAAI,OAAO,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjC,MAAM;YACR,CAAC;YAED,4CAA4C;YAC5C,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACnD,oDAAoD;gBACpD,uDAAuD;gBACvD,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;oBAC3B,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,YAAY,GAAG,IAAI,KAAK,CAC5B,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,SAAS,IAAI,KAAK;oBACtD,CAAC,CAAC,MAAM,CAAE,KAA8B,CAAC,OAAO,CAAC;oBACjD,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAClB,CAAC;gBACF,6DAA6D;gBAC7D,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBACvC,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;gBACrC,CAAC;gBACD,MAAM,YAAY,CAAC;YACrB,CAAC;YAED,2EAA2E;YAC3E,2EAA2E;YAC3E,oCAAoC;YACpC,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACzE,MAAM,IAAA,aAAK,EAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChD,CAAC;IACD,uDAAuD;IACvD,IAAI,SAAS,YAAY,KAAK,EAAE,CAAC;QAC/B,MAAM,SAAS,CAAC;IAClB,CAAC;IACD,+CAA+C;IAC/C,MAAM,YAAY,GAAG,IAAI,KAAK,CAC5B,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,IAAI,SAAS;QAClE,CAAC,CAAC,MAAM,CAAE,SAAkC,CAAC,OAAO,CAAC;QACrD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CACtB,CAAC;IACF,IAAI,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,YAAY,CAAC;AACrB,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,iBAAiB,CACrC,EAAoB,EACpB,SAAiC;IAEjC,OAAO,SAAS,CAAC,EAAE,EAAE;QACnB,WAAW,EAAE,CAAC;QACd,WAAW,EAAE,GAAG;QAChB,UAAU,EAAE,IAAI;QAChB,GAAG,SAAS;KACb,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -23,7 +23,11 @@ export interface S3OffloadConfig {
|
|
|
23
23
|
keyPrefix?: string;
|
|
24
24
|
/** Payload size threshold in bytes that triggers offloading (default: 358400 = 350KB) */
|
|
25
25
|
thresholdBytes?: number;
|
|
26
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* Server-side encryption algorithm (`'AES256'` or `'aws:kms'`).
|
|
28
|
+
* Defaults to `'AES256'` — pass an explicit value to override (e.g. `'aws:kms'`
|
|
29
|
+
* together with `sseKmsKeyId`).
|
|
30
|
+
*/
|
|
27
31
|
serverSideEncryption?: string;
|
|
28
32
|
/** KMS key ID or ARN. Only used when serverSideEncryption is 'aws:kms'. */
|
|
29
33
|
sseKmsKeyId?: string;
|
|
@@ -63,7 +67,7 @@ export declare class S3Offloader {
|
|
|
63
67
|
private readonly bucketName;
|
|
64
68
|
private readonly keyPrefix;
|
|
65
69
|
private readonly thresholdBytes;
|
|
66
|
-
private readonly serverSideEncryption
|
|
70
|
+
private readonly serverSideEncryption;
|
|
67
71
|
private readonly sseKmsKeyId?;
|
|
68
72
|
private readonly clientConfig?;
|
|
69
73
|
constructor(config: S3OffloadConfig);
|
|
@@ -134,7 +138,10 @@ export declare class S3Offloader {
|
|
|
134
138
|
/**
|
|
135
139
|
* Generate a deterministic lifecycle rule ID for idempotency.
|
|
136
140
|
*
|
|
137
|
-
*
|
|
141
|
+
* The ID is scoped to the configured key prefix but intentionally TTL-independent:
|
|
142
|
+
* changing ttlDays updates the existing rule in place instead of leaving a stale
|
|
143
|
+
* rule behind that would still expire objects at the old cadence.
|
|
144
|
+
*
|
|
138
145
|
* @returns Rule ID string
|
|
139
146
|
*/
|
|
140
147
|
private buildLifecycleRuleId;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3-offloader.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/s3-offloader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;
|
|
1
|
+
{"version":3,"file":"s3-offloader.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/s3-offloader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA+BH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yFAAyF;IACzF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,YAAY,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;CACH;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,CAA6D;IAC9E,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAkC;gBAEpD,MAAM,EAAE,eAAe;IASnC;;;OAGG;IACH,OAAO,CAAC,SAAS;IAQjB;;;;;OAKG;IACH,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO;IAIxC;;;;;;;OAOG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;IAIvE;;;;;;OAMG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAmB5D;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAoBhD;;;;OAIG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxC;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgChD;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;;;;;;;;OASG;IACG,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8EzD;;;;;;;;OAQG;IACH,OAAO,CAAC,oBAAoB;IAM5B;;OAEG;IACH,OAAO,IAAI,IAAI;CAGhB"}
|
|
@@ -21,6 +21,13 @@ const logger_1 = require("./logger");
|
|
|
21
21
|
const DEFAULT_THRESHOLD_BYTES = 350 * 1024;
|
|
22
22
|
/** Default S3 key prefix for offloaded payloads */
|
|
23
23
|
const DEFAULT_KEY_PREFIX = 'langgraph-checkpoints/';
|
|
24
|
+
/**
|
|
25
|
+
* Default server-side encryption. Matches S3's own default (SSE-S3, AES256) since
|
|
26
|
+
* January 2023, but asserting it on the PutObject makes the intent explicit for
|
|
27
|
+
* compliance reviews and protects users running against older buckets or custom
|
|
28
|
+
* endpoints that don't inherit the modern default.
|
|
29
|
+
*/
|
|
30
|
+
const DEFAULT_SERVER_SIDE_ENCRYPTION = 'AES256';
|
|
24
31
|
/* eslint-disable @typescript-eslint/no-require-imports -- TODO: migrate to dynamic import() when Jest ESM support is stable */
|
|
25
32
|
// Lazy-loaded S3 SDK — uses require() for Jest CJS compatibility.
|
|
26
33
|
// Replace with import() when the test runner supports --experimental-vm-modules.
|
|
@@ -63,7 +70,7 @@ class S3Offloader {
|
|
|
63
70
|
this.bucketName = config.bucketName;
|
|
64
71
|
this.keyPrefix = config.keyPrefix ?? DEFAULT_KEY_PREFIX;
|
|
65
72
|
this.thresholdBytes = config.thresholdBytes ?? DEFAULT_THRESHOLD_BYTES;
|
|
66
|
-
this.serverSideEncryption = config.serverSideEncryption;
|
|
73
|
+
this.serverSideEncryption = config.serverSideEncryption ?? DEFAULT_SERVER_SIDE_ENCRYPTION;
|
|
67
74
|
this.sseKmsKeyId = config.sseKmsKeyId;
|
|
68
75
|
this.clientConfig = config.clientConfig;
|
|
69
76
|
}
|
|
@@ -113,10 +120,8 @@ class S3Offloader {
|
|
|
113
120
|
Key: key,
|
|
114
121
|
Body: data,
|
|
115
122
|
ContentType: 'application/octet-stream',
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
.serverSideEncryption,
|
|
119
|
-
}),
|
|
123
|
+
ServerSideEncryption: this
|
|
124
|
+
.serverSideEncryption,
|
|
120
125
|
...(this.sseKmsKeyId && { SSEKMSKeyId: this.sseKmsKeyId }),
|
|
121
126
|
}));
|
|
122
127
|
return key;
|
|
@@ -202,7 +207,7 @@ class S3Offloader {
|
|
|
202
207
|
async ensureLifecycleRule(ttlDays) {
|
|
203
208
|
const { GetBucketLifecycleConfigurationCommand, PutBucketLifecycleConfigurationCommand } = getS3Sdk();
|
|
204
209
|
const client = this.getClient();
|
|
205
|
-
const ruleId = this.buildLifecycleRuleId(
|
|
210
|
+
const ruleId = this.buildLifecycleRuleId();
|
|
206
211
|
// 1. Read existing lifecycle configuration
|
|
207
212
|
let existingRules = [];
|
|
208
213
|
try {
|
|
@@ -254,11 +259,16 @@ class S3Offloader {
|
|
|
254
259
|
/**
|
|
255
260
|
* Generate a deterministic lifecycle rule ID for idempotency.
|
|
256
261
|
*
|
|
257
|
-
*
|
|
262
|
+
* The ID is scoped to the configured key prefix but intentionally TTL-independent:
|
|
263
|
+
* changing ttlDays updates the existing rule in place instead of leaving a stale
|
|
264
|
+
* rule behind that would still expire objects at the old cadence.
|
|
265
|
+
*
|
|
258
266
|
* @returns Rule ID string
|
|
259
267
|
*/
|
|
260
|
-
buildLifecycleRuleId(
|
|
261
|
-
|
|
268
|
+
buildLifecycleRuleId() {
|
|
269
|
+
// Normalize the prefix into a lifecycle-rule-safe slug, trimming the trailing slash.
|
|
270
|
+
const slug = this.keyPrefix.replace(/\/+$/, '').replace(/[^a-zA-Z0-9-]/g, '-') || 'default';
|
|
271
|
+
return `langgraph-ttl-${slug}`;
|
|
262
272
|
}
|
|
263
273
|
/**
|
|
264
274
|
* Release underlying S3 client resources
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3-offloader.js","sourceRoot":"","sources":["../../../src/shared/utils/s3-offloader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAEH,qCAAqC;AAErC,mFAAmF;AACnF,MAAM,uBAAuB,GAAG,GAAG,GAAG,IAAI,CAAC;AAE3C,mDAAmD;AACnD,MAAM,kBAAkB,GAAG,wBAAwB,CAAC;AAEpD,+HAA+H;AAE/H,kEAAkE;AAClE,iFAAiF;AACjF,IAAI,KAAsD,CAAC;AAE3D,SAAS,QAAQ;IACf,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAM,CAAC;AAChB,CAAC;
|
|
1
|
+
{"version":3,"file":"s3-offloader.js","sourceRoot":"","sources":["../../../src/shared/utils/s3-offloader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAEH,qCAAqC;AAErC,mFAAmF;AACnF,MAAM,uBAAuB,GAAG,GAAG,GAAG,IAAI,CAAC;AAE3C,mDAAmD;AACnD,MAAM,kBAAkB,GAAG,wBAAwB,CAAC;AAEpD;;;;;GAKG;AACH,MAAM,8BAA8B,GAAG,QAAQ,CAAC;AAEhD,+HAA+H;AAE/H,kEAAkE;AAClE,iFAAiF;AACjF,IAAI,KAAsD,CAAC;AAE3D,SAAS,QAAQ;IACf,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAM,CAAC;AAChB,CAAC;AAgCD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAa,WAAW;IACd,QAAQ,CAA8D;IAC7D,UAAU,CAAS;IACnB,SAAS,CAAS;IAClB,cAAc,CAAS;IACvB,oBAAoB,CAAS;IAC7B,WAAW,CAAU;IACrB,YAAY,CAAmC;IAEhE,YAAY,MAAuB;QACjC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;QACxD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,uBAAuB,CAAC;QACvE,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACK,SAAS;QACf,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAA4B,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACH,aAAa,CAAC,IAAgB;QAC5B,OAAO,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC;IAC5C,CAAC;IAED;;;;;;;OAOG;IACH,QAAQ,CAAC,QAAgB,EAAE,YAAoB,EAAE,KAAa;QAC5D,OAAO,GAAG,IAAI,CAAC,SAAS,GAAG,QAAQ,IAAI,YAAY,IAAI,KAAK,MAAM,CAAC;IACrE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,IAAgB;QACxC,MAAM,EAAE,gBAAgB,EAAE,GAAG,QAAQ,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,gBAAgB,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,GAAG,EAAE,GAAG;YACR,IAAI,EAAE,IAAI;YACV,WAAW,EAAE,0BAA0B;YACvC,oBAAoB,EAAE,IAAI;iBACvB,oBAAyE;YAC5E,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;SAC3D,CAAC,CACH,CAAC;QAEF,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW;QACxB,MAAM,EAAE,gBAAgB,EAAE,GAAG,QAAQ,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,gBAAgB,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,GAAG,EAAE,GAAG;SACT,CAAC,CACH,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,4CAA4C;QAC5C,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC7D,OAAO,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,EAAE,mBAAmB,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,mBAAmB,CAAC;YACtB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,GAAG,EAAE,GAAG;SACT,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,IAAc;QAC9B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE9B,MAAM,EAAE,oBAAoB,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,wDAAwD;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,oBAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,UAAU;gBACvB,MAAM,EAAE;oBACN,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC3C,KAAK,EAAE,IAAI;iBACZ;aACF,CAAC,CACH,CAAC;YAEF,6DAA6D;YAC7D,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CACpC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,GAAG,CACpD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACb,IAAA,kBAAS,GAAE,CAAC,IAAI,CACd,mBAAmB,QAAQ,CAAC,MAAM,CAAC,MAAM,8BAA8B,UAAU,EAAE,CACpF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,mBAAmB,CAAC,OAAe;QACvC,MAAM,EAAE,sCAAsC,EAAE,sCAAsC,EAAE,GACtF,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE3C,2CAA2C;QAC3C,IAAI,aAAa,GAMZ,EAAE,CAAC;QAER,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,sCAAsC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CACxE,CAAC;YACF,aAAa,GAAI,QAAQ,CAAC,KAA8B,IAAI,EAAE,CAAC;QACjE,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,qFAAqF;YACrF,IACE,KAAK;gBACL,OAAO,KAAK,KAAK,QAAQ;gBACzB,MAAM,IAAI,KAAK;gBACd,KAA0B,CAAC,IAAI,KAAK,8BAA8B,EACnE,CAAC;gBACD,aAAa,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC;QAChE,IACE,YAAY;YACZ,YAAY,CAAC,MAAM,KAAK,SAAS;YACjC,YAAY,CAAC,UAAU;YACtB,YAAY,CAAC,UAAgC,CAAC,IAAI,KAAK,OAAO,EAC/D,CAAC;YACD,4DAA4D;YAC5D,IAAA,kBAAS,GAAE,CAAC,IAAI,CACd,sBAAsB,MAAM,yBAAyB,OAAO,4BAA4B,CACzF,CAAC;YACF,OAAO;QACT,CAAC;QAED,wBAAwB;QACxB,MAAM,OAAO,GAAG;YACd,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;YAClC,MAAM,EAAE,SAAkB;YAC1B,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;SAC9B,CAAC;QAEF,yDAAyD;QACzD,MAAM,WAAW,GAAG,YAAY;YAC9B,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,CAAC,CAAC,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,CAAC;QAEhC,gBAAgB;QAChB,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,sCAAsC,CAAC;YACzC,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,sBAAsB,EAAE;gBACtB,KAAK,EAAE,WAAsE;aAC9E;SACF,CAAC,CACH,CAAC;QAEF,IAAA,kBAAS,GAAE,CAAC,IAAI,CACd,sBAAsB,MAAM,iBAAiB,OAAO,8BAA8B,IAAI,CAAC,SAAS,GAAG,CACpG,CAAC;IACJ,CAAC;IAED;;;;;;;;OAQG;IACK,oBAAoB;QAC1B,qFAAqF;QACrF,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;QAC5F,OAAO,iBAAiB,IAAI,EAAE,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAnRD,kCAmRC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Best-effort S3 orphan cleanup shared by checkpointer writers.
|
|
3
|
+
*
|
|
4
|
+
* When a DynamoDB write fails *after* payloads have already been uploaded to S3,
|
|
5
|
+
* this helper deletes the uploaded objects so they don't linger until the
|
|
6
|
+
* lifecycle rule sweeps them.
|
|
7
|
+
*
|
|
8
|
+
* Transient S3 errors (503 SlowDown, 500 InternalError, throttling, socket
|
|
9
|
+
* blips) get bounded retries with full-jitter backoff so a single bad minute
|
|
10
|
+
* doesn't leak objects. Persistent failures are only logged — the lifecycle
|
|
11
|
+
* rule is the ultimate backstop and failing the caller's write *after* DDB has
|
|
12
|
+
* already rolled back would add confusion, not safety.
|
|
13
|
+
*/
|
|
14
|
+
import type { S3Offloader } from './s3-offloader';
|
|
15
|
+
export declare function cleanUpS3Orphans(offloader: S3Offloader, keys: ReadonlyArray<string | undefined>, context: string): Promise<void>;
|
|
16
|
+
//# sourceMappingURL=s3-orphans.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3-orphans.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/s3-orphans.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAqClD,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,WAAW,EACtB,IAAI,EAAE,aAAa,CAAC,MAAM,GAAG,SAAS,CAAC,EACvC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CA4Bf"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Best-effort S3 orphan cleanup shared by checkpointer writers.
|
|
4
|
+
*
|
|
5
|
+
* When a DynamoDB write fails *after* payloads have already been uploaded to S3,
|
|
6
|
+
* this helper deletes the uploaded objects so they don't linger until the
|
|
7
|
+
* lifecycle rule sweeps them.
|
|
8
|
+
*
|
|
9
|
+
* Transient S3 errors (503 SlowDown, 500 InternalError, throttling, socket
|
|
10
|
+
* blips) get bounded retries with full-jitter backoff so a single bad minute
|
|
11
|
+
* doesn't leak objects. Persistent failures are only logged — the lifecycle
|
|
12
|
+
* rule is the ultimate backstop and failing the caller's write *after* DDB has
|
|
13
|
+
* already rolled back would add confusion, not safety.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.cleanUpS3Orphans = cleanUpS3Orphans;
|
|
17
|
+
const logger_1 = require("./logger");
|
|
18
|
+
const sleep_1 = require("./sleep");
|
|
19
|
+
const ORPHAN_CLEANUP_MAX_ATTEMPTS = 3;
|
|
20
|
+
const ORPHAN_CLEANUP_BASE_DELAY_MS = 100;
|
|
21
|
+
const ORPHAN_CLEANUP_MAX_DELAY_MS = 2_000;
|
|
22
|
+
/** Error codes/names that indicate an S3 request is worth re-issuing. */
|
|
23
|
+
const RETRYABLE_S3_SIGNALS = [
|
|
24
|
+
'SlowDown',
|
|
25
|
+
'InternalError',
|
|
26
|
+
'ServiceUnavailable',
|
|
27
|
+
'RequestTimeout',
|
|
28
|
+
'ThrottlingException',
|
|
29
|
+
'ProvisionedThroughputExceededException',
|
|
30
|
+
// Node-level network transients (also caught by @aws-sdk/client-s3 wrappers).
|
|
31
|
+
'ECONNRESET',
|
|
32
|
+
'ECONNREFUSED',
|
|
33
|
+
'ETIMEDOUT',
|
|
34
|
+
'EPIPE',
|
|
35
|
+
'NetworkingError',
|
|
36
|
+
'TimeoutError',
|
|
37
|
+
];
|
|
38
|
+
function isTransientS3Error(err) {
|
|
39
|
+
if (!err || typeof err !== 'object')
|
|
40
|
+
return false;
|
|
41
|
+
const signals = [];
|
|
42
|
+
const e = err;
|
|
43
|
+
if (typeof e.name === 'string')
|
|
44
|
+
signals.push(e.name);
|
|
45
|
+
if (typeof e.code === 'string')
|
|
46
|
+
signals.push(e.code);
|
|
47
|
+
const status = e.$metadata?.httpStatusCode;
|
|
48
|
+
if (typeof status === 'number' && (status === 429 || (status >= 500 && status < 600))) {
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
return RETRYABLE_S3_SIGNALS.some((s) => signals.some((sig) => sig.includes(s)));
|
|
52
|
+
}
|
|
53
|
+
async function cleanUpS3Orphans(offloader, keys, context) {
|
|
54
|
+
const orphans = keys.filter((k) => typeof k === 'string' && k.length > 0);
|
|
55
|
+
if (orphans.length === 0)
|
|
56
|
+
return;
|
|
57
|
+
let delay = ORPHAN_CLEANUP_BASE_DELAY_MS;
|
|
58
|
+
let lastErr;
|
|
59
|
+
for (let attempt = 1; attempt <= ORPHAN_CLEANUP_MAX_ATTEMPTS; attempt++) {
|
|
60
|
+
try {
|
|
61
|
+
await offloader.deleteBatch(orphans);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
lastErr = err;
|
|
66
|
+
if (attempt >= ORPHAN_CLEANUP_MAX_ATTEMPTS || !isTransientS3Error(err)) {
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
await (0, sleep_1.sleep)((0, sleep_1.fullJitter)(delay));
|
|
70
|
+
delay = Math.min(delay * 2, ORPHAN_CLEANUP_MAX_DELAY_MS);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const msg = lastErr && typeof lastErr === 'object' && 'message' in lastErr
|
|
74
|
+
? String(lastErr.message)
|
|
75
|
+
: String(lastErr);
|
|
76
|
+
(0, logger_1.getLogger)().warn(`Failed to clean up orphaned S3 objects after ${context} ` +
|
|
77
|
+
`(${ORPHAN_CLEANUP_MAX_ATTEMPTS} attempts, lifecycle rule will sweep): ${msg}`);
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=s3-orphans.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3-orphans.js","sourceRoot":"","sources":["../../../src/shared/utils/s3-orphans.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG;;AAwCH,4CAgCC;AAtED,qCAAqC;AAErC,mCAA4C;AAE5C,MAAM,2BAA2B,GAAG,CAAC,CAAC;AACtC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AACzC,MAAM,2BAA2B,GAAG,KAAK,CAAC;AAE1C,yEAAyE;AACzE,MAAM,oBAAoB,GAAsB;IAC9C,UAAU;IACV,eAAe;IACf,oBAAoB;IACpB,gBAAgB;IAChB,qBAAqB;IACrB,wCAAwC;IACxC,8EAA8E;IAC9E,YAAY;IACZ,cAAc;IACd,WAAW;IACX,OAAO;IACP,iBAAiB;IACjB,cAAc;CACf,CAAC;AAEF,SAAS,kBAAkB,CAAC,GAAY;IACtC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,CAAC,GAAG,GAAkF,CAAC;IAC7F,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,CAAC,CAAC,SAAS,EAAE,cAAc,CAAC;IAC3C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;QACtF,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClF,CAAC;AAEM,KAAK,UAAU,gBAAgB,CACpC,SAAsB,EACtB,IAAuC,EACvC,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEjC,IAAI,KAAK,GAAG,4BAA4B,CAAC;IACzC,IAAI,OAAgB,CAAC;IACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,2BAA2B,EAAE,OAAO,EAAE,EAAE,CAAC;QACxE,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YACrC,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,GAAG,GAAG,CAAC;YACd,IAAI,OAAO,IAAI,2BAA2B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvE,MAAM;YACR,CAAC;YACD,MAAM,IAAA,aAAK,EAAC,IAAA,kBAAU,EAAC,KAAK,CAAC,CAAC,CAAC;YAC/B,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GACP,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,SAAS,IAAI,OAAO;QAC5D,CAAC,CAAC,MAAM,CAAE,OAAgC,CAAC,OAAO,CAAC;QACnD,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtB,IAAA,kBAAS,GAAE,CAAC,IAAI,CACd,gDAAgD,OAAO,GAAG;QACxD,IAAI,2BAA2B,0CAA0C,GAAG,EAAE,CACjF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backoff helper shared by the UnprocessedItems / UnprocessedKeys retry loops
|
|
3
|
+
* in batch-write and BatchGetItem paths. Centralized so both call sites use
|
|
4
|
+
* the same cap, growth, and sleep semantics.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Sleep for `ms` milliseconds, optionally cancellable via an `AbortSignal`.
|
|
8
|
+
*
|
|
9
|
+
* If `signal` is already aborted the returned promise rejects synchronously with
|
|
10
|
+
* the signal's abort reason. Otherwise the timer resolves on the normal path and
|
|
11
|
+
* the abort listener is removed to avoid leaking handlers.
|
|
12
|
+
*
|
|
13
|
+
* The `settled` flag is a belt-and-braces guard: `{ once: true }` already
|
|
14
|
+
* prevents the listener from firing twice, but the guard also ensures we never
|
|
15
|
+
* resolve *and* reject (which would produce an unhandled rejection if the timer
|
|
16
|
+
* and abort event fire in the same microtask turn).
|
|
17
|
+
*/
|
|
18
|
+
export declare function sleep(ms: number, signal?: AbortSignal): Promise<void>;
|
|
19
|
+
/**
|
|
20
|
+
* Exponential-backoff delay calculator. Given the *previous* delay, returns the
|
|
21
|
+
* next delay — double it, capped at `max`. Use with {@link sleep} and
|
|
22
|
+
* {@link fullJitter} so concurrent retriers don't synchronize.
|
|
23
|
+
*
|
|
24
|
+
* let delay = INITIAL_DELAY_MS;
|
|
25
|
+
* while (keep_going) {
|
|
26
|
+
* await sleep(fullJitter(delay));
|
|
27
|
+
* delay = nextBackoffDelay(delay);
|
|
28
|
+
* }
|
|
29
|
+
*/
|
|
30
|
+
export declare function nextBackoffDelay(currentMs: number, maxMs?: number): number;
|
|
31
|
+
/**
|
|
32
|
+
* Apply full jitter (AWS recommended) to a backoff delay.
|
|
33
|
+
*
|
|
34
|
+
* Returns a uniform random value in `[0, delayMs)`. Spreading retries across the
|
|
35
|
+
* backoff window prevents the thundering-herd pattern where every throttled
|
|
36
|
+
* client wakes up at the same instant and re-throttles the service.
|
|
37
|
+
* See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
|
|
38
|
+
*/
|
|
39
|
+
export declare function fullJitter(delayMs: number): number;
|
|
40
|
+
/**
|
|
41
|
+
* Starting delay for UnprocessedItems / UnprocessedKeys retry loops.
|
|
42
|
+
*/
|
|
43
|
+
export declare const INITIAL_BACKOFF_DELAY_MS = 100;
|
|
44
|
+
/**
|
|
45
|
+
* Maximum delay for UnprocessedItems / UnprocessedKeys retry loops.
|
|
46
|
+
*/
|
|
47
|
+
export declare const MAX_BACKOFF_DELAY_MS = 5000;
|
|
48
|
+
//# sourceMappingURL=sleep.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sleep.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/sleep.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH;;;;;;;;;;;GAWG;AACH,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAoBrE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,GAAE,MAA6B,GAAG,MAAM,CAEhG;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAElD;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,MAA2B,CAAC;AAEjE;;GAEG;AACH,eAAO,MAAM,oBAAoB,OAAuB,CAAC"}
|