@farukada/aws-langgraph-dynamodb-ts 0.0.11 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +353 -242
- package/dist/checkpointer/actions/delete-thread.d.ts +3 -3
- package/dist/checkpointer/actions/delete-thread.d.ts.map +1 -1
- package/dist/checkpointer/actions/delete-thread.js +95 -52
- package/dist/checkpointer/actions/delete-thread.js.map +1 -1
- package/dist/checkpointer/actions/get-tuple.d.ts +8 -1
- package/dist/checkpointer/actions/get-tuple.d.ts.map +1 -1
- package/dist/checkpointer/actions/get-tuple.js +72 -20
- package/dist/checkpointer/actions/get-tuple.js.map +1 -1
- package/dist/checkpointer/actions/put-writes.d.ts.map +1 -1
- package/dist/checkpointer/actions/put-writes.js +27 -22
- package/dist/checkpointer/actions/put-writes.js.map +1 -1
- package/dist/checkpointer/actions/put.d.ts +6 -2
- package/dist/checkpointer/actions/put.d.ts.map +1 -1
- package/dist/checkpointer/actions/put.js +80 -19
- package/dist/checkpointer/actions/put.js.map +1 -1
- package/dist/checkpointer/index.d.ts +43 -2
- package/dist/checkpointer/index.d.ts.map +1 -1
- package/dist/checkpointer/index.js +226 -11
- package/dist/checkpointer/index.js.map +1 -1
- package/dist/checkpointer/types/index.d.ts +66 -4
- package/dist/checkpointer/types/index.d.ts.map +1 -1
- package/dist/checkpointer/types/index.js +10 -0
- package/dist/checkpointer/types/index.js.map +1 -1
- package/dist/checkpointer/utils/deserialization.d.ts +8 -3
- package/dist/checkpointer/utils/deserialization.d.ts.map +1 -1
- package/dist/checkpointer/utils/deserialization.js +23 -5
- package/dist/checkpointer/utils/deserialization.js.map +1 -1
- package/dist/checkpointer/utils/validation.d.ts +1 -1
- package/dist/checkpointer/utils/validation.js +1 -1
- package/dist/checkpointer/utils/validation.js.map +1 -1
- package/dist/factory.d.ts +8 -6
- package/dist/factory.d.ts.map +1 -1
- package/dist/factory.js +16 -4
- package/dist/factory.js.map +1 -1
- package/dist/history/actions/add-message.d.ts +2 -1
- package/dist/history/actions/add-message.d.ts.map +1 -1
- package/dist/history/actions/add-message.js +22 -18
- package/dist/history/actions/add-message.js.map +1 -1
- package/dist/history/actions/add-messages.d.ts +3 -2
- package/dist/history/actions/add-messages.d.ts.map +1 -1
- package/dist/history/actions/add-messages.js +30 -20
- package/dist/history/actions/add-messages.js.map +1 -1
- package/dist/history/actions/clear.d.ts +4 -3
- package/dist/history/actions/clear.d.ts.map +1 -1
- package/dist/history/actions/clear.js +47 -12
- package/dist/history/actions/clear.js.map +1 -1
- package/dist/history/actions/get-messages.d.ts +2 -1
- package/dist/history/actions/get-messages.d.ts.map +1 -1
- package/dist/history/actions/get-messages.js +41 -15
- package/dist/history/actions/get-messages.js.map +1 -1
- package/dist/history/actions/list-sessions.d.ts +11 -0
- package/dist/history/actions/list-sessions.d.ts.map +1 -1
- package/dist/history/actions/list-sessions.js +23 -3
- package/dist/history/actions/list-sessions.js.map +1 -1
- package/dist/history/index.d.ts +8 -0
- package/dist/history/index.d.ts.map +1 -1
- package/dist/history/index.js +22 -2
- package/dist/history/index.js.map +1 -1
- package/dist/history/types/index.d.ts +27 -5
- package/dist/history/types/index.d.ts.map +1 -1
- package/dist/history/types/index.js +1 -0
- package/dist/history/types/index.js.map +1 -1
- package/dist/history/utils/title-generator.js.map +1 -1
- package/dist/history/utils/update-expression-builder.d.ts +45 -21
- package/dist/history/utils/update-expression-builder.d.ts.map +1 -1
- package/dist/history/utils/update-expression-builder.js +105 -38
- package/dist/history/utils/update-expression-builder.js.map +1 -1
- package/dist/history/utils/validation.d.ts +8 -0
- package/dist/history/utils/validation.d.ts.map +1 -1
- package/dist/history/utils/validation.js +22 -0
- package/dist/history/utils/validation.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/shared/utils/batch-write.d.ts +36 -0
- package/dist/shared/utils/batch-write.d.ts.map +1 -0
- package/dist/shared/utils/batch-write.js +62 -0
- package/dist/shared/utils/batch-write.js.map +1 -0
- package/dist/shared/utils/compressor.d.ts +48 -0
- package/dist/shared/utils/compressor.d.ts.map +1 -0
- package/dist/shared/utils/compressor.js +68 -0
- package/dist/shared/utils/compressor.js.map +1 -0
- package/dist/shared/utils/constants.d.ts +2 -0
- package/dist/shared/utils/constants.d.ts.map +1 -1
- package/dist/shared/utils/constants.js +3 -7
- package/dist/shared/utils/constants.js.map +1 -1
- package/dist/shared/utils/index.d.ts +4 -0
- package/dist/shared/utils/index.d.ts.map +1 -1
- package/dist/shared/utils/index.js +4 -0
- package/dist/shared/utils/index.js.map +1 -1
- package/dist/shared/utils/logger.d.ts +48 -0
- package/dist/shared/utils/logger.d.ts.map +1 -0
- package/dist/shared/utils/logger.js +62 -0
- package/dist/shared/utils/logger.js.map +1 -0
- package/dist/shared/utils/retry.d.ts +1 -1
- package/dist/shared/utils/retry.d.ts.map +1 -1
- package/dist/shared/utils/retry.js +13 -6
- package/dist/shared/utils/retry.js.map +1 -1
- package/dist/shared/utils/s3-offloader.d.ts +146 -0
- package/dist/shared/utils/s3-offloader.d.ts.map +1 -0
- package/dist/shared/utils/s3-offloader.js +271 -0
- package/dist/shared/utils/s3-offloader.js.map +1 -0
- package/dist/shared/utils/ttl.d.ts +8 -1
- package/dist/shared/utils/ttl.d.ts.map +1 -1
- package/dist/shared/utils/ttl.js +12 -8
- package/dist/shared/utils/ttl.js.map +1 -1
- package/dist/store/actions/get-operation.js +2 -2
- package/dist/store/actions/get-operation.js.map +1 -1
- package/dist/store/actions/put-operation.d.ts.map +1 -1
- package/dist/store/actions/put-operation.js +2 -1
- package/dist/store/actions/put-operation.js.map +1 -1
- package/dist/store/actions/search-operation.d.ts +6 -0
- package/dist/store/actions/search-operation.d.ts.map +1 -1
- package/dist/store/actions/search-operation.js +41 -21
- package/dist/store/actions/search-operation.js.map +1 -1
- package/dist/store/index.d.ts +10 -2
- package/dist/store/index.d.ts.map +1 -1
- package/dist/store/index.js +28 -5
- package/dist/store/index.js.map +1 -1
- package/dist/store/types/index.d.ts +7 -5
- package/dist/store/types/index.d.ts.map +1 -1
- package/dist/store/utils/index.d.ts +0 -1
- package/dist/store/utils/index.d.ts.map +1 -1
- package/dist/store/utils/index.js +0 -1
- package/dist/store/utils/index.js.map +1 -1
- package/dist/store/utils/validation.js +1 -1
- package/dist/store/utils/validation.js.map +1 -1
- package/package.json +43 -26
- package/dist/store/utils/result.d.ts +0 -42
- package/dist/store/utils/result.d.ts.map +0 -1
- package/dist/store/utils/result.js +0 -51
- package/dist/store/utils/result.js.map +0 -1
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configurable logger for the library
|
|
3
|
+
*
|
|
4
|
+
* Provides a pluggable logging interface so consumers can integrate
|
|
5
|
+
* with their preferred logging framework (pino, winston, etc.)
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```TypeScript
|
|
9
|
+
* import { setGlobalLogger } from '@farukada/aws-langgraph-dynamodb-ts';
|
|
10
|
+
*
|
|
11
|
+
* // Use a custom logger
|
|
12
|
+
* setGlobalLogger({
|
|
13
|
+
* info: (msg, ...args) => myLogger.info(msg, ...args),
|
|
14
|
+
* warn: (msg, ...args) => myLogger.warn(msg, ...args),
|
|
15
|
+
* error: (msg, ...args) => myLogger.error(msg, ...args),
|
|
16
|
+
* debug: (msg, ...args) => myLogger.debug(msg, ...args),
|
|
17
|
+
* });
|
|
18
|
+
*
|
|
19
|
+
* // Disable logging entirely
|
|
20
|
+
* setGlobalLogger({ info: () => {}, warn: () => {}, error: () => {}, debug: () => {} });
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
/**
|
|
24
|
+
* Logger interface - consumers can provide their own implementation
|
|
25
|
+
*/
|
|
26
|
+
export interface Logger {
|
|
27
|
+
info(message: string, ...args: unknown[]): void;
|
|
28
|
+
warn(message: string, ...args: unknown[]): void;
|
|
29
|
+
error(message: string, ...args: unknown[]): void;
|
|
30
|
+
debug(message: string, ...args: unknown[]): void;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Set a custom global logger for the library
|
|
34
|
+
*
|
|
35
|
+
* @param logger - Custom logger implementation
|
|
36
|
+
*/
|
|
37
|
+
export declare function setGlobalLogger(logger: Logger): void;
|
|
38
|
+
/**
|
|
39
|
+
* Get the current global logger
|
|
40
|
+
*
|
|
41
|
+
* @returns The currently configured logger
|
|
42
|
+
*/
|
|
43
|
+
export declare function getLogger(): Logger;
|
|
44
|
+
/**
|
|
45
|
+
* Reset the logger to the default console-based implementation
|
|
46
|
+
*/
|
|
47
|
+
export declare function resetLogger(): void;
|
|
48
|
+
//# sourceMappingURL=logger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAChD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAChD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IACjD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;CAClD;AAuBD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAEpD;AAED;;;;GAIG;AACH,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED;;GAEG;AACH,wBAAgB,WAAW,IAAI,IAAI,CAElC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Configurable logger for the library
|
|
4
|
+
*
|
|
5
|
+
* Provides a pluggable logging interface so consumers can integrate
|
|
6
|
+
* with their preferred logging framework (pino, winston, etc.)
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```TypeScript
|
|
10
|
+
* import { setGlobalLogger } from '@farukada/aws-langgraph-dynamodb-ts';
|
|
11
|
+
*
|
|
12
|
+
* // Use a custom logger
|
|
13
|
+
* setGlobalLogger({
|
|
14
|
+
* info: (msg, ...args) => myLogger.info(msg, ...args),
|
|
15
|
+
* warn: (msg, ...args) => myLogger.warn(msg, ...args),
|
|
16
|
+
* error: (msg, ...args) => myLogger.error(msg, ...args),
|
|
17
|
+
* debug: (msg, ...args) => myLogger.debug(msg, ...args),
|
|
18
|
+
* });
|
|
19
|
+
*
|
|
20
|
+
* // Disable logging entirely
|
|
21
|
+
* setGlobalLogger({ info: () => {}, warn: () => {}, error: () => {}, debug: () => {} });
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
exports.setGlobalLogger = setGlobalLogger;
|
|
26
|
+
exports.getLogger = getLogger;
|
|
27
|
+
exports.resetLogger = resetLogger;
|
|
28
|
+
/**
|
|
29
|
+
* Default logger using console
|
|
30
|
+
*/
|
|
31
|
+
/* eslint-disable no-console -- default logger intentionally uses console; override via setGlobalLogger() */
|
|
32
|
+
const defaultLogger = {
|
|
33
|
+
info: (message, ...args) => console.info(`[langgraph-dynamodb] ${message}`, ...args),
|
|
34
|
+
warn: (message, ...args) => console.warn(`[langgraph-dynamodb] ${message}`, ...args),
|
|
35
|
+
error: (message, ...args) => console.error(`[langgraph-dynamodb] ${message}`, ...args),
|
|
36
|
+
debug: (message, ...args) => console.debug(`[langgraph-dynamodb] ${message}`, ...args),
|
|
37
|
+
};
|
|
38
|
+
/* eslint-enable no-console */
|
|
39
|
+
let globalLogger = defaultLogger;
|
|
40
|
+
/**
|
|
41
|
+
* Set a custom global logger for the library
|
|
42
|
+
*
|
|
43
|
+
* @param logger - Custom logger implementation
|
|
44
|
+
*/
|
|
45
|
+
function setGlobalLogger(logger) {
|
|
46
|
+
globalLogger = logger;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Get the current global logger
|
|
50
|
+
*
|
|
51
|
+
* @returns The currently configured logger
|
|
52
|
+
*/
|
|
53
|
+
function getLogger() {
|
|
54
|
+
return globalLogger;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Reset the logger to the default console-based implementation
|
|
58
|
+
*/
|
|
59
|
+
function resetLogger() {
|
|
60
|
+
globalLogger = defaultLogger;
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../../src/shared/utils/logger.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;;AAsCH,0CAEC;AAOD,8BAEC;AAKD,kCAEC;AA5CD;;GAEG;AACH,4GAA4G;AAC5G,MAAM,aAAa,GAAW;IAC5B,IAAI,EAAE,CAAC,OAAe,EAAE,GAAG,IAAe,EAAE,EAAE,CAC5C,OAAO,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC;IAE1D,IAAI,EAAE,CAAC,OAAe,EAAE,GAAG,IAAe,EAAE,EAAE,CAC5C,OAAO,CAAC,IAAI,CAAC,wBAAwB,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC;IAE1D,KAAK,EAAE,CAAC,OAAe,EAAE,GAAG,IAAe,EAAE,EAAE,CAC7C,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC;IAE3D,KAAK,EAAE,CAAC,OAAe,EAAE,GAAG,IAAe,EAAE,EAAE,CAC7C,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC;CAC5D,CAAC;AACF,8BAA8B;AAE9B,IAAI,YAAY,GAAW,aAAa,CAAC;AAEzC;;;;GAIG;AACH,SAAgB,eAAe,CAAC,MAAc;IAC5C,YAAY,GAAG,MAAM,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,SAAgB,SAAS;IACvB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW;IACzB,YAAY,GAAG,aAAa,CAAC;AAC/B,CAAC"}
|
|
@@ -15,5 +15,5 @@ export declare function withRetry<T>(fn: () => Promise<T>, options?: RetryOption
|
|
|
15
15
|
/**
|
|
16
16
|
* Specialized retry for DynamoDB operations
|
|
17
17
|
*/
|
|
18
|
-
export declare function withDynamoDBRetry<T>(fn: () => Promise<T>): Promise<T>;
|
|
18
|
+
export declare function withDynamoDBRetry<T>(fn: () => Promise<T>, overrides?: Partial<RetryOptions>): Promise<T>;
|
|
19
19
|
//# sourceMappingURL=retry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/retry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;
|
|
1
|
+
{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/retry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AA2CD;;GAEG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,CAAC,CAAC,CA0D/F;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,CAAC,EACvC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,SAAS,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAChC,OAAO,CAAC,CAAC,CAAC,CAOZ"}
|
|
@@ -7,7 +7,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
7
7
|
exports.withRetry = withRetry;
|
|
8
8
|
exports.withDynamoDBRetry = withDynamoDBRetry;
|
|
9
9
|
const DEFAULT_OPTIONS = {
|
|
10
|
-
maxAttempts:
|
|
10
|
+
maxAttempts: 5,
|
|
11
11
|
baseDelayMs: 100,
|
|
12
12
|
maxDelayMs: 5000,
|
|
13
13
|
retryableErrors: [
|
|
@@ -16,6 +16,7 @@ const DEFAULT_OPTIONS = {
|
|
|
16
16
|
'RequestLimitExceeded',
|
|
17
17
|
'InternalServerError',
|
|
18
18
|
'ServiceUnavailable',
|
|
19
|
+
'TransactionCanceledException',
|
|
19
20
|
],
|
|
20
21
|
};
|
|
21
22
|
/**
|
|
@@ -38,7 +39,8 @@ function calculateDelay(attempt, baseDelayMs, maxDelayMs) {
|
|
|
38
39
|
function isRetryableError(error, retryableErrors) {
|
|
39
40
|
if (!error)
|
|
40
41
|
return false;
|
|
41
|
-
const
|
|
42
|
+
const errObj = error;
|
|
43
|
+
const errorName = String(errObj.name ?? errObj.code ?? '');
|
|
42
44
|
return retryableErrors.some((retryable) => errorName.includes(retryable));
|
|
43
45
|
}
|
|
44
46
|
/**
|
|
@@ -64,7 +66,9 @@ async function withRetry(fn, options = {}) {
|
|
|
64
66
|
if (error instanceof Error) {
|
|
65
67
|
throw error;
|
|
66
68
|
}
|
|
67
|
-
const wrappedError = new Error(error
|
|
69
|
+
const wrappedError = new Error(error && typeof error === 'object' && 'message' in error
|
|
70
|
+
? String(error.message)
|
|
71
|
+
: String(error));
|
|
68
72
|
// Preserve name and other properties from the original error
|
|
69
73
|
if (error && typeof error === 'object') {
|
|
70
74
|
Object.assign(wrappedError, error);
|
|
@@ -85,7 +89,9 @@ async function withRetry(fn, options = {}) {
|
|
|
85
89
|
throw lastError;
|
|
86
90
|
}
|
|
87
91
|
// Convert to Error while preserving properties
|
|
88
|
-
const wrappedError = new Error(lastError
|
|
92
|
+
const wrappedError = new Error(lastError && typeof lastError === 'object' && 'message' in lastError
|
|
93
|
+
? String(lastError.message)
|
|
94
|
+
: String(lastError));
|
|
89
95
|
if (lastError && typeof lastError === 'object') {
|
|
90
96
|
Object.assign(wrappedError, lastError);
|
|
91
97
|
}
|
|
@@ -94,11 +100,12 @@ async function withRetry(fn, options = {}) {
|
|
|
94
100
|
/**
|
|
95
101
|
* Specialized retry for DynamoDB operations
|
|
96
102
|
*/
|
|
97
|
-
async function withDynamoDBRetry(fn) {
|
|
103
|
+
async function withDynamoDBRetry(fn, overrides) {
|
|
98
104
|
return withRetry(fn, {
|
|
99
|
-
maxAttempts:
|
|
105
|
+
maxAttempts: 5,
|
|
100
106
|
baseDelayMs: 100,
|
|
101
107
|
maxDelayMs: 5000,
|
|
108
|
+
...overrides,
|
|
102
109
|
});
|
|
103
110
|
}
|
|
104
111
|
//# sourceMappingURL=retry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../../src/shared/utils/retry.ts"],"names":[],"mappings":";AAAA;;;GAGG;;
|
|
1
|
+
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../../src/shared/utils/retry.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAqDH,8BA0DC;AAKD,8CAUC;AArHD,MAAM,eAAe,GAA2B;IAC9C,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,GAAG;IAChB,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE;QACf,wCAAwC;QACxC,qBAAqB;QACrB,sBAAsB;QACtB,qBAAqB;QACrB,oBAAoB;QACpB,8BAA8B;KAC/B;CACF,CAAC;AAEF;;GAEG;AACH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,OAAe,EAAE,WAAmB,EAAE,UAAkB;IAC9E,MAAM,gBAAgB,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,gBAAgB,CAAC,CAAC,uBAAuB;IAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,GAAG,MAAM,EAAE,UAAU,CAAC,CAAC;AACzD,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,KAAc,EAAE,eAAyB;IACjE,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IAEzB,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3D,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,SAAS,CAAI,EAAoB,EAAE,UAAwB,EAAE;IACjF,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChD,IAAI,SAAkB,CAAC;IAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QAC7D,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAElB,0CAA0C;YAC1C,IAAI,OAAO,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjC,MAAM;YACR,CAAC;YAED,4CAA4C;YAC5C,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACnD,oDAAoD;gBACpD,uDAAuD;gBACvD,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;oBAC3B,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,YAAY,GAAG,IAAI,KAAK,CAC5B,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,SAAS,IAAI,KAAK;oBACtD,CAAC,CAAC,MAAM,CAAE,KAA8B,CAAC,OAAO,CAAC;oBACjD,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAClB,CAAC;gBACF,6DAA6D;gBAC7D,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBACvC,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;gBACrC,CAAC;gBACD,MAAM,YAAY,CAAC;YACrB,CAAC;YAED,2BAA2B;YAC3B,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChD,CAAC;IACD,uDAAuD;IACvD,IAAI,SAAS,YAAY,KAAK,EAAE,CAAC;QAC/B,MAAM,SAAS,CAAC;IAClB,CAAC;IACD,+CAA+C;IAC/C,MAAM,YAAY,GAAG,IAAI,KAAK,CAC5B,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,IAAI,SAAS;QAClE,CAAC,CAAC,MAAM,CAAE,SAAkC,CAAC,OAAO,CAAC;QACrD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CACtB,CAAC;IACF,IAAI,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,YAAY,CAAC;AACrB,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,iBAAiB,CACrC,EAAoB,EACpB,SAAiC;IAEjC,OAAO,SAAS,CAAC,EAAE,EAAE;QACnB,WAAW,EAAE,CAAC;QACd,WAAW,EAAE,GAAG;QAChB,UAAU,EAAE,IAAI;QAChB,GAAG,SAAS;KACb,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* S3 offloader for large checkpoint payloads
|
|
3
|
+
*
|
|
4
|
+
* DynamoDB has a hard 400KB item size limit. This utility transparently
|
|
5
|
+
* offloads payloads exceeding a configurable threshold (default: 350KB)
|
|
6
|
+
* to S3, storing only an S3 key reference in DynamoDB.
|
|
7
|
+
*
|
|
8
|
+
* Requires `@aws-sdk/client-s3` as an optional peer dependency.
|
|
9
|
+
*
|
|
10
|
+
* @remarks
|
|
11
|
+
* This module uses synchronous `require()` for lazy loading of the S3 SDK
|
|
12
|
+
* to avoid making the DynamoDBSaver constructor async. This means it
|
|
13
|
+
* requires a CommonJS-compatible environment (Node.js with CJS, or a
|
|
14
|
+
* bundler that polyfills require). Pure ESM environments are not supported.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Configuration for S3 offloading
|
|
18
|
+
*/
|
|
19
|
+
export interface S3OffloadConfig {
|
|
20
|
+
/** S3 bucket name for storing offloaded payloads (required) */
|
|
21
|
+
bucketName: string;
|
|
22
|
+
/** Key prefix for S3 objects (default: 'langgraph-checkpoints/') */
|
|
23
|
+
keyPrefix?: string;
|
|
24
|
+
/** Payload size threshold in bytes that triggers offloading (default: 358400 = 350KB) */
|
|
25
|
+
thresholdBytes?: number;
|
|
26
|
+
/** Server-side encryption algorithm (e.g., 'AES256' or 'aws:kms'). Not set by default. */
|
|
27
|
+
serverSideEncryption?: string;
|
|
28
|
+
/** KMS key ID or ARN. Only used when serverSideEncryption is 'aws:kms'. */
|
|
29
|
+
sseKmsKeyId?: string;
|
|
30
|
+
/**
|
|
31
|
+
* Optional S3 client configuration (region, credentials, endpoint, etc.).
|
|
32
|
+
* Accepts any valid S3ClientConfig properties from @aws-sdk/client-s3.
|
|
33
|
+
*/
|
|
34
|
+
clientConfig?: {
|
|
35
|
+
region?: string;
|
|
36
|
+
endpoint?: string;
|
|
37
|
+
credentials?: unknown;
|
|
38
|
+
[key: string]: unknown;
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* S3 offloader for large checkpoint payloads
|
|
43
|
+
*
|
|
44
|
+
* Transparently uploads large payloads to S3 and returns S3 key references
|
|
45
|
+
* for storage in DynamoDB. On read, downloads from S3 when a reference is present.
|
|
46
|
+
*
|
|
47
|
+
* @example
|
|
48
|
+
* ```typescript
|
|
49
|
+
* const offloader = new S3Offloader({
|
|
50
|
+
* bucketName: 'my-checkpoints-bucket',
|
|
51
|
+
* keyPrefix: 'langgraph/',
|
|
52
|
+
* thresholdBytes: 350 * 1024,
|
|
53
|
+
* });
|
|
54
|
+
*
|
|
55
|
+
* if (offloader.shouldOffload(data)) {
|
|
56
|
+
* const key = await offloader.upload('thread/checkpoint/field.bin', data);
|
|
57
|
+
* // Store `key` in DynamoDB instead of `data`
|
|
58
|
+
* }
|
|
59
|
+
* ```
|
|
60
|
+
*/
|
|
61
|
+
export declare class S3Offloader {
|
|
62
|
+
private s3Client?;
|
|
63
|
+
private readonly bucketName;
|
|
64
|
+
private readonly keyPrefix;
|
|
65
|
+
private readonly thresholdBytes;
|
|
66
|
+
private readonly serverSideEncryption?;
|
|
67
|
+
private readonly sseKmsKeyId?;
|
|
68
|
+
private readonly clientConfig?;
|
|
69
|
+
constructor(config: S3OffloadConfig);
|
|
70
|
+
/**
|
|
71
|
+
* Get or lazily create the S3 client on first use.
|
|
72
|
+
* Defers S3Client construction so the constructor remains lightweight and sync.
|
|
73
|
+
*/
|
|
74
|
+
private getClient;
|
|
75
|
+
/**
|
|
76
|
+
* Check whether a payload should be offloaded to S3
|
|
77
|
+
*
|
|
78
|
+
* @param data - Serialized payload
|
|
79
|
+
* @returns true if the payload exceeds the threshold
|
|
80
|
+
*/
|
|
81
|
+
shouldOffload(data: Uint8Array): boolean;
|
|
82
|
+
/**
|
|
83
|
+
* Build a fully-qualified S3 key for a checkpoint field
|
|
84
|
+
*
|
|
85
|
+
* @param threadId - Thread identifier
|
|
86
|
+
* @param checkpointId - Checkpoint identifier
|
|
87
|
+
* @param field - Field name (e.g., 'checkpoint', 'metadata', 'write-0')
|
|
88
|
+
* @returns Fully-qualified S3 key
|
|
89
|
+
*/
|
|
90
|
+
buildKey(threadId: string, checkpointId: string, field: string): string;
|
|
91
|
+
/**
|
|
92
|
+
* Upload data to S3
|
|
93
|
+
*
|
|
94
|
+
* @param key - S3 object key
|
|
95
|
+
* @param data - Data to upload
|
|
96
|
+
* @returns The S3 key that was used
|
|
97
|
+
*/
|
|
98
|
+
upload(key: string, data: Uint8Array): Promise<string>;
|
|
99
|
+
/**
|
|
100
|
+
* Download data from S3
|
|
101
|
+
*
|
|
102
|
+
* @param key - S3 object key to download
|
|
103
|
+
* @returns The downloaded data as Uint8Array
|
|
104
|
+
* @throws Error if the download fails or body is empty
|
|
105
|
+
*/
|
|
106
|
+
download(key: string): Promise<Uint8Array>;
|
|
107
|
+
/**
|
|
108
|
+
* Delete a single object from S3
|
|
109
|
+
*
|
|
110
|
+
* @param key - S3 object key to delete
|
|
111
|
+
*/
|
|
112
|
+
delete(key: string): Promise<void>;
|
|
113
|
+
/**
|
|
114
|
+
* Batch delete multiple objects from S3
|
|
115
|
+
*
|
|
116
|
+
* @param keys - Array of S3 object keys to delete
|
|
117
|
+
*/
|
|
118
|
+
deleteBatch(keys: string[]): Promise<void>;
|
|
119
|
+
/**
|
|
120
|
+
* Get the configured key prefix (useful for logging)
|
|
121
|
+
*/
|
|
122
|
+
getKeyPrefix(): string;
|
|
123
|
+
/**
|
|
124
|
+
* Ensure an S3 lifecycle rule exists for automatic object expiration.
|
|
125
|
+
*
|
|
126
|
+
* This method is **idempotent**: it reads existing lifecycle rules and only
|
|
127
|
+
* adds or updates the langgraph-specific rule. User-defined rules on the
|
|
128
|
+
* bucket are preserved.
|
|
129
|
+
*
|
|
130
|
+
* @param ttlDays - Expiration in days (S3 lifecycle uses day-level precision)
|
|
131
|
+
* @throws Error if the S3 API calls fail (callers should catch gracefully)
|
|
132
|
+
*/
|
|
133
|
+
ensureLifecycleRule(ttlDays: number): Promise<void>;
|
|
134
|
+
/**
|
|
135
|
+
* Generate a deterministic lifecycle rule ID for idempotency.
|
|
136
|
+
*
|
|
137
|
+
* @param ttlDays - Expiration in days
|
|
138
|
+
* @returns Rule ID string
|
|
139
|
+
*/
|
|
140
|
+
private buildLifecycleRuleId;
|
|
141
|
+
/**
|
|
142
|
+
* Release underlying S3 client resources
|
|
143
|
+
*/
|
|
144
|
+
destroy(): void;
|
|
145
|
+
}
|
|
146
|
+
//# sourceMappingURL=s3-offloader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3-offloader.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/s3-offloader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAuBH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yFAAyF;IACzF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0FAA0F;IAC1F,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,YAAY,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACxB,CAAC;CACH;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,CAA6D;IAC9E,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAkC;gBAEpD,MAAM,EAAE,eAAe;IASnC;;;OAGG;IACH,OAAO,CAAC,SAAS;IAQjB;;;;;OAKG;IACH,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO;IAIxC;;;;;;;OAOG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM;IAIvE;;;;;;OAMG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAqB5D;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAoBhD;;;;OAIG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYxC;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgChD;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;;;;;;;;OASG;IACG,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8EzD;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;OAEG;IACH,OAAO,IAAI,IAAI;CAGhB"}
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* S3 offloader for large checkpoint payloads
|
|
4
|
+
*
|
|
5
|
+
* DynamoDB has a hard 400KB item size limit. This utility transparently
|
|
6
|
+
* offloads payloads exceeding a configurable threshold (default: 350KB)
|
|
7
|
+
* to S3, storing only an S3 key reference in DynamoDB.
|
|
8
|
+
*
|
|
9
|
+
* Requires `@aws-sdk/client-s3` as an optional peer dependency.
|
|
10
|
+
*
|
|
11
|
+
* @remarks
|
|
12
|
+
* This module uses synchronous `require()` for lazy loading of the S3 SDK
|
|
13
|
+
* to avoid making the DynamoDBSaver constructor async. This means it
|
|
14
|
+
* requires a CommonJS-compatible environment (Node.js with CJS, or a
|
|
15
|
+
* bundler that polyfills require). Pure ESM environments are not supported.
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.S3Offloader = void 0;
|
|
19
|
+
const logger_1 = require("./logger");
|
|
20
|
+
/** Default threshold: 350KB (leaves 50KB headroom below DynamoDB's 400KB limit) */
|
|
21
|
+
const DEFAULT_THRESHOLD_BYTES = 350 * 1024;
|
|
22
|
+
/** Default S3 key prefix for offloaded payloads */
|
|
23
|
+
const DEFAULT_KEY_PREFIX = 'langgraph-checkpoints/';
|
|
24
|
+
/* eslint-disable @typescript-eslint/no-require-imports -- TODO: migrate to dynamic import() when Jest ESM support is stable */
|
|
25
|
+
// Lazy-loaded S3 SDK — uses require() for Jest CJS compatibility.
|
|
26
|
+
// Replace with import() when the test runner supports --experimental-vm-modules.
|
|
27
|
+
let s3Sdk;
|
|
28
|
+
function getS3Sdk() {
|
|
29
|
+
if (!s3Sdk) {
|
|
30
|
+
s3Sdk = require('@aws-sdk/client-s3');
|
|
31
|
+
}
|
|
32
|
+
return s3Sdk;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* S3 offloader for large checkpoint payloads
|
|
36
|
+
*
|
|
37
|
+
* Transparently uploads large payloads to S3 and returns S3 key references
|
|
38
|
+
* for storage in DynamoDB. On read, downloads from S3 when a reference is present.
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```typescript
|
|
42
|
+
* const offloader = new S3Offloader({
|
|
43
|
+
* bucketName: 'my-checkpoints-bucket',
|
|
44
|
+
* keyPrefix: 'langgraph/',
|
|
45
|
+
* thresholdBytes: 350 * 1024,
|
|
46
|
+
* });
|
|
47
|
+
*
|
|
48
|
+
* if (offloader.shouldOffload(data)) {
|
|
49
|
+
* const key = await offloader.upload('thread/checkpoint/field.bin', data);
|
|
50
|
+
* // Store `key` in DynamoDB instead of `data`
|
|
51
|
+
* }
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
class S3Offloader {
|
|
55
|
+
s3Client;
|
|
56
|
+
bucketName;
|
|
57
|
+
keyPrefix;
|
|
58
|
+
thresholdBytes;
|
|
59
|
+
serverSideEncryption;
|
|
60
|
+
sseKmsKeyId;
|
|
61
|
+
clientConfig;
|
|
62
|
+
constructor(config) {
|
|
63
|
+
this.bucketName = config.bucketName;
|
|
64
|
+
this.keyPrefix = config.keyPrefix ?? DEFAULT_KEY_PREFIX;
|
|
65
|
+
this.thresholdBytes = config.thresholdBytes ?? DEFAULT_THRESHOLD_BYTES;
|
|
66
|
+
this.serverSideEncryption = config.serverSideEncryption;
|
|
67
|
+
this.sseKmsKeyId = config.sseKmsKeyId;
|
|
68
|
+
this.clientConfig = config.clientConfig;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Get or lazily create the S3 client on first use.
|
|
72
|
+
* Defers S3Client construction so the constructor remains lightweight and sync.
|
|
73
|
+
*/
|
|
74
|
+
getClient() {
|
|
75
|
+
if (!this.s3Client) {
|
|
76
|
+
const { S3Client } = getS3Sdk();
|
|
77
|
+
this.s3Client = new S3Client((this.clientConfig ?? {}));
|
|
78
|
+
}
|
|
79
|
+
return this.s3Client;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Check whether a payload should be offloaded to S3
|
|
83
|
+
*
|
|
84
|
+
* @param data - Serialized payload
|
|
85
|
+
* @returns true if the payload exceeds the threshold
|
|
86
|
+
*/
|
|
87
|
+
shouldOffload(data) {
|
|
88
|
+
return data.length >= this.thresholdBytes;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Build a fully-qualified S3 key for a checkpoint field
|
|
92
|
+
*
|
|
93
|
+
* @param threadId - Thread identifier
|
|
94
|
+
* @param checkpointId - Checkpoint identifier
|
|
95
|
+
* @param field - Field name (e.g., 'checkpoint', 'metadata', 'write-0')
|
|
96
|
+
* @returns Fully-qualified S3 key
|
|
97
|
+
*/
|
|
98
|
+
buildKey(threadId, checkpointId, field) {
|
|
99
|
+
return `${this.keyPrefix}${threadId}/${checkpointId}/${field}.bin`;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Upload data to S3
|
|
103
|
+
*
|
|
104
|
+
* @param key - S3 object key
|
|
105
|
+
* @param data - Data to upload
|
|
106
|
+
* @returns The S3 key that was used
|
|
107
|
+
*/
|
|
108
|
+
async upload(key, data) {
|
|
109
|
+
const { PutObjectCommand } = getS3Sdk();
|
|
110
|
+
const client = this.getClient();
|
|
111
|
+
await client.send(new PutObjectCommand({
|
|
112
|
+
Bucket: this.bucketName,
|
|
113
|
+
Key: key,
|
|
114
|
+
Body: data,
|
|
115
|
+
ContentType: 'application/octet-stream',
|
|
116
|
+
...(this.serverSideEncryption && {
|
|
117
|
+
ServerSideEncryption: this
|
|
118
|
+
.serverSideEncryption,
|
|
119
|
+
}),
|
|
120
|
+
...(this.sseKmsKeyId && { SSEKMSKeyId: this.sseKmsKeyId }),
|
|
121
|
+
}));
|
|
122
|
+
return key;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Download data from S3
|
|
126
|
+
*
|
|
127
|
+
* @param key - S3 object key to download
|
|
128
|
+
* @returns The downloaded data as Uint8Array
|
|
129
|
+
* @throws Error if the download fails or body is empty
|
|
130
|
+
*/
|
|
131
|
+
async download(key) {
|
|
132
|
+
const { GetObjectCommand } = getS3Sdk();
|
|
133
|
+
const client = this.getClient();
|
|
134
|
+
const response = await client.send(new GetObjectCommand({
|
|
135
|
+
Bucket: this.bucketName,
|
|
136
|
+
Key: key,
|
|
137
|
+
}));
|
|
138
|
+
if (!response.Body) {
|
|
139
|
+
throw new Error(`S3 object body is empty for key: ${key}`);
|
|
140
|
+
}
|
|
141
|
+
// Convert the readable stream to Uint8Array
|
|
142
|
+
const byteArray = await response.Body.transformToByteArray();
|
|
143
|
+
return new Uint8Array(byteArray);
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Delete a single object from S3
|
|
147
|
+
*
|
|
148
|
+
* @param key - S3 object key to delete
|
|
149
|
+
*/
|
|
150
|
+
async delete(key) {
|
|
151
|
+
const { DeleteObjectCommand } = getS3Sdk();
|
|
152
|
+
const client = this.getClient();
|
|
153
|
+
await client.send(new DeleteObjectCommand({
|
|
154
|
+
Bucket: this.bucketName,
|
|
155
|
+
Key: key,
|
|
156
|
+
}));
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Batch delete multiple objects from S3
|
|
160
|
+
*
|
|
161
|
+
* @param keys - Array of S3 object keys to delete
|
|
162
|
+
*/
|
|
163
|
+
async deleteBatch(keys) {
|
|
164
|
+
if (keys.length === 0)
|
|
165
|
+
return;
|
|
166
|
+
const { DeleteObjectsCommand } = getS3Sdk();
|
|
167
|
+
const client = this.getClient();
|
|
168
|
+
// S3 DeleteObjects supports up to 1000 keys per request
|
|
169
|
+
const batchSize = 1000;
|
|
170
|
+
for (let i = 0; i < keys.length; i += batchSize) {
|
|
171
|
+
const batch = keys.slice(i, i + batchSize);
|
|
172
|
+
const response = await client.send(new DeleteObjectsCommand({
|
|
173
|
+
Bucket: this.bucketName,
|
|
174
|
+
Delete: {
|
|
175
|
+
Objects: batch.map((key) => ({ Key: key })),
|
|
176
|
+
Quiet: true,
|
|
177
|
+
},
|
|
178
|
+
}));
|
|
179
|
+
// Log any failed deletions (Quiet mode still reports errors)
|
|
180
|
+
if (response.Errors && response.Errors.length > 0) {
|
|
181
|
+
const failedKeys = response.Errors.map((err) => `${err.Key} (${err.Code}: ${err.Message})`).join(', ');
|
|
182
|
+
(0, logger_1.getLogger)().warn(`S3 deleteBatch: ${response.Errors.length} objects failed to delete: ${failedKeys}`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Get the configured key prefix (useful for logging)
|
|
188
|
+
*/
|
|
189
|
+
getKeyPrefix() {
|
|
190
|
+
return this.keyPrefix;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Ensure an S3 lifecycle rule exists for automatic object expiration.
|
|
194
|
+
*
|
|
195
|
+
* This method is **idempotent**: it reads existing lifecycle rules and only
|
|
196
|
+
* adds or updates the langgraph-specific rule. User-defined rules on the
|
|
197
|
+
* bucket are preserved.
|
|
198
|
+
*
|
|
199
|
+
* @param ttlDays - Expiration in days (S3 lifecycle uses day-level precision)
|
|
200
|
+
* @throws Error if the S3 API calls fail (callers should catch gracefully)
|
|
201
|
+
*/
|
|
202
|
+
async ensureLifecycleRule(ttlDays) {
|
|
203
|
+
const { GetBucketLifecycleConfigurationCommand, PutBucketLifecycleConfigurationCommand } = getS3Sdk();
|
|
204
|
+
const client = this.getClient();
|
|
205
|
+
const ruleId = this.buildLifecycleRuleId(ttlDays);
|
|
206
|
+
// 1. Read existing lifecycle configuration
|
|
207
|
+
let existingRules = [];
|
|
208
|
+
try {
|
|
209
|
+
const existing = await client.send(new GetBucketLifecycleConfigurationCommand({ Bucket: this.bucketName }));
|
|
210
|
+
existingRules = existing.Rules ?? [];
|
|
211
|
+
}
|
|
212
|
+
catch (error) {
|
|
213
|
+
// NoSuchLifecycleConfiguration means bucket has no lifecycle config — treat as empty
|
|
214
|
+
if (error &&
|
|
215
|
+
typeof error === 'object' &&
|
|
216
|
+
'name' in error &&
|
|
217
|
+
error.name === 'NoSuchLifecycleConfiguration') {
|
|
218
|
+
existingRules = [];
|
|
219
|
+
}
|
|
220
|
+
else {
|
|
221
|
+
throw error;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
// 2. Check if a matching rule already exists (by ID)
|
|
225
|
+
const existingRule = existingRules.find((r) => r.ID === ruleId);
|
|
226
|
+
if (existingRule &&
|
|
227
|
+
existingRule.Status === 'Enabled' &&
|
|
228
|
+
existingRule.Expiration &&
|
|
229
|
+
existingRule.Expiration.Days === ttlDays) {
|
|
230
|
+
// Rule already exists with correct settings — nothing to do
|
|
231
|
+
(0, logger_1.getLogger)().info(`S3 lifecycle rule '${ruleId}' already exists with ${ttlDays}-day expiration — skipping`);
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
// 3. Build the new rule
|
|
235
|
+
const newRule = {
|
|
236
|
+
ID: ruleId,
|
|
237
|
+
Filter: { Prefix: this.keyPrefix },
|
|
238
|
+
Status: 'Enabled',
|
|
239
|
+
Expiration: { Days: ttlDays },
|
|
240
|
+
};
|
|
241
|
+
// 4. Merge: replace existing rule with same ID or append
|
|
242
|
+
const mergedRules = existingRule
|
|
243
|
+
? existingRules.map((r) => (r.ID === ruleId ? newRule : r))
|
|
244
|
+
: [...existingRules, newRule];
|
|
245
|
+
// 5. Write back
|
|
246
|
+
await client.send(new PutBucketLifecycleConfigurationCommand({
|
|
247
|
+
Bucket: this.bucketName,
|
|
248
|
+
LifecycleConfiguration: {
|
|
249
|
+
Rules: mergedRules,
|
|
250
|
+
},
|
|
251
|
+
}));
|
|
252
|
+
(0, logger_1.getLogger)().info(`S3 lifecycle rule '${ruleId}' configured: ${ttlDays}-day expiration on prefix '${this.keyPrefix}'`);
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Generate a deterministic lifecycle rule ID for idempotency.
|
|
256
|
+
*
|
|
257
|
+
* @param ttlDays - Expiration in days
|
|
258
|
+
* @returns Rule ID string
|
|
259
|
+
*/
|
|
260
|
+
buildLifecycleRuleId(ttlDays) {
|
|
261
|
+
return `langgraph-ttl-${ttlDays}d`;
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Release underlying S3 client resources
|
|
265
|
+
*/
|
|
266
|
+
destroy() {
|
|
267
|
+
this.s3Client?.destroy();
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
exports.S3Offloader = S3Offloader;
|
|
271
|
+
//# sourceMappingURL=s3-offloader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3-offloader.js","sourceRoot":"","sources":["../../../src/shared/utils/s3-offloader.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAEH,qCAAqC;AAErC,mFAAmF;AACnF,MAAM,uBAAuB,GAAG,GAAG,GAAG,IAAI,CAAC;AAE3C,mDAAmD;AACnD,MAAM,kBAAkB,GAAG,wBAAwB,CAAC;AAEpD,+HAA+H;AAE/H,kEAAkE;AAClE,iFAAiF;AACjF,IAAI,KAAsD,CAAC;AAE3D,SAAS,QAAQ;IACf,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAM,CAAC;AAChB,CAAC;AA4BD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAa,WAAW;IACd,QAAQ,CAA8D;IAC7D,UAAU,CAAS;IACnB,SAAS,CAAS;IAClB,cAAc,CAAS;IACvB,oBAAoB,CAAU;IAC9B,WAAW,CAAU;IACrB,YAAY,CAAmC;IAEhE,YAAY,MAAuB;QACjC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;QACxD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,uBAAuB,CAAC;QACvE,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,CAAC;QACxD,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACK,SAAS;QACf,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAA4B,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACH,aAAa,CAAC,IAAgB;QAC5B,OAAO,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC;IAC5C,CAAC;IAED;;;;;;;OAOG;IACH,QAAQ,CAAC,QAAgB,EAAE,YAAoB,EAAE,KAAa;QAC5D,OAAO,GAAG,IAAI,CAAC,SAAS,GAAG,QAAQ,IAAI,YAAY,IAAI,KAAK,MAAM,CAAC;IACrE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,IAAgB;QACxC,MAAM,EAAE,gBAAgB,EAAE,GAAG,QAAQ,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,gBAAgB,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,GAAG,EAAE,GAAG;YACR,IAAI,EAAE,IAAI;YACV,WAAW,EAAE,0BAA0B;YACvC,GAAG,CAAC,IAAI,CAAC,oBAAoB,IAAI;gBAC/B,oBAAoB,EAAE,IAAI;qBACvB,oBAAyE;aAC7E,CAAC;YACF,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;SAC3D,CAAC,CACH,CAAC;QAEF,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW;QACxB,MAAM,EAAE,gBAAgB,EAAE,GAAG,QAAQ,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,gBAAgB,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,GAAG,EAAE,GAAG;SACT,CAAC,CACH,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,4CAA4C;QAC5C,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC7D,OAAO,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,EAAE,mBAAmB,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,mBAAmB,CAAC;YACtB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,GAAG,EAAE,GAAG;SACT,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,IAAc;QAC9B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE9B,MAAM,EAAE,oBAAoB,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,wDAAwD;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,oBAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,UAAU;gBACvB,MAAM,EAAE;oBACN,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC3C,KAAK,EAAE,IAAI;iBACZ;aACF,CAAC,CACH,CAAC;YAEF,6DAA6D;YAC7D,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CACpC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,GAAG,CACpD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACb,IAAA,kBAAS,GAAE,CAAC,IAAI,CACd,mBAAmB,QAAQ,CAAC,MAAM,CAAC,MAAM,8BAA8B,UAAU,EAAE,CACpF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,mBAAmB,CAAC,OAAe;QACvC,MAAM,EAAE,sCAAsC,EAAE,sCAAsC,EAAE,GACtF,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAElD,2CAA2C;QAC3C,IAAI,aAAa,GAMZ,EAAE,CAAC;QAER,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAChC,IAAI,sCAAsC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CACxE,CAAC;YACF,aAAa,GAAI,QAAQ,CAAC,KAA8B,IAAI,EAAE,CAAC;QACjE,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,qFAAqF;YACrF,IACE,KAAK;gBACL,OAAO,KAAK,KAAK,QAAQ;gBACzB,MAAM,IAAI,KAAK;gBACd,KAA0B,CAAC,IAAI,KAAK,8BAA8B,EACnE,CAAC;gBACD,aAAa,GAAG,EAAE,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC;QAChE,IACE,YAAY;YACZ,YAAY,CAAC,MAAM,KAAK,SAAS;YACjC,YAAY,CAAC,UAAU;YACtB,YAAY,CAAC,UAAgC,CAAC,IAAI,KAAK,OAAO,EAC/D,CAAC;YACD,4DAA4D;YAC5D,IAAA,kBAAS,GAAE,CAAC,IAAI,CACd,sBAAsB,MAAM,yBAAyB,OAAO,4BAA4B,CACzF,CAAC;YACF,OAAO;QACT,CAAC;QAED,wBAAwB;QACxB,MAAM,OAAO,GAAG;YACd,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;YAClC,MAAM,EAAE,SAAkB;YAC1B,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;SAC9B,CAAC;QAEF,yDAAyD;QACzD,MAAM,WAAW,GAAG,YAAY;YAC9B,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,CAAC,CAAC,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,CAAC;QAEhC,gBAAgB;QAChB,MAAM,MAAM,CAAC,IAAI,CACf,IAAI,sCAAsC,CAAC;YACzC,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,sBAAsB,EAAE;gBACtB,KAAK,EAAE,WAAsE;aAC9E;SACF,CAAC,CACH,CAAC;QAEF,IAAA,kBAAS,GAAE,CAAC,IAAI,CACd,sBAAsB,MAAM,iBAAiB,OAAO,8BAA8B,IAAI,CAAC,SAAS,GAAG,CACpG,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACK,oBAAoB,CAAC,OAAe;QAC1C,OAAO,iBAAiB,OAAO,GAAG,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAhRD,kCAgRC"}
|
|
@@ -3,7 +3,14 @@
|
|
|
3
3
|
* Provide centralized TTL calculation for DynamoDB items
|
|
4
4
|
*/
|
|
5
5
|
/**
|
|
6
|
-
* Calculate Unix timestamp for TTL expiration
|
|
6
|
+
* Calculate Unix timestamp for TTL expiration from seconds
|
|
7
|
+
*
|
|
8
|
+
* @param ttlSeconds - Number of seconds until expiration
|
|
9
|
+
* @returns Unix timestamp in seconds for DynamoDB TTL attribute
|
|
10
|
+
*/
|
|
11
|
+
export declare function calculateTTLTimestampFromSeconds(ttlSeconds: number): number;
|
|
12
|
+
/**
|
|
13
|
+
* Calculate Unix timestamp for TTL expiration from days
|
|
7
14
|
*
|
|
8
15
|
* @param ttlDays - Number of days until expiration
|
|
9
16
|
* @returns Unix timestamp in seconds for DynamoDB TTL attribute
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ttl.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/ttl.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"ttl.d.ts","sourceRoot":"","sources":["../../../src/shared/utils/ttl.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AACH,wBAAgB,gCAAgC,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE3E;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE7D"}
|