ml-cache 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +599 -0
- package/dist/client.d.ts +175 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +411 -0
- package/dist/client.js.map +1 -0
- package/dist/constants.d.ts +184 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +236 -0
- package/dist/constants.js.map +1 -0
- package/dist/esm/client.d.ts +175 -0
- package/dist/esm/client.d.ts.map +1 -0
- package/dist/esm/client.js +407 -0
- package/dist/esm/client.js.map +1 -0
- package/dist/esm/constants.d.ts +184 -0
- package/dist/esm/constants.d.ts.map +1 -0
- package/dist/esm/constants.js +233 -0
- package/dist/esm/constants.js.map +1 -0
- package/dist/esm/index.d.ts +58 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/queue.d.ts +106 -0
- package/dist/esm/queue.d.ts.map +1 -0
- package/dist/esm/queue.js +207 -0
- package/dist/esm/queue.js.map +1 -0
- package/dist/esm/storage/glacier-storage.d.ts +77 -0
- package/dist/esm/storage/glacier-storage.d.ts.map +1 -0
- package/dist/esm/storage/glacier-storage.js +133 -0
- package/dist/esm/storage/glacier-storage.js.map +1 -0
- package/dist/esm/storage/index.d.ts +7 -0
- package/dist/esm/storage/index.d.ts.map +1 -0
- package/dist/esm/storage/index.js +7 -0
- package/dist/esm/storage/index.js.map +1 -0
- package/dist/esm/storage/s3-storage.d.ts +89 -0
- package/dist/esm/storage/s3-storage.d.ts.map +1 -0
- package/dist/esm/storage/s3-storage.js +161 -0
- package/dist/esm/storage/s3-storage.js.map +1 -0
- package/dist/esm/types/index.d.ts +337 -0
- package/dist/esm/types/index.d.ts.map +1 -0
- package/dist/esm/types/index.js +6 -0
- package/dist/esm/types/index.js.map +1 -0
- package/dist/esm/utils/helpers.d.ts +132 -0
- package/dist/esm/utils/helpers.d.ts.map +1 -0
- package/dist/esm/utils/helpers.js +198 -0
- package/dist/esm/utils/helpers.js.map +1 -0
- package/dist/esm/utils/index.d.ts +8 -0
- package/dist/esm/utils/index.d.ts.map +1 -0
- package/dist/esm/utils/index.js +8 -0
- package/dist/esm/utils/index.js.map +1 -0
- package/dist/esm/utils/logger.d.ts +110 -0
- package/dist/esm/utils/logger.d.ts.map +1 -0
- package/dist/esm/utils/logger.js +177 -0
- package/dist/esm/utils/logger.js.map +1 -0
- package/dist/esm/utils/validator.d.ts +77 -0
- package/dist/esm/utils/validator.d.ts.map +1 -0
- package/dist/esm/utils/validator.js +208 -0
- package/dist/esm/utils/validator.js.map +1 -0
- package/dist/index.d.ts +58 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +100 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +72 -0
- package/dist/queue.d.ts +106 -0
- package/dist/queue.d.ts.map +1 -0
- package/dist/queue.js +211 -0
- package/dist/queue.js.map +1 -0
- package/dist/storage/glacier-storage.d.ts +77 -0
- package/dist/storage/glacier-storage.d.ts.map +1 -0
- package/dist/storage/glacier-storage.js +137 -0
- package/dist/storage/glacier-storage.js.map +1 -0
- package/dist/storage/index.d.ts +7 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +12 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/s3-storage.d.ts +89 -0
- package/dist/storage/s3-storage.d.ts.map +1 -0
- package/dist/storage/s3-storage.js +165 -0
- package/dist/storage/s3-storage.js.map +1 -0
- package/dist/types/index.d.ts +337 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +7 -0
- package/dist/types/index.js.map +1 -0
- package/dist/utils/helpers.d.ts +132 -0
- package/dist/utils/helpers.d.ts.map +1 -0
- package/dist/utils/helpers.js +215 -0
- package/dist/utils/helpers.js.map +1 -0
- package/dist/utils/index.d.ts +8 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +35 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/logger.d.ts +110 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +181 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/validator.d.ts +77 -0
- package/dist/utils/validator.d.ts.map +1 -0
- package/dist/utils/validator.js +221 -0
- package/dist/utils/validator.js.map +1 -0
- package/package.json +64 -0
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @fileoverview Glacier storage provider for ml-cache SDK
|
|
4
|
+
* @module ml-cache/storage/glacier-storage
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.GlacierStorage = void 0;
|
|
8
|
+
const client_glacier_1 = require("@aws-sdk/client-glacier");
|
|
9
|
+
const constants_1 = require("../constants");
|
|
10
|
+
const utils_1 = require("../utils");
|
|
11
|
+
/**
|
|
12
|
+
* Glacier storage provider for long-term event archival
|
|
13
|
+
* @class GlacierStorage
|
|
14
|
+
*/
|
|
15
|
+
class GlacierStorage {
|
|
16
|
+
/**
|
|
17
|
+
* Creates a new GlacierStorage instance
|
|
18
|
+
* @param {GlacierConfig} config - Glacier configuration
|
|
19
|
+
* @param {AWSCredentials} credentials - AWS credentials
|
|
20
|
+
* @param {Logger} [logger] - Logger instance
|
|
21
|
+
*/
|
|
22
|
+
constructor(config, credentials, logger) {
|
|
23
|
+
this.config = config;
|
|
24
|
+
this.accountId = config.accountId || constants_1.DEFAULT_GLACIER_ACCOUNT_ID;
|
|
25
|
+
this.logger = logger || new utils_1.Logger();
|
|
26
|
+
this.client = new client_glacier_1.GlacierClient({
|
|
27
|
+
region: config.region,
|
|
28
|
+
credentials: {
|
|
29
|
+
accessKeyId: credentials.accessKeyId,
|
|
30
|
+
secretAccessKey: credentials.secretAccessKey,
|
|
31
|
+
sessionToken: credentials.sessionToken,
|
|
32
|
+
},
|
|
33
|
+
requestHandler: {
|
|
34
|
+
requestTimeout: constants_1.REQUEST_TIMEOUT_MS,
|
|
35
|
+
connectionTimeout: constants_1.CONNECTION_TIMEOUT_MS,
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Tests the connection to Glacier
|
|
41
|
+
* @returns {Promise<boolean>} Whether the connection is successful
|
|
42
|
+
*/
|
|
43
|
+
async testConnection() {
|
|
44
|
+
try {
|
|
45
|
+
const command = new client_glacier_1.DescribeVaultCommand({
|
|
46
|
+
accountId: this.accountId,
|
|
47
|
+
vaultName: this.config.vaultName,
|
|
48
|
+
});
|
|
49
|
+
await this.client.send(command);
|
|
50
|
+
this.logger.info('Glacier connection test successful', {
|
|
51
|
+
vaultName: this.config.vaultName,
|
|
52
|
+
});
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
this.logger.error('Glacier connection test failed', {
|
|
57
|
+
vaultName: this.config.vaultName,
|
|
58
|
+
error: error instanceof Error ? error.message : String(error),
|
|
59
|
+
});
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Archives events in Glacier
|
|
65
|
+
* @param {MLCacheEvent[]} events - Events to archive
|
|
66
|
+
* @param {string} [batchId] - Optional batch ID
|
|
67
|
+
* @returns {Promise<GlacierStorageResult>} Storage result
|
|
68
|
+
*/
|
|
69
|
+
async archiveEvents(events, batchId) {
|
|
70
|
+
const result = {
|
|
71
|
+
success: false,
|
|
72
|
+
vaultName: this.config.vaultName,
|
|
73
|
+
};
|
|
74
|
+
try {
|
|
75
|
+
const id = batchId || (0, utils_1.generateBatchId)();
|
|
76
|
+
const body = (0, utils_1.serializeEventsToNDJSON)(events);
|
|
77
|
+
const description = (0, utils_1.generateGlacierDescription)({
|
|
78
|
+
batchId: id,
|
|
79
|
+
eventCount: events.length,
|
|
80
|
+
});
|
|
81
|
+
const command = new client_glacier_1.UploadArchiveCommand({
|
|
82
|
+
accountId: this.accountId,
|
|
83
|
+
vaultName: this.config.vaultName,
|
|
84
|
+
body: Buffer.from(body, 'utf-8'),
|
|
85
|
+
archiveDescription: description,
|
|
86
|
+
});
|
|
87
|
+
const response = await this.client.send(command);
|
|
88
|
+
result.success = true;
|
|
89
|
+
result.archiveId = response.archiveId;
|
|
90
|
+
result.checksum = response.checksum;
|
|
91
|
+
result.location = response.location;
|
|
92
|
+
this.logger.debug('Events archived in Glacier', {
|
|
93
|
+
archiveId: response.archiveId,
|
|
94
|
+
eventCount: events.length,
|
|
95
|
+
checksum: response.checksum,
|
|
96
|
+
});
|
|
97
|
+
return result;
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
result.error = error instanceof Error ? error.message : String(error);
|
|
101
|
+
this.logger.error('Failed to archive events in Glacier', {
|
|
102
|
+
eventCount: events.length,
|
|
103
|
+
error: result.error,
|
|
104
|
+
});
|
|
105
|
+
return result;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Gets the vault name
|
|
110
|
+
* @returns {string} Vault name
|
|
111
|
+
*/
|
|
112
|
+
getVaultName() {
|
|
113
|
+
return this.config.vaultName;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Gets the region
|
|
117
|
+
* @returns {string} Region
|
|
118
|
+
*/
|
|
119
|
+
getRegion() {
|
|
120
|
+
return this.config.region;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Gets the account ID
|
|
124
|
+
* @returns {string} Account ID
|
|
125
|
+
*/
|
|
126
|
+
getAccountId() {
|
|
127
|
+
return this.accountId;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Destroys the Glacier client
|
|
131
|
+
*/
|
|
132
|
+
destroy() {
|
|
133
|
+
this.client.destroy();
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
exports.GlacierStorage = GlacierStorage;
|
|
137
|
+
//# sourceMappingURL=glacier-storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"glacier-storage.js","sourceRoot":"","sources":["../../src/storage/glacier-storage.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,4DAIiC;AAEjC,4CAIsB;AACtB,oCAKkB;AAqBlB;;;GAGG;AACH,MAAa,cAAc;IAazB;;;;;OAKG;IACH,YACE,MAAqB,EACrB,WAA2B,EAC3B,MAAe;QAEf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,sCAA0B,CAAC;QAChE,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,IAAI,cAAM,EAAE,CAAC;QAErC,IAAI,CAAC,MAAM,GAAG,IAAI,8BAAa,CAAC;YAC9B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE;gBACX,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,eAAe,EAAE,WAAW,CAAC,eAAe;gBAC5C,YAAY,EAAE,WAAW,CAAC,YAAY;aACvC;YACD,cAAc,EAAE;gBACd,cAAc,EAAE,8BAAkB;gBAClC,iBAAiB,EAAE,iCAAqB;aAChC;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,qCAAoB,CAAC;gBACvC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;aACjC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBACrD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;aACjC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;gBAClD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;YACH,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa,CACjB,MAAsB,EACtB,OAAgB;QAEhB,MAAM,MAAM,GAAyB;YACnC,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;SACjC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,OAAO,IAAI,IAAA,uBAAe,GAAE,CAAC;YACxC,MAAM,IAAI,GAAG,IAAA,+BAAuB,EAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,WAAW,GAAG,IAAA,kCAA0B,EAAC;gBAC7C,OAAO,EAAE,EAAE;gBACX,UAAU,EAAE,MAAM,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,qCAAoB,CAAC;gBACvC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC;gBAChC,kBAAkB,EAAE,WAAW;aAChC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEjD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,MAAM,CAAC,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;YACtC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;YACpC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAEpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE;gBAC9C,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEtE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE;gBACvD,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;CACF;AAzJD,wCAyJC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @fileoverview Storage barrel export
|
|
4
|
+
* @module ml-cache/storage
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.GlacierStorage = exports.S3Storage = void 0;
|
|
8
|
+
var s3_storage_1 = require("./s3-storage");
|
|
9
|
+
Object.defineProperty(exports, "S3Storage", { enumerable: true, get: function () { return s3_storage_1.S3Storage; } });
|
|
10
|
+
var glacier_storage_1 = require("./glacier-storage");
|
|
11
|
+
Object.defineProperty(exports, "GlacierStorage", { enumerable: true, get: function () { return glacier_storage_1.GlacierStorage; } });
|
|
12
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,2CAA+D;AAAtD,uGAAA,SAAS,OAAA;AAClB,qDAA8E;AAArE,iHAAA,cAAc,OAAA"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview S3 storage provider for ml-cache SDK
|
|
3
|
+
* @module ml-cache/storage/s3-storage
|
|
4
|
+
*/
|
|
5
|
+
import type { S3Config, AWSCredentials, MLCacheEvent, S3StorageClass } from '../types';
|
|
6
|
+
import { Logger } from '../utils';
|
|
7
|
+
/**
|
|
8
|
+
* S3 storage result
|
|
9
|
+
* @interface S3StorageResult
|
|
10
|
+
*/
|
|
11
|
+
export interface S3StorageResult {
|
|
12
|
+
/** Whether the operation was successful */
|
|
13
|
+
success: boolean;
|
|
14
|
+
/** S3 object key */
|
|
15
|
+
key: string;
|
|
16
|
+
/** S3 bucket name */
|
|
17
|
+
bucket: string;
|
|
18
|
+
/** ETag of stored object */
|
|
19
|
+
etag?: string;
|
|
20
|
+
/** Error message if failed */
|
|
21
|
+
error?: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* S3 storage provider for event persistence
|
|
25
|
+
* @class S3Storage
|
|
26
|
+
*/
|
|
27
|
+
export declare class S3Storage {
|
|
28
|
+
/** @private */
|
|
29
|
+
private client;
|
|
30
|
+
/** @private */
|
|
31
|
+
private config;
|
|
32
|
+
/** @private */
|
|
33
|
+
private storageClass;
|
|
34
|
+
/** @private */
|
|
35
|
+
private logger;
|
|
36
|
+
/**
|
|
37
|
+
* Creates a new S3Storage instance
|
|
38
|
+
* @param {S3Config} config - S3 configuration
|
|
39
|
+
* @param {AWSCredentials} credentials - AWS credentials
|
|
40
|
+
* @param {Logger} [logger] - Logger instance
|
|
41
|
+
*/
|
|
42
|
+
constructor(config: S3Config, credentials: AWSCredentials, logger?: Logger);
|
|
43
|
+
/**
|
|
44
|
+
* Tests the connection to S3
|
|
45
|
+
* @returns {Promise<boolean>} Whether the connection is successful
|
|
46
|
+
*/
|
|
47
|
+
testConnection(): Promise<boolean>;
|
|
48
|
+
/**
|
|
49
|
+
* Stores a single event in S3
|
|
50
|
+
* @param {MLCacheEvent} event - Event to store
|
|
51
|
+
* @returns {Promise<S3StorageResult>} Storage result
|
|
52
|
+
*/
|
|
53
|
+
storeEvent(event: MLCacheEvent): Promise<S3StorageResult>;
|
|
54
|
+
/**
|
|
55
|
+
* Stores a batch of events in S3
|
|
56
|
+
* @param {MLCacheEvent[]} events - Events to store
|
|
57
|
+
* @param {string} [batchId] - Optional batch ID
|
|
58
|
+
* @returns {Promise<S3StorageResult>} Storage result
|
|
59
|
+
*/
|
|
60
|
+
storeBatch(events: MLCacheEvent[], batchId?: string): Promise<S3StorageResult>;
|
|
61
|
+
/**
|
|
62
|
+
* Stores events in S3 with a specific key
|
|
63
|
+
* @private
|
|
64
|
+
* @param {MLCacheEvent[]} events - Events to store
|
|
65
|
+
* @param {string} key - S3 object key
|
|
66
|
+
* @returns {Promise<S3StorageResult>} Storage result
|
|
67
|
+
*/
|
|
68
|
+
private store;
|
|
69
|
+
/**
|
|
70
|
+
* Gets the S3 bucket name
|
|
71
|
+
* @returns {string} Bucket name
|
|
72
|
+
*/
|
|
73
|
+
getBucket(): string;
|
|
74
|
+
/**
|
|
75
|
+
* Gets the S3 region
|
|
76
|
+
* @returns {string} Region
|
|
77
|
+
*/
|
|
78
|
+
getRegion(): string;
|
|
79
|
+
/**
|
|
80
|
+
* Gets the storage class
|
|
81
|
+
* @returns {S3StorageClass} Storage class
|
|
82
|
+
*/
|
|
83
|
+
getStorageClass(): S3StorageClass;
|
|
84
|
+
/**
|
|
85
|
+
* Destroys the S3 client
|
|
86
|
+
*/
|
|
87
|
+
destroy(): void;
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=s3-storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3-storage.d.ts","sourceRoot":"","sources":["../../src/storage/s3-storage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,cAAc,EACf,MAAM,UAAU,CAAC;AAOlB,OAAO,EACL,MAAM,EAIP,MAAM,UAAU,CAAC;AAElB;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,2CAA2C;IAC3C,OAAO,EAAE,OAAO,CAAC;IACjB,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,4BAA4B;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,qBAAa,SAAS;IACpB,eAAe;IACf,OAAO,CAAC,MAAM,CAAW;IAEzB,eAAe;IACf,OAAO,CAAC,MAAM,CAAW;IAEzB,eAAe;IACf,OAAO,CAAC,YAAY,CAAiB;IAErC,eAAe;IACf,OAAO,CAAC,MAAM,CAAS;IAEvB;;;;;OAKG;gBAED,MAAM,EAAE,QAAQ,EAChB,WAAW,EAAE,cAAc,EAC3B,MAAM,CAAC,EAAE,MAAM;IAoBjB;;;OAGG;IACG,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAoBxC;;;;OAIG;IACG,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IAS/D;;;;;OAKG;IACG,UAAU,CACd,MAAM,EAAE,YAAY,EAAE,EACtB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC;IAU3B;;;;;;OAMG;YACW,KAAK;IAoDnB;;;OAGG;IACH,SAAS,IAAI,MAAM;IAInB;;;OAGG;IACH,SAAS,IAAI,MAAM;IAInB;;;OAGG;IACH,eAAe,IAAI,cAAc;IAIjC;;OAEG;IACH,OAAO,IAAI,IAAI;CAGhB"}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* @fileoverview S3 storage provider for ml-cache SDK
|
|
4
|
+
* @module ml-cache/storage/s3-storage
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.S3Storage = void 0;
|
|
8
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
9
|
+
const constants_1 = require("../constants");
|
|
10
|
+
const utils_1 = require("../utils");
|
|
11
|
+
/**
|
|
12
|
+
* S3 storage provider for event persistence
|
|
13
|
+
* @class S3Storage
|
|
14
|
+
*/
|
|
15
|
+
class S3Storage {
|
|
16
|
+
/**
|
|
17
|
+
* Creates a new S3Storage instance
|
|
18
|
+
* @param {S3Config} config - S3 configuration
|
|
19
|
+
* @param {AWSCredentials} credentials - AWS credentials
|
|
20
|
+
* @param {Logger} [logger] - Logger instance
|
|
21
|
+
*/
|
|
22
|
+
constructor(config, credentials, logger) {
|
|
23
|
+
this.config = config;
|
|
24
|
+
this.storageClass = config.storageClass || constants_1.DEFAULT_S3_STORAGE_CLASS;
|
|
25
|
+
this.logger = logger || new utils_1.Logger();
|
|
26
|
+
this.client = new client_s3_1.S3Client({
|
|
27
|
+
region: config.region,
|
|
28
|
+
credentials: {
|
|
29
|
+
accessKeyId: credentials.accessKeyId,
|
|
30
|
+
secretAccessKey: credentials.secretAccessKey,
|
|
31
|
+
sessionToken: credentials.sessionToken,
|
|
32
|
+
},
|
|
33
|
+
requestHandler: {
|
|
34
|
+
requestTimeout: constants_1.REQUEST_TIMEOUT_MS,
|
|
35
|
+
connectionTimeout: constants_1.CONNECTION_TIMEOUT_MS,
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Tests the connection to S3
|
|
41
|
+
* @returns {Promise<boolean>} Whether the connection is successful
|
|
42
|
+
*/
|
|
43
|
+
async testConnection() {
|
|
44
|
+
try {
|
|
45
|
+
const command = new client_s3_1.HeadBucketCommand({
|
|
46
|
+
Bucket: this.config.bucket,
|
|
47
|
+
});
|
|
48
|
+
await this.client.send(command);
|
|
49
|
+
this.logger.info('S3 connection test successful', {
|
|
50
|
+
bucket: this.config.bucket,
|
|
51
|
+
});
|
|
52
|
+
return true;
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
this.logger.error('S3 connection test failed', {
|
|
56
|
+
bucket: this.config.bucket,
|
|
57
|
+
error: error instanceof Error ? error.message : String(error),
|
|
58
|
+
});
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Stores a single event in S3
|
|
64
|
+
* @param {MLCacheEvent} event - Event to store
|
|
65
|
+
* @returns {Promise<S3StorageResult>} Storage result
|
|
66
|
+
*/
|
|
67
|
+
async storeEvent(event) {
|
|
68
|
+
const key = (0, utils_1.generateS3Key)({
|
|
69
|
+
prefix: this.config.prefix,
|
|
70
|
+
eventId: event.eventId,
|
|
71
|
+
});
|
|
72
|
+
return this.store([event], key);
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Stores a batch of events in S3
|
|
76
|
+
* @param {MLCacheEvent[]} events - Events to store
|
|
77
|
+
* @param {string} [batchId] - Optional batch ID
|
|
78
|
+
* @returns {Promise<S3StorageResult>} Storage result
|
|
79
|
+
*/
|
|
80
|
+
async storeBatch(events, batchId) {
|
|
81
|
+
const id = batchId || (0, utils_1.generateBatchId)();
|
|
82
|
+
const key = (0, utils_1.generateS3Key)({
|
|
83
|
+
prefix: this.config.prefix,
|
|
84
|
+
batchId: id,
|
|
85
|
+
});
|
|
86
|
+
return this.store(events, key);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Stores events in S3 with a specific key
|
|
90
|
+
* @private
|
|
91
|
+
* @param {MLCacheEvent[]} events - Events to store
|
|
92
|
+
* @param {string} key - S3 object key
|
|
93
|
+
* @returns {Promise<S3StorageResult>} Storage result
|
|
94
|
+
*/
|
|
95
|
+
async store(events, key) {
|
|
96
|
+
const result = {
|
|
97
|
+
success: false,
|
|
98
|
+
key,
|
|
99
|
+
bucket: this.config.bucket,
|
|
100
|
+
};
|
|
101
|
+
try {
|
|
102
|
+
const body = (0, utils_1.serializeEventsToNDJSON)(events);
|
|
103
|
+
const params = {
|
|
104
|
+
Bucket: this.config.bucket,
|
|
105
|
+
Key: key,
|
|
106
|
+
Body: body,
|
|
107
|
+
ContentType: constants_1.CONTENT_TYPE_NDJSON,
|
|
108
|
+
StorageClass: this.storageClass,
|
|
109
|
+
Metadata: {
|
|
110
|
+
'event-count': String(events.length),
|
|
111
|
+
'sdk-version': '1.0.0',
|
|
112
|
+
'created-at': new Date().toISOString(),
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
const command = new client_s3_1.PutObjectCommand(params);
|
|
116
|
+
const response = await this.client.send(command);
|
|
117
|
+
result.success = true;
|
|
118
|
+
result.etag = response.ETag;
|
|
119
|
+
this.logger.debug('Events stored in S3', {
|
|
120
|
+
key,
|
|
121
|
+
eventCount: events.length,
|
|
122
|
+
etag: response.ETag,
|
|
123
|
+
});
|
|
124
|
+
return result;
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
result.error = error instanceof Error ? error.message : String(error);
|
|
128
|
+
this.logger.error('Failed to store events in S3', {
|
|
129
|
+
key,
|
|
130
|
+
eventCount: events.length,
|
|
131
|
+
error: result.error,
|
|
132
|
+
});
|
|
133
|
+
return result;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Gets the S3 bucket name
|
|
138
|
+
* @returns {string} Bucket name
|
|
139
|
+
*/
|
|
140
|
+
getBucket() {
|
|
141
|
+
return this.config.bucket;
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Gets the S3 region
|
|
145
|
+
* @returns {string} Region
|
|
146
|
+
*/
|
|
147
|
+
getRegion() {
|
|
148
|
+
return this.config.region;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Gets the storage class
|
|
152
|
+
* @returns {S3StorageClass} Storage class
|
|
153
|
+
*/
|
|
154
|
+
getStorageClass() {
|
|
155
|
+
return this.storageClass;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Destroys the S3 client
|
|
159
|
+
*/
|
|
160
|
+
destroy() {
|
|
161
|
+
this.client.destroy();
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
exports.S3Storage = S3Storage;
|
|
165
|
+
//# sourceMappingURL=s3-storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3-storage.js","sourceRoot":"","sources":["../../src/storage/s3-storage.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,kDAK4B;AAO5B,4CAKsB;AACtB,oCAKkB;AAmBlB;;;GAGG;AACH,MAAa,SAAS;IAapB;;;;;OAKG;IACH,YACE,MAAgB,EAChB,WAA2B,EAC3B,MAAe;QAEf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,oCAAwB,CAAC;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,IAAI,cAAM,EAAE,CAAC;QAErC,IAAI,CAAC,MAAM,GAAG,IAAI,oBAAQ,CAAC;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE;gBACX,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,eAAe,EAAE,WAAW,CAAC,eAAe;gBAC5C,YAAY,EAAE,WAAW,CAAC,YAAY;aACvC;YACD,cAAc,EAAE;gBACd,cAAc,EAAE,8BAAkB;gBAClC,iBAAiB,EAAE,iCAAqB;aAChC;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,6BAAiB,CAAC;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;aAC3B,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;gBAChD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;aAC3B,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE;gBAC7C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;YACH,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,KAAmB;QAClC,MAAM,GAAG,GAAG,IAAA,qBAAa,EAAC;YACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CACd,MAAsB,EACtB,OAAgB;QAEhB,MAAM,EAAE,GAAG,OAAO,IAAI,IAAA,uBAAe,GAAE,CAAC;QACxC,MAAM,GAAG,GAAG,IAAA,qBAAa,EAAC;YACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,OAAO,EAAE,EAAE;SACZ,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,KAAK,CACjB,MAAsB,EACtB,GAAW;QAEX,MAAM,MAAM,GAAoB;YAC9B,OAAO,EAAE,KAAK;YACd,GAAG;YACH,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;SAC3B,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAA,+BAAuB,EAAC,MAAM,CAAC,CAAC;YAE7C,MAAM,MAAM,GAA0B;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,GAAG,EAAE,GAAG;gBACR,IAAI,EAAE,IAAI;gBACV,WAAW,EAAE,+BAAmB;gBAChC,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,QAAQ,EAAE;oBACR,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;oBACpC,aAAa,EAAE,OAAO;oBACtB,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACvC;aACF,CAAC;YAEF,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEjD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,MAAM,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAE5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;gBACvC,GAAG;gBACH,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;aACpB,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEtE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;gBAChD,GAAG;gBACH,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;CACF;AA5LD,8BA4LC"}
|