ml-cache 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +599 -0
  3. package/dist/client.d.ts +175 -0
  4. package/dist/client.d.ts.map +1 -0
  5. package/dist/client.js +411 -0
  6. package/dist/client.js.map +1 -0
  7. package/dist/constants.d.ts +184 -0
  8. package/dist/constants.d.ts.map +1 -0
  9. package/dist/constants.js +236 -0
  10. package/dist/constants.js.map +1 -0
  11. package/dist/esm/client.d.ts +175 -0
  12. package/dist/esm/client.d.ts.map +1 -0
  13. package/dist/esm/client.js +407 -0
  14. package/dist/esm/client.js.map +1 -0
  15. package/dist/esm/constants.d.ts +184 -0
  16. package/dist/esm/constants.d.ts.map +1 -0
  17. package/dist/esm/constants.js +233 -0
  18. package/dist/esm/constants.js.map +1 -0
  19. package/dist/esm/index.d.ts +58 -0
  20. package/dist/esm/index.d.ts.map +1 -0
  21. package/dist/esm/index.js.map +1 -0
  22. package/dist/esm/queue.d.ts +106 -0
  23. package/dist/esm/queue.d.ts.map +1 -0
  24. package/dist/esm/queue.js +207 -0
  25. package/dist/esm/queue.js.map +1 -0
  26. package/dist/esm/storage/glacier-storage.d.ts +77 -0
  27. package/dist/esm/storage/glacier-storage.d.ts.map +1 -0
  28. package/dist/esm/storage/glacier-storage.js +133 -0
  29. package/dist/esm/storage/glacier-storage.js.map +1 -0
  30. package/dist/esm/storage/index.d.ts +7 -0
  31. package/dist/esm/storage/index.d.ts.map +1 -0
  32. package/dist/esm/storage/index.js +7 -0
  33. package/dist/esm/storage/index.js.map +1 -0
  34. package/dist/esm/storage/s3-storage.d.ts +89 -0
  35. package/dist/esm/storage/s3-storage.d.ts.map +1 -0
  36. package/dist/esm/storage/s3-storage.js +161 -0
  37. package/dist/esm/storage/s3-storage.js.map +1 -0
  38. package/dist/esm/types/index.d.ts +337 -0
  39. package/dist/esm/types/index.d.ts.map +1 -0
  40. package/dist/esm/types/index.js +6 -0
  41. package/dist/esm/types/index.js.map +1 -0
  42. package/dist/esm/utils/helpers.d.ts +132 -0
  43. package/dist/esm/utils/helpers.d.ts.map +1 -0
  44. package/dist/esm/utils/helpers.js +198 -0
  45. package/dist/esm/utils/helpers.js.map +1 -0
  46. package/dist/esm/utils/index.d.ts +8 -0
  47. package/dist/esm/utils/index.d.ts.map +1 -0
  48. package/dist/esm/utils/index.js +8 -0
  49. package/dist/esm/utils/index.js.map +1 -0
  50. package/dist/esm/utils/logger.d.ts +110 -0
  51. package/dist/esm/utils/logger.d.ts.map +1 -0
  52. package/dist/esm/utils/logger.js +177 -0
  53. package/dist/esm/utils/logger.js.map +1 -0
  54. package/dist/esm/utils/validator.d.ts +77 -0
  55. package/dist/esm/utils/validator.d.ts.map +1 -0
  56. package/dist/esm/utils/validator.js +208 -0
  57. package/dist/esm/utils/validator.js.map +1 -0
  58. package/dist/index.d.ts +58 -0
  59. package/dist/index.d.ts.map +1 -0
  60. package/dist/index.js +100 -0
  61. package/dist/index.js.map +1 -0
  62. package/dist/index.mjs +72 -0
  63. package/dist/queue.d.ts +106 -0
  64. package/dist/queue.d.ts.map +1 -0
  65. package/dist/queue.js +211 -0
  66. package/dist/queue.js.map +1 -0
  67. package/dist/storage/glacier-storage.d.ts +77 -0
  68. package/dist/storage/glacier-storage.d.ts.map +1 -0
  69. package/dist/storage/glacier-storage.js +137 -0
  70. package/dist/storage/glacier-storage.js.map +1 -0
  71. package/dist/storage/index.d.ts +7 -0
  72. package/dist/storage/index.d.ts.map +1 -0
  73. package/dist/storage/index.js +12 -0
  74. package/dist/storage/index.js.map +1 -0
  75. package/dist/storage/s3-storage.d.ts +89 -0
  76. package/dist/storage/s3-storage.d.ts.map +1 -0
  77. package/dist/storage/s3-storage.js +165 -0
  78. package/dist/storage/s3-storage.js.map +1 -0
  79. package/dist/types/index.d.ts +337 -0
  80. package/dist/types/index.d.ts.map +1 -0
  81. package/dist/types/index.js +7 -0
  82. package/dist/types/index.js.map +1 -0
  83. package/dist/utils/helpers.d.ts +132 -0
  84. package/dist/utils/helpers.d.ts.map +1 -0
  85. package/dist/utils/helpers.js +215 -0
  86. package/dist/utils/helpers.js.map +1 -0
  87. package/dist/utils/index.d.ts +8 -0
  88. package/dist/utils/index.d.ts.map +1 -0
  89. package/dist/utils/index.js +35 -0
  90. package/dist/utils/index.js.map +1 -0
  91. package/dist/utils/logger.d.ts +110 -0
  92. package/dist/utils/logger.d.ts.map +1 -0
  93. package/dist/utils/logger.js +181 -0
  94. package/dist/utils/logger.js.map +1 -0
  95. package/dist/utils/validator.d.ts +77 -0
  96. package/dist/utils/validator.d.ts.map +1 -0
  97. package/dist/utils/validator.js +221 -0
  98. package/dist/utils/validator.js.map +1 -0
  99. package/package.json +64 -0
@@ -0,0 +1,77 @@
1
+ /**
2
+ * @fileoverview Glacier storage provider for ml-cache SDK
3
+ * @module ml-cache/storage/glacier-storage
4
+ */
5
+ import type { GlacierConfig, AWSCredentials, MLCacheEvent } from '../types';
6
+ import { Logger } from '../utils';
7
+ /**
8
+ * Glacier storage result
9
+ * @interface GlacierStorageResult
10
+ */
11
+ export interface GlacierStorageResult {
12
+ /** Whether the operation was successful */
13
+ success: boolean;
14
+ /** Glacier archive ID */
15
+ archiveId?: string;
16
+ /** Vault name */
17
+ vaultName: string;
18
+ /** Archive checksum */
19
+ checksum?: string;
20
+ /** Archive location */
21
+ location?: string;
22
+ /** Error message if failed */
23
+ error?: string;
24
+ }
25
+ /**
26
+ * Glacier storage provider for long-term event archival
27
+ * @class GlacierStorage
28
+ */
29
+ export declare class GlacierStorage {
30
+ /** @private */
31
+ private client;
32
+ /** @private */
33
+ private config;
34
+ /** @private */
35
+ private accountId;
36
+ /** @private */
37
+ private logger;
38
+ /**
39
+ * Creates a new GlacierStorage instance
40
+ * @param {GlacierConfig} config - Glacier configuration
41
+ * @param {AWSCredentials} credentials - AWS credentials
42
+ * @param {Logger} [logger] - Logger instance
43
+ */
44
+ constructor(config: GlacierConfig, credentials: AWSCredentials, logger?: Logger);
45
+ /**
46
+ * Tests the connection to Glacier
47
+ * @returns {Promise<boolean>} Whether the connection is successful
48
+ */
49
+ testConnection(): Promise<boolean>;
50
+ /**
51
+ * Archives events in Glacier
52
+ * @param {MLCacheEvent[]} events - Events to archive
53
+ * @param {string} [batchId] - Optional batch ID
54
+ * @returns {Promise<GlacierStorageResult>} Storage result
55
+ */
56
+ archiveEvents(events: MLCacheEvent[], batchId?: string): Promise<GlacierStorageResult>;
57
+ /**
58
+ * Gets the vault name
59
+ * @returns {string} Vault name
60
+ */
61
+ getVaultName(): string;
62
+ /**
63
+ * Gets the region
64
+ * @returns {string} Region
65
+ */
66
+ getRegion(): string;
67
+ /**
68
+ * Gets the account ID
69
+ * @returns {string} Account ID
70
+ */
71
+ getAccountId(): string;
72
+ /**
73
+ * Destroys the Glacier client
74
+ */
75
+ destroy(): void;
76
+ }
77
+ //# sourceMappingURL=glacier-storage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"glacier-storage.d.ts","sourceRoot":"","sources":["../../../src/storage/glacier-storage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAM5E,OAAO,EACL,MAAM,EAIP,MAAM,UAAU,CAAC;AAElB;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,2CAA2C;IAC3C,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,qBAAa,cAAc;IACzB,eAAe;IACf,OAAO,CAAC,MAAM,CAAgB;IAE9B,eAAe;IACf,OAAO,CAAC,MAAM,CAAgB;IAE9B,eAAe;IACf,OAAO,CAAC,SAAS,CAAS;IAE1B,eAAe;IACf,OAAO,CAAC,MAAM,CAAS;IAEvB;;;;;OAKG;gBAED,MAAM,EAAE,aAAa,EACrB,WAAW,EAAE,cAAc,EAC3B,MAAM,CAAC,EAAE,MAAM;IAoBjB;;;OAGG;IACG,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAqBxC;;;;;OAKG;IACG,aAAa,CACjB,MAAM,EAAE,YAAY,EAAE,EACtB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,oBAAoB,CAAC;IA+ChC;;;OAGG;IACH,YAAY,IAAI,MAAM;IAItB;;;OAGG;IACH,SAAS,IAAI,MAAM;IAInB;;;OAGG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,OAAO,IAAI,IAAI;CAGhB"}
@@ -0,0 +1,133 @@
1
+ /**
2
+ * @fileoverview Glacier storage provider for ml-cache SDK
3
+ * @module ml-cache/storage/glacier-storage
4
+ */
5
+ import { GlacierClient, UploadArchiveCommand, DescribeVaultCommand, } from '@aws-sdk/client-glacier';
6
+ import { DEFAULT_GLACIER_ACCOUNT_ID, CONNECTION_TIMEOUT_MS, REQUEST_TIMEOUT_MS, } from '../constants';
7
+ import { Logger, generateBatchId, generateGlacierDescription, serializeEventsToNDJSON, } from '../utils';
8
+ /**
9
+ * Glacier storage provider for long-term event archival
10
+ * @class GlacierStorage
11
+ */
12
+ export class GlacierStorage {
13
+ /**
14
+ * Creates a new GlacierStorage instance
15
+ * @param {GlacierConfig} config - Glacier configuration
16
+ * @param {AWSCredentials} credentials - AWS credentials
17
+ * @param {Logger} [logger] - Logger instance
18
+ */
19
+ constructor(config, credentials, logger) {
20
+ this.config = config;
21
+ this.accountId = config.accountId || DEFAULT_GLACIER_ACCOUNT_ID;
22
+ this.logger = logger || new Logger();
23
+ this.client = new GlacierClient({
24
+ region: config.region,
25
+ credentials: {
26
+ accessKeyId: credentials.accessKeyId,
27
+ secretAccessKey: credentials.secretAccessKey,
28
+ sessionToken: credentials.sessionToken,
29
+ },
30
+ requestHandler: {
31
+ requestTimeout: REQUEST_TIMEOUT_MS,
32
+ connectionTimeout: CONNECTION_TIMEOUT_MS,
33
+ },
34
+ });
35
+ }
36
+ /**
37
+ * Tests the connection to Glacier
38
+ * @returns {Promise<boolean>} Whether the connection is successful
39
+ */
40
+ async testConnection() {
41
+ try {
42
+ const command = new DescribeVaultCommand({
43
+ accountId: this.accountId,
44
+ vaultName: this.config.vaultName,
45
+ });
46
+ await this.client.send(command);
47
+ this.logger.info('Glacier connection test successful', {
48
+ vaultName: this.config.vaultName,
49
+ });
50
+ return true;
51
+ }
52
+ catch (error) {
53
+ this.logger.error('Glacier connection test failed', {
54
+ vaultName: this.config.vaultName,
55
+ error: error instanceof Error ? error.message : String(error),
56
+ });
57
+ return false;
58
+ }
59
+ }
60
+ /**
61
+ * Archives events in Glacier
62
+ * @param {MLCacheEvent[]} events - Events to archive
63
+ * @param {string} [batchId] - Optional batch ID
64
+ * @returns {Promise<GlacierStorageResult>} Storage result
65
+ */
66
+ async archiveEvents(events, batchId) {
67
+ const result = {
68
+ success: false,
69
+ vaultName: this.config.vaultName,
70
+ };
71
+ try {
72
+ const id = batchId || generateBatchId();
73
+ const body = serializeEventsToNDJSON(events);
74
+ const description = generateGlacierDescription({
75
+ batchId: id,
76
+ eventCount: events.length,
77
+ });
78
+ const command = new UploadArchiveCommand({
79
+ accountId: this.accountId,
80
+ vaultName: this.config.vaultName,
81
+ body: Buffer.from(body, 'utf-8'),
82
+ archiveDescription: description,
83
+ });
84
+ const response = await this.client.send(command);
85
+ result.success = true;
86
+ result.archiveId = response.archiveId;
87
+ result.checksum = response.checksum;
88
+ result.location = response.location;
89
+ this.logger.debug('Events archived in Glacier', {
90
+ archiveId: response.archiveId,
91
+ eventCount: events.length,
92
+ checksum: response.checksum,
93
+ });
94
+ return result;
95
+ }
96
+ catch (error) {
97
+ result.error = error instanceof Error ? error.message : String(error);
98
+ this.logger.error('Failed to archive events in Glacier', {
99
+ eventCount: events.length,
100
+ error: result.error,
101
+ });
102
+ return result;
103
+ }
104
+ }
105
+ /**
106
+ * Gets the vault name
107
+ * @returns {string} Vault name
108
+ */
109
+ getVaultName() {
110
+ return this.config.vaultName;
111
+ }
112
+ /**
113
+ * Gets the region
114
+ * @returns {string} Region
115
+ */
116
+ getRegion() {
117
+ return this.config.region;
118
+ }
119
+ /**
120
+ * Gets the account ID
121
+ * @returns {string} Account ID
122
+ */
123
+ getAccountId() {
124
+ return this.accountId;
125
+ }
126
+ /**
127
+ * Destroys the Glacier client
128
+ */
129
+ destroy() {
130
+ this.client.destroy();
131
+ }
132
+ }
133
+ //# sourceMappingURL=glacier-storage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"glacier-storage.js","sourceRoot":"","sources":["../../../src/storage/glacier-storage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,0BAA0B,EAC1B,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,MAAM,EACN,eAAe,EACf,0BAA0B,EAC1B,uBAAuB,GACxB,MAAM,UAAU,CAAC;AAqBlB;;;GAGG;AACH,MAAM,OAAO,cAAc;IAazB;;;;;OAKG;IACH,YACE,MAAqB,EACrB,WAA2B,EAC3B,MAAe;QAEf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,0BAA0B,CAAC;QAChE,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAErC,IAAI,CAAC,MAAM,GAAG,IAAI,aAAa,CAAC;YAC9B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE;gBACX,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,eAAe,EAAE,WAAW,CAAC,eAAe;gBAC5C,YAAY,EAAE,WAAW,CAAC,YAAY;aACvC;YACD,cAAc,EAAE;gBACd,cAAc,EAAE,kBAAkB;gBAClC,iBAAiB,EAAE,qBAAqB;aAChC;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC;gBACvC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;aACjC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBACrD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;aACjC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;gBAClD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;YACH,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa,CACjB,MAAsB,EACtB,OAAgB;QAEhB,MAAM,MAAM,GAAyB;YACnC,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;SACjC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,OAAO,IAAI,eAAe,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,WAAW,GAAG,0BAA0B,CAAC;gBAC7C,OAAO,EAAE,EAAE;gBACX,UAAU,EAAE,MAAM,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC;gBACvC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBAChC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC;gBAChC,kBAAkB,EAAE,WAAW;aAChC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEjD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,MAAM,CAAC,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;YACtC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;YACpC,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAEpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE;gBAC9C,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEtE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE;gBACvD,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;CACF"}
@@ -0,0 +1,7 @@
1
+ /**
2
+ * @fileoverview Storage barrel export
3
+ * @module ml-cache/storage
4
+ */
5
+ export { S3Storage, type S3StorageResult } from './s3-storage';
6
+ export { GlacierStorage, type GlacierStorageResult } from './glacier-storage';
7
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,KAAK,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC"}
@@ -0,0 +1,7 @@
1
+ /**
2
+ * @fileoverview Storage barrel export
3
+ * @module ml-cache/storage
4
+ */
5
+ export { S3Storage } from './s3-storage';
6
+ export { GlacierStorage } from './glacier-storage';
7
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAwB,MAAM,cAAc,CAAC;AAC/D,OAAO,EAAE,cAAc,EAA6B,MAAM,mBAAmB,CAAC"}
@@ -0,0 +1,89 @@
1
+ /**
2
+ * @fileoverview S3 storage provider for ml-cache SDK
3
+ * @module ml-cache/storage/s3-storage
4
+ */
5
+ import type { S3Config, AWSCredentials, MLCacheEvent, S3StorageClass } from '../types';
6
+ import { Logger } from '../utils';
7
+ /**
8
+ * S3 storage result
9
+ * @interface S3StorageResult
10
+ */
11
+ export interface S3StorageResult {
12
+ /** Whether the operation was successful */
13
+ success: boolean;
14
+ /** S3 object key */
15
+ key: string;
16
+ /** S3 bucket name */
17
+ bucket: string;
18
+ /** ETag of stored object */
19
+ etag?: string;
20
+ /** Error message if failed */
21
+ error?: string;
22
+ }
23
+ /**
24
+ * S3 storage provider for event persistence
25
+ * @class S3Storage
26
+ */
27
+ export declare class S3Storage {
28
+ /** @private */
29
+ private client;
30
+ /** @private */
31
+ private config;
32
+ /** @private */
33
+ private storageClass;
34
+ /** @private */
35
+ private logger;
36
+ /**
37
+ * Creates a new S3Storage instance
38
+ * @param {S3Config} config - S3 configuration
39
+ * @param {AWSCredentials} credentials - AWS credentials
40
+ * @param {Logger} [logger] - Logger instance
41
+ */
42
+ constructor(config: S3Config, credentials: AWSCredentials, logger?: Logger);
43
+ /**
44
+ * Tests the connection to S3
45
+ * @returns {Promise<boolean>} Whether the connection is successful
46
+ */
47
+ testConnection(): Promise<boolean>;
48
+ /**
49
+ * Stores a single event in S3
50
+ * @param {MLCacheEvent} event - Event to store
51
+ * @returns {Promise<S3StorageResult>} Storage result
52
+ */
53
+ storeEvent(event: MLCacheEvent): Promise<S3StorageResult>;
54
+ /**
55
+ * Stores a batch of events in S3
56
+ * @param {MLCacheEvent[]} events - Events to store
57
+ * @param {string} [batchId] - Optional batch ID
58
+ * @returns {Promise<S3StorageResult>} Storage result
59
+ */
60
+ storeBatch(events: MLCacheEvent[], batchId?: string): Promise<S3StorageResult>;
61
+ /**
62
+ * Stores events in S3 with a specific key
63
+ * @private
64
+ * @param {MLCacheEvent[]} events - Events to store
65
+ * @param {string} key - S3 object key
66
+ * @returns {Promise<S3StorageResult>} Storage result
67
+ */
68
+ private store;
69
+ /**
70
+ * Gets the S3 bucket name
71
+ * @returns {string} Bucket name
72
+ */
73
+ getBucket(): string;
74
+ /**
75
+ * Gets the S3 region
76
+ * @returns {string} Region
77
+ */
78
+ getRegion(): string;
79
+ /**
80
+ * Gets the storage class
81
+ * @returns {S3StorageClass} Storage class
82
+ */
83
+ getStorageClass(): S3StorageClass;
84
+ /**
85
+ * Destroys the S3 client
86
+ */
87
+ destroy(): void;
88
+ }
89
+ //# sourceMappingURL=s3-storage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"s3-storage.d.ts","sourceRoot":"","sources":["../../../src/storage/s3-storage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,cAAc,EACf,MAAM,UAAU,CAAC;AAOlB,OAAO,EACL,MAAM,EAIP,MAAM,UAAU,CAAC;AAElB;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,2CAA2C;IAC3C,OAAO,EAAE,OAAO,CAAC;IACjB,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,4BAA4B;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,qBAAa,SAAS;IACpB,eAAe;IACf,OAAO,CAAC,MAAM,CAAW;IAEzB,eAAe;IACf,OAAO,CAAC,MAAM,CAAW;IAEzB,eAAe;IACf,OAAO,CAAC,YAAY,CAAiB;IAErC,eAAe;IACf,OAAO,CAAC,MAAM,CAAS;IAEvB;;;;;OAKG;gBAED,MAAM,EAAE,QAAQ,EAChB,WAAW,EAAE,cAAc,EAC3B,MAAM,CAAC,EAAE,MAAM;IAoBjB;;;OAGG;IACG,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAoBxC;;;;OAIG;IACG,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IAS/D;;;;;OAKG;IACG,UAAU,CACd,MAAM,EAAE,YAAY,EAAE,EACtB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC;IAU3B;;;;;;OAMG;YACW,KAAK;IAoDnB;;;OAGG;IACH,SAAS,IAAI,MAAM;IAInB;;;OAGG;IACH,SAAS,IAAI,MAAM;IAInB;;;OAGG;IACH,eAAe,IAAI,cAAc;IAIjC;;OAEG;IACH,OAAO,IAAI,IAAI;CAGhB"}
@@ -0,0 +1,161 @@
1
+ /**
2
+ * @fileoverview S3 storage provider for ml-cache SDK
3
+ * @module ml-cache/storage/s3-storage
4
+ */
5
+ import { S3Client, PutObjectCommand, HeadBucketCommand, } from '@aws-sdk/client-s3';
6
+ import { DEFAULT_S3_STORAGE_CLASS, CONTENT_TYPE_NDJSON, CONNECTION_TIMEOUT_MS, REQUEST_TIMEOUT_MS, } from '../constants';
7
+ import { Logger, generateS3Key, generateBatchId, serializeEventsToNDJSON, } from '../utils';
8
+ /**
9
+ * S3 storage provider for event persistence
10
+ * @class S3Storage
11
+ */
12
+ export class S3Storage {
13
+ /**
14
+ * Creates a new S3Storage instance
15
+ * @param {S3Config} config - S3 configuration
16
+ * @param {AWSCredentials} credentials - AWS credentials
17
+ * @param {Logger} [logger] - Logger instance
18
+ */
19
+ constructor(config, credentials, logger) {
20
+ this.config = config;
21
+ this.storageClass = config.storageClass || DEFAULT_S3_STORAGE_CLASS;
22
+ this.logger = logger || new Logger();
23
+ this.client = new S3Client({
24
+ region: config.region,
25
+ credentials: {
26
+ accessKeyId: credentials.accessKeyId,
27
+ secretAccessKey: credentials.secretAccessKey,
28
+ sessionToken: credentials.sessionToken,
29
+ },
30
+ requestHandler: {
31
+ requestTimeout: REQUEST_TIMEOUT_MS,
32
+ connectionTimeout: CONNECTION_TIMEOUT_MS,
33
+ },
34
+ });
35
+ }
36
+ /**
37
+ * Tests the connection to S3
38
+ * @returns {Promise<boolean>} Whether the connection is successful
39
+ */
40
+ async testConnection() {
41
+ try {
42
+ const command = new HeadBucketCommand({
43
+ Bucket: this.config.bucket,
44
+ });
45
+ await this.client.send(command);
46
+ this.logger.info('S3 connection test successful', {
47
+ bucket: this.config.bucket,
48
+ });
49
+ return true;
50
+ }
51
+ catch (error) {
52
+ this.logger.error('S3 connection test failed', {
53
+ bucket: this.config.bucket,
54
+ error: error instanceof Error ? error.message : String(error),
55
+ });
56
+ return false;
57
+ }
58
+ }
59
+ /**
60
+ * Stores a single event in S3
61
+ * @param {MLCacheEvent} event - Event to store
62
+ * @returns {Promise<S3StorageResult>} Storage result
63
+ */
64
+ async storeEvent(event) {
65
+ const key = generateS3Key({
66
+ prefix: this.config.prefix,
67
+ eventId: event.eventId,
68
+ });
69
+ return this.store([event], key);
70
+ }
71
+ /**
72
+ * Stores a batch of events in S3
73
+ * @param {MLCacheEvent[]} events - Events to store
74
+ * @param {string} [batchId] - Optional batch ID
75
+ * @returns {Promise<S3StorageResult>} Storage result
76
+ */
77
+ async storeBatch(events, batchId) {
78
+ const id = batchId || generateBatchId();
79
+ const key = generateS3Key({
80
+ prefix: this.config.prefix,
81
+ batchId: id,
82
+ });
83
+ return this.store(events, key);
84
+ }
85
+ /**
86
+ * Stores events in S3 with a specific key
87
+ * @private
88
+ * @param {MLCacheEvent[]} events - Events to store
89
+ * @param {string} key - S3 object key
90
+ * @returns {Promise<S3StorageResult>} Storage result
91
+ */
92
+ async store(events, key) {
93
+ const result = {
94
+ success: false,
95
+ key,
96
+ bucket: this.config.bucket,
97
+ };
98
+ try {
99
+ const body = serializeEventsToNDJSON(events);
100
+ const params = {
101
+ Bucket: this.config.bucket,
102
+ Key: key,
103
+ Body: body,
104
+ ContentType: CONTENT_TYPE_NDJSON,
105
+ StorageClass: this.storageClass,
106
+ Metadata: {
107
+ 'event-count': String(events.length),
108
+ 'sdk-version': '1.0.0',
109
+ 'created-at': new Date().toISOString(),
110
+ },
111
+ };
112
+ const command = new PutObjectCommand(params);
113
+ const response = await this.client.send(command);
114
+ result.success = true;
115
+ result.etag = response.ETag;
116
+ this.logger.debug('Events stored in S3', {
117
+ key,
118
+ eventCount: events.length,
119
+ etag: response.ETag,
120
+ });
121
+ return result;
122
+ }
123
+ catch (error) {
124
+ result.error = error instanceof Error ? error.message : String(error);
125
+ this.logger.error('Failed to store events in S3', {
126
+ key,
127
+ eventCount: events.length,
128
+ error: result.error,
129
+ });
130
+ return result;
131
+ }
132
+ }
133
+ /**
134
+ * Gets the S3 bucket name
135
+ * @returns {string} Bucket name
136
+ */
137
+ getBucket() {
138
+ return this.config.bucket;
139
+ }
140
+ /**
141
+ * Gets the S3 region
142
+ * @returns {string} Region
143
+ */
144
+ getRegion() {
145
+ return this.config.region;
146
+ }
147
+ /**
148
+ * Gets the storage class
149
+ * @returns {S3StorageClass} Storage class
150
+ */
151
+ getStorageClass() {
152
+ return this.storageClass;
153
+ }
154
+ /**
155
+ * Destroys the S3 client
156
+ */
157
+ destroy() {
158
+ this.client.destroy();
159
+ }
160
+ }
161
+ //# sourceMappingURL=s3-storage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"s3-storage.js","sourceRoot":"","sources":["../../../src/storage/s3-storage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,QAAQ,EACR,gBAAgB,EAChB,iBAAiB,GAElB,MAAM,oBAAoB,CAAC;AAO5B,OAAO,EACL,wBAAwB,EACxB,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,MAAM,EACN,aAAa,EACb,eAAe,EACf,uBAAuB,GACxB,MAAM,UAAU,CAAC;AAmBlB;;;GAGG;AACH,MAAM,OAAO,SAAS;IAapB;;;;;OAKG;IACH,YACE,MAAgB,EAChB,WAA2B,EAC3B,MAAe;QAEf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,wBAAwB,CAAC;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAErC,IAAI,CAAC,MAAM,GAAG,IAAI,QAAQ,CAAC;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,WAAW,EAAE;gBACX,WAAW,EAAE,WAAW,CAAC,WAAW;gBACpC,eAAe,EAAE,WAAW,CAAC,eAAe;gBAC5C,YAAY,EAAE,WAAW,CAAC,YAAY;aACvC;YACD,cAAc,EAAE;gBACd,cAAc,EAAE,kBAAkB;gBAClC,iBAAiB,EAAE,qBAAqB;aAChC;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;aAC3B,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;gBAChD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;aAC3B,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE;gBAC7C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;YACH,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,KAAmB;QAClC,MAAM,GAAG,GAAG,aAAa,CAAC;YACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CACd,MAAsB,EACtB,OAAgB;QAEhB,MAAM,EAAE,GAAG,OAAO,IAAI,eAAe,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,aAAa,CAAC;YACxB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,OAAO,EAAE,EAAE;SACZ,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,KAAK,CACjB,MAAsB,EACtB,GAAW;QAEX,MAAM,MAAM,GAAoB;YAC9B,OAAO,EAAE,KAAK;YACd,GAAG;YACH,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;SAC3B,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;YAE7C,MAAM,MAAM,GAA0B;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBAC1B,GAAG,EAAE,GAAG;gBACR,IAAI,EAAE,IAAI;gBACV,WAAW,EAAE,mBAAmB;gBAChC,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,QAAQ,EAAE;oBACR,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;oBACpC,aAAa,EAAE,OAAO;oBACtB,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACvC;aACF,CAAC;YAEF,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAEjD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,MAAM,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAE5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;gBACvC,GAAG;gBACH,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;aACpB,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEtE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;gBAChD,GAAG;gBACH,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IACxB,CAAC;CACF"}