@onlineapps/conn-base-storage 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API.md +0 -0
- package/README.md +53 -172
- package/SHARED_URL_ADDRESSING.md +0 -0
- package/coverage/base.css +0 -0
- package/coverage/block-navigation.js +0 -0
- package/coverage/clover.xml +0 -0
- package/coverage/coverage-final.json +0 -0
- package/coverage/favicon.png +0 -0
- package/coverage/index.html +0 -0
- package/coverage/index.js.html +0 -0
- package/coverage/internal-url-adapter.js.html +0 -0
- package/coverage/lcov-report/base.css +0 -0
- package/coverage/lcov-report/block-navigation.js +0 -0
- package/coverage/lcov-report/favicon.png +0 -0
- package/coverage/lcov-report/index.html +0 -0
- package/coverage/lcov-report/index.js.html +0 -0
- package/coverage/lcov-report/internal-url-adapter.js.html +0 -0
- package/coverage/lcov-report/prettify.css +0 -0
- package/coverage/lcov-report/prettify.js +0 -0
- package/coverage/lcov-report/sort-arrow-sprite.png +0 -0
- package/coverage/lcov-report/sorter.js +0 -0
- package/coverage/lcov.info +0 -0
- package/coverage/prettify.css +0 -0
- package/coverage/prettify.js +0 -0
- package/coverage/sort-arrow-sprite.png +0 -0
- package/coverage/sorter.js +0 -0
- package/jest.config.js +2 -2
- package/jest.integration.config.js +0 -0
- package/onlineapps-conn-base-storage-1.0.0.tgz +0 -0
- package/package.json +1 -1
- package/src/index.js +0 -0
- package/src/internal-url-adapter.js +0 -0
- package/src/sharedUrlAdapter.js +0 -0
- package/{test → tests}/component/storage.component.test.js +1 -1
- package/{test → tests}/integration/setup.js +0 -0
- package/{test → tests}/integration/storage.integration.test.js +1 -1
- package/{test → tests}/unit/internal-url-adapter.test.js +1 -1
- package/{test → tests}/unit/legacy.storage.test.js.bak +0 -0
- package/{test → tests}/unit/storage.extended.unit.test.js +6 -5
- package/{test → tests}/unit/storage.unit.test.js +4 -3
package/API.md
CHANGED
|
File without changes
|
package/README.md
CHANGED
|
@@ -1,33 +1,33 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @onlineapps/conn-base-storage
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Unified MinIO client for all OA Drive services. Provides direct S3-compatible access to object storage with fingerprint-based immutable content handling.
|
|
4
4
|
|
|
5
|
-
##
|
|
5
|
+
## Why Direct Access (Not MQ Wrapper)?
|
|
6
6
|
|
|
7
|
-
1. **Performance** - MinIO
|
|
8
|
-
2. **Standard** - S3 API
|
|
9
|
-
3. **
|
|
10
|
-
4. **Best Practice** - Object storage
|
|
7
|
+
1. **Performance** - MinIO is optimized for direct streaming
|
|
8
|
+
2. **Standard** - S3 API is industry standard
|
|
9
|
+
3. **Efficiency** - Direct streaming for large files
|
|
10
|
+
4. **Best Practice** - Object storage always accessed directly
|
|
11
11
|
|
|
12
|
-
##
|
|
12
|
+
## Installation
|
|
13
13
|
|
|
14
14
|
```bash
|
|
15
|
-
npm install @
|
|
15
|
+
npm install @onlineapps/conn-base-storage
|
|
16
16
|
```
|
|
17
17
|
|
|
18
|
-
##
|
|
18
|
+
## Quick Start
|
|
19
19
|
|
|
20
20
|
```javascript
|
|
21
|
-
const StorageConnector = require('@
|
|
21
|
+
const StorageConnector = require('@onlineapps/conn-base-storage');
|
|
22
22
|
|
|
23
23
|
const storage = new StorageConnector({
|
|
24
24
|
endpoint: process.env.MINIO_HOST || 'api_services_storage',
|
|
25
25
|
port: 9000,
|
|
26
|
-
accessKey: process.env.MINIO_ACCESS_KEY
|
|
27
|
-
secretKey: process.env.MINIO_SECRET_KEY
|
|
26
|
+
accessKey: process.env.MINIO_ACCESS_KEY,
|
|
27
|
+
secretKey: process.env.MINIO_SECRET_KEY
|
|
28
28
|
});
|
|
29
29
|
|
|
30
|
-
// Upload
|
|
30
|
+
// Upload with fingerprint
|
|
31
31
|
const result = await storage.uploadWithFingerprint(
|
|
32
32
|
'registry',
|
|
33
33
|
content,
|
|
@@ -35,14 +35,14 @@ const result = await storage.uploadWithFingerprint(
|
|
|
35
35
|
);
|
|
36
36
|
// Returns: { path: 'specs/invoicing/abc123.json', fingerprint: 'abc123', size: 1234 }
|
|
37
37
|
|
|
38
|
-
// Download
|
|
38
|
+
// Download with verification
|
|
39
39
|
const content = await storage.downloadWithVerification(
|
|
40
40
|
'registry',
|
|
41
41
|
'specs/invoicing/abc123.json',
|
|
42
42
|
'abc123' // expected fingerprint
|
|
43
43
|
);
|
|
44
44
|
|
|
45
|
-
// Get pre-signed URL
|
|
45
|
+
// Get pre-signed URL for external access
|
|
46
46
|
const url = await storage.getPresignedUrl(
|
|
47
47
|
'registry',
|
|
48
48
|
'specs/invoicing/abc123.json',
|
|
@@ -50,145 +50,30 @@ const url = await storage.getPresignedUrl(
|
|
|
50
50
|
);
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
##
|
|
53
|
+
## Core Methods
|
|
54
54
|
|
|
55
|
-
###
|
|
55
|
+
### `uploadWithFingerprint(bucket, content, basePath)`
|
|
56
|
+
Uploads content with automatic fingerprint in filename.
|
|
56
57
|
|
|
57
|
-
|
|
58
|
-
|
|
58
|
+
### `downloadWithVerification(bucket, path, expectedFingerprint)`
|
|
59
|
+
Downloads file and verifies fingerprint.
|
|
59
60
|
|
|
60
|
-
|
|
61
|
-
|
|
61
|
+
### `exists(bucket, path)`
|
|
62
|
+
Checks if file exists.
|
|
62
63
|
|
|
63
|
-
|
|
64
|
-
|
|
64
|
+
### `getPresignedUrl(bucket, path, expiry)`
|
|
65
|
+
Creates temporary URL for external access.
|
|
65
66
|
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
### `calculateFingerprint(content)`
|
|
68
|
+
Calculates SHA256 fingerprint.
|
|
68
69
|
|
|
69
|
-
###
|
|
70
|
+
### `listByPrefix(bucket, prefix)`
|
|
71
|
+
Lists all objects with given prefix.
|
|
70
72
|
|
|
71
|
-
|
|
72
|
-
Spočítá SHA256 fingerprint obsahu.
|
|
73
|
-
|
|
74
|
-
#### `listByPrefix(bucket, prefix)`
|
|
75
|
-
Vypíše všechny objekty s daným prefixem.
|
|
76
|
-
|
|
77
|
-
## Implementace
|
|
78
|
-
|
|
79
|
-
```javascript
|
|
80
|
-
// connector-storage/index.js
|
|
81
|
-
const Minio = require('minio');
|
|
82
|
-
const crypto = require('crypto');
|
|
83
|
-
|
|
84
|
-
class StorageConnector {
|
|
85
|
-
constructor(config = {}) {
|
|
86
|
-
this.client = new Minio.Client({
|
|
87
|
-
endPoint: config.endpoint || 'api_services_storage',
|
|
88
|
-
port: config.port || 9000,
|
|
89
|
-
useSSL: config.useSSL || false,
|
|
90
|
-
accessKey: config.accessKey || 'minioadmin',
|
|
91
|
-
secretKey: config.secretKey || 'minioadmin'
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
// Ensure buckets exist
|
|
95
|
-
this.initBuckets();
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
async initBuckets() {
|
|
99
|
-
const buckets = ['registry', 'workflow', 'services'];
|
|
100
|
-
for (const bucket of buckets) {
|
|
101
|
-
const exists = await this.client.bucketExists(bucket);
|
|
102
|
-
if (!exists) {
|
|
103
|
-
await this.client.makeBucket(bucket);
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
calculateFingerprint(content) {
|
|
109
|
-
return crypto
|
|
110
|
-
.createHash('sha256')
|
|
111
|
-
.update(content)
|
|
112
|
-
.digest('hex');
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
async uploadWithFingerprint(bucket, content, basePath) {
|
|
116
|
-
const fingerprint = this.calculateFingerprint(content);
|
|
117
|
-
const path = `${basePath}/${fingerprint}.json`;
|
|
118
|
-
|
|
119
|
-
await this.client.putObject(
|
|
120
|
-
bucket,
|
|
121
|
-
path,
|
|
122
|
-
content,
|
|
123
|
-
content.length,
|
|
124
|
-
{
|
|
125
|
-
'Content-Type': 'application/json',
|
|
126
|
-
'X-Fingerprint': fingerprint
|
|
127
|
-
}
|
|
128
|
-
);
|
|
129
|
-
|
|
130
|
-
return {
|
|
131
|
-
path,
|
|
132
|
-
fingerprint,
|
|
133
|
-
size: content.length,
|
|
134
|
-
bucket
|
|
135
|
-
};
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
async downloadWithVerification(bucket, path, expectedFingerprint) {
|
|
139
|
-
const stream = await this.client.getObject(bucket, path);
|
|
140
|
-
const chunks = [];
|
|
141
|
-
|
|
142
|
-
for await (const chunk of stream) {
|
|
143
|
-
chunks.push(chunk);
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
const content = Buffer.concat(chunks).toString();
|
|
147
|
-
const actualFingerprint = this.calculateFingerprint(content);
|
|
148
|
-
|
|
149
|
-
if (expectedFingerprint && actualFingerprint !== expectedFingerprint) {
|
|
150
|
-
throw new Error(`Fingerprint mismatch! Expected: ${expectedFingerprint}, Got: ${actualFingerprint}`);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
return content;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
async exists(bucket, path) {
|
|
157
|
-
try {
|
|
158
|
-
await this.client.statObject(bucket, path);
|
|
159
|
-
return true;
|
|
160
|
-
} catch (err) {
|
|
161
|
-
if (err.code === 'NotFound') {
|
|
162
|
-
return false;
|
|
163
|
-
}
|
|
164
|
-
throw err;
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
async getPresignedUrl(bucket, path, expiry = 3600) {
|
|
169
|
-
return await this.client.presignedGetObject(bucket, path, expiry);
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
async listByPrefix(bucket, prefix) {
|
|
173
|
-
const objects = [];
|
|
174
|
-
const stream = this.client.listObjectsV2(bucket, prefix);
|
|
175
|
-
|
|
176
|
-
for await (const obj of stream) {
|
|
177
|
-
objects.push(obj);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
return objects;
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
module.exports = StorageConnector;
|
|
185
|
-
```
|
|
186
|
-
|
|
187
|
-
## Použití v Registry
|
|
73
|
+
## Usage in Registry
|
|
188
74
|
|
|
189
75
|
```javascript
|
|
190
|
-
|
|
191
|
-
const StorageConnector = require('@shared/connector-storage');
|
|
76
|
+
const StorageConnector = require('@onlineapps/conn-base-storage');
|
|
192
77
|
|
|
193
78
|
class SpecPublisher {
|
|
194
79
|
constructor() {
|
|
@@ -217,11 +102,10 @@ class SpecPublisher {
|
|
|
217
102
|
}
|
|
218
103
|
```
|
|
219
104
|
|
|
220
|
-
##
|
|
105
|
+
## Usage in Services
|
|
221
106
|
|
|
222
107
|
```javascript
|
|
223
|
-
|
|
224
|
-
const StorageConnector = require('@shared/connector-storage');
|
|
108
|
+
const StorageConnector = require('@onlineapps/conn-base-storage');
|
|
225
109
|
|
|
226
110
|
class InvoicingService {
|
|
227
111
|
constructor() {
|
|
@@ -256,22 +140,21 @@ class InvoicingService {
|
|
|
256
140
|
}
|
|
257
141
|
```
|
|
258
142
|
|
|
259
|
-
##
|
|
143
|
+
## Security
|
|
260
144
|
|
|
261
|
-
###
|
|
262
|
-
- `registry/` - Read-only
|
|
263
|
-
- `workflow/` - Read/write
|
|
264
|
-
- `services/` - Read/write
|
|
145
|
+
### Bucket Access Rights
|
|
146
|
+
- `registry/` - Read-only for services, write for Registry
|
|
147
|
+
- `workflow/` - Read/write for workflow components
|
|
148
|
+
- `services/` - Read/write for services (each has own prefix)
|
|
265
149
|
|
|
266
|
-
### Network
|
|
267
|
-
- MinIO
|
|
268
|
-
-
|
|
150
|
+
### Network Isolation
|
|
151
|
+
- MinIO runs only in internal Docker network
|
|
152
|
+
- External access only via pre-signed URLs
|
|
269
153
|
|
|
270
154
|
## Testing
|
|
271
155
|
|
|
272
156
|
```javascript
|
|
273
|
-
|
|
274
|
-
const StorageConnector = require('../index');
|
|
157
|
+
const StorageConnector = require('@onlineapps/conn-base-storage');
|
|
275
158
|
|
|
276
159
|
describe('StorageConnector', () => {
|
|
277
160
|
let storage;
|
|
@@ -286,7 +169,6 @@ describe('StorageConnector', () => {
|
|
|
286
169
|
test('upload and download with fingerprint', async () => {
|
|
287
170
|
const content = JSON.stringify({ test: 'data' });
|
|
288
171
|
|
|
289
|
-
// Upload
|
|
290
172
|
const upload = await storage.uploadWithFingerprint(
|
|
291
173
|
'test',
|
|
292
174
|
content,
|
|
@@ -295,7 +177,6 @@ describe('StorageConnector', () => {
|
|
|
295
177
|
|
|
296
178
|
expect(upload.fingerprint).toBeDefined();
|
|
297
179
|
|
|
298
|
-
// Download and verify
|
|
299
180
|
const downloaded = await storage.downloadWithVerification(
|
|
300
181
|
'test',
|
|
301
182
|
upload.path,
|
|
@@ -322,20 +203,20 @@ MINIO_HOST=localhost
|
|
|
322
203
|
MINIO_PORT=33025
|
|
323
204
|
```
|
|
324
205
|
|
|
325
|
-
##
|
|
206
|
+
## Benefits
|
|
326
207
|
|
|
327
|
-
1. **
|
|
328
|
-
2. **
|
|
329
|
-
3. **Verification** -
|
|
330
|
-
4. **Pre-signed URLs** -
|
|
331
|
-
5. **
|
|
332
|
-
6. **Standard** - S3 API
|
|
208
|
+
1. **Unified API** - All services use same library
|
|
209
|
+
2. **Built-in Fingerprints** - Automatic immutable content management
|
|
210
|
+
3. **Verification** - Integrity check on download
|
|
211
|
+
4. **Pre-signed URLs** - Secure external access
|
|
212
|
+
5. **Efficient** - Direct streaming, no MQ overhead
|
|
213
|
+
6. **Standard** - S3 API is industry standard
|
|
333
214
|
|
|
334
215
|
---
|
|
335
216
|
|
|
336
|
-
|
|
337
|
-
## 📚 Documentation
|
|
217
|
+
**This is the correct way to handle object storage in microservices architecture.**
|
|
338
218
|
|
|
339
|
-
|
|
340
|
-
- [Testing Standards](../../../docs/modules/connector.md#testing-standards)
|
|
219
|
+
## 📚 Documentation
|
|
341
220
|
|
|
221
|
+
- [Complete Connectors Documentation](../../../docs/architecture/connector.md)
|
|
222
|
+
- [MinIO Infrastructure](../../../docs/guides/INFRASTRUCTURE.md#minio-object-storage)
|
package/SHARED_URL_ADDRESSING.md
CHANGED
|
File without changes
|
package/coverage/base.css
CHANGED
|
File without changes
|
|
File without changes
|
package/coverage/clover.xml
CHANGED
|
File without changes
|
|
File without changes
|
package/coverage/favicon.png
CHANGED
|
File without changes
|
package/coverage/index.html
CHANGED
|
File without changes
|
package/coverage/index.js.html
CHANGED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
package/coverage/lcov.info
CHANGED
|
File without changes
|
package/coverage/prettify.css
CHANGED
|
File without changes
|
package/coverage/prettify.js
CHANGED
|
File without changes
|
|
File without changes
|
package/coverage/sorter.js
CHANGED
|
File without changes
|
package/jest.config.js
CHANGED
|
File without changes
|
|
Binary file
|
package/package.json
CHANGED
package/src/index.js
CHANGED
|
File without changes
|
|
File without changes
|
package/src/sharedUrlAdapter.js
CHANGED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -9,7 +9,7 @@ const Minio = require('minio');
|
|
|
9
9
|
// Mock MinIO client
|
|
10
10
|
jest.mock('minio');
|
|
11
11
|
|
|
12
|
-
describe('StorageConnector Extended Unit Tests', () => {
|
|
12
|
+
describe('StorageConnector Extended Unit Tests @unit', () => {
|
|
13
13
|
let storage;
|
|
14
14
|
let mockMinioClient;
|
|
15
15
|
|
|
@@ -24,6 +24,7 @@ describe('StorageConnector Extended Unit Tests', () => {
|
|
|
24
24
|
statObject: jest.fn(),
|
|
25
25
|
listObjectsV2: jest.fn(),
|
|
26
26
|
presignedGetUrl: jest.fn(),
|
|
27
|
+
presignedUrl: jest.fn(), // Add missing mock method
|
|
27
28
|
removeBucket: jest.fn()
|
|
28
29
|
};
|
|
29
30
|
|
|
@@ -291,12 +292,12 @@ describe('StorageConnector Extended Unit Tests', () => {
|
|
|
291
292
|
|
|
292
293
|
test('should generate presigned URL', async () => {
|
|
293
294
|
const mockUrl = 'https://minio.example.com/bucket/object?signature=xyz';
|
|
294
|
-
mockMinioClient.
|
|
295
|
+
mockMinioClient.presignedUrl.mockResolvedValue(mockUrl);
|
|
295
296
|
|
|
296
297
|
const url = await storage.getPresignedUrl('test-bucket', 'test-object', 3600);
|
|
297
298
|
|
|
298
299
|
expect(url).toBe(mockUrl);
|
|
299
|
-
expect(mockMinioClient.
|
|
300
|
+
expect(mockMinioClient.presignedUrl).toHaveBeenCalledWith(
|
|
300
301
|
'GET',
|
|
301
302
|
'test-bucket',
|
|
302
303
|
'test-object',
|
|
@@ -305,11 +306,11 @@ describe('StorageConnector Extended Unit Tests', () => {
|
|
|
305
306
|
});
|
|
306
307
|
|
|
307
308
|
test('should use default expiry', async () => {
|
|
308
|
-
mockMinioClient.
|
|
309
|
+
mockMinioClient.presignedUrl.mockResolvedValue('https://example.com');
|
|
309
310
|
|
|
310
311
|
await storage.getPresignedUrl('bucket', 'object');
|
|
311
312
|
|
|
312
|
-
expect(mockMinioClient.
|
|
313
|
+
expect(mockMinioClient.presignedUrl).toHaveBeenCalledWith(
|
|
313
314
|
'GET',
|
|
314
315
|
'bucket',
|
|
315
316
|
'object',
|
|
@@ -27,6 +27,7 @@ jest.mock('minio', () => {
|
|
|
27
27
|
listObjectsV2: jest.fn(),
|
|
28
28
|
presignedGetObject: jest.fn(),
|
|
29
29
|
presignedGetUrl: jest.fn(),
|
|
30
|
+
presignedUrl: jest.fn(), // Add missing mock method
|
|
30
31
|
useSSL: false,
|
|
31
32
|
port: 9000,
|
|
32
33
|
host: 'localhost'
|
|
@@ -34,7 +35,7 @@ jest.mock('minio', () => {
|
|
|
34
35
|
};
|
|
35
36
|
});
|
|
36
37
|
|
|
37
|
-
describe('StorageConnector Unit Tests', () => {
|
|
38
|
+
describe('StorageConnector Unit Tests @unit', () => {
|
|
38
39
|
describe('Initialization', () => {
|
|
39
40
|
test('should create instance with default config', () => {
|
|
40
41
|
const storage = new StorageConnector();
|
|
@@ -328,12 +329,12 @@ describe('StorageConnector Unit Tests', () => {
|
|
|
328
329
|
|
|
329
330
|
test('should generate presigned URL with mocked client', async () => {
|
|
330
331
|
const expectedUrl = 'http://localhost:9000/bucket/object?signature=xyz';
|
|
331
|
-
mockClient.
|
|
332
|
+
mockClient.presignedUrl.mockResolvedValue(expectedUrl);
|
|
332
333
|
|
|
333
334
|
const url = await storage.getPresignedUrl('test-bucket', 'test-object');
|
|
334
335
|
|
|
335
336
|
expect(url).toBe(expectedUrl);
|
|
336
|
-
expect(mockClient.
|
|
337
|
+
expect(mockClient.presignedUrl).toHaveBeenCalledWith(
|
|
337
338
|
'GET',
|
|
338
339
|
'test-bucket',
|
|
339
340
|
'test-object',
|