@onlineapps/conn-base-storage 1.0.0

package/README.md

@@ -0,0 +1,341 @@
+# Connector Storage - Sdílený přístup k MinIO
+
+Jednotná knihovna pro všechny služby pro práci s api_services_storage (MinIO).
+
+## Proč přímý přístup (ne MQ wrapper)?
+
+1. **Performance** - MinIO je optimalizované pro přímý přístup
+2. **Standard** - S3 API je průmyslový standard
+3. **Efektivita** - Streaming velkých souborů
+4. **Best Practice** - Object storage se vždy používá přímo
+
+## Instalace
+
+```bash
+npm install @shared/connector-storage
+```
+
+## Použití
+
+```javascript
+const StorageConnector = require('@shared/connector-storage');
+
+const storage = new StorageConnector({
+  endpoint: process.env.MINIO_HOST || 'api_services_storage',
+  port: 9000,
+  accessKey: process.env.MINIO_ACCESS_KEY || 'minioadmin',
+  secretKey: process.env.MINIO_SECRET_KEY || 'minioadmin'
+});
+
+// Upload s fingerprintem
+const result = await storage.uploadWithFingerprint(
+  'registry',
+  content,
+  'specs/invoicing'
+);
+// Returns: { path: 'specs/invoicing/abc123.json', fingerprint: 'abc123', size: 1234 }
+
+// Download s ověřením
+const content = await storage.downloadWithVerification(
+  'registry',
+  'specs/invoicing/abc123.json',
+  'abc123'  // expected fingerprint
+);
+
+// Get pre-signed URL pro externí přístup
+const url = await storage.getPresignedUrl(
+  'registry',
+  'specs/invoicing/abc123.json',
+  3600  // expiration in seconds
+);
+```
+
+## API
+
+### Core Methods
+
+#### `uploadWithFingerprint(bucket, content, basePath)`
+Nahraje obsah a automaticky přidá fingerprint do názvu souboru.
+
+#### `downloadWithVerification(bucket, path, expectedFingerprint)`
+Stáhne soubor a ověří jeho fingerprint.
+
+#### `exists(bucket, path)`
+Zkontroluje existenci souboru.
+
+#### `getPresignedUrl(bucket, path, expiry)`
+Vytvoří dočasnou URL pro externí přístup.
+
+### Helper Methods
+
+#### `calculateFingerprint(content)`
+Spočítá SHA256 fingerprint obsahu.
+
+#### `listByPrefix(bucket, prefix)`
+Vypíše všechny objekty s daným prefixem.
+
+## Implementace
+
+```javascript
+// connector-storage/index.js
+const Minio = require('minio');
+const crypto = require('crypto');
+
+class StorageConnector {
+  constructor(config = {}) {
+    this.client = new Minio.Client({
+      endPoint: config.endpoint || 'api_services_storage',
+      port: config.port || 9000,
+      useSSL: config.useSSL || false,
+      accessKey: config.accessKey || 'minioadmin',
+      secretKey: config.secretKey || 'minioadmin'
+    });
+
+    // Ensure buckets exist
+    this.initBuckets();
+  }
+
+  async initBuckets() {
+    const buckets = ['registry', 'workflow', 'services'];
+    for (const bucket of buckets) {
+      const exists = await this.client.bucketExists(bucket);
+      if (!exists) {
+        await this.client.makeBucket(bucket);
+      }
+    }
+  }
+
+  calculateFingerprint(content) {
+    return crypto
+      .createHash('sha256')
+      .update(content)
+      .digest('hex');
+  }
+
+  async uploadWithFingerprint(bucket, content, basePath) {
+    const fingerprint = this.calculateFingerprint(content);
+    const path = `${basePath}/${fingerprint}.json`;
+
+    await this.client.putObject(
+      bucket,
+      path,
+      content,
+      content.length,
+      {
+        'Content-Type': 'application/json',
+        'X-Fingerprint': fingerprint
+      }
+    );
+
+    return {
+      path,
+      fingerprint,
+      size: content.length,
+      bucket
+    };
+  }
+
+  async downloadWithVerification(bucket, path, expectedFingerprint) {
+    const stream = await this.client.getObject(bucket, path);
+    const chunks = [];
+
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+
+    const content = Buffer.concat(chunks).toString();
+    const actualFingerprint = this.calculateFingerprint(content);
+
+    if (expectedFingerprint && actualFingerprint !== expectedFingerprint) {
+      throw new Error(`Fingerprint mismatch! Expected: ${expectedFingerprint}, Got: ${actualFingerprint}`);
+    }
+
+    return content;
+  }
+
+  async exists(bucket, path) {
+    try {
+      await this.client.statObject(bucket, path);
+      return true;
+    } catch (err) {
+      if (err.code === 'NotFound') {
+        return false;
+      }
+      throw err;
+    }
+  }
+
+  async getPresignedUrl(bucket, path, expiry = 3600) {
+    return await this.client.presignedGetObject(bucket, path, expiry);
+  }
+
+  async listByPrefix(bucket, prefix) {
+    const objects = [];
+    const stream = this.client.listObjectsV2(bucket, prefix);
+
+    for await (const obj of stream) {
+      objects.push(obj);
+    }
+
+    return objects;
+  }
+}
+
+module.exports = StorageConnector;
+```
+
+## Použití v Registry
+
+```javascript
+// api_services_registry/src/services/specPublisher.js
+const StorageConnector = require('@shared/connector-storage');
+
+class SpecPublisher {
+  constructor() {
+    this.storage = new StorageConnector();
+  }
+
+  async publishSpec(serviceName, spec) {
+    // Upload spec to MinIO
+    const result = await this.storage.uploadWithFingerprint(
+      'registry',
+      JSON.stringify(spec),
+      `specs/${serviceName}`
+    );
+
+    // Publish event with reference
+    await this.mq.publish('registry.changes', {
+      type: 'SPEC_PUBLISHED',
+      service: serviceName,
+      fingerprint: result.fingerprint,
+      path: result.path,
+      bucket: result.bucket
+    });
+
+    return result;
+  }
+}
+```
+
+## Použití ve službě
+
+```javascript
+// services/invoicing/src/index.js
+const StorageConnector = require('@shared/connector-storage');
+
+class InvoicingService {
+  constructor() {
+    this.storage = new StorageConnector();
+  }
+
+  async saveInvoice(invoice) {
+    // Save invoice PDF to storage
+    const result = await this.storage.uploadWithFingerprint(
+      'services',
+      invoice.pdfBuffer,
+      'invoicing/invoices'
+    );
+
+    // Return reference, not content
+    return {
+      invoiceId: invoice.id,
+      pdfUrl: result.path,
+      fingerprint: result.fingerprint,
+      size: result.size
+    };
+  }
+
+  async getInvoice(path, fingerprint) {
+    // Download and verify
+    return await this.storage.downloadWithVerification(
+      'services',
+      path,
+      fingerprint
+    );
+  }
+}
+```
+
+## Bezpečnost
+
+### Přístupová práva (buckets)
+- `registry/` - Read-only pro služby, write pro Registry
+- `workflow/` - Read/write pro workflow komponenty
+- `services/` - Read/write pro služby (každá má svůj prefix)
+
+### Network isolation
+- MinIO běží pouze v internal Docker network
+- Externí přístup pouze přes pre-signed URLs
+
+## Testing
+
+```javascript
+// test/connector-storage.test.js
+const StorageConnector = require('../index');
+
+describe('StorageConnector', () => {
+  let storage;
+
+  beforeAll(() => {
+    storage = new StorageConnector({
+      endpoint: 'localhost',
+      port: 33025  // Host port
+    });
+  });
+
+  test('upload and download with fingerprint', async () => {
+    const content = JSON.stringify({ test: 'data' });
+
+    // Upload
+    const upload = await storage.uploadWithFingerprint(
+      'test',
+      content,
+      'specs/test'
+    );
+
+    expect(upload.fingerprint).toBeDefined();
+
+    // Download and verify
+    const downloaded = await storage.downloadWithVerification(
+      'test',
+      upload.path,
+      upload.fingerprint
+    );
+
+    expect(downloaded).toBe(content);
+  });
+});
+```
+
+## Environment Variables
+
+```env
+# MinIO Configuration
+MINIO_HOST=api_services_storage
+MINIO_PORT=9000
+MINIO_ACCESS_KEY=minioadmin
+MINIO_SECRET_KEY=minioadmin
+MINIO_USE_SSL=false
+
+# From host (for testing)
+MINIO_HOST=localhost
+MINIO_PORT=33025
+```
+
+## Výhody tohoto přístupu
+
+1. **Jednotné API** - Všechny služby používají stejnou knihovnu
+2. **Fingerprint built-in** - Automatická správa immutable obsahu
+3. **Verification** - Ověření integrity při stahování
+4. **Pre-signed URLs** - Bezpečný externí přístup
+5. **Efektivní** - Přímý streaming, žádné MQ overhead
+6. **Standard** - S3 API je průmyslový standard
+
+---
+
+Toto je správný způsob práce s object storage v microservices architektuře.
+## 📚 Documentation
+
+- [Complete Connectors Documentation](../../../docs/modules/connector.md)
+- [Testing Standards](../../../docs/modules/connector.md#testing-standards)
+

package/SHARED_URL_ADDRESSING.md

@@ -0,0 +1,258 @@
+# Shared URL Addressing System
+
+## Overview
+
+The shared URL addressing system provides a unified way to reference resources across the OA Drive platform using the `shared://` protocol. This abstraction layer allows both backend services and frontend applications to use the same addressing scheme for accessing resources.
+
+## URL Format
+
+```
+shared://<namespace>/<path>
+```
+
+### Available Namespaces
+
+- **registry** - Service specifications and metadata
+- **workflow** - Workflow cookbooks and definitions
+- **storage** - General file storage
+- **cache** - Temporary cache storage
+- **logs** - Centralized logging storage
+
+## Examples
+
+### Registry Specifications
+```javascript
+// Service specification
+shared://registry/hello-service/v1.0.0/spec.json
+
+// Service metadata
+shared://registry/hello-service/v1.0.0/metadata.json
+```
+
+### Workflow Cookbooks
+```javascript
+// Workflow definition
+shared://workflow/cookbook-123/definition.json
+
+// Workflow step
+shared://workflow/cookbook-123/steps/step-1.json
+```
+
+### General Storage
+```javascript
+// User upload
+shared://storage/users/user-123/profile-photo.jpg
+
+// Document storage
+shared://storage/documents/report-2025.pdf
+```
+
+## API Usage
+
+### Upload Content
+
+```javascript
+const storage = new StorageConnector(config);
+await storage.initialize();
+
+// Upload to shared URL
+const result = await storage.sharedUrl.upload(
+  'shared://registry/my-service/v1.0.0/spec.json',
+  JSON.stringify(serviceSpec)
+);
+
+console.log(`Uploaded with fingerprint: ${result.fingerprint}`);
+```
+
+### Download Content
+
+```javascript
+// Download from shared URL
+const content = await storage.sharedUrl.download(
+  'shared://registry/my-service/v1.0.0/spec.json'
+);
+
+// Download with fingerprint verification
+const verified = await storage.sharedUrl.download(
+  'shared://registry/my-service/v1.0.0/spec.json',
+  expectedFingerprint
+);
+```
+
+### Generate HTTP URL
+
+```javascript
+// Get presigned HTTP URL for external access
+const httpUrl = await storage.sharedUrl.toHttpUrl(
+  'shared://registry/my-service/v1.0.0/spec.json',
+  3600 // expiry in seconds
+);
+```
+
+### List Resources
+
+```javascript
+// List all services in registry
+const services = await storage.sharedUrl.list('shared://registry/');
+
+// List workflow cookbooks
+const cookbooks = await storage.sharedUrl.list('shared://workflow/');
+```
+
+### Check Existence
+
+```javascript
+const exists = await storage.sharedUrl.exists(
+  'shared://registry/my-service/v1.0.0/spec.json'
+);
+```
+
+### Get Metadata
+
+```javascript
+const metadata = await storage.sharedUrl.getMetadata(
+  'shared://registry/my-service/v1.0.0/spec.json'
+);
+
+console.log(`Size: ${metadata.size} bytes`);
+console.log(`Last Modified: ${metadata.lastModified}`);
+console.log(`ETag: ${metadata.etag}`);
+```
+
+## Frontend Usage
+
+Frontend applications can use the same shared URLs and convert them to HTTP URLs for browser access:
+
+```javascript
+// In React/Vue/Angular component
+const sharedUrl = 'shared://registry/my-service/v1.0.0/spec.json';
+
+// Get HTTP URL for browser
+const httpUrl = await api.getPresignedUrl(sharedUrl);
+
+// Use in component
+<img src={httpUrl} />
+<a href={httpUrl}>Download Spec</a>
+```
+
+## Benefits
+
+1. **Unified Addressing** - Same URLs work for backend and frontend
+2. **Location Independence** - Resources can be moved without changing code
+3. **Access Control** - Centralized permission management
+4. **Caching** - Built-in caching support
+5. **Fingerprinting** - Automatic content verification
+6. **Presigned URLs** - Secure temporary access for external clients
+
+## Implementation Details
+
+### Backend Mapping
+
+The shared URL adapter maps namespaces to MinIO buckets:
+
+```javascript
+{
+  'registry': 'registry',      // Service specifications
+  'workflow': 'workflow',      // Workflow definitions
+  'storage': 'api-storage',    // General storage
+  'cache': 'cache',           // Temporary cache
+  'logs': 'logs'              // Centralized logs
+}
+```
+
+### Security
+
+- All uploads include automatic fingerprinting
+- Downloads can verify fingerprints
+- Presigned URLs expire after specified time
+- Access control at bucket level
+
+### Performance
+
+- Built-in caching for frequently accessed resources
+- Direct MinIO access for optimal performance
+- Lazy loading support for large resources
+
+## Migration Guide
+
+### From Direct MinIO Access
+
+```javascript
+// Before
+const url = await minioClient.presignedUrl(
+  'GET',
+  'registry',
+  'service/spec.json',
+  86400
+);
+
+// After
+const url = await storage.sharedUrl.toHttpUrl(
+  'shared://registry/service/spec.json',
+  86400
+);
+```
+
+### From File Paths
+
+```javascript
+// Before
+const spec = fs.readFileSync('./specs/service.json');
+
+// After
+const spec = await storage.sharedUrl.download(
+  'shared://registry/service/spec.json'
+);
+```
+
+## Best Practices
+
+1. **Use Namespaces Appropriately**
+   - registry: Only for service specifications
+   - workflow: Only for workflow definitions
+   - storage: General purpose files
+
+2. **Include Versions in Paths**
+   ```
+   shared://registry/service/v1.0.0/spec.json  ✓
+   shared://registry/service/spec.json         ✗
+   ```
+
+3. **Verify Fingerprints for Critical Data**
+   ```javascript
+   await storage.sharedUrl.download(url, expectedFingerprint);
+   ```
+
+4. **Set Appropriate Expiry Times**
+   - Public resources: 86400 (24 hours)
+   - Private resources: 3600 (1 hour)
+   - Sensitive data: 300 (5 minutes)
+
+## Troubleshooting
+
+### Invalid URL Format
+```
+Error: Invalid shared URL format. Must start with 'shared://'
+```
+Solution: Ensure URL starts with `shared://`
+
+### Unknown Namespace
+```
+Error: Unknown namespace 'custom'. Available: registry, workflow, storage, cache, logs
+```
+Solution: Use one of the predefined namespaces
+
+### Object Not Found
+```
+Error: The specified key does not exist
+```
+Solution: Check if the resource exists using `exists()` method
+
+## Future Enhancements
+
+- [ ] Custom namespace registration
+- [ ] URL aliases and shortcuts
+- [ ] Batch operations support
+- [ ] Real-time change notifications
+- [ ] Cross-region replication
+- [ ] CDN integration for global access