@undefineds.co/xpod 0.1.0-local.202602081751

package/dist/storage/accessors/TieredMinioDataAccessor.js

@@ -0,0 +1,582 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TieredMinioDataAccessor = void 0;
+const node_fs_1 = require("node:fs");
+const global_logger_factory_1 = require("global-logger-factory");
+const node_path_1 = require("node:path");
+const minio_1 = require("minio");
+const community_server_1 = require("@solid/community-server");
+/**
+ * TieredMinioDataAccessor extends MinioDataAccessor with a local cache layer.
+ *
+ * Read: Check local cache first, if miss then fetch from COS and cache locally.
+ * Write: Write to COS first (ensure durability), then cache locally.
+ *
+ * Uses LRU eviction when cache exceeds maxSize.
+ *
+ * Supports cross-region migration via MigratableDataAccessor interface.
+ */
+class TieredMinioDataAccessor {
+    constructor(config) {
+        this.logger = (0, global_logger_factory_1.getLoggerFor)(this);
+        // LRU tracking: Map<cacheFilePath, lastAccessTime>
+        this.cacheEntries = new Map();
+        this.currentCacheSize = 0;
+        // Active sync subscriptions for migration
+        this.activeSyncs = new Map();
+        this.resourceMapper = config.resourceMapper;
+        this.client = new minio_1.Client({
+            accessKey: config.accessKey,
+            secretKey: config.secretKey,
+            endPoint: config.endpoint,
+            useSSL: true,
+        });
+        this.bucketName = config.bucketName;
+        this.cachePath = config.cachePath;
+        this.cacheMaxSize = config.cacheMaxSize;
+        this.region = config.region;
+        this.regionBuckets = config.regionBuckets ?? {};
+        // Ensure cache directory exists
+        if (!(0, node_fs_1.existsSync)(this.cachePath)) {
+            (0, node_fs_1.mkdirSync)(this.cachePath, { recursive: true });
+        }
+        // Initialize cache tracking from existing files
+        this.initializeCacheTracking();
+        this.logger.info(`TieredMinioDataAccessor initialized with endpoint: ${config.endpoint}, cache: ${config.cachePath}, maxSize: ${this.formatBytes(config.cacheMaxSize)}`);
+    }
+    /**
+     * Scan existing cache directory and populate cache entries map.
+     */
+    initializeCacheTracking() {
+        try {
+            this.scanCacheDir(this.cachePath);
+            this.logger.info(`Cache initialized: ${this.cacheEntries.size} files, ${this.formatBytes(this.currentCacheSize)}`);
+        }
+        catch (error) {
+            this.logger.warn(`Failed to scan cache directory: ${error}`);
+        }
+    }
+    scanCacheDir(dir) {
+        if (!(0, node_fs_1.existsSync)(dir))
+            return;
+        const entries = (0, node_fs_1.readdirSync)(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = (0, node_path_1.join)(dir, entry.name);
+            if (entry.isDirectory()) {
+                this.scanCacheDir(fullPath);
+            }
+            else if (entry.isFile()) {
+                try {
+                    const stats = (0, node_fs_1.statSync)(fullPath);
+                    this.cacheEntries.set(fullPath, {
+                        path: fullPath,
+                        size: stats.size,
+                        lastAccess: stats.atimeMs,
+                    });
+                    this.currentCacheSize += stats.size;
+                }
+                catch {
+                    // File may have been deleted
+                }
+            }
+        }
+    }
+    /**
+     * Convert URL path to local cache file path.
+     */
+    getCacheFilePath(identifier) {
+        const url = new URL(identifier.path);
+        // Remove leading slash and encode special chars
+        const relativePath = url.pathname.slice(1).replace(/[<>:"|?*]/g, '_');
+        return (0, node_path_1.join)(this.cachePath, relativePath);
+    }
+    /**
+     * Check if file exists in local cache.
+     */
+    isCached(cacheFilePath) {
+        return (0, node_fs_1.existsSync)(cacheFilePath);
+    }
+    /**
+     * Update LRU access time for a cache entry.
+     */
+    touchCache(cacheFilePath) {
+        const entry = this.cacheEntries.get(cacheFilePath);
+        if (entry) {
+            entry.lastAccess = Date.now();
+        }
+    }
+    /**
+     * Add file to cache tracking.
+     */
+    addToCacheTracking(cacheFilePath, size) {
+        this.cacheEntries.set(cacheFilePath, {
+            path: cacheFilePath,
+            size,
+            lastAccess: Date.now(),
+        });
+        this.currentCacheSize += size;
+        // Trigger eviction if needed
+        this.evictIfNeeded();
+    }
+    /**
+     * Remove file from cache tracking.
+     */
+    removeFromCacheTracking(cacheFilePath) {
+        const entry = this.cacheEntries.get(cacheFilePath);
+        if (entry) {
+            this.currentCacheSize -= entry.size;
+            this.cacheEntries.delete(cacheFilePath);
+        }
+    }
+    /**
+     * Evict least recently used files until cache is under maxSize.
+     */
+    evictIfNeeded() {
+        if (this.currentCacheSize <= this.cacheMaxSize) {
+            return;
+        }
+        // Sort by lastAccess (oldest first)
+        const sortedEntries = [...this.cacheEntries.entries()]
+            .sort((a, b) => a[1].lastAccess - b[1].lastAccess);
+        for (const [path, entry] of sortedEntries) {
+            if (this.currentCacheSize <= this.cacheMaxSize * 0.8) {
+                // Evict until 80% of max to avoid frequent evictions
+                break;
+            }
+            try {
+                (0, node_fs_1.unlinkSync)(path);
+                this.currentCacheSize -= entry.size;
+                this.cacheEntries.delete(path);
+                this.logger.debug(`Evicted from cache: ${path}`);
+            }
+            catch (error) {
+                this.logger.warn(`Failed to evict cache file ${path}: ${error}`);
+            }
+        }
+    }
+    formatBytes(bytes) {
+        if (bytes >= 1024 * 1024 * 1024) {
+            return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
+        }
+        if (bytes >= 1024 * 1024) {
+            return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+        }
+        if (bytes >= 1024) {
+            return `${(bytes / 1024).toFixed(2)} KB`;
+        }
+        return `${bytes} B`;
+    }
+    // ============== DataAccessor Interface ==============
+    async canHandle(representation) {
+        if (!representation.binary) {
+            throw new community_server_1.UnsupportedMediaTypeHttpError('Only binary data is supported.');
+        }
+    }
+    /**
+     * Get data with cache-first strategy and cross-region fallback.
+     *
+     * Read order:
+     * 1. Local cache (fastest)
+     * 2. Local bucket (current region)
+     * 3. Fallback buckets (other regions) - enables instant migration
+     *
+     * When reading from fallback bucket, optionally copy to local bucket (lazy migration).
+     */
+    async getData(identifier) {
+        const cacheFilePath = this.getCacheFilePath(identifier);
+        const url = new URL(identifier.path);
+        // 1. Check local cache first
+        if (this.isCached(cacheFilePath)) {
+            this.logger.debug(`Cache hit: ${identifier.path}`);
+            this.touchCache(cacheFilePath);
+            const stream = (0, node_fs_1.createReadStream)(cacheFilePath);
+            return (0, community_server_1.guardStream)(stream);
+        }
+        // 2. Try local bucket first
+        this.logger.debug(`Cache miss: ${identifier.path}`);
+        let data = null;
+        let sourceLocation = 'local';
+        try {
+            data = await this.fetchFromBucket(this.bucketName, url.pathname);
+        }
+        catch (error) {
+            // Local bucket failed, try fallback buckets
+            if (this.supportsMigration()) {
+                const fallbackResult = await this.fetchFromFallbackBuckets(url.pathname);
+                if (fallbackResult) {
+                    data = fallbackResult.data;
+                    sourceLocation = fallbackResult.bucket;
+                    this.logger.debug(`Fallback read from ${sourceLocation}: ${identifier.path}`);
+                }
+            }
+        }
+        if (!data) {
+            throw new community_server_1.NotFoundHttpError(`Resource not found: ${identifier.path}`);
+        }
+        // 3. Write to local cache
+        await this.writeToCache(cacheFilePath, data, identifier.path);
+        // 4. Lazy copy to local bucket if read from fallback
+        if (sourceLocation !== 'local' && sourceLocation !== this.bucketName) {
+            this.lazyCopyToLocalBucket(url.pathname, data).catch(err => {
+                this.logger.warn(`Lazy copy failed for ${identifier.path}: ${err.message}`);
+            });
+        }
+        // Return data as stream
+        const { Readable } = require('node:stream');
+        const readable = Readable.from(data);
+        return (0, community_server_1.guardStream)(readable);
+    }
+    /**
+     * Fetch data from a specific bucket.
+     */
+    async fetchFromBucket(bucket, path) {
+        const stream = await this.client.getObject(bucket, path);
+        const chunks = [];
+        for await (const chunk of stream) {
+            chunks.push(chunk);
+        }
+        return Buffer.concat(chunks);
+    }
+    /**
+     * Try to fetch data from fallback buckets (other regions).
+     * Returns the data and source bucket name, or null if not found.
+     */
+    async fetchFromFallbackBuckets(path) {
+        // Try each region bucket except the current one
+        for (const [region, bucket] of Object.entries(this.regionBuckets)) {
+            if (bucket === this.bucketName) {
+                continue; // Skip local bucket
+            }
+            try {
+                this.logger.debug(`Trying fallback bucket: ${bucket} (region: ${region})`);
+                const data = await this.fetchFromBucket(bucket, path);
+                return { data, bucket };
+            }
+            catch {
+                // Not found in this bucket, try next
+                continue;
+            }
+        }
+        return null;
+    }
+    /**
+     * Write data to local cache.
+     */
+    async writeToCache(cacheFilePath, data, path) {
+        try {
+            const cacheDir = (0, node_path_1.dirname)(cacheFilePath);
+            if (!(0, node_fs_1.existsSync)(cacheDir)) {
+                (0, node_fs_1.mkdirSync)(cacheDir, { recursive: true });
+            }
+            const writeStream = (0, node_fs_1.createWriteStream)(cacheFilePath);
+            await new Promise((resolve, reject) => {
+                writeStream.write(data, (err) => {
+                    if (err)
+                        reject(err);
+                    else {
+                        writeStream.end();
+                        resolve();
+                    }
+                });
+            });
+            this.addToCacheTracking(cacheFilePath, data.length);
+        }
+        catch (error) {
+            this.logger.warn(`Failed to cache ${path}: ${error}`);
+        }
+    }
+    /**
+     * Lazily copy data to local bucket for future reads.
+     * This runs in the background and doesn't block the read.
+     */
+    async lazyCopyToLocalBucket(path, data) {
+        const { Readable } = require('node:stream');
+        const stream = Readable.from(data);
+        await this.client.putObject(this.bucketName, path, stream, data.length);
+        this.logger.debug(`Lazy copied to local bucket: ${path}`);
+    }
+    async getMetadata(identifier) {
+        const url = new URL(identifier.path);
+        const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
+        const isDirectory = identifier.path.endsWith('/');
+        const objectName = isDirectory ? `${url.pathname}/.container` : url.pathname;
+        let stats;
+        try {
+            stats = await this.client.statObject(this.bucketName, objectName);
+        }
+        catch (error) {
+            throw new community_server_1.NotFoundHttpError();
+        }
+        if (!(0, community_server_1.isContainerIdentifier)(identifier) && !isDirectory) {
+            return this.getFileMetadata(link, stats);
+        }
+        if ((0, community_server_1.isContainerIdentifier)(identifier) && isDirectory) {
+            return this.getDirectoryMetadata(link, stats);
+        }
+        throw new community_server_1.NotFoundHttpError();
+    }
+    async *getChildren(identifier) {
+        const url = new URL(identifier.path);
+        const objects = this.client.listObjectsV2(this.bucketName, url.pathname);
+        for await (const object of objects) {
+            const metadata = await this.getMetadata(object);
+            yield metadata;
+        }
+    }
+    /**
+     * Write document: COS first (durability), then cache.
+     */
+    async writeDocument(identifier, data, metadata) {
+        const url = new URL(identifier.path);
+        const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
+        const itemMetadata = this.encodeMetadata(link, metadata);
+        // Collect data for both COS and cache
+        const chunks = [];
+        for await (const chunk of data) {
+            chunks.push(chunk);
+        }
+        const buffer = Buffer.concat(chunks);
+        // Write to COS first (ensure durability)
+        try {
+            const { Readable } = require('node:stream');
+            const cosStream = Readable.from(buffer);
+            await this.client.putObject(this.bucketName, url.pathname, cosStream, buffer.length, itemMetadata || undefined);
+        }
+        catch (error) {
+            this.logger.error(`Error writing to COS: ${identifier.path} ${error}`);
+            throw error;
+        }
+        // Write to local cache
+        const cacheFilePath = this.getCacheFilePath(identifier);
+        try {
+            const cacheDir = (0, node_path_1.dirname)(cacheFilePath);
+            if (!(0, node_fs_1.existsSync)(cacheDir)) {
+                (0, node_fs_1.mkdirSync)(cacheDir, { recursive: true });
+            }
+            // Remove old cache entry if exists
+            if (this.isCached(cacheFilePath)) {
+                this.removeFromCacheTracking(cacheFilePath);
+            }
+            const writeStream = (0, node_fs_1.createWriteStream)(cacheFilePath);
+            await new Promise((resolve, reject) => {
+                writeStream.write(buffer, (err) => {
+                    if (err)
+                        reject(err);
+                    else {
+                        writeStream.end();
+                        resolve();
+                    }
+                });
+            });
+            this.addToCacheTracking(cacheFilePath, buffer.length);
+        }
+        catch (error) {
+            this.logger.warn(`Failed to write cache ${identifier.path}: ${error}`);
+            // Cache failure is non-fatal
+        }
+        // Replicate to sync targets (for active migrations)
+        const syncTargets = this.getActiveSyncTargets(url.pathname);
+        for (const targetBucket of syncTargets) {
+            try {
+                const { Readable } = require('node:stream');
+                const syncStream = Readable.from(buffer);
+                await this.client.putObject(targetBucket, url.pathname, syncStream, buffer.length, itemMetadata || undefined);
+                this.logger.debug(`Synced to ${targetBucket}: ${url.pathname}`);
+            }
+            catch (error) {
+                this.logger.warn(`Failed to sync ${url.pathname} to ${targetBucket}: ${error}`);
+                // Sync failure is non-fatal, migration will catch up
+            }
+        }
+    }
+    async writeContainer(identifier, metadata) {
+        const url = new URL(identifier.path);
+        const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
+        await this.client.putObject(this.bucketName, `${url.pathname}/.container`, Buffer.from(''), 0, this.encodeMetadata(link, metadata) || undefined);
+    }
+    async writeMetadata(identifier, metadata) {
+        throw new Error('TieredMinioDataAccessor does not support writing metadata for a resource.');
+    }
+    async deleteResource(identifier) {
+        const url = new URL(identifier.path);
+        // Delete from COS
+        await this.client.removeObject(this.bucketName, url.pathname);
+        // Delete from cache
+        const cacheFilePath = this.getCacheFilePath(identifier);
+        if (this.isCached(cacheFilePath)) {
+            try {
+                (0, node_fs_1.unlinkSync)(cacheFilePath);
+                this.removeFromCacheTracking(cacheFilePath);
+            }
+            catch (error) {
+                this.logger.warn(`Failed to delete cache ${identifier.path}: ${error}`);
+            }
+        }
+        // Sync delete to migration targets
+        const syncTargets = this.getActiveSyncTargets(url.pathname);
+        for (const targetBucket of syncTargets) {
+            try {
+                await this.client.removeObject(targetBucket, url.pathname);
+                this.logger.debug(`Synced delete to ${targetBucket}: ${url.pathname}`);
+            }
+            catch (error) {
+                this.logger.warn(`Failed to sync delete ${url.pathname} to ${targetBucket}: ${error}`);
+            }
+        }
+    }
+    // ============== Metadata Helpers ==============
+    async getFileMetadata(link, stats) {
+        const metadata = await this.getBaseMetadata(link, stats, false);
+        if (typeof metadata.contentType === 'undefined') {
+            metadata.set(community_server_1.CONTENT_TYPE_TERM, link.contentType);
+        }
+        return metadata;
+    }
+    async getDirectoryMetadata(link, stats) {
+        return this.getBaseMetadata(link, stats, true);
+    }
+    async getBaseMetadata(link, stats, isContainer) {
+        const metadata = this.decodeMetadata(link, stats.metaData);
+        (0, community_server_1.addResourceMetadata)(metadata, isContainer);
+        this.addPosixMetadata(metadata, stats, isContainer);
+        return metadata;
+    }
+    addPosixMetadata(metadata, stats, isDirectory) {
+        (0, community_server_1.updateModifiedDate)(metadata, stats.lastModified);
+        metadata.add(community_server_1.POSIX.terms.mtime, (0, community_server_1.toLiteral)(Math.floor(stats.lastModified.getTime() / 1000), community_server_1.XSD.terms.integer), community_server_1.SOLID_META.terms.ResponseMetadata);
+        if (!isDirectory) {
+            metadata.add(community_server_1.POSIX.terms.size, (0, community_server_1.toLiteral)(stats.size, community_server_1.XSD.terms.integer), community_server_1.SOLID_META.terms.ResponseMetadata);
+        }
+    }
+    encodeMetadata(link, metadata) {
+        metadata.remove(community_server_1.RDF.terms.type, community_server_1.LDP.terms.Resource);
+        metadata.remove(community_server_1.RDF.terms.type, community_server_1.LDP.terms.Container);
+        metadata.remove(community_server_1.RDF.terms.type, community_server_1.LDP.terms.BasicContainer);
+        metadata.removeAll(community_server_1.DC.terms.modified);
+        if ((0, community_server_1.isContainerPath)(link.filePath) || typeof metadata.contentType !== 'undefined') {
+            metadata.removeAll(community_server_1.CONTENT_TYPE_TERM);
+        }
+        const contentTypeObject = metadata.contentTypeObject;
+        if (contentTypeObject === undefined || Object.keys(contentTypeObject.parameters).length === 0) {
+            return null;
+        }
+        return contentTypeObject.parameters;
+    }
+    decodeMetadata(link, metadata) {
+        return new community_server_1.RepresentationMetadata(link.identifier, metadata);
+    }
+    // ============== Cache Stats (for monitoring) ==============
+    getCacheStats() {
+        return {
+            entries: this.cacheEntries.size,
+            size: this.currentCacheSize,
+            maxSize: this.cacheMaxSize,
+        };
+    }
+    // ============== MigratableDataAccessor Implementation ==============
+    /**
+     * Check if migration is supported.
+     * Migration requires region configuration and region-to-bucket mapping.
+     */
+    supportsMigration() {
+        return this.region !== undefined && Object.keys(this.regionBuckets).length > 0;
+    }
+    /**
+     * Migrate all objects under the given prefix to a target region's bucket.
+     * Uses Minio server-side copy for efficiency (data doesn't pass through this node).
+     */
+    async migrateToRegion(prefix, targetRegion, onProgress) {
+        if (!this.supportsMigration()) {
+            throw new Error('Migration not supported: region configuration missing');
+        }
+        const targetBucket = this.regionBuckets[targetRegion];
+        if (!targetBucket) {
+            throw new Error(`Unknown target region: ${targetRegion}. Available regions: ${Object.keys(this.regionBuckets).join(', ')}`);
+        }
+        if (targetBucket === this.bucketName) {
+            this.logger.info(`Source and target bucket are the same (${this.bucketName}), skipping migration`);
+            onProgress?.({ copied: 0, total: 0, bytesTransferred: 0 });
+            return;
+        }
+        this.logger.info(`Starting migration: prefix=${prefix}, source=${this.bucketName}, target=${targetBucket}`);
+        // Normalize prefix (remove leading slash for Minio)
+        const objectPrefix = prefix.startsWith('/') ? prefix.slice(1) : prefix;
+        // 1. List all objects to migrate
+        const objects = [];
+        const stream = this.client.listObjectsV2(this.bucketName, objectPrefix, true);
+        for await (const obj of stream) {
+            if (obj.name) {
+                objects.push({ name: obj.name, size: obj.size ?? 0 });
+            }
+        }
+        this.logger.info(`Found ${objects.length} objects to migrate`);
+        if (objects.length === 0) {
+            onProgress?.({ copied: 0, total: 0, bytesTransferred: 0 });
+            return;
+        }
+        // 2. Copy each object using server-side copy
+        let copied = 0;
+        let bytesTransferred = 0;
+        for (const obj of objects) {
+            try {
+                // Minio copyObject uses server-side copy when source and target are on same cluster
+                const copySource = `/${this.bucketName}/${obj.name}`;
+                await this.client.copyObject(targetBucket, obj.name, copySource, new minio_1.CopyConditions());
+                copied++;
+                bytesTransferred += obj.size;
+                onProgress?.({
+                    copied,
+                    total: objects.length,
+                    bytesTransferred,
+                });
+                this.logger.debug(`Copied: ${obj.name} (${copied}/${objects.length})`);
+            }
+            catch (error) {
+                this.logger.error(`Failed to copy object ${obj.name}: ${error.message}`);
+                throw error;
+            }
+        }
+        this.logger.info(`Migration completed: ${copied} objects, ${this.formatBytes(bytesTransferred)}`);
+    }
+    /**
+     * Set up real-time sync during migration.
+     *
+     * Note: Full implementation would use Minio Bucket Notifications to replicate
+     * new writes to the target bucket. For now, we track active syncs and replicate
+     * writes in writeDocument().
+     */
+    async setupRealtimeSync(prefix, targetRegion) {
+        if (!this.supportsMigration()) {
+            throw new Error('Migration not supported: region configuration missing');
+        }
+        const targetBucket = this.regionBuckets[targetRegion];
+        if (!targetBucket) {
+            throw new Error(`Unknown target region: ${targetRegion}`);
+        }
+        const syncKey = `${prefix}:${targetRegion}`;
+        this.activeSyncs.set(syncKey, { prefix, targetBucket });
+        this.logger.info(`Real-time sync enabled: prefix=${prefix}, target=${targetBucket}`);
+    }
+    /**
+     * Stop real-time sync after migration completes.
+     */
+    async stopRealtimeSync(prefix, targetRegion) {
+        const syncKey = `${prefix}:${targetRegion}`;
+        this.activeSyncs.delete(syncKey);
+        this.logger.info(`Real-time sync disabled: prefix=${prefix}, targetRegion=${targetRegion}`);
+    }
+    /**
+     * Check if a write should be replicated to sync targets.
+     */
+    getActiveSyncTargets(path) {
+        const targets = [];
+        for (const [, sync] of this.activeSyncs) {
+            if (path.startsWith(sync.prefix)) {
+                targets.push(sync.targetBucket);
+            }
+        }
+        return targets;
+    }
+}
+exports.TieredMinioDataAccessor = TieredMinioDataAccessor;
+//# sourceMappingURL=TieredMinioDataAccessor.js.map