@undefineds.co/xpod 0.1.6 → 0.2.0-preview.2

package/dist/storage/accessors/TieredMinioDataAccessor.js

@@ -1,582 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TieredMinioDataAccessor = void 0;
-const node_fs_1 = require("node:fs");
-const global_logger_factory_1 = require("global-logger-factory");
-const node_path_1 = require("node:path");
-const minio_1 = require("minio");
-const community_server_1 = require("@solid/community-server");
-/**
- * TieredMinioDataAccessor extends MinioDataAccessor with a local cache layer.
- *
- * Read: Check local cache first, if miss then fetch from COS and cache locally.
- * Write: Write to COS first (ensure durability), then cache locally.
- *
- * Uses LRU eviction when cache exceeds maxSize.
- *
- * Supports cross-region migration via MigratableDataAccessor interface.
- */
-class TieredMinioDataAccessor {
-    constructor(config) {
-        this.logger = (0, global_logger_factory_1.getLoggerFor)(this);
-        // LRU tracking: Map<cacheFilePath, lastAccessTime>
-        this.cacheEntries = new Map();
-        this.currentCacheSize = 0;
-        // Active sync subscriptions for migration
-        this.activeSyncs = new Map();
-        this.resourceMapper = config.resourceMapper;
-        this.client = new minio_1.Client({
-            accessKey: config.accessKey,
-            secretKey: config.secretKey,
-            endPoint: config.endpoint,
-            useSSL: true,
-        });
-        this.bucketName = config.bucketName;
-        this.cachePath = config.cachePath;
-        this.cacheMaxSize = config.cacheMaxSize;
-        this.region = config.region;
-        this.regionBuckets = config.regionBuckets ?? {};
-        // Ensure cache directory exists
-        if (!(0, node_fs_1.existsSync)(this.cachePath)) {
-            (0, node_fs_1.mkdirSync)(this.cachePath, { recursive: true });
-        }
-        // Initialize cache tracking from existing files
-        this.initializeCacheTracking();
-        this.logger.info(`TieredMinioDataAccessor initialized with endpoint: ${config.endpoint}, cache: ${config.cachePath}, maxSize: ${this.formatBytes(config.cacheMaxSize)}`);
-    }
-    /**
-     * Scan existing cache directory and populate cache entries map.
-     */
-    initializeCacheTracking() {
-        try {
-            this.scanCacheDir(this.cachePath);
-            this.logger.info(`Cache initialized: ${this.cacheEntries.size} files, ${this.formatBytes(this.currentCacheSize)}`);
-        }
-        catch (error) {
-            this.logger.warn(`Failed to scan cache directory: ${error}`);
-        }
-    }
-    scanCacheDir(dir) {
-        if (!(0, node_fs_1.existsSync)(dir))
-            return;
-        const entries = (0, node_fs_1.readdirSync)(dir, { withFileTypes: true });
-        for (const entry of entries) {
-            const fullPath = (0, node_path_1.join)(dir, entry.name);
-            if (entry.isDirectory()) {
-                this.scanCacheDir(fullPath);
-            }
-            else if (entry.isFile()) {
-                try {
-                    const stats = (0, node_fs_1.statSync)(fullPath);
-                    this.cacheEntries.set(fullPath, {
-                        path: fullPath,
-                        size: stats.size,
-                        lastAccess: stats.atimeMs,
-                    });
-                    this.currentCacheSize += stats.size;
-                }
-                catch {
-                    // File may have been deleted
-                }
-            }
-        }
-    }
-    /**
-     * Convert URL path to local cache file path.
-     */
-    getCacheFilePath(identifier) {
-        const url = new URL(identifier.path);
-        // Remove leading slash and encode special chars
-        const relativePath = url.pathname.slice(1).replace(/[<>:"|?*]/g, '_');
-        return (0, node_path_1.join)(this.cachePath, relativePath);
-    }
-    /**
-     * Check if file exists in local cache.
-     */
-    isCached(cacheFilePath) {
-        return (0, node_fs_1.existsSync)(cacheFilePath);
-    }
-    /**
-     * Update LRU access time for a cache entry.
-     */
-    touchCache(cacheFilePath) {
-        const entry = this.cacheEntries.get(cacheFilePath);
-        if (entry) {
-            entry.lastAccess = Date.now();
-        }
-    }
-    /**
-     * Add file to cache tracking.
-     */
-    addToCacheTracking(cacheFilePath, size) {
-        this.cacheEntries.set(cacheFilePath, {
-            path: cacheFilePath,
-            size,
-            lastAccess: Date.now(),
-        });
-        this.currentCacheSize += size;
-        // Trigger eviction if needed
-        this.evictIfNeeded();
-    }
-    /**
-     * Remove file from cache tracking.
-     */
-    removeFromCacheTracking(cacheFilePath) {
-        const entry = this.cacheEntries.get(cacheFilePath);
-        if (entry) {
-            this.currentCacheSize -= entry.size;
-            this.cacheEntries.delete(cacheFilePath);
-        }
-    }
-    /**
-     * Evict least recently used files until cache is under maxSize.
-     */
-    evictIfNeeded() {
-        if (this.currentCacheSize <= this.cacheMaxSize) {
-            return;
-        }
-        // Sort by lastAccess (oldest first)
-        const sortedEntries = [...this.cacheEntries.entries()]
-            .sort((a, b) => a[1].lastAccess - b[1].lastAccess);
-        for (const [path, entry] of sortedEntries) {
-            if (this.currentCacheSize <= this.cacheMaxSize * 0.8) {
-                // Evict until 80% of max to avoid frequent evictions
-                break;
-            }
-            try {
-                (0, node_fs_1.unlinkSync)(path);
-                this.currentCacheSize -= entry.size;
-                this.cacheEntries.delete(path);
-                this.logger.debug(`Evicted from cache: ${path}`);
-            }
-            catch (error) {
-                this.logger.warn(`Failed to evict cache file ${path}: ${error}`);
-            }
-        }
-    }
-    formatBytes(bytes) {
-        if (bytes >= 1024 * 1024 * 1024) {
-            return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
-        }
-        if (bytes >= 1024 * 1024) {
-            return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
-        }
-        if (bytes >= 1024) {
-            return `${(bytes / 1024).toFixed(2)} KB`;
-        }
-        return `${bytes} B`;
-    }
-    // ============== DataAccessor Interface ==============
-    async canHandle(representation) {
-        if (!representation.binary) {
-            throw new community_server_1.UnsupportedMediaTypeHttpError('Only binary data is supported.');
-        }
-    }
-    /**
-     * Get data with cache-first strategy and cross-region fallback.
-     *
-     * Read order:
-     * 1. Local cache (fastest)
-     * 2. Local bucket (current region)
-     * 3. Fallback buckets (other regions) - enables instant migration
-     *
-     * When reading from fallback bucket, optionally copy to local bucket (lazy migration).
-     */
-    async getData(identifier) {
-        const cacheFilePath = this.getCacheFilePath(identifier);
-        const url = new URL(identifier.path);
-        // 1. Check local cache first
-        if (this.isCached(cacheFilePath)) {
-            this.logger.debug(`Cache hit: ${identifier.path}`);
-            this.touchCache(cacheFilePath);
-            const stream = (0, node_fs_1.createReadStream)(cacheFilePath);
-            return (0, community_server_1.guardStream)(stream);
-        }
-        // 2. Try local bucket first
-        this.logger.debug(`Cache miss: ${identifier.path}`);
-        let data = null;
-        let sourceLocation = 'local';
-        try {
-            data = await this.fetchFromBucket(this.bucketName, url.pathname);
-        }
-        catch (error) {
-            // Local bucket failed, try fallback buckets
-            if (this.supportsMigration()) {
-                const fallbackResult = await this.fetchFromFallbackBuckets(url.pathname);
-                if (fallbackResult) {
-                    data = fallbackResult.data;
-                    sourceLocation = fallbackResult.bucket;
-                    this.logger.debug(`Fallback read from ${sourceLocation}: ${identifier.path}`);
-                }
-            }
-        }
-        if (!data) {
-            throw new community_server_1.NotFoundHttpError(`Resource not found: ${identifier.path}`);
-        }
-        // 3. Write to local cache
-        await this.writeToCache(cacheFilePath, data, identifier.path);
-        // 4. Lazy copy to local bucket if read from fallback
-        if (sourceLocation !== 'local' && sourceLocation !== this.bucketName) {
-            this.lazyCopyToLocalBucket(url.pathname, data).catch(err => {
-                this.logger.warn(`Lazy copy failed for ${identifier.path}: ${err.message}`);
-            });
-        }
-        // Return data as stream
-        const { Readable } = require('node:stream');
-        const readable = Readable.from(data);
-        return (0, community_server_1.guardStream)(readable);
-    }
-    /**
-     * Fetch data from a specific bucket.
-     */
-    async fetchFromBucket(bucket, path) {
-        const stream = await this.client.getObject(bucket, path);
-        const chunks = [];
-        for await (const chunk of stream) {
-            chunks.push(chunk);
-        }
-        return Buffer.concat(chunks);
-    }
-    /**
-     * Try to fetch data from fallback buckets (other regions).
-     * Returns the data and source bucket name, or null if not found.
-     */
-    async fetchFromFallbackBuckets(path) {
-        // Try each region bucket except the current one
-        for (const [region, bucket] of Object.entries(this.regionBuckets)) {
-            if (bucket === this.bucketName) {
-                continue; // Skip local bucket
-            }
-            try {
-                this.logger.debug(`Trying fallback bucket: ${bucket} (region: ${region})`);
-                const data = await this.fetchFromBucket(bucket, path);
-                return { data, bucket };
-            }
-            catch {
-                // Not found in this bucket, try next
-                continue;
-            }
-        }
-        return null;
-    }
-    /**
-     * Write data to local cache.
-     */
-    async writeToCache(cacheFilePath, data, path) {
-        try {
-            const cacheDir = (0, node_path_1.dirname)(cacheFilePath);
-            if (!(0, node_fs_1.existsSync)(cacheDir)) {
-                (0, node_fs_1.mkdirSync)(cacheDir, { recursive: true });
-            }
-            const writeStream = (0, node_fs_1.createWriteStream)(cacheFilePath);
-            await new Promise((resolve, reject) => {
-                writeStream.write(data, (err) => {
-                    if (err)
-                        reject(err);
-                    else {
-                        writeStream.end();
-                        resolve();
-                    }
-                });
-            });
-            this.addToCacheTracking(cacheFilePath, data.length);
-        }
-        catch (error) {
-            this.logger.warn(`Failed to cache ${path}: ${error}`);
-        }
-    }
-    /**
-     * Lazily copy data to local bucket for future reads.
-     * This runs in the background and doesn't block the read.
-     */
-    async lazyCopyToLocalBucket(path, data) {
-        const { Readable } = require('node:stream');
-        const stream = Readable.from(data);
-        await this.client.putObject(this.bucketName, path, stream, data.length);
-        this.logger.debug(`Lazy copied to local bucket: ${path}`);
-    }
-    async getMetadata(identifier) {
-        const url = new URL(identifier.path);
-        const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
-        const isDirectory = identifier.path.endsWith('/');
-        const objectName = isDirectory ? `${url.pathname}/.container` : url.pathname;
-        let stats;
-        try {
-            stats = await this.client.statObject(this.bucketName, objectName);
-        }
-        catch (error) {
-            throw new community_server_1.NotFoundHttpError();
-        }
-        if (!(0, community_server_1.isContainerIdentifier)(identifier) && !isDirectory) {
-            return this.getFileMetadata(link, stats);
-        }
-        if ((0, community_server_1.isContainerIdentifier)(identifier) && isDirectory) {
-            return this.getDirectoryMetadata(link, stats);
-        }
-        throw new community_server_1.NotFoundHttpError();
-    }
-    async *getChildren(identifier) {
-        const url = new URL(identifier.path);
-        const objects = this.client.listObjectsV2(this.bucketName, url.pathname);
-        for await (const object of objects) {
-            const metadata = await this.getMetadata(object);
-            yield metadata;
-        }
-    }
-    /**
-     * Write document: COS first (durability), then cache.
-     */
-    async writeDocument(identifier, data, metadata) {
-        const url = new URL(identifier.path);
-        const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
-        const itemMetadata = this.encodeMetadata(link, metadata);
-        // Collect data for both COS and cache
-        const chunks = [];
-        for await (const chunk of data) {
-            chunks.push(chunk);
-        }
-        const buffer = Buffer.concat(chunks);
-        // Write to COS first (ensure durability)
-        try {
-            const { Readable } = require('node:stream');
-            const cosStream = Readable.from(buffer);
-            await this.client.putObject(this.bucketName, url.pathname, cosStream, buffer.length, itemMetadata || undefined);
-        }
-        catch (error) {
-            this.logger.error(`Error writing to COS: ${identifier.path} ${error}`);
-            throw error;
-        }
-        // Write to local cache
-        const cacheFilePath = this.getCacheFilePath(identifier);
-        try {
-            const cacheDir = (0, node_path_1.dirname)(cacheFilePath);
-            if (!(0, node_fs_1.existsSync)(cacheDir)) {
-                (0, node_fs_1.mkdirSync)(cacheDir, { recursive: true });
-            }
-            // Remove old cache entry if exists
-            if (this.isCached(cacheFilePath)) {
-                this.removeFromCacheTracking(cacheFilePath);
-            }
-            const writeStream = (0, node_fs_1.createWriteStream)(cacheFilePath);
-            await new Promise((resolve, reject) => {
-                writeStream.write(buffer, (err) => {
-                    if (err)
-                        reject(err);
-                    else {
-                        writeStream.end();
-                        resolve();
-                    }
-                });
-            });
-            this.addToCacheTracking(cacheFilePath, buffer.length);
-        }
-        catch (error) {
-            this.logger.warn(`Failed to write cache ${identifier.path}: ${error}`);
-            // Cache failure is non-fatal
-        }
-        // Replicate to sync targets (for active migrations)
-        const syncTargets = this.getActiveSyncTargets(url.pathname);
-        for (const targetBucket of syncTargets) {
-            try {
-                const { Readable } = require('node:stream');
-                const syncStream = Readable.from(buffer);
-                await this.client.putObject(targetBucket, url.pathname, syncStream, buffer.length, itemMetadata || undefined);
-                this.logger.debug(`Synced to ${targetBucket}: ${url.pathname}`);
-            }
-            catch (error) {
-                this.logger.warn(`Failed to sync ${url.pathname} to ${targetBucket}: ${error}`);
-                // Sync failure is non-fatal, migration will catch up
-            }
-        }
-    }
-    async writeContainer(identifier, metadata) {
-        const url = new URL(identifier.path);
-        const link = await this.resourceMapper.mapUrlToFilePath(identifier, false);
-        await this.client.putObject(this.bucketName, `${url.pathname}/.container`, Buffer.from(''), 0, this.encodeMetadata(link, metadata) || undefined);
-    }
-    async writeMetadata(identifier, metadata) {
-        throw new Error('TieredMinioDataAccessor does not support writing metadata for a resource.');
-    }
-    async deleteResource(identifier) {
-        const url = new URL(identifier.path);
-        // Delete from COS
-        await this.client.removeObject(this.bucketName, url.pathname);
-        // Delete from cache
-        const cacheFilePath = this.getCacheFilePath(identifier);
-        if (this.isCached(cacheFilePath)) {
-            try {
-                (0, node_fs_1.unlinkSync)(cacheFilePath);
-                this.removeFromCacheTracking(cacheFilePath);
-            }
-            catch (error) {
-                this.logger.warn(`Failed to delete cache ${identifier.path}: ${error}`);
-            }
-        }
-        // Sync delete to migration targets
-        const syncTargets = this.getActiveSyncTargets(url.pathname);
-        for (const targetBucket of syncTargets) {
-            try {
-                await this.client.removeObject(targetBucket, url.pathname);
-                this.logger.debug(`Synced delete to ${targetBucket}: ${url.pathname}`);
-            }
-            catch (error) {
-                this.logger.warn(`Failed to sync delete ${url.pathname} to ${targetBucket}: ${error}`);
-            }
-        }
-    }
-    // ============== Metadata Helpers ==============
-    async getFileMetadata(link, stats) {
-        const metadata = await this.getBaseMetadata(link, stats, false);
-        if (typeof metadata.contentType === 'undefined') {
-            metadata.set(community_server_1.CONTENT_TYPE_TERM, link.contentType);
-        }
-        return metadata;
-    }
-    async getDirectoryMetadata(link, stats) {
-        return this.getBaseMetadata(link, stats, true);
-    }
-    async getBaseMetadata(link, stats, isContainer) {
-        const metadata = this.decodeMetadata(link, stats.metaData);
-        (0, community_server_1.addResourceMetadata)(metadata, isContainer);
-        this.addPosixMetadata(metadata, stats, isContainer);
-        return metadata;
-    }
-    addPosixMetadata(metadata, stats, isDirectory) {
-        (0, community_server_1.updateModifiedDate)(metadata, stats.lastModified);
-        metadata.add(community_server_1.POSIX.terms.mtime, (0, community_server_1.toLiteral)(Math.floor(stats.lastModified.getTime() / 1000), community_server_1.XSD.terms.integer), community_server_1.SOLID_META.terms.ResponseMetadata);
-        if (!isDirectory) {
-            metadata.add(community_server_1.POSIX.terms.size, (0, community_server_1.toLiteral)(stats.size, community_server_1.XSD.terms.integer), community_server_1.SOLID_META.terms.ResponseMetadata);
-        }
-    }
-    encodeMetadata(link, metadata) {
-        metadata.remove(community_server_1.RDF.terms.type, community_server_1.LDP.terms.Resource);
-        metadata.remove(community_server_1.RDF.terms.type, community_server_1.LDP.terms.Container);
-        metadata.remove(community_server_1.RDF.terms.type, community_server_1.LDP.terms.BasicContainer);
-        metadata.removeAll(community_server_1.DC.terms.modified);
-        if ((0, community_server_1.isContainerPath)(link.filePath) || typeof metadata.contentType !== 'undefined') {
-            metadata.removeAll(community_server_1.CONTENT_TYPE_TERM);
-        }
-        const contentTypeObject = metadata.contentTypeObject;
-        if (contentTypeObject === undefined || Object.keys(contentTypeObject.parameters).length === 0) {
-            return null;
-        }
-        return contentTypeObject.parameters;
-    }
-    decodeMetadata(link, metadata) {
-        return new community_server_1.RepresentationMetadata(link.identifier, metadata);
-    }
-    // ============== Cache Stats (for monitoring) ==============
-    getCacheStats() {
-        return {
-            entries: this.cacheEntries.size,
-            size: this.currentCacheSize,
-            maxSize: this.cacheMaxSize,
-        };
-    }
-    // ============== MigratableDataAccessor Implementation ==============
-    /**
-     * Check if migration is supported.
-     * Migration requires region configuration and region-to-bucket mapping.
-     */
-    supportsMigration() {
-        return this.region !== undefined && Object.keys(this.regionBuckets).length > 0;
-    }
-    /**
-     * Migrate all objects under the given prefix to a target region's bucket.
-     * Uses Minio server-side copy for efficiency (data doesn't pass through this node).
-     */
-    async migrateToRegion(prefix, targetRegion, onProgress) {
-        if (!this.supportsMigration()) {
-            throw new Error('Migration not supported: region configuration missing');
-        }
-        const targetBucket = this.regionBuckets[targetRegion];
-        if (!targetBucket) {
-            throw new Error(`Unknown target region: ${targetRegion}. Available regions: ${Object.keys(this.regionBuckets).join(', ')}`);
-        }
-        if (targetBucket === this.bucketName) {
-            this.logger.info(`Source and target bucket are the same (${this.bucketName}), skipping migration`);
-            onProgress?.({ copied: 0, total: 0, bytesTransferred: 0 });
-            return;
-        }
-        this.logger.info(`Starting migration: prefix=${prefix}, source=${this.bucketName}, target=${targetBucket}`);
-        // Normalize prefix (remove leading slash for Minio)
-        const objectPrefix = prefix.startsWith('/') ? prefix.slice(1) : prefix;
-        // 1. List all objects to migrate
-        const objects = [];
-        const stream = this.client.listObjectsV2(this.bucketName, objectPrefix, true);
-        for await (const obj of stream) {
-            if (obj.name) {
-                objects.push({ name: obj.name, size: obj.size ?? 0 });
-            }
-        }
-        this.logger.info(`Found ${objects.length} objects to migrate`);
-        if (objects.length === 0) {
-            onProgress?.({ copied: 0, total: 0, bytesTransferred: 0 });
-            return;
-        }
-        // 2. Copy each object using server-side copy
-        let copied = 0;
-        let bytesTransferred = 0;
-        for (const obj of objects) {
-            try {
-                // Minio copyObject uses server-side copy when source and target are on same cluster
-                const copySource = `/${this.bucketName}/${obj.name}`;
-                await this.client.copyObject(targetBucket, obj.name, copySource, new minio_1.CopyConditions());
-                copied++;
-                bytesTransferred += obj.size;
-                onProgress?.({
-                    copied,
-                    total: objects.length,
-                    bytesTransferred,
-                });
-                this.logger.debug(`Copied: ${obj.name} (${copied}/${objects.length})`);
-            }
-            catch (error) {
-                this.logger.error(`Failed to copy object ${obj.name}: ${error.message}`);
-                throw error;
-            }
-        }
-        this.logger.info(`Migration completed: ${copied} objects, ${this.formatBytes(bytesTransferred)}`);
-    }
-    /**
-     * Set up real-time sync during migration.
-     *
-     * Note: Full implementation would use Minio Bucket Notifications to replicate
-     * new writes to the target bucket. For now, we track active syncs and replicate
-     * writes in writeDocument().
-     */
-    async setupRealtimeSync(prefix, targetRegion) {
-        if (!this.supportsMigration()) {
-            throw new Error('Migration not supported: region configuration missing');
-        }
-        const targetBucket = this.regionBuckets[targetRegion];
-        if (!targetBucket) {
-            throw new Error(`Unknown target region: ${targetRegion}`);
-        }
-        const syncKey = `${prefix}:${targetRegion}`;
-        this.activeSyncs.set(syncKey, { prefix, targetBucket });
-        this.logger.info(`Real-time sync enabled: prefix=${prefix}, target=${targetBucket}`);
-    }
-    /**
-     * Stop real-time sync after migration completes.
-     */
-    async stopRealtimeSync(prefix, targetRegion) {
-        const syncKey = `${prefix}:${targetRegion}`;
-        this.activeSyncs.delete(syncKey);
-        this.logger.info(`Real-time sync disabled: prefix=${prefix}, targetRegion=${targetRegion}`);
-    }
-    /**
-     * Check if a write should be replicated to sync targets.
-     */
-    getActiveSyncTargets(path) {
-        const targets = [];
-        for (const [, sync] of this.activeSyncs) {
-            if (path.startsWith(sync.prefix)) {
-                targets.push(sync.targetBucket);
-            }
-        }
-        return targets;
-    }
-}
-exports.TieredMinioDataAccessor = TieredMinioDataAccessor;
-//# sourceMappingURL=TieredMinioDataAccessor.js.map