@did-space/core 0.5.78 → 0.5.80

package/dist/es/space/object-space.js

@@ -0,0 +1,684 @@
+import Debug from 'debug';
+import { Op, QueryTypes } from 'sequelize';
+import jsYaml from 'js-yaml';
+import { basename, join } from 'path';
+import isEmpty from 'lodash/isEmpty';
+import isUndefined from 'lodash/isUndefined';
+import omit from 'lodash/omit';
+import { isNumber } from 'lodash';
+import EventEmitter from 'events';
+import { Scopes } from '../configuration';
+import { TreeModelType, } from '../model';
+import { getHash, getHashPath, getSize, isDirectory, logger, isStream, streamToString } from '../utils';
+import { DeleteAsOwnerOptionsSchema, OwnerOperatorKeySchema, ObjectSpaceOptionsSchema, WriteAsOwnerOptionsSchema, WriteOptionsSchema, } from '../schemas';
+import { APPS } from '../constants';
+import { GlobalSpace } from './global-space';
+const debug = Debug('@did-space/core:ObjectSpace');
+export class ObjectSpace extends EventEmitter {
+    options;
+    globalSpace;
+    driver;
+    static READONLY_OBJECT_KEYS = ['/config.yml'];
+    constructor(options) {
+        super();
+        const { error } = ObjectSpaceOptionsSchema.validate(options, {
+            allowUnknown: true,
+            stripUnknown: true,
+        });
+        if (error) {
+            throw error;
+        }
+        this.driver = options.driver;
+        this.options = options;
+        this.globalSpace = new GlobalSpace({
+            driver: options.driver,
+            treeRepository: options.treeRepository,
+            objectRepository: options.objectRepository,
+            objectCollectionRepository: options.objectCollectionRepository,
+        });
+    }
+    async createSpace(spaceConfig) {
+        debug('createSpace.before', JSON.stringify(spaceConfig));
+        await this.writeAsOwner('/', null);
+        await this.createConfig(spaceConfig);
+        debug('createSpace.after', JSON.stringify(spaceConfig));
+    }
+    async isSpaceCreated() {
+        const where = {
+            did: this.options.spaceDid,
+        };
+        debug('isSpaceCreated.before', JSON.stringify(where));
+        const created = await this.options.spaceRepository.count({
+            where,
+        });
+        debug('isSpaceCreated.after', JSON.stringify({ created }));
+        return Boolean(created);
+    }
+    /**
+     *
+     * @description
+     * @return {*}  {Promise<void>}
+     * @memberof ObjectSpace
+     */
+    async destroySpace() {
+        await this.deleteAsOwner('/');
+        const { spaceDid } = this.options;
+        await this.options.appSpaceApplyRepository.destroy({
+            where: {
+                storageDid: spaceDid,
+            },
+        });
+        await this.options.didObjectMappingRepository.destroy({
+            where: {
+                spaceDid,
+            },
+        });
+        await this.options.spaceRepository.destroy({
+            where: {
+                did: spaceDid,
+            },
+        });
+    }
+    async getSpaceSize() {
+        const { size } = await this.getStatusAsOwner('/');
+        return size;
+    }
+    async createAppSpace(options) {
+        debug('createAppSpace.before', JSON.stringify({ options }));
+        await this.writeAsOwner(await this.getAppSpacePath(options), null);
+        debug('createAppSpace.after', JSON.stringify({ options }));
+    }
+    /**
+     * @description
+     * @param {AppSpaceOptions} options
+     * @return {Promise<void>}  {Promise<void>}
+     * @memberof ObjectSpace
+     */
+    async destroyAppSpace(options) {
+        debug('destroyAppSpace.before', JSON.stringify({ options }));
+        const permissionOptions = {
+            fromAppDid: options.appDid,
+            toAppDid: options.appDid,
+        };
+        await this.setListable(permissionOptions, false);
+        await this.setReadable(permissionOptions, false);
+        await this.setWritable(permissionOptions, false);
+        await this.deleteAsOwner(await this.getAppSpacePath(options));
+        debug('destroyAppSpace.after', JSON.stringify({ options }));
+    }
+    getAppSpacePath(options) {
+        // @ts-expect-error
+        return join(`/${APPS}`, `${options.appDid}/`);
+    }
+    async write(options) {
+        debug('write.before', JSON.stringify(omit(options, 'data')));
+        const { error } = WriteOptionsSchema.validate(options, {
+            allowUnknown: true,
+            stripUnknown: true,
+        });
+        if (error) {
+            throw error;
+        }
+        const key = join(await this.getAppSpacePath(options), options.key);
+        // @ts-expect-error
+        await this.writeAsOwner(key, options.data, options);
+        debug('write.after', JSON.stringify({ key }));
+    }
+    async delete(options) {
+        debug('delete.before', JSON.stringify({ options }));
+        const key = join(await this.getAppSpacePath(options), options.key);
+        await this.deleteAsOwner(key);
+        debug('delete.after', JSON.stringify({ key }));
+    }
+    async read(options) {
+        debug('read.before', JSON.stringify({ options }));
+        const key = join(await this.getAppSpacePath(options), options.key);
+        debug('read.$key', key);
+        return this.readAsOwner(key);
+    }
+    async getHash(options) {
+        debug('getHash.before', JSON.stringify({ options }));
+        const key = join(await this.getAppSpacePath(options), options.key);
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key,
+        };
+        debug('getHash.$key', key);
+        debug('getHash.$where', JSON.stringify({ where }));
+        if (!(await this.exists(options))) {
+            throw new Error(`Object(${key}) not exists`);
+        }
+        const tree = await this.options.treeRepository.findOne({
+            where,
+            attributes: ['objectId'],
+            raw: true,
+        });
+        if (!tree) {
+            throw new Error(`Object(${key}) not exists`);
+        }
+        return tree.objectId;
+    }
+    async exists(options) {
+        debug('exists.before', JSON.stringify({ options: omit(options, 'data') }));
+        const key = join(await this.getAppSpacePath(options), options.key);
+        return this.existsAsOwner(key);
+    }
+    async listsOneLevelAsOwner(options) {
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key: options.key,
+        };
+        debug('listsOneLevel.before', JSON.stringify(options));
+        const parentTree = await this.options.treeRepository.findOne({
+            where,
+            attributes: ['id'],
+            raw: true,
+        });
+        if (!parentTree) {
+            return [];
+        }
+        debug('listsOneLevel.$parentTree', JSON.stringify(parentTree, null, 2));
+        const trees = await this.options.treeRepository.findAll({
+            where: {
+                spaceDid: this.options.spaceDid,
+                parentId: parentTree.id,
+            },
+            include: this.options.objectRepository,
+        });
+        return Promise.all(trees.map(async (x) => {
+            const absolutePath = join(x.spaceDid, x.key);
+            if (x.type === TreeModelType.FOLDER) {
+                const { size, lastModified } = await this.getStatusAsOwner(x.key);
+                return {
+                    key: x.key,
+                    name: basename(x.key),
+                    isDir: true,
+                    size,
+                    lastModified: lastModified || new Date(x.updatedAt).getTime(),
+                    editable: true,
+                    mimeType: null,
+                    absolutePath,
+                    metadata: x.meta,
+                };
+            }
+            return {
+                key: x.key,
+                name: basename(x.key),
+                isDir: false,
+                size: x.Object.size,
+                lastModified: new Date(x.updatedAt).getTime(),
+                editable: true,
+                mimeType: x.Object.meta.mimeType,
+                hash: x.objectId,
+                absolutePath,
+                metadata: x.meta,
+            };
+        }));
+    }
+    async listsRecursiveAsOwner(options) {
+        debug('listsRecursiveAsOwner.before', JSON.stringify(options));
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key: {
+                [Op.like]: `${options.key}%`,
+            },
+            type: TreeModelType.FILE,
+        };
+        const trees = await this.options.treeRepository.findAll({
+            where,
+            include: this.options.objectRepository,
+        });
+        return trees.map((x) => {
+            return {
+                key: x.key,
+                name: basename(x.key),
+                isDir: x.type === TreeModelType.FOLDER,
+                size: x.Object.size,
+                lastModified: new Date(x.updatedAt).getTime(),
+                editable: true,
+                mimeType: x.Object.meta.mimeType,
+                hash: x.objectId,
+                absolutePath: join(x.spaceDid, x.key),
+                metadata: x.meta,
+            };
+        });
+    }
+    /**
+     * @description 后续改名为 listsAsOwner 更合适
+     * @param {ListsOptions} options
+     * @return {*}  {Promise<Object[]>}
+     * @memberof ObjectSpace
+     */
+    async lists(options) {
+        debug('lists.before', JSON.stringify(options));
+        if (!(await this.existsAsOwner(options.key))) {
+            return [];
+        }
+        // 以 非递归的方式 && 显示文件夹 的方式列出当前文件
+        if (!options.recursive) {
+            return this.listsOneLevelAsOwner(options);
+        }
+        if (options.recursive && options.ignoreDirectories) {
+            const objects = await this.listsRecursiveAsOwner(options);
+            return objects.map((x) => {
+                return {
+                    ...x,
+                    key: x.key.replace(options.key, '/'),
+                };
+            });
+        }
+        debug('lists.after', JSON.stringify(options));
+        throw new Error(`Filter methods not yet supported:\n ${JSON.stringify(options, null, 2)}`);
+    }
+    /**
+     * @FIXME: 接口的功能实际应该是 listAsOwner 实现的，属于技术债 @jianchao
+     * @description
+     * @param {ListOptions} options
+     * @return {*}  {Promise<Object>}
+     * @memberof ObjectSpace
+     */
+    async list(options) {
+        debug('list.before', JSON.stringify(options));
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key: options.key,
+        };
+        const tree = await this.options.treeRepository.findOne({
+            where,
+            include: this.options.objectRepository,
+        });
+        if (!tree) {
+            return null;
+        }
+        return {
+            key: tree.key,
+            name: basename(tree.key),
+            isDir: tree.type === TreeModelType.FOLDER,
+            size: tree.type === TreeModelType.FOLDER ? 0 : tree.Object.size,
+            lastModified: new Date(tree.updatedAt).getTime(),
+            editable: true,
+            mimeType: tree.type === TreeModelType.FOLDER ? null : tree.Object.meta.mimeType,
+            absolutePath: join(tree.spaceDid, tree.key),
+            metadata: tree.meta,
+        };
+    }
+    /**
+     * @refactor: @jianchao 后续让 write 的接口参数保持一致，即 key, data 都在 options 里面，目前改动收益比较小
+     * @description
+     * @param {string} key
+     * @param {Data} data
+     * @param {WriteAsOwnerOptions} options
+     * @return {*}  {Promise<void>}
+     * @memberof ObjectSpace
+     */
+    async writeAsOwner(key, data, options = { hash: null, size: 0 }) {
+        debug('writeAsOwner.before', JSON.stringify({ key, options: omit(options, 'data') }));
+        const { error } = WriteAsOwnerOptionsSchema.validate({
+            ...options,
+            key,
+            data,
+        }, {
+            allowUnknown: true,
+            stripUnknown: true,
+        });
+        if (error) {
+            throw error;
+        }
+        if (!isDirectory(key) && !isStream(data)) {
+            options.hash = options.hash || (await getHash(data));
+            options.size = options.size || (await getSize(data));
+            // @note: size 可能是空的
+            if (!options.hash || !isNumber(options.size)) {
+                throw new Error('Hash and size cannot be empty');
+            }
+        }
+        try {
+            const exists = await this.existsAsOwner(key);
+            debug('writeAsOwner.$exists', exists);
+            if (exists) {
+                // 更新对象
+                await this._updateAsOwner(key, data, options);
+            }
+            else {
+                // 添加新对象
+                await this._createAsOwner(key, data, options);
+            }
+        }
+        catch (err) {
+            logger.error('writeAsOwner.$options', options);
+            logger.error('writeAsOwner.$error', err);
+            throw err;
+        }
+        finally {
+            this.emit('space.writeAsOwner.after', {
+                spaceDid: this.options.spaceDid,
+                key,
+            });
+            debug('writeAsOwner.after', JSON.stringify({ key, options: omit(options, 'data') }));
+        }
+    }
+    async _updateAsOwner(key, data, options) {
+        debug('updateAsOwner.before', { key, options: omit(options, 'data') });
+        await this.updateMetadata(key, data, options);
+        if (isUndefined(data)) {
+            return;
+        }
+        if (isDirectory(key)) {
+            return;
+        }
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key,
+        };
+        const oldTree = await this.options.treeRepository.findOne({
+            where,
+            include: this.options.objectRepository,
+        });
+        if (!oldTree) {
+            return;
+        }
+        const listObject = await this.driver.list({
+            key: getHashPath(options.hash),
+            useGlobal: true,
+        });
+        // @note: 运行时的纠错机制,我们发现 size 有时候保存出错(修复后将不可能出现),必须 hash 和 size 都相同,才不需要修改底层存储
+        if (oldTree.objectId === options.hash &&
+            oldTree.Object.size === options.size &&
+            listObject?.size === options.size) {
+            // 内容不变，不需要再次存储对象，只需要更新对象的更新时间即可
+            await this.options.treeRepository.update({
+                updatedAt: new Date().toISOString(),
+                size: options.size,
+            }, {
+                where: {
+                    id: oldTree.id,
+                },
+            });
+            return;
+        }
+        await this.globalSpace.add(key, data, options);
+        // 更新 tree 记录
+        await this.options.treeRepository.update({
+            objectId: options.hash,
+            size: options.size,
+            updatedAt: new Date().toISOString(),
+        }, {
+            where: {
+                id: oldTree.id,
+            },
+        });
+        // 标记清除
+        await this.globalSpace.mark(oldTree.objectId);
+    }
+    async _createAsOwner(key, data, options) {
+        debug('createAsOwner.before', JSON.stringify({ key, options: omit(options, 'data') }));
+        const isDir = key.endsWith('/');
+        if (isDir) {
+            // 添加 tree 记录
+            await this.options.treeRepository.deepCreate({
+                spaceDid: this.options.spaceDid,
+                objectId: null,
+                parentId: null,
+                key,
+                type: TreeModelType.FOLDER,
+                count: 0,
+                size: 0,
+            });
+        }
+        else {
+            // 存储新对象到全局存储区
+            await this.globalSpace.add(key, data, options);
+            // 添加 tree 记录
+            await this.options.treeRepository.deepCreate({
+                spaceDid: this.options.spaceDid,
+                objectId: options.hash,
+                parentId: null,
+                key,
+                type: TreeModelType.FILE,
+                count: 1,
+                size: options.size,
+            });
+        }
+        await this.updateMetadata(key, data, options);
+    }
+    async updateMetadata(key, _data, options) {
+        debug('updateMetadata.before', { key, options: omit(options, 'data') });
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key,
+            type: key.endsWith('/') ? TreeModelType.FOLDER : TreeModelType.FILE,
+        };
+        debug('updateMetadata.before', JSON.stringify({ key, options: omit(options, 'data'), where }));
+        const oldTree = await this.options.treeRepository.findOne({
+            where,
+            attributes: ['id', 'meta'],
+        });
+        debug('updateMetadata.$oldTree', JSON.stringify(oldTree));
+        if (!oldTree) {
+            return;
+        }
+        if (!isEmpty(options.metadata)) {
+            await oldTree.update({
+                meta: {
+                    ...oldTree.meta,
+                    ...options.metadata,
+                },
+                updatedAt: new Date().toISOString(),
+            });
+        }
+    }
+    async deleteAsOwner(key) {
+        debug('deleteAsOwner.before', JSON.stringify({ key }));
+        const { error } = DeleteAsOwnerOptionsSchema.validate({
+            key,
+        });
+        if (error) {
+            throw error;
+        }
+        if (ObjectSpace.READONLY_OBJECT_KEYS.includes(key)) {
+            throw new Error(`Object ${key} cannot be deleted`);
+        }
+        const isDir = key.endsWith('/');
+        try {
+            if (isDir) {
+                // 删除文件夹
+                const whereForDeleteFolder = {
+                    spaceDid: this.options.spaceDid,
+                    // 删除以 key 开头的记录
+                    key: {
+                        [Op.like]: `${key}%`,
+                    },
+                };
+                debug('deleteAsOwner.$whereForDeleteFolder', JSON.stringify(whereForDeleteFolder));
+                await this.options.objectCollectionRepository.sequelize.query(`INSERT OR REPLACE INTO ${this.options.objectCollectionRepository.tableName} (id, createdAt)
+              SELECT distinct objectId, strftime('%Y-%m-%d %H:%M:%f +00:00',DATETIME('now'))
+              FROM ${this.options.treeRepository.tableName} where spaceDid = '${this.options.spaceDid}' and type = ${TreeModelType.FILE} and key like '${key}%';`, {
+                    type: QueryTypes.INSERT,
+                });
+                await this.options.treeRepository.destroy({
+                    where: whereForDeleteFolder,
+                });
+            }
+            else {
+                const where = {
+                    spaceDid: this.options.spaceDid,
+                    key,
+                };
+                const tree = await this.options.treeRepository.findOne({
+                    where,
+                    attributes: ['objectId'],
+                    raw: true,
+                });
+                if (!tree) {
+                    return;
+                }
+                await this.options.treeRepository.destroy({
+                    where,
+                });
+                await this.globalSpace.mark(tree.objectId);
+            }
+        }
+        catch (err) {
+            logger.error('deleteAsOwner.$key', key);
+            logger.error('deleteAsOwner.$error', err);
+            throw err;
+        }
+        finally {
+            this.emit('space.deleteAsOwner.after', {
+                spaceDid: this.options.spaceDid,
+                key,
+            });
+        }
+    }
+    async readAsOwner(key, options) {
+        debug('readAsOwner.before', JSON.stringify({ key, options }));
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key,
+        };
+        debug('readAsOwner.$where', JSON.stringify(where));
+        const tree = await this.options.treeRepository.findOne({
+            where,
+            attributes: ['objectId'],
+            raw: true,
+        });
+        if (!tree) {
+            throw new Error(`Object(${key}) not exists`);
+        }
+        return this.driver.readAsOwner(getHashPath(tree.objectId), {
+            useGlobal: true,
+        });
+    }
+    async existsAsOwner(key) {
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key,
+        };
+        debug('existsAsOwner.before', JSON.stringify({ where }));
+        const exists = Boolean(await this.options.treeRepository.count({
+            where,
+        }));
+        debug('existsAsOwner.after', JSON.stringify({ exists }));
+        return exists;
+    }
+    async getStatusAsOwner(key) {
+        const { error } = OwnerOperatorKeySchema.validate(key);
+        if (error) {
+            throw error;
+        }
+        const where = {
+            spaceDid: this.options.spaceDid,
+            key,
+            type: TreeModelType.FOLDER,
+        };
+        const tree = await this.options.treeRepository.findOne({
+            where,
+            attributes: ['updatedAt', 'size', 'count'],
+            raw: true,
+        });
+        if (!tree) {
+            return {
+                objects: 0,
+                size: 0,
+                lastModified: 0,
+            };
+        }
+        // @note: 我们采用了逐级冒泡的方式，从叶子节点自下而上的更新 folder 的更新时间,对象数,size
+        const objects = tree.count;
+        const lastModified = new Date(tree.updatedAt).getTime();
+        const { size } = tree;
+        return {
+            objects,
+            size,
+            lastModified,
+        };
+    }
+    async createConfig(spaceConfig) {
+        debug('createConfig.before', { spaceConfig });
+        const data = jsYaml.dump(spaceConfig);
+        const hash = await getHash(data);
+        const size = await getSize(data);
+        debug('createConfig.$data', data);
+        debug('createConfig.$hash', hash);
+        debug('createConfig.$size', size);
+        await this.writeAsOwner('/config.yml', data, {
+            hash,
+            size,
+            useGlobal: true,
+        });
+    }
+    destroyConfig() {
+        return this.deleteAsOwner('/config.yml');
+    }
+    async set(key, value) {
+        debug('set.before', JSON.stringify({ key, value }));
+        const configData = await this.readAsOwner('/config.yml');
+        const data = jsYaml.load(await streamToString(configData));
+        data[key] = value;
+        await this.createConfig(data);
+    }
+    async get(key, defaultValue = {}) {
+        debug('get.before', JSON.stringify({ key }));
+        const configData = await this.readAsOwner('/config.yml');
+        const data = jsYaml.load(await streamToString(configData));
+        return data[key] ?? defaultValue;
+    }
+    async getPermission({ fromAppDid, toAppDid }) {
+        const permissions = await this.get('permissions');
+        return permissions?.[fromAppDid]?.[toAppDid] || 0;
+    }
+    /**
+     *
+     * @see https://blog.csdn.net/a1173537204/article/details/89765932
+     * @private
+     * @param {PermissionOptions} { fromAppDid, toAppDid }
+     * @param {number} permission
+     * @param {boolean} status
+     * @return {*}  {Promise<void>}
+     * @memberof S3SpaceConfig
+     */
+    async setPermission({ fromAppDid, toAppDid }, permission, status) {
+        const permissions = await this.get('permissions', {});
+        // 我想知道原来的权限是啥样的？
+        const oldPermission = permissions?.[fromAppDid]?.[toAppDid] || 0;
+        permissions[fromAppDid] = Object.assign(permissions?.[fromAppDid] ?? {}, {
+            ...permissions?.[fromAppDid],
+            // eslint-disable-next-line no-bitwise
+            [toAppDid]: status ? oldPermission | permission : oldPermission & ~permission,
+        });
+        await this.set('permissions', permissions);
+    }
+    /**
+     *
+     * @refactor 这部分的实现大同小异，之后可以抽象到上层实现，这样可以减少代码量 @jianchao
+     * @description
+     * @param {PermissionOptions} options
+     * @param {boolean} value
+     * @return {*}  {Promise<void>}
+     * @memberof ObjectSpace
+     */
+    setListable(options, value) {
+        return this.setPermission(options, Scopes['list:object'], value);
+    }
+    setReadable(options, value) {
+        return this.setPermission(options, Scopes['read:object'], value);
+    }
+    setWritable(options, value) {
+        return this.setPermission(options, Scopes['write:object'], value);
+    }
+    async isListable(options) {
+        const permission = await this.getPermission(options);
+        return Boolean((permission & Scopes['list:object']) === Scopes['list:object']);
+    }
+    async isReadable(options) {
+        const permission = await this.getPermission(options);
+        return Boolean((permission & Scopes['read:object']) === Scopes['read:object']);
+    }
+    async isWritable(options) {
+        const permission = await this.getPermission(options);
+        return Boolean((permission & Scopes['write:object']) === Scopes['write:object']);
+    }
+    static editable(key) {
+        return !ObjectSpace.READONLY_OBJECT_KEYS.includes(key);
+    }
+}