@did-space/core 0.5.78 → 0.5.80

package/dist/es/space/global-space.d.ts

@@ -0,0 +1,43 @@
+import { Data } from '../meta';
+import { DriverProtocol, WriteAsOwnerOptions } from '../protocols';
+import { ObjectCollectionRepository, ObjectRepository, TreeRepository } from '../model';
+export type GlobalSpaceOptions = {
+    driver: DriverProtocol;
+    treeRepository: TreeRepository;
+    objectRepository: ObjectRepository;
+    objectCollectionRepository: ObjectCollectionRepository;
+    gcMaxCount?: 100 | number;
+    gcMaxConcurrency?: 8 | number;
+};
+export declare class GlobalSpace {
+    readonly driver: DriverProtocol;
+    readonly treeRepository: TreeRepository;
+    readonly objectRepository: ObjectRepository;
+    readonly objectCollectionRepository: ObjectCollectionRepository;
+    readonly gcMaxCount: number;
+    readonly gcMaxConcurrency: number;
+    constructor(options: GlobalSpaceOptions);
+    /**
+     * @description 存储新对象到全局存储区
+     * @param {string} key
+     * @param {Data} data
+     * @param {WriteAsOwnerOptions} options
+     * @return {*}  {Promise<void>}
+     * @memberof GlobalSpace
+     */
+    add(key: string, data: Data, options: WriteAsOwnerOptions): Promise<void>;
+    delete(hash: string): Promise<void>;
+    /**
+     * @description 清理未被使用的对象
+     * @return {Promise<void>}  {Promise<void>}
+     * @memberof GlobalSpace
+     */
+    gc(): Promise<void>;
+    /**
+     * @description 标记一个对象，以备之后回收
+     * @param {...string[]} hashList
+     * @return {Promise<void>}  {Promise<void>}
+     * @memberof GlobalSpace
+     */
+    mark(...hashList: string[]): Promise<void>;
+}

package/dist/es/space/global-space.js

@@ -0,0 +1,169 @@
+import { merge, omit, pick } from 'lodash';
+import { Op } from 'sequelize';
+import mimeTypes from 'mime-types';
+import Debug from 'debug';
+import pAll from 'p-all';
+import dayjs from 'dayjs';
+import { getHashPath } from '../utils';
+import { GlobalSpaceOptionsSchema, HashSchema } from '../schemas/global-space';
+import { WriteAsOwnerOptionsSchema } from '../schemas';
+const debug = Debug('@did-space/core:GlobalSpace');
+export class GlobalSpace {
+    driver;
+    treeRepository;
+    objectRepository;
+    objectCollectionRepository;
+    gcMaxCount;
+    gcMaxConcurrency;
+    constructor(options) {
+        debug('GlobalSpace.constructor', JSON.stringify({ options: pick(options, 'gcMaxCount', 'gcMaxConcurrency') }));
+        const { error, value } = GlobalSpaceOptionsSchema.validate(options);
+        if (error) {
+            throw error;
+        }
+        this.driver = value.driver;
+        this.treeRepository = value.treeRepository;
+        this.objectRepository = value.objectRepository;
+        this.objectCollectionRepository = value.objectCollectionRepository;
+        this.gcMaxCount = value.gcMaxCount ?? 100;
+        this.gcMaxConcurrency = value.gcMaxConcurrency ?? 8;
+        debug('GlobalSpace.constructor', JSON.stringify({ gcMaxCount: value.gcMaxCount, gcMaxConcurrency: value.gcMaxConcurrency }));
+    }
+    /**
+     * @description 存储新对象到全局存储区
+     * @param {string} key
+     * @param {Data} data
+     * @param {WriteAsOwnerOptions} options
+     * @return {*}  {Promise<void>}
+     * @memberof GlobalSpace
+     */
+    async add(key, data, options) {
+        // @note: 注意此处必须存储到全局存储区
+        // eslint-disable-next-line no-param-reassign
+        options = merge(options, { useGlobal: true });
+        debug('add.before', JSON.stringify({ key, options: omit(options, 'data') }));
+        const { error } = WriteAsOwnerOptionsSchema.validate({
+            ...options,
+            key,
+            data,
+        }, { allowUnknown: true, stripUnknown: true });
+        if (error) {
+            throw error;
+        }
+        const where = {
+            id: options.hash,
+        };
+        debug('add.$where', JSON.stringify(where));
+        const objectRecord = await this.objectRepository.findOne({
+            where,
+            raw: true,
+        });
+        debug('add.$objectRecord', JSON.stringify(objectRecord));
+        const listObject = await this.driver.list({
+            key: getHashPath(options.hash),
+            useGlobal: true,
+        });
+        // @note: 运行时的纠错机制,我们发现 size 有时候保存出错(修复后将不可能出现),必须 hash 和 size 都相同,才不需要修改底层存储
+        if (objectRecord && objectRecord.size === options.size && listObject?.size === options.size) {
+            return;
+        }
+        // 存储新对象到全局存储区
+        const newKey = getHashPath(options.hash);
+        await this.driver.writeAsOwner(newKey, data, options);
+        // 添加 object 记录
+        if (objectRecord) {
+            // @note: 运行时的纠错机制, 我们发现 size 有时候保存出错(修复后将不可能出现),这个是一个兜底方案,但是这是一个稳健的做法
+            await this.objectRepository.update({
+                size: options.size,
+                meta: {
+                    mimeType: mimeTypes.lookup(key) || null,
+                    key,
+                },
+            }, {
+                where: {
+                    id: options.hash,
+                },
+            });
+        }
+        else {
+            await this.objectRepository.create({
+                id: options.hash,
+                size: options.size,
+                meta: {
+                    mimeType: mimeTypes.lookup(key) || null,
+                    key,
+                },
+            }, {
+                ignoreDuplicates: true,
+            });
+        }
+        debug('add.after', JSON.stringify({ objectRecord }));
+    }
+    async delete(hash) {
+        const { error } = HashSchema.validate(hash);
+        if (error) {
+            throw error;
+        }
+        debug('delete.before', JSON.stringify({ hash }));
+        const where = {
+            objectId: hash,
+        };
+        debug('delete.$where', JSON.stringify(where));
+        const treeRefCount = await this.treeRepository.count({
+            where,
+        });
+        debug('delete.$treeRefCount', treeRefCount);
+        if (treeRefCount) {
+            await this.objectCollectionRepository.destroy({ where: { id: hash } });
+            return;
+        }
+        await this.driver.deleteAsOwner(getHashPath(hash), { useGlobal: true });
+        await this.objectRepository.destroy({ where: { id: hash } });
+        await this.objectCollectionRepository.destroy({ where: { id: hash } });
+        debug('delete.after', { hash });
+    }
+    /**
+     * @description 清理未被使用的对象
+     * @return {Promise<void>}  {Promise<void>}
+     * @memberof GlobalSpace
+     */
+    async gc() {
+        debug('gc.before', 'start gc...');
+        const objects = await this.objectCollectionRepository.findAll({
+            where: {
+                createdAt: {
+                    [Op.lt]: dayjs().subtract(10, 'm').toDate(),
+                },
+            },
+            attributes: ['id'],
+            order: [['createdAt', 'ASC']],
+            limit: this.gcMaxCount,
+        });
+        debug('gc.$objects.length', objects.length);
+        const actions = objects.map((x) => {
+            return () => this.delete(x.id);
+        });
+        // 限制并发数量
+        await pAll(actions, { concurrency: this.gcMaxConcurrency });
+        debug('gc.after', true);
+    }
+    /**
+     * @description 标记一个对象，以备之后回收
+     * @param {...string[]} hashList
+     * @return {Promise<void>}  {Promise<void>}
+     * @memberof GlobalSpace
+     */
+    async mark(...hashList) {
+        debug('mark.before', true);
+        debug('mark.$hashList.length', hashList.length);
+        await this.objectCollectionRepository.bulkCreate(hashList.map((hash) => {
+            return {
+                id: hash,
+            };
+        }), {
+            ignoreDuplicates: true,
+            updateOnDuplicate: ['createdAt'],
+        });
+        debug('mark.after', true);
+    }
+}

package/dist/es/space/index.d.ts

@@ -0,0 +1,3 @@
+export * from './space';
+export * from './object-space';
+export * from './global-space';

package/dist/es/space/index.js

@@ -0,0 +1,3 @@
+export * from './space';
+export * from './object-space';
+export * from './global-space';

package/dist/es/space/object-space.d.ts

@@ -0,0 +1,123 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { Stream } from 'stream';
+import EventEmitter from 'events';
+import { SpaceConfig } from '../configuration';
+import type { Object, Data } from '../meta';
+import { AppSpaceOptions, DeleteOptions, DriverProtocol, GetHashOptions, KeyStatus, ListOptions, ListsOptions, PermissionOptions, ReadAsOwnerOptions, ReadOptions, SpaceProtocol, WriteAsOwnerOptions, WriteOptions } from '../protocols';
+import { ObjectCollectionRepository, ObjectRepository, TreeRepository, type AppSpaceApplyRepository } from '../model';
+import { GlobalSpace } from './global-space';
+import type { SpaceRepository } from '../model/space';
+import type { DidObjectMappingRepository } from '../model/did-object-mapping';
+export type ObjectSpaceOptions = {
+    driver: DriverProtocol;
+    spaceDid: string;
+    treeRepository: TreeRepository;
+    objectRepository: ObjectRepository;
+    objectCollectionRepository: ObjectCollectionRepository;
+    spaceRepository: SpaceRepository;
+    appSpaceApplyRepository: typeof AppSpaceApplyRepository;
+    didObjectMappingRepository: typeof DidObjectMappingRepository;
+    /**
+     * @description 忽略存储容量限制吗?
+     * @type {(false | true)}
+     */
+    ignoreStorageLimit?: false | true;
+};
+export declare class ObjectSpace extends EventEmitter implements SpaceProtocol {
+    readonly options: ObjectSpaceOptions;
+    readonly globalSpace: GlobalSpace;
+    readonly driver: DriverProtocol;
+    static readonly READONLY_OBJECT_KEYS: string[];
+    constructor(options: ObjectSpaceOptions);
+    createSpace(spaceConfig: SpaceConfig): Promise<void>;
+    isSpaceCreated(): Promise<boolean>;
+    /**
+     *
+     * @description
+     * @return {*}  {Promise<void>}
+     * @memberof ObjectSpace
+     */
+    destroySpace(): Promise<void>;
+    getSpaceSize(): Promise<number>;
+    createAppSpace(options: AppSpaceOptions): Promise<void>;
+    /**
+     * @description
+     * @param {AppSpaceOptions} options
+     * @return {Promise<void>}  {Promise<void>}
+     * @memberof ObjectSpace
+     */
+    destroyAppSpace(options: AppSpaceOptions): Promise<void>;
+    getAppSpacePath(options: AppSpaceOptions): Promise<string>;
+    write(options: WriteOptions): Promise<void>;
+    delete(options: DeleteOptions): Promise<void>;
+    read(options: ReadOptions): Promise<Stream>;
+    getHash(options: GetHashOptions): Promise<string>;
+    exists(options: ReadOptions): Promise<boolean>;
+    private listsOneLevelAsOwner;
+    private listsRecursiveAsOwner;
+    /**
+     * @description 后续改名为 listsAsOwner 更合适
+     * @param {ListsOptions} options
+     * @return {*}  {Promise<Object[]>}
+     * @memberof ObjectSpace
+     */
+    lists(options: ListsOptions): Promise<Object[]>;
+    /**
+     * @FIXME: 接口的功能实际应该是 listAsOwner 实现的，属于技术债 @jianchao
+     * @description
+     * @param {ListOptions} options
+     * @return {*}  {Promise<Object>}
+     * @memberof ObjectSpace
+     */
+    list(options: ListOptions): Promise<Object | null>;
+    /**
+     * @refactor: @jianchao 后续让 write 的接口参数保持一致，即 key, data 都在 options 里面，目前改动收益比较小
+     * @description
+     * @param {string} key
+     * @param {Data} data
+     * @param {WriteAsOwnerOptions} options
+     * @return {*}  {Promise<void>}
+     * @memberof ObjectSpace
+     */
+    writeAsOwner(key: string, data: Data, options?: WriteAsOwnerOptions): Promise<void>;
+    private _updateAsOwner;
+    private _createAsOwner;
+    private updateMetadata;
+    deleteAsOwner(key: string): Promise<void>;
+    readAsOwner(key: string, options?: ReadAsOwnerOptions): Promise<Stream>;
+    existsAsOwner(key: string): Promise<boolean>;
+    getStatusAsOwner(key: string): Promise<KeyStatus>;
+    createConfig(spaceConfig: SpaceConfig): Promise<void>;
+    destroyConfig(): Promise<void>;
+    set<T = any>(key: string, value: T): Promise<void>;
+    get<T = any>(key: string, defaultValue?: T): Promise<T>;
+    private getPermission;
+    /**
+     *
+     * @see https://blog.csdn.net/a1173537204/article/details/89765932
+     * @private
+     * @param {PermissionOptions} { fromAppDid, toAppDid }
+     * @param {number} permission
+     * @param {boolean} status
+     * @return {*}  {Promise<void>}
+     * @memberof S3SpaceConfig
+     */
+    private setPermission;
+    /**
+     *
+     * @refactor 这部分的实现大同小异，之后可以抽象到上层实现，这样可以减少代码量 @jianchao
+     * @description
+     * @param {PermissionOptions} options
+     * @param {boolean} value
+     * @return {*}  {Promise<void>}
+     * @memberof ObjectSpace
+     */
+    setListable(options: PermissionOptions, value: boolean): Promise<void>;
+    setReadable(options: PermissionOptions, value: boolean): Promise<void>;
+    setWritable(options: PermissionOptions, value: boolean): Promise<void>;
+    isListable(options: PermissionOptions): Promise<boolean>;
+    isReadable(options: PermissionOptions): Promise<boolean>;
+    isWritable(options: PermissionOptions): Promise<boolean>;
+    static editable(key: string): boolean;
+}