@ibgib/node-gib 0.0.2

package/src/witness/space/node-filesystem-space/node-filesystem-space-v1.mts

@@ -0,0 +1,672 @@
+import { existsSync, mkdirSync } from 'node:fs';
+import { writeFile, rm, readdir } from 'node:fs/promises';
+import { default as pathUtils } from 'path';
+
+import { clone, extractErrorMsg, } from '@ibgib/helper-gib/dist/helpers/utils-helper.mjs';
+import { getIbAndGib, getIbGibAddr, } from '@ibgib/ts-gib/dist/helper.mjs';
+import { getGib, getGibInfo, } from '@ibgib/ts-gib/dist/V1/transforms/transform-helper.mjs';
+import {
+    IbGib_V1, IbGibRel8ns_V1,
+} from '@ibgib/ts-gib/dist/V1/types.mjs';
+import { Gib, Ib, IbGibAddr } from '@ibgib/ts-gib/dist/types.mjs';
+import { validateIbGibAddr } from '@ibgib/ts-gib/dist/V1/validate-helper.mjs';
+import { IBGIB_DELIMITER } from '@ibgib/ts-gib/dist/V1/constants.mjs';
+
+import { GLOBAL_LOG_A_LOT } from '@ibgib/core-gib/dist/core-constants.mjs';
+import { isBinary, parseBinIb, toDto, } from '@ibgib/core-gib/dist/common/other/ibgib-helper.mjs';
+import { getSpaceIb, } from '@ibgib/core-gib/dist/witness/space/space-helper.mjs';
+import {
+    DeleteIbGibFileOpts, DeleteIbGibFilesResult,
+    GetIbGibFileOpts,
+    GetIbGibFileResult,
+    PutIbGibFileOpts,
+    PutIbGibFileResult
+} from '@ibgib/core-gib/dist/witness/space/filesystem-space/filesystem-types.mjs';
+import {
+    IBGIB_BASE_SUBPATH, IBGIB_SPACE_SUBPATH_DEFAULT, IBGIB_BASE_DIR,
+    IBGIB_ENCODING, IBGIB_IBGIBS_SUBPATH, IBGIB_META_SUBPATH,
+    IBGIB_BIN_SUBPATH,
+    IBGIB_DNA_SUBPATH,
+    DEFAULT_LONG_PATH_LENGTH,
+    IBGIB_LONG_SUBPATH,
+} from '@ibgib/core-gib/dist/witness/space/filesystem-space/filesystem-constants.mjs';
+import { FilesystemSpace_V1, } from '@ibgib/core-gib/dist/witness/space/filesystem-space/filesystem-space-v1.mjs';
+import { tryRead, tryRead_bin } from './node-filesystem-space-helper.mjs';
+import {
+    NodeFilesystemSpaceData_V1, NodeFilesystemSpaceRel8ns_V1,
+    NodeFilesystemSpaceOptionsData, NodeFilesystemSpaceOptionsRel8ns, NodeFilesystemSpaceOptionsIbGib,
+    NodeFilesystemSpaceResultData, NodeFilesystemSpaceResultRel8ns,
+    DEFAULT_NODE_FILESYSTEM_SPACE_DATA_V1,
+    FileDataInfo,
+} from './node-filesystem-space-types.mjs';
+import { META_STONE_ATOM } from '@ibgib/core-gib/dist/common/meta-stone/meta-stone-constants.mjs';
+import { isMetaStone, validateCommonMetaStoneIb } from '@ibgib/core-gib/dist/common/meta-stone/meta-stone-helper.mjs';
+import { BinIbGib_V1 } from '@ibgib/core-gib/dist/common/bin/bin-types.mjs';
+
+const logalot = GLOBAL_LOG_A_LOT;
+
+/**
+ * Base class convenience for a local space with V1 ibgibs working with the node
+ * filesystem.
+ *
+ * Unfortunately, file systems have short file name requirements, where 255 is
+ * often the max length of a filename. So this cannot store ibgibs directly by
+ * their address.
+ *
+ * This naively caches ibGibs in memory. When not found there, will looks in
+ * files using Ionic `FileSystem`.
+ */
+export class NodeFilesystemSpace_V1<
+    TData extends NodeFilesystemSpaceData_V1 = NodeFilesystemSpaceData_V1,
+    TRel8ns extends NodeFilesystemSpaceRel8ns_V1 = NodeFilesystemSpaceRel8ns_V1
+> extends FilesystemSpace_V1<
+    NodeFilesystemSpaceOptionsData,
+    NodeFilesystemSpaceOptionsRel8ns,
+    NodeFilesystemSpaceResultData,
+    NodeFilesystemSpaceResultRel8ns,
+    TData,
+    TRel8ns
+> {
+
+    /**
+     * Log context for convenience with logging. (Ignore if you don't want to use this.)
+     */
+    protected lc: string = `[${NodeFilesystemSpace_V1.name}]`;
+
+    /**
+     * in case the caller tries to override the classname. This should be overridden in any descendant classes
+     */
+    get classname(): string { return NodeFilesystemSpace_V1.name; }
+
+    constructor(
+        // /**
+        //  * Default predicate value when putting an unknown ibGib.
+        //  *
+        //  * ## notes
+        //  *
+        //  * So when a repo witnesses another ibGib, it either defaults to
+        //  * storing that ibGib or not storing that ibGib. This is what that
+        //  * is referring to. If it's optimistic, then it stores any ibGib by
+        //  * default and it passes its put predicate.
+        //  */
+        // public optimisticPut: boolean = true,
+        initialData?: TData,
+        initialRel8ns?: TRel8ns,
+    ) {
+        super(initialData ?? clone(DEFAULT_NODE_FILESYSTEM_SPACE_DATA_V1), initialRel8ns);
+        const lc = `${this.lc}[ctor]`;
+
+        // try {
+        //     if (logalot) { console.log(`${lc} starting...`); }
+        //     this.initialize();
+        // } catch (error) {
+        //     console.error(`${lc} ${extractErrorMsg(error)}`);
+        //     throw error;
+        // } finally {
+        //     if (logalot) { console.log(`${lc} complete.`); }
+        // }
+    }
+
+    /**
+     * Factory static method to create the space with the given
+     * `dto` param's ibGib properties.
+     *
+     * We do this because when we persist this space (and its settings
+     * located in `data`), we do not save the actual class instantiation
+     * but just the ibgib properties. Use this factory method to
+     * create a new space instance and rehydrate from that saved dto.
+     *
+     * ## notes
+     *
+     * * DTO stands for data transfer object.
+     *
+     * @param dto space ibGib dto that we're going to load from
+     * @returns newly created space built upon `dto`
+     */
+    static async createFromDto<
+        TData extends NodeFilesystemSpaceData_V1 = NodeFilesystemSpaceData_V1,
+        TRel8ns extends IbGibRel8ns_V1 = IbGibRel8ns_V1
+    >(dto: IbGib_V1<TData, TRel8ns>): Promise<NodeFilesystemSpace_V1<TData, TRel8ns>> {
+        const lc = `[${FilesystemSpace_V1.name}][${this.createFromDto.name}]`;
+        if (logalot) { console.log(`${lc}`); }
+        const space = new NodeFilesystemSpace_V1<TData, TRel8ns>();
+        await space.initialized;
+        await space.loadIbGibDto(dto);
+        return space;
+    }
+
+    protected async validateWitnessArg(arg: NodeFilesystemSpaceOptionsIbGib): Promise<string[]> {
+        const lc = `${this.lc}[${this.validateWitnessArg.name}]`;
+        let errors: string[] = [];
+        try {
+            errors = (await super.validateWitnessArg(arg)) || [];
+            if (arg.data?.cmd === 'put' && (arg.ibGibs || []).length === 0) {
+                errors.push(`when "put" cmd is called, ibGibs required.`);
+            }
+            if (arg.data?.cmd === 'get' && (arg.data?.ibGibAddrs || []).length === 0) {
+                errors.push(`when "get" cmd is called, ibGibAddrs required.`);
+            }
+        } catch (error) {
+            console.error(`${lc} ${extractErrorMsg(error)}`);
+            throw error;
+        } finally {
+            if (errors?.length > 0) { console.error(`${lc} errors: ${errors}`); }
+        }
+
+        return errors;
+    }
+
+    /**
+     * Initializes to default space values.
+     */
+    protected async initialize(): Promise<void> {
+        const lc = `${this.lc}[${this.initialize.name}]`;
+        try {
+            if (logalot) { console.log(`${lc} starting...`); }
+            if (!this.data) {
+                this.data = clone(DEFAULT_NODE_FILESYSTEM_SPACE_DATA_V1);
+                this.data = this.data!; // why does ts compiler need this?
+            }
+            // if (!this.data?.classname) { this.data!.classname = NodeFilesystemSpace_V1.name }
+            if (this.data!.classname !== this.classname) {
+                if (this.data.classname) {
+                    // only warn if the consumer has explicitly set a different classname
+                    debugger; // how are we hitting this in space-gib server init
+                    console.warn(`${lc} initializer tried to init this data with this.data.classname (${this.data.classname}) !== ${this.classname}. overriding this to be ${this.classname}. (W: 0f53162c43a440c2a790c19dd223c6b6)`);
+                }
+                this.data!.classname = this.classname; // always set?
+            }
+            if (!this.data.baseDir) { this.data.baseDir = IBGIB_BASE_DIR; }
+            if (!this.data.encoding) { this.data.encoding = IBGIB_ENCODING; }
+            if (!this.data.baseSubPath) { this.data.baseSubPath = IBGIB_BASE_SUBPATH; }
+            if (!this.data.spaceSubPath) { this.data.spaceSubPath = IBGIB_SPACE_SUBPATH_DEFAULT; }
+            if (!this.data.ibgibsSubPath) { this.data.ibgibsSubPath = IBGIB_IBGIBS_SUBPATH; }
+            if (!this.data.metaSubPath) { this.data.metaSubPath = IBGIB_META_SUBPATH; }
+            if (!this.data.binSubPath) { this.data.binSubPath = IBGIB_BIN_SUBPATH; }
+            if (!this.data.dnaSubPath) { this.data.dnaSubPath = IBGIB_DNA_SUBPATH; }
+
+            // do nothing, allow falsy
+            if (this.data.longSubPath === undefined) { this.data.longSubPath = IBGIB_LONG_SUBPATH; }
+            if (this.data.longPathLength === undefined) { this.data.longPathLength = DEFAULT_LONG_PATH_LENGTH; }
+            if (this.data.mitigateLongPaths === undefined) { this.data.mitigateLongPaths = true; }
+
+            this.ib = getSpaceIb({ space: this, classname: this.data!.classname });
+
+            this.gib = await getGib({ ibGib: this });
+        } catch (error) {
+            console.error(`${lc} ${extractErrorMsg(error)}`);
+        } finally {
+            if (logalot) { console.log(`${lc} complete.`); }
+        }
+    }
+
+    protected async putFile({
+        ibGib,
+        isDna,
+    }: PutIbGibFileOpts): Promise<PutIbGibFileResult> {
+        const lc = `${this.lc}[${this.putFile.name}]`;
+
+        let result: PutIbGibFileResult = {};
+
+        try {
+            if (!ibGib) { throw new Error(`ibGib required. (E: b019eedc22094687a83dca8f7a98ba35)`) };
+
+            const thisData = this.data!;
+            // await this.ensureAllDirsExist();
+            let path: string = "";
+            let data: any = "";
+
+            const addr = getIbGibAddr({ ibGib });
+            const isBin = isBinary({ addr });
+            path = await this.buildPath({
+                addr, isDna: isDna ?? false, isBin, ensureMetaStonePaths: true,
+            });
+            await this.ensureDirsImpl([pathUtils.dirname(path)]);
+
+            let fullPath = pathUtils.join(thisData.baseDir, path);
+
+            if (logalot) { console.log(`${lc} fullPath: ${fullPath} (I: 2441a896f7e6418db57c0341d5c65ef2)`); }
+            if (logalot) { console.log(`${lc} path: ${path}, directory: ${thisData.baseDir}, thisData.encoding: ${thisData.encoding} (I: d4440ececbe6c859f8fcf6d7ece12522)`); }
+
+
+            let encoding: string | undefined;
+
+            if (!isBin) {
+                // not binary - most ibgibs will not be bins.
+
+                // we only want to persist the ibGib protocol properties (not
+                // any functions or other properties that might exist on the
+                // incoming ibGib arg)
+                const bareIbGib = toDto({ ibGib })
+                data = JSON.stringify(bareIbGib);
+
+                encoding = thisData.encoding || 'utf8';
+                await writeFile(fullPath, data, encoding as BufferEncoding);
+            } else {
+                // binary
+                const binIbGib = ibGib as BinIbGib_V1;
+                data = binIbGib.data;
+                if (!data) { throw new Error(`(UNEXPECTED) binIbGib.data falsy? I'm trying to have data always populated for bin ibgibs even if it is an empty Uint8Array. (E: 8c9c3650ec97792b1913754ed0877824)`); }
+
+                // if (!Buffer.isBuffer(data)) {
+                //     debugger; // checking to see if !Buffer.isBuffer(data) fails for Uint8Array
+                //     throw new Error(`(UNEXPECTED) !Buffer.isBuffer(data)? I'm trying to have binIbGib.data always populated and it should be a Buffer. (E: 179c97da10ac082b77376c0db9077d24)`);
+                // }
+                // encoding = parseBinIb({ addr }).binEncoding ?? undefined
+
+                // write bins directly as buffers
+                // const dataBuffer = Buffer.from(data);
+                await writeFile(fullPath, data);
+            }
+
+            result.success = true;
+        } catch (error) {
+            const errorMsg = `${lc} ${extractErrorMsg(error)}`;
+            console.error(errorMsg);
+            result.errorMsg = errorMsg;
+        }
+
+        return result;
+    }
+
+    protected async deleteFile({
+        addr,
+        isDna,
+    }: DeleteIbGibFileOpts): Promise<DeleteIbGibFilesResult> {
+        const lc = `${this.lc}[${this.deleteFile.name}]`;
+
+        const result: DeleteIbGibFilesResult = {};
+
+        try {
+            if (!addr) { throw new Error(`addr required. (E: 98ff4fb4067a4a8281e21756aca4bf82)`) };
+
+            if (!this.data) { throw new Error(`this.data required (E: 27e1f5206d96d635e617f5ba69141423)`); }
+            const data = this.data;
+            let path: string = "";
+
+            if (!isBinary({ addr })) {
+                // regular ibGib
+                path = await this.buildPath({
+                    addr, isDna: isDna ?? false, ensureMetaStonePaths: true,
+                });
+            } else {
+                path = await this.buildPath({
+                    addr, isDna: false, isBin: true, ensureMetaStonePaths: false,
+                });
+            }
+            if (logalot) { console.log(`${lc} path: ${path}, directory: ${data.baseDir}`); }
+            const fullPath = pathUtils.join(this.data.baseDir, path);
+            await rm(fullPath, { force: true });
+            if (logalot) { console.log(`${lc} deleted. path: ${path}`); }
+            result.success = true;
+        } catch (error) {
+            const errorMsg = `${lc} ${extractErrorMsg(error)}`;
+            if (!errorMsg.includes('File does not exist')) {
+                console.error(errorMsg);
+            } else {
+                if (logalot) { console.log(`${lc} attempted to delete non-existent file. ${errorMsg} (I: 8953b51a5f14960e5ea2e86f6c6a7622)`); }
+            }
+            result.errorMsg = errorMsg;
+        }
+
+        return result;
+    }
+
+    protected async getFile({
+        addr,
+        isDna,
+    }: GetIbGibFileOpts): Promise<GetIbGibFileResult> {
+        let lc = `${this.lc}[${this.getFile.name}(${addr})]`;
+
+        const result: GetIbGibFileResult = {};
+        try {
+            if (logalot) { console.log(`${lc} starting... (I: f69b58063ff7cafaaab341e633002a23)`); }
+            if (!addr) { throw new Error(`addr required`) };
+
+            const data = this.data!;
+            const { ib } = getIbAndGib({ ibGibAddr: addr });
+
+            const addrIsBin = isBinary({ addr });
+            const knownTransformIbs = ['fork', 'mut8', 'rel8', 'plan']; // hack/kluge here
+            const addrMightBeDna =
+                knownTransformIbs.some(x => x === ib || x.startsWith(ib + ' '));
+
+            const pathsToLook: string[] = [];
+
+            if (isDna || addrMightBeDna) {
+                pathsToLook.push(await this.buildPath({
+                    addr, isDna: true, ensureMetaStonePaths: false
+                }));
+            } else if (addrIsBin) {
+                pathsToLook.push(await this.buildPath({
+                    addr, isDna: false, isBin: true, ensureMetaStonePaths: false
+                }));
+            }
+
+            // always look in the main location as a last resort
+            pathsToLook.push(await this.buildPath({
+                addr, isDna: false, ensureMetaStonePaths: false
+            }));
+
+            // ...(pretend shortened as well as an absolute last last resort)
+            if (!isMetaStone({ addr })) {
+                pathsToLook.push(await this.buildPath({
+                    addr, isDna: false, ensureMetaStonePaths: false,
+                    pretendItsALongPath: true,
+                }));
+            }
+
+            // this is ugly atow (02/2024) because i'm struggling a bit of
+            // changing bin ibgibs to actually support binaries. i apologize.
+
+            if (!addrIsBin) {
+                // non-bin ibGib(s) retrieved (most ibgibs are non-bin)
+                let resRead: any = null;
+                for (const tryPath of pathsToLook) {
+                    const x = await tryRead({ path: tryPath, directory: data.baseDir, encoding: data.encoding });
+                    if (x) { resRead = x; break; }
+                }
+                if (resRead) {
+                    result.ibGib = JSON.parse(resRead) as IbGib_V1;
+                } else {
+                    if (logalot) { console.log(`${lc} paths not found: ${JSON.stringify(pathsToLook)} (I: 7fc6c550b1a047d8a14392185de1b06e)`); }
+                    // will return success since it's not really an error, but ibgib
+                    // will not be populated, indicating the addr was not found.
+                }
+            } else {
+                // bin
+                let resRead: FileDataInfo | null = null;
+                let { binEncoding, binHash } = parseBinIb({ addr });
+                if (binHash === '0') {
+                    // special case: if the binHash is 0, then it is an empty file.
+                    const { ib, gib } = getIbAndGib({ ibGibAddr: addr });
+                    result.ibGib = {
+                        ib,
+                        gib,
+                        data: new Uint8Array(0),
+                        // data: '',
+                        // rawData: Buffer.from([]),
+                    } as BinIbGib_V1;
+                } else {
+                    for (const tryPath of pathsToLook) {
+                        const x = await tryRead_bin({
+                            path: tryPath, directory: data.baseDir, encoding: binEncoding ?? data.encoding,
+                        });
+                        if (x) { resRead = x; break; }
+                    }
+
+                    if (resRead) {
+                        // all bins are buffers
+                        const { ib, gib } = getIbAndGib({ ibGibAddr: addr });
+                        const binIbGib: BinIbGib_V1 = {
+                            ib, gib, data: resRead.dataBuffer,
+                            // rawData: resRead.dataBuffer,
+                        }
+                        result.ibGib = binIbGib;
+                    } else {
+                        if (logalot) { console.log(`${lc} paths not found: ${JSON.stringify(pathsToLook)} (I: 6a3bd3b125619da7a944200b14e7e922)`); }
+                        // will return success since it's not really an error, but ibgib
+                        // will not be populated, indicating the addr was not found.
+                    }
+                }
+            }
+
+            result.success = true;
+        } catch (error) {
+            const errorMsg = `${lc} ${extractErrorMsg(error)}`;
+            console.error(errorMsg);
+            result.errorMsg = errorMsg;
+        } finally {
+            if (logalot) { console.log(`${lc} complete. (I: 413374e48f0a3cb8fd25de0d82134123)`); }
+        }
+
+        return result;
+    }
+
+    protected async ensurePermissions(): Promise<boolean> {
+        const lc = `${this.lc}[${this.ensurePermissions.name}]`;
+        if (logalot) { console.log(`${lc} always returns true in base class (I: 0a036a097093c622167e40f81e03d923)`); }
+        return true;
+    }
+
+    protected async ensureAllDirsExist(): Promise<void> {
+        const lc = `${this.lc}[${this.ensureAllDirsExist.name}]`;
+        try {
+            if (logalot) { console.log(`${lc} starting... (I: f9f1bc18215fd591cf9dcaea4deaa323)`); }
+
+            if (!this.data) { throw new Error(`this.data required (E: ccf51541c1d5714ddafb67a8c3289823)`); }
+            if (!this.data.uuid) { throw new Error(`this.data.uuid required (E: 33691c4c97e2033bf13d2c2d26a8f823)`); }
+
+            const data = this.data!;
+
+            /** these are the paths we're ensuring exist. all ibgibs are stored here. */
+            const paths = [
+                data.baseSubPath, // = 'ibgib';
+                data.baseSubPath + '/' + data.spaceSubPath,
+                data.baseSubPath + '/' + data.spaceSubPath + '/' + data.ibgibsSubPath,
+                data.baseSubPath + '/' + data.spaceSubPath + '/' + data.metaSubPath,
+                data.baseSubPath + '/' + data.spaceSubPath + '/' + data.binSubPath,
+                data.baseSubPath + '/' + data.spaceSubPath + '/' + data.dnaSubPath,
+            ];
+
+            if (data.mitigateLongPaths) {
+                paths.push(data.baseSubPath + '/' + data.spaceSubPath + '/' + data.longSubPath);
+            }
+
+            await this.ensureDirsImpl(paths);
+        } catch (error) {
+            console.error(`${lc} ${extractErrorMsg(error)}`);
+            throw error;
+        } finally {
+            if (logalot) { console.log(`${lc} complete.`); }
+        }
+    }
+
+    /**
+     * actually executes the ensure functionality for given `paths`. If they
+     * don't exist, then this tries to make the dirs.
+     * @param paths to ensure exist.
+     */
+    protected async ensureDirsImpl(paths: string[]): Promise<void> {
+        const lc = `${this.lc}[${this.ensureDirsImpl.name}]`;
+        try {
+            if (logalot) { console.log(`${lc} starting... (I: ad355a80dafa5873ac8ef31bf1f27923)`); }
+            if (paths.length === 0) {
+                console.warn(`${lc} paths empty? returning early. (W: a089ab1c9c0a4275a49341fb9f4c01c1)`)
+                return; /* <<<< returns early */
+            }
+
+            const directory = this.data!.baseDir;
+
+            const getPathKey = (p: string) => { return directory.toString() + '/' + p; }
+
+            let allExist = paths.every(p => this.pathExistsMap[getPathKey(p)]);
+            if (allExist) {
+                if (logalot) { console.log(`${lc} allExist (I: f14ad6db1d29e368c37c3117fee1cb22)`); }
+                return; /* <<<< returns early */
+            }
+
+            const permitted = await this.ensurePermissions();
+            if (!permitted) {
+                console.error(`${lc} permission not granted.`);
+                return; /* <<<< returns early */
+            }
+
+            for (let i = 0; i < paths.length; i++) {
+                const path = paths[i];
+                const lc2 = `${lc}[(path: ${path}, directory: ${directory})]`;
+                const fullPath = pathUtils.join(directory, path);
+
+                // check if we've already ensured for this path
+                const pathExistsKey = getPathKey(path);
+                let exists = this.pathExistsMap[pathExistsKey] || false;
+
+                if (!exists) {
+                    // we've not checked this path (or it didn't exist)
+                    try {
+                        exists = existsSync(fullPath);
+                        this.pathExistsMap[pathExistsKey] = exists;
+                    } catch (error) {
+                        if (logalot) { console.log(`${lc2} Did not exist`); }
+                    }
+                }
+
+                if (!exists) {
+                    // try full path
+                    if (logalot) { console.log(`${lc2} creating...`); }
+                    try {
+                        mkdirSync(fullPath, { recursive: true });
+                        exists = existsSync(fullPath);
+                        if (logalot) { console.log(`${lc} exists: ${exists}`); }
+                        this.pathExistsMap[pathExistsKey] = exists;
+                    } catch (error) {
+                        if (logalot) { console.log(`${lc2} Error creating. Trying next`); }
+                    } finally {
+                        if (logalot) { console.log(`${lc2} complete.`); }
+                    }
+                }
+            }
+
+        } catch (error) {
+            console.error(`${lc} ${extractErrorMsg(error)}`);
+            throw error;
+        } finally {
+            if (logalot) { console.log(`${lc} complete.`); }
+        }
+    }
+
+    /** */
+    protected async getMetaStoneAddrs({
+        ibGibAddr,
+        tjpGib,
+        fnFilterIb,
+    }: {
+        /**
+         * addr of ibGib for which we're getting the metastones.
+         */
+        ibGibAddr: IbGibAddr,
+        /**
+         * tjpGib of the metastone's target ibgib
+         */
+        tjpGib: Gib,
+        fnFilterIb?: (ib: Ib) => boolean,
+    }): Promise<string[]> {
+        const lc = `${this.lc}[${this.getMetaStoneAddrs.name}]`;
+        try {
+            if (logalot) { console.log(`${lc} starting... (I: 510ba658195aa0aed9b8836f616d7823)`); }
+
+            // first build the path with the given tjpGib
+            // debugger;
+            let pathSansBaseDir = await this.buildPath({
+                addr: ibGibAddr, ensureMetaStonePaths: false, isDna: false,
+                addrIsForAMetaStone: true,
+            });
+            let fullPath = pathUtils.join(this.data!.baseDir, pathSansBaseDir);
+            if (logalot) { console.log(`${lc} fullPath: ${fullPath} (I: 9fc7113d6c9649f9b25f227ddad57f69)`); }
+
+            // the given ibgib's containing dir is the one that should contain
+            // the metastones.
+            let gibInfo = getGibInfo({ ibGibAddr });
+
+            let containingDir = !!gibInfo.tjpGib ?
+                pathUtils.dirname(pathUtils.join(fullPath, '..')) :
+                pathUtils.dirname(fullPath);
+            if (logalot) { console.log(`${lc} containingDir: ${containingDir} (I: 2780437a6778a4633f49d81ac26ab723)`); }
+
+            // iterate through the files for metastones.
+            const fnFilter = fnFilterIb ?
+                (s: string) => s.startsWith(META_STONE_ATOM) && fnFilterIb(s) :
+                (s: string) => s.startsWith(META_STONE_ATOM);
+            let filenames: string[] = [];
+            try {
+                if (logalot) { console.log(`${lc} calling readdir(containingDir) (I: 682d129a44d1101a0a7c2be999921923)`); }
+                filenames = await readdir(containingDir, {
+                    recursive: false,
+                    withFileTypes: false,
+                });
+            } catch (error) {
+                let emsg = extractErrorMsg(error);
+                if (!emsg.includes('ENOENT')) { // no such file or directory
+                    console.warn(`${lc} readdir error did not contain ENOENT...not sure what this is...maybe a different OS no dir exists emsg? Treating this as if dir does not exist. (W: cde1cab3f64c47559782da8da55ed48a)`);
+                } else {
+                    if (logalot) { console.log(`${lc} readdir errors: ${extractErrorMsg(error)} (I: 62bd4ec398b86ca84f7b3866dd83c123)`); }
+                }
+                filenames = [];
+            }
+            if (filenames.length === 0) {
+                if (logalot) { console.log(`${lc} filenames: [empty]. no metastones found. (I: da9fef3f2edde037ca1c340bc38b7723)`); }
+                return []; /* <<<< returns early */
+            }
+            if (logalot) { console.log(`${lc} filenames: ${filenames} (I: cb2948540a031522d184b1611f914323)`); }
+
+            /**
+             * atow (11/2023) extensions are hard-coded to ".json",
+             */
+            const dotExt = '.json';
+            const dotExtLength = 5;
+            const metaStoneAddrs = filenames
+                .filter(x => {
+                    // some filenames may be adjusted and not the full ibs.
+                    // however, the metastones are never shortened (atow
+                    // 11/2023), so those that don't have the delimiter will be
+                    // stored in ibIsh but they should get filtered out.
+                    const [ibIsh, gibPlusExt] = x.split(IBGIB_DELIMITER);
+                    return !!gibPlusExt?.endsWith(dotExt) && fnFilter(ibIsh);
+                }).map(addrPlusExt => {
+                    // strip the extension
+                    return addrPlusExt.substring(0, addrPlusExt.length - dotExtLength);
+                }).filter(metaStoneAddr => {
+                    // atow (11/2023) I'm not sure what other metastones are
+                    // going to be in this dir, so I'm saying it has to
+                    // explicitly contain the tjpGib.
+                    return isMetaStone({ addr: metaStoneAddr }) && metaStoneAddr.includes(tjpGib);
+                });
+
+            if (logalot) { console.log(`${lc} filtered/mapped filenames -> metaStoneAddrs: ${metaStoneAddrs} (I: ed062833a65730c04c7f9523e31a5a23)`); }
+
+            // do some basic validation
+            // throw (which could cripple in the future...), or silent fail
+            // (which could lead to corrupt data)...  hmm...going to throw for
+            // now.
+            for (let i = 0; i < metaStoneAddrs.length; i++) {
+                const metaStoneAddr = metaStoneAddrs[i];
+                let basicErrors = validateIbGibAddr({ addr: metaStoneAddr });
+                if ((basicErrors ?? []).length > 0) {
+                    throw new Error(`(UNEXPECTED) invalid metastone found? metaStoneAddr (${metaStoneAddr}) had basic validation errors: ${basicErrors!.join('|')} (E: 318c164fc2014c036c91b179c8d7f623)`);
+                }
+                const { ib } = getIbAndGib({ ibGibAddr: metaStoneAddr });
+                let metaStoneErrors = validateCommonMetaStoneIb({ ib });
+                if (metaStoneErrors.length > 0) {
+                    throw new Error(`(UNEXPECTED) metaStoneIb had validation errors? metaStoneErrors: ${metaStoneErrors.join('|')} (E: 5a07b4dcd817f72a1a70e4b2d18f7323)`);
+                }
+            }
+
+            // all good, return 'em!
+            if (logalot) { console.log(`${lc} returning metaStoneAddrs: ${metaStoneAddrs} (I: c235e1005d02a53219e032f8818e8223)`); }
+            return metaStoneAddrs;
+        } catch (error) {
+            console.error(`${lc} ${extractErrorMsg(error)}`);
+            throw error;
+        } finally {
+            if (logalot) { console.log(`${lc} complete.`); }
+        }
+    }
+}
+export async function convertData({
+    data,
+    encoding,
+}: {
+    data: string,
+    encoding: any,
+}): Promise<any> {
+    const lc = `[${convertData.name}]`;
+    try {
+        if (logalot) { console.log(`${lc} starting... (I: 77ef782249fdc9588c5b5b810126cc24)`); }
+
+    } catch (error) {
+        console.error(`${lc} ${extractErrorMsg(error)}`);
+        throw error;
+    } finally {
+        if (logalot) { console.log(`${lc} complete.`); }
+    }
+}

package/src/witness/space/node-filesystem-space/node-indexed-filesystem-space/node-indexed-filesystem-space-constants.mts

@@ -0,0 +1,6 @@
+/**
+ * @module node-indexed-filesystem-space-constants
+ */
+
+
+export const DEFAULT_INDEXED_FILESYSTEM_SPACE_DESCRIPTION = `This is an indexed-based filesystem space. The ibgib data contains settings for the space itself, and the witness ibgib object interfaces with a back end that is hierarchical like a filesystem. This uses a custom indexing system to shorten path lengths.`;