mindoodb 0.0.23 → 0.0.32

package/dist/core/appendonlystores/BasicOnDiskContentAddressedStore.js

@@ -1,659 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BasicOnDiskContentAddressedStoreFactory = exports.BasicOnDiskContentAddressedStore = void 0;
-const promises_1 = require("fs/promises");
-const fs_1 = require("fs");
-const path = __importStar(require("path"));
-const bloom_1 = require("./bloom");
-const logging_1 = require("../logging");
-function serializeMetadata(metadata) {
-    return {
-        ...metadata,
-        signature: Buffer.from(metadata.signature).toString("base64"),
-    };
-}
-function deserializeMetadata(serialized) {
-    return {
-        ...serialized,
-        signature: new Uint8Array(Buffer.from(serialized.signature, "base64")),
-    };
-}
-function toEntryFileName(id) {
-    return `${encodeURIComponent(id)}.json`;
-}
-function fromEntryFileName(fileName) {
-    const withoutExt = fileName.endsWith(".json") ? fileName.slice(0, -5) : fileName;
-    return decodeURIComponent(withoutExt);
-}
-function tempPathFor(finalPath) {
-    return `${finalPath}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
-}
-/**
- * Simple persistent on-disk ContentAddressedStore implementation.
- * Uses metadata files by id and binary content files by contentHash.
- * Supports reset-on-start and optional in-memory indexing.
- */
-class BasicOnDiskContentAddressedStore {
-    constructor(dbId, logger, options) {
-        // In-memory indexes (only used when indexingEnabled=true)
-        this.entries = new Map();
-        this.docIndex = new Map();
-        this.contentRefCount = new Map();
-        this.orderedMetadata = [];
-        this.dbId = dbId;
-        this.logger =
-            logger ||
-                new logging_1.MindooLogger((0, logging_1.getDefaultLogLevel)(), `BasicOnDiskStore:${dbId}`, true);
-        this.storeRoot = path.join(options?.basePath?.toString() || ".mindoodb-store", dbId);
-        this.entriesDir = path.join(this.storeRoot, "entries");
-        this.contentDir = path.join(this.storeRoot, "content");
-        this.metadataIndexPath = path.join(this.storeRoot, "metadata-index.json");
-        this.indexingEnabled = options?.indexingEnabled !== false;
-        this.indexStatus = {
-            phase: this.indexingEnabled ? "building" : "ready",
-            indexingEnabled: this.indexingEnabled,
-            progress01: this.indexingEnabled ? 0 : 1,
-        };
-        this.initPromise = this.initialize(options);
-    }
-    async initialize(options) {
-        if (options?.clearLocalDataOnStartup) {
-            await this.clearAllLocalDataInternal();
-        }
-        await (0, promises_1.mkdir)(this.entriesDir, { recursive: true });
-        await (0, promises_1.mkdir)(this.contentDir, { recursive: true });
-        if (!this.indexingEnabled) {
-            this.indexStatus = {
-                phase: "ready",
-                indexingEnabled: false,
-                progress01: 1,
-            };
-            return;
-        }
-        const loaded = await this.tryLoadPersistedIndex();
-        const indexCoverageOk = loaded ? await this.validatePersistedIndexCoverage() : false;
-        if (!loaded || !indexCoverageOk) {
-            await this.rebuildInMemoryIndexes();
-            await this.persistMetadataIndex();
-        }
-        this.indexStatus = {
-            phase: "ready",
-            indexingEnabled: true,
-            progress01: 1,
-        };
-    }
-    async ensureInitialized() {
-        await this.initPromise;
-    }
-    async clearAllLocalDataInternal() {
-        await (0, promises_1.rm)(this.storeRoot, { recursive: true, force: true });
-        this.entries.clear();
-        this.docIndex.clear();
-        this.contentRefCount.clear();
-        this.orderedMetadata = [];
-    }
-    async rebuildInMemoryIndexes() {
-        this.entries.clear();
-        this.docIndex.clear();
-        this.contentRefCount.clear();
-        this.orderedMetadata = [];
-        const files = await this.listEntryFiles();
-        for (const fileName of files) {
-            const metadata = await this.readMetadataByFileName(fileName);
-            if (!metadata) {
-                continue;
-            }
-            this.entries.set(metadata.id, metadata);
-            if (!this.docIndex.has(metadata.docId)) {
-                this.docIndex.set(metadata.docId, new Set());
-            }
-            this.docIndex.get(metadata.docId).add(metadata.id);
-            this.contentRefCount.set(metadata.contentHash, (this.contentRefCount.get(metadata.contentHash) || 0) + 1);
-            this.orderedMetadata.push(metadata);
-        }
-        this.orderedMetadata.sort((a, b) => (a.createdAt === b.createdAt ? a.id.localeCompare(b.id) : a.createdAt - b.createdAt));
-    }
-    async tryLoadPersistedIndex() {
-        if (!(await this.fileExists(this.metadataIndexPath))) {
-            return false;
-        }
-        try {
-            const raw = await (0, promises_1.readFile)(this.metadataIndexPath, "utf-8");
-            const serialized = JSON.parse(raw);
-            const loaded = serialized.map((s) => deserializeMetadata(s));
-            this.entries.clear();
-            this.docIndex.clear();
-            this.contentRefCount.clear();
-            this.orderedMetadata = [];
-            for (const metadata of loaded) {
-                this.entries.set(metadata.id, metadata);
-                if (!this.docIndex.has(metadata.docId)) {
-                    this.docIndex.set(metadata.docId, new Set());
-                }
-                this.docIndex.get(metadata.docId).add(metadata.id);
-                this.contentRefCount.set(metadata.contentHash, (this.contentRefCount.get(metadata.contentHash) || 0) + 1);
-                this.orderedMetadata.push(metadata);
-            }
-            return true;
-        }
-        catch (err) {
-            this.logger.warn(`Failed to load metadata index, rebuilding from entry files: ${String(err)}`);
-            return false;
-        }
-    }
-    async validatePersistedIndexCoverage() {
-        if (!this.indexingEnabled) {
-            return true;
-        }
-        const files = await this.listEntryFiles();
-        if (files.length !== this.entries.size) {
-            return false;
-        }
-        for (const fileName of files) {
-            const entryId = fromEntryFileName(fileName);
-            if (!this.entries.has(entryId)) {
-                return false;
-            }
-        }
-        return true;
-    }
-    async persistMetadataIndex() {
-        if (!this.indexingEnabled) {
-            return;
-        }
-        const serialized = JSON.stringify(this.orderedMetadata.map((m) => serializeMetadata(m)));
-        await this.writeFileAtomic(this.metadataIndexPath, serialized);
-    }
-    compareMetadata(a, b) {
-        if (a.createdAt !== b.createdAt) {
-            return a.createdAt - b.createdAt;
-        }
-        return a.id.localeCompare(b.id);
-    }
-    lowerBoundForCursor(cursor) {
-        if (cursor === null) {
-            return 0;
-        }
-        let left = 0;
-        let right = this.orderedMetadata.length;
-        while (left < right) {
-            const mid = Math.floor((left + right) / 2);
-            const candidate = this.orderedMetadata[mid];
-            const greater = candidate.createdAt > cursor.createdAt ||
-                (candidate.createdAt === cursor.createdAt && candidate.id > cursor.id);
-            if (greater) {
-                right = mid;
-            }
-            else {
-                left = mid + 1;
-            }
-        }
-        return left;
-    }
-    metadataMatchesFilters(meta, filters) {
-        if (filters?.docId && meta.docId !== filters.docId) {
-            return false;
-        }
-        if (filters?.entryTypes && filters.entryTypes.length > 0 && !filters.entryTypes.includes(meta.entryType)) {
-            return false;
-        }
-        if (filters?.creationDateFrom !== undefined && filters.creationDateFrom !== null && meta.createdAt < filters.creationDateFrom) {
-            return false;
-        }
-        if (filters?.creationDateUntil !== undefined && filters.creationDateUntil !== null && meta.createdAt >= filters.creationDateUntil) {
-            return false;
-        }
-        return true;
-    }
-    insertOrderedMetadata(meta) {
-        let left = 0;
-        let right = this.orderedMetadata.length;
-        while (left < right) {
-            const mid = Math.floor((left + right) / 2);
-            if (this.compareMetadata(this.orderedMetadata[mid], meta) < 0) {
-                left = mid + 1;
-            }
-            else {
-                right = mid;
-            }
-        }
-        this.orderedMetadata.splice(left, 0, meta);
-    }
-    removeOrderedMetadataById(id) {
-        const idx = this.orderedMetadata.findIndex((m) => m.id === id);
-        if (idx >= 0) {
-            this.orderedMetadata.splice(idx, 1);
-        }
-    }
-    async listEntryFiles() {
-        try {
-            const files = await (0, promises_1.readdir)(this.entriesDir);
-            return files.filter((f) => f.endsWith(".json"));
-        }
-        catch {
-            return [];
-        }
-    }
-    metadataPathForId(id) {
-        return path.join(this.entriesDir, toEntryFileName(id));
-    }
-    contentPathForHash(contentHash) {
-        return path.join(this.contentDir, `${contentHash}.bin`);
-    }
-    async fileExists(filePath) {
-        try {
-            await (0, promises_1.access)(filePath, fs_1.constants.F_OK);
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    async writeFileAtomic(filePath, data) {
-        const tmpPath = tempPathFor(filePath);
-        const tmpHandle = await (0, promises_1.open)(tmpPath, "w");
-        try {
-            await tmpHandle.writeFile(data);
-            await tmpHandle.sync();
-        }
-        finally {
-            await tmpHandle.close();
-        }
-        await (0, promises_1.rename)(tmpPath, filePath);
-        await this.syncParentDirectory(filePath);
-    }
-    async syncParentDirectory(filePath) {
-        const dirPath = path.dirname(filePath);
-        try {
-            const dirHandle = await (0, promises_1.open)(dirPath, "r");
-            try {
-                await dirHandle.sync();
-            }
-            finally {
-                await dirHandle.close();
-            }
-        }
-        catch (err) {
-            this.logger.debug(`Directory fsync skipped for ${dirPath}: ${String(err)}`);
-        }
-    }
-    async readMetadataById(id) {
-        if (this.indexingEnabled) {
-            return this.entries.get(id) || null;
-        }
-        const filePath = this.metadataPathForId(id);
-        if (!(await this.fileExists(filePath))) {
-            return null;
-        }
-        const raw = await (0, promises_1.readFile)(filePath, "utf-8");
-        return deserializeMetadata(JSON.parse(raw));
-    }
-    async readMetadataByFileName(fileName) {
-        const filePath = path.join(this.entriesDir, fileName);
-        try {
-            const raw = await (0, promises_1.readFile)(filePath, "utf-8");
-            const metadata = deserializeMetadata(JSON.parse(raw));
-            return metadata;
-        }
-        catch (err) {
-            this.logger.warn(`Failed to read metadata file ${fileName}: ${String(err)}`);
-            return null;
-        }
-    }
-    async listAllMetadata() {
-        if (this.indexingEnabled) {
-            return [...this.orderedMetadata];
-        }
-        const files = await this.listEntryFiles();
-        const result = [];
-        for (const fileName of files) {
-            const metadata = await this.readMetadataByFileName(fileName);
-            if (metadata) {
-                result.push(metadata);
-            }
-        }
-        return result;
-    }
-    getId() {
-        return this.dbId;
-    }
-    async putEntries(entries) {
-        await this.ensureInitialized();
-        let hasMutation = false;
-        for (const entry of entries) {
-            const metadataPath = this.metadataPathForId(entry.id);
-            if (await this.fileExists(metadataPath)) {
-                continue;
-            }
-            const contentPath = this.contentPathForHash(entry.contentHash);
-            if (!(await this.fileExists(contentPath))) {
-                await this.writeFileAtomic(contentPath, entry.encryptedData);
-            }
-            const { encryptedData, ...metadata } = entry;
-            const serialized = JSON.stringify(serializeMetadata(metadata));
-            await this.writeFileAtomic(metadataPath, serialized);
-            hasMutation = true;
-            if (this.indexingEnabled) {
-                this.entries.set(entry.id, metadata);
-                if (!this.docIndex.has(entry.docId)) {
-                    this.docIndex.set(entry.docId, new Set());
-                }
-                this.docIndex.get(entry.docId).add(entry.id);
-                this.contentRefCount.set(entry.contentHash, (this.contentRefCount.get(entry.contentHash) || 0) + 1);
-                this.insertOrderedMetadata(metadata);
-            }
-        }
-        if (hasMutation && this.indexingEnabled) {
-            await this.persistMetadataIndex();
-        }
-    }
-    async getEntries(ids) {
-        await this.ensureInitialized();
-        const result = [];
-        for (const id of ids) {
-            const metadata = await this.readMetadataById(id);
-            if (!metadata) {
-                continue;
-            }
-            const contentPath = this.contentPathForHash(metadata.contentHash);
-            if (!(await this.fileExists(contentPath))) {
-                this.logger.warn(`Content ${metadata.contentHash} not found for entry ${id}`);
-                continue;
-            }
-            const encryptedData = new Uint8Array(await (0, promises_1.readFile)(contentPath));
-            result.push({ ...metadata, encryptedData });
-        }
-        return result;
-    }
-    async hasEntries(ids) {
-        await this.ensureInitialized();
-        const existing = [];
-        for (const id of ids) {
-            if (this.indexingEnabled) {
-                if (this.entries.has(id)) {
-                    existing.push(id);
-                }
-            }
-            else {
-                if (await this.fileExists(this.metadataPathForId(id))) {
-                    existing.push(id);
-                }
-            }
-        }
-        return existing;
-    }
-    async findNewEntries(knownIds) {
-        await this.ensureInitialized();
-        const known = new Set(knownIds);
-        const all = await this.listAllMetadata();
-        return all.filter((meta) => !known.has(meta.id));
-    }
-    async findNewEntriesForDoc(knownIds, docId) {
-        await this.ensureInitialized();
-        const known = new Set(knownIds);
-        if (this.indexingEnabled) {
-            const ids = this.docIndex.get(docId) || new Set();
-            const result = [];
-            for (const id of ids) {
-                if (known.has(id)) {
-                    continue;
-                }
-                const metadata = this.entries.get(id);
-                if (metadata) {
-                    result.push(metadata);
-                }
-            }
-            return result;
-        }
-        const all = await this.listAllMetadata();
-        return all.filter((meta) => meta.docId === docId && !known.has(meta.id));
-    }
-    async findEntries(type, creationDateFrom, creationDateUntil) {
-        await this.ensureInitialized();
-        const all = await this.listAllMetadata();
-        return all.filter((meta) => {
-            if (meta.entryType !== type) {
-                return false;
-            }
-            if (creationDateFrom !== null && meta.createdAt < creationDateFrom) {
-                return false;
-            }
-            if (creationDateUntil !== null && meta.createdAt >= creationDateUntil) {
-                return false;
-            }
-            return true;
-        });
-    }
-    async getAllIds() {
-        await this.ensureInitialized();
-        if (this.indexingEnabled) {
-            return this.orderedMetadata.map((m) => m.id);
-        }
-        const files = await this.listEntryFiles();
-        return files.map(fromEntryFileName);
-    }
-    /**
-     * Cursor-based metadata scan for high-volume synchronization.
-     * Ordering: createdAt ASC, id ASC.
-     */
-    async scanEntriesSince(cursor, limit = Number.MAX_SAFE_INTEGER, filters) {
-        await this.ensureInitialized();
-        if (this.indexingEnabled) {
-            const startIndex = this.lowerBoundForCursor(cursor);
-            if (!filters || Object.keys(filters).length === 0) {
-                const page = this.orderedMetadata.slice(startIndex, startIndex + limit);
-                const last = page.length > 0 ? page[page.length - 1] : null;
-                return {
-                    entries: page,
-                    nextCursor: last ? { createdAt: last.createdAt, id: last.id } : cursor,
-                    hasMore: startIndex + page.length < this.orderedMetadata.length,
-                };
-            }
-            const page = [];
-            let idx = startIndex;
-            while (idx < this.orderedMetadata.length && page.length < limit) {
-                const meta = this.orderedMetadata[idx];
-                if (this.metadataMatchesFilters(meta, filters)) {
-                    page.push(meta);
-                }
-                idx++;
-            }
-            let hasMore = false;
-            while (idx < this.orderedMetadata.length) {
-                if (this.metadataMatchesFilters(this.orderedMetadata[idx], filters)) {
-                    hasMore = true;
-                    break;
-                }
-                idx++;
-            }
-            const last = page.length > 0 ? page[page.length - 1] : null;
-            return {
-                entries: page,
-                nextCursor: last ? { createdAt: last.createdAt, id: last.id } : cursor,
-                hasMore,
-            };
-        }
-        const all = (await this.listAllMetadata())
-            .filter((meta) => {
-            if (filters?.docId && meta.docId !== filters.docId) {
-                return false;
-            }
-            if (filters?.entryTypes && filters.entryTypes.length > 0 && !filters.entryTypes.includes(meta.entryType)) {
-                return false;
-            }
-            if (filters?.creationDateFrom !== undefined && filters.creationDateFrom !== null && meta.createdAt < filters.creationDateFrom) {
-                return false;
-            }
-            if (filters?.creationDateUntil !== undefined && filters.creationDateUntil !== null && meta.createdAt >= filters.creationDateUntil) {
-                return false;
-            }
-            return true;
-        })
-            .sort((a, b) => (a.createdAt === b.createdAt ? a.id.localeCompare(b.id) : a.createdAt - b.createdAt));
-        const startIndex = cursor === null
-            ? 0
-            : all.findIndex((meta) => meta.createdAt > cursor.createdAt || (meta.createdAt === cursor.createdAt && meta.id > cursor.id));
-        if (startIndex === -1) {
-            return { entries: [], nextCursor: cursor, hasMore: false };
-        }
-        const page = all.slice(startIndex, startIndex + limit);
-        const last = page.length > 0 ? page[page.length - 1] : null;
-        return {
-            entries: page,
-            nextCursor: last ? { createdAt: last.createdAt, id: last.id } : cursor,
-            hasMore: startIndex + page.length < all.length,
-        };
-    }
-    async getIdBloomSummary() {
-        await this.ensureInitialized();
-        const ids = await this.getAllIds();
-        return (0, bloom_1.createIdBloomSummary)(ids);
-    }
-    async resolveDependencies(startId, options) {
-        await this.ensureInitialized();
-        const stopAtEntryType = options?.stopAtEntryType;
-        const maxDepth = options?.maxDepth;
-        const includeStart = options?.includeStart !== false;
-        const result = [];
-        const visited = new Set();
-        const queue = [{ id: startId, depth: 0 }];
-        while (queue.length > 0) {
-            const { id, depth } = queue.shift();
-            if (visited.has(id)) {
-                continue;
-            }
-            visited.add(id);
-            if (maxDepth !== undefined && depth > maxDepth) {
-                continue;
-            }
-            const entry = await this.readMetadataById(id);
-            if (!entry) {
-                continue;
-            }
-            if (stopAtEntryType && entry.entryType === stopAtEntryType && id !== startId) {
-                result.push(id);
-                continue;
-            }
-            if (id !== startId || includeStart) {
-                result.push(id);
-            }
-            for (const depId of entry.dependencyIds) {
-                if (!visited.has(depId)) {
-                    queue.push({ id: depId, depth: depth + 1 });
-                }
-            }
-        }
-        result.reverse();
-        return result;
-    }
-    async purgeDocHistory(docId) {
-        await this.ensureInitialized();
-        const all = await this.listAllMetadata();
-        const toDelete = all.filter((meta) => meta.docId === docId);
-        if (toDelete.length === 0) {
-            return;
-        }
-        const deletedHashes = new Set();
-        for (const meta of toDelete) {
-            deletedHashes.add(meta.contentHash);
-            const metadataPath = this.metadataPathForId(meta.id);
-            await (0, promises_1.unlink)(metadataPath).catch(() => {
-                // Ignore races or already-deleted files.
-            });
-            await this.syncParentDirectory(metadataPath);
-            if (this.indexingEnabled) {
-                this.entries.delete(meta.id);
-                this.docIndex.get(docId)?.delete(meta.id);
-                this.removeOrderedMetadataById(meta.id);
-                const nextRef = (this.contentRefCount.get(meta.contentHash) || 1) - 1;
-                if (nextRef <= 0) {
-                    this.contentRefCount.delete(meta.contentHash);
-                }
-                else {
-                    this.contentRefCount.set(meta.contentHash, nextRef);
-                }
-            }
-        }
-        if (this.indexingEnabled && (this.docIndex.get(docId)?.size || 0) === 0) {
-            this.docIndex.delete(docId);
-        }
-        const remaining = await this.listAllMetadata();
-        const remainingHashes = new Set(remaining.map((m) => m.contentHash));
-        for (const hash of deletedHashes) {
-            if (!remainingHashes.has(hash)) {
-                const contentPath = this.contentPathForHash(hash);
-                await (0, promises_1.unlink)(contentPath).catch(() => {
-                    // Ignore races or already-deleted files.
-                });
-                await this.syncParentDirectory(contentPath);
-            }
-        }
-        if (this.indexingEnabled) {
-            await this.persistMetadataIndex();
-        }
-    }
-    async clearAllLocalData() {
-        await this.ensureInitialized();
-        this.indexStatus = {
-            phase: this.indexingEnabled ? "building" : "ready",
-            indexingEnabled: this.indexingEnabled,
-            progress01: this.indexingEnabled ? 0 : 1,
-        };
-        await this.clearAllLocalDataInternal();
-        await (0, promises_1.mkdir)(this.entriesDir, { recursive: true });
-        await (0, promises_1.mkdir)(this.contentDir, { recursive: true });
-        if (this.indexingEnabled) {
-            this.indexStatus = { phase: "ready", indexingEnabled: true, progress01: 1 };
-        }
-    }
-    async awaitIndexReady(_options) {
-        await this.ensureInitialized();
-        return this.indexStatus;
-    }
-    getIndexBuildStatus() {
-        return this.indexStatus;
-    }
-}
-exports.BasicOnDiskContentAddressedStore = BasicOnDiskContentAddressedStore;
-class BasicOnDiskContentAddressedStoreFactory {
-    createStore(dbId, options) {
-        return {
-            docStore: new BasicOnDiskContentAddressedStore(dbId, undefined, options),
-        };
-    }
-}
-exports.BasicOnDiskContentAddressedStoreFactory = BasicOnDiskContentAddressedStoreFactory;
-//# sourceMappingURL=BasicOnDiskContentAddressedStore.js.map