mindoodb 0.0.2 → 0.0.3

package/dist/core/appendonlystores/BasicOnDiskContentAddressedStore.js

@@ -0,0 +1,659 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BasicOnDiskContentAddressedStoreFactory = exports.BasicOnDiskContentAddressedStore = void 0;
+const promises_1 = require("fs/promises");
+const fs_1 = require("fs");
+const path = __importStar(require("path"));
+const bloom_1 = require("./bloom");
+const logging_1 = require("../logging");
+function serializeMetadata(metadata) {
+    return {
+        ...metadata,
+        signature: Buffer.from(metadata.signature).toString("base64"),
+    };
+}
+function deserializeMetadata(serialized) {
+    return {
+        ...serialized,
+        signature: new Uint8Array(Buffer.from(serialized.signature, "base64")),
+    };
+}
+function toEntryFileName(id) {
+    return `${encodeURIComponent(id)}.json`;
+}
+function fromEntryFileName(fileName) {
+    const withoutExt = fileName.endsWith(".json") ? fileName.slice(0, -5) : fileName;
+    return decodeURIComponent(withoutExt);
+}
+function tempPathFor(finalPath) {
+    return `${finalPath}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+}
+/**
+ * Simple persistent on-disk ContentAddressedStore implementation.
+ * Uses metadata files by id and binary content files by contentHash.
+ * Supports reset-on-start and optional in-memory indexing.
+ */
+class BasicOnDiskContentAddressedStore {
+    constructor(dbId, logger, options) {
+        // In-memory indexes (only used when indexingEnabled=true)
+        this.entries = new Map();
+        this.docIndex = new Map();
+        this.contentRefCount = new Map();
+        this.orderedMetadata = [];
+        this.dbId = dbId;
+        this.logger =
+            logger ||
+                new logging_1.MindooLogger((0, logging_1.getDefaultLogLevel)(), `BasicOnDiskStore:${dbId}`, true);
+        this.storeRoot = path.join(options?.basePath?.toString() || ".mindoodb-store", dbId);
+        this.entriesDir = path.join(this.storeRoot, "entries");
+        this.contentDir = path.join(this.storeRoot, "content");
+        this.metadataIndexPath = path.join(this.storeRoot, "metadata-index.json");
+        this.indexingEnabled = options?.indexingEnabled !== false;
+        this.indexStatus = {
+            phase: this.indexingEnabled ? "building" : "ready",
+            indexingEnabled: this.indexingEnabled,
+            progress01: this.indexingEnabled ? 0 : 1,
+        };
+        this.initPromise = this.initialize(options);
+    }
+    async initialize(options) {
+        if (options?.clearLocalDataOnStartup) {
+            await this.clearAllLocalDataInternal();
+        }
+        await (0, promises_1.mkdir)(this.entriesDir, { recursive: true });
+        await (0, promises_1.mkdir)(this.contentDir, { recursive: true });
+        if (!this.indexingEnabled) {
+            this.indexStatus = {
+                phase: "ready",
+                indexingEnabled: false,
+                progress01: 1,
+            };
+            return;
+        }
+        const loaded = await this.tryLoadPersistedIndex();
+        const indexCoverageOk = loaded ? await this.validatePersistedIndexCoverage() : false;
+        if (!loaded || !indexCoverageOk) {
+            await this.rebuildInMemoryIndexes();
+            await this.persistMetadataIndex();
+        }
+        this.indexStatus = {
+            phase: "ready",
+            indexingEnabled: true,
+            progress01: 1,
+        };
+    }
+    async ensureInitialized() {
+        await this.initPromise;
+    }
+    async clearAllLocalDataInternal() {
+        await (0, promises_1.rm)(this.storeRoot, { recursive: true, force: true });
+        this.entries.clear();
+        this.docIndex.clear();
+        this.contentRefCount.clear();
+        this.orderedMetadata = [];
+    }
+    async rebuildInMemoryIndexes() {
+        this.entries.clear();
+        this.docIndex.clear();
+        this.contentRefCount.clear();
+        this.orderedMetadata = [];
+        const files = await this.listEntryFiles();
+        for (const fileName of files) {
+            const metadata = await this.readMetadataByFileName(fileName);
+            if (!metadata) {
+                continue;
+            }
+            this.entries.set(metadata.id, metadata);
+            if (!this.docIndex.has(metadata.docId)) {
+                this.docIndex.set(metadata.docId, new Set());
+            }
+            this.docIndex.get(metadata.docId).add(metadata.id);
+            this.contentRefCount.set(metadata.contentHash, (this.contentRefCount.get(metadata.contentHash) || 0) + 1);
+            this.orderedMetadata.push(metadata);
+        }
+        this.orderedMetadata.sort((a, b) => (a.createdAt === b.createdAt ? a.id.localeCompare(b.id) : a.createdAt - b.createdAt));
+    }
+    async tryLoadPersistedIndex() {
+        if (!(await this.fileExists(this.metadataIndexPath))) {
+            return false;
+        }
+        try {
+            const raw = await (0, promises_1.readFile)(this.metadataIndexPath, "utf-8");
+            const serialized = JSON.parse(raw);
+            const loaded = serialized.map((s) => deserializeMetadata(s));
+            this.entries.clear();
+            this.docIndex.clear();
+            this.contentRefCount.clear();
+            this.orderedMetadata = [];
+            for (const metadata of loaded) {
+                this.entries.set(metadata.id, metadata);
+                if (!this.docIndex.has(metadata.docId)) {
+                    this.docIndex.set(metadata.docId, new Set());
+                }
+                this.docIndex.get(metadata.docId).add(metadata.id);
+                this.contentRefCount.set(metadata.contentHash, (this.contentRefCount.get(metadata.contentHash) || 0) + 1);
+                this.orderedMetadata.push(metadata);
+            }
+            return true;
+        }
+        catch (err) {
+            this.logger.warn(`Failed to load metadata index, rebuilding from entry files: ${String(err)}`);
+            return false;
+        }
+    }
+    async validatePersistedIndexCoverage() {
+        if (!this.indexingEnabled) {
+            return true;
+        }
+        const files = await this.listEntryFiles();
+        if (files.length !== this.entries.size) {
+            return false;
+        }
+        for (const fileName of files) {
+            const entryId = fromEntryFileName(fileName);
+            if (!this.entries.has(entryId)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    async persistMetadataIndex() {
+        if (!this.indexingEnabled) {
+            return;
+        }
+        const serialized = JSON.stringify(this.orderedMetadata.map((m) => serializeMetadata(m)));
+        await this.writeFileAtomic(this.metadataIndexPath, serialized);
+    }
+    compareMetadata(a, b) {
+        if (a.createdAt !== b.createdAt) {
+            return a.createdAt - b.createdAt;
+        }
+        return a.id.localeCompare(b.id);
+    }
+    lowerBoundForCursor(cursor) {
+        if (cursor === null) {
+            return 0;
+        }
+        let left = 0;
+        let right = this.orderedMetadata.length;
+        while (left < right) {
+            const mid = Math.floor((left + right) / 2);
+            const candidate = this.orderedMetadata[mid];
+            const greater = candidate.createdAt > cursor.createdAt ||
+                (candidate.createdAt === cursor.createdAt && candidate.id > cursor.id);
+            if (greater) {
+                right = mid;
+            }
+            else {
+                left = mid + 1;
+            }
+        }
+        return left;
+    }
+    metadataMatchesFilters(meta, filters) {
+        if (filters?.docId && meta.docId !== filters.docId) {
+            return false;
+        }
+        if (filters?.entryTypes && filters.entryTypes.length > 0 && !filters.entryTypes.includes(meta.entryType)) {
+            return false;
+        }
+        if (filters?.creationDateFrom !== undefined && filters.creationDateFrom !== null && meta.createdAt < filters.creationDateFrom) {
+            return false;
+        }
+        if (filters?.creationDateUntil !== undefined && filters.creationDateUntil !== null && meta.createdAt >= filters.creationDateUntil) {
+            return false;
+        }
+        return true;
+    }
+    insertOrderedMetadata(meta) {
+        let left = 0;
+        let right = this.orderedMetadata.length;
+        while (left < right) {
+            const mid = Math.floor((left + right) / 2);
+            if (this.compareMetadata(this.orderedMetadata[mid], meta) < 0) {
+                left = mid + 1;
+            }
+            else {
+                right = mid;
+            }
+        }
+        this.orderedMetadata.splice(left, 0, meta);
+    }
+    removeOrderedMetadataById(id) {
+        const idx = this.orderedMetadata.findIndex((m) => m.id === id);
+        if (idx >= 0) {
+            this.orderedMetadata.splice(idx, 1);
+        }
+    }
+    async listEntryFiles() {
+        try {
+            const files = await (0, promises_1.readdir)(this.entriesDir);
+            return files.filter((f) => f.endsWith(".json"));
+        }
+        catch {
+            return [];
+        }
+    }
+    metadataPathForId(id) {
+        return path.join(this.entriesDir, toEntryFileName(id));
+    }
+    contentPathForHash(contentHash) {
+        return path.join(this.contentDir, `${contentHash}.bin`);
+    }
+    async fileExists(filePath) {
+        try {
+            await (0, promises_1.access)(filePath, fs_1.constants.F_OK);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async writeFileAtomic(filePath, data) {
+        const tmpPath = tempPathFor(filePath);
+        const tmpHandle = await (0, promises_1.open)(tmpPath, "w");
+        try {
+            await tmpHandle.writeFile(data);
+            await tmpHandle.sync();
+        }
+        finally {
+            await tmpHandle.close();
+        }
+        await (0, promises_1.rename)(tmpPath, filePath);
+        await this.syncParentDirectory(filePath);
+    }
+    async syncParentDirectory(filePath) {
+        const dirPath = path.dirname(filePath);
+        try {
+            const dirHandle = await (0, promises_1.open)(dirPath, "r");
+            try {
+                await dirHandle.sync();
+            }
+            finally {
+                await dirHandle.close();
+            }
+        }
+        catch (err) {
+            this.logger.debug(`Directory fsync skipped for ${dirPath}: ${String(err)}`);
+        }
+    }
+    async readMetadataById(id) {
+        if (this.indexingEnabled) {
+            return this.entries.get(id) || null;
+        }
+        const filePath = this.metadataPathForId(id);
+        if (!(await this.fileExists(filePath))) {
+            return null;
+        }
+        const raw = await (0, promises_1.readFile)(filePath, "utf-8");
+        return deserializeMetadata(JSON.parse(raw));
+    }
+    async readMetadataByFileName(fileName) {
+        const filePath = path.join(this.entriesDir, fileName);
+        try {
+            const raw = await (0, promises_1.readFile)(filePath, "utf-8");
+            const metadata = deserializeMetadata(JSON.parse(raw));
+            return metadata;
+        }
+        catch (err) {
+            this.logger.warn(`Failed to read metadata file ${fileName}: ${String(err)}`);
+            return null;
+        }
+    }
+    async listAllMetadata() {
+        if (this.indexingEnabled) {
+            return [...this.orderedMetadata];
+        }
+        const files = await this.listEntryFiles();
+        const result = [];
+        for (const fileName of files) {
+            const metadata = await this.readMetadataByFileName(fileName);
+            if (metadata) {
+                result.push(metadata);
+            }
+        }
+        return result;
+    }
+    getId() {
+        return this.dbId;
+    }
+    async putEntries(entries) {
+        await this.ensureInitialized();
+        let hasMutation = false;
+        for (const entry of entries) {
+            const metadataPath = this.metadataPathForId(entry.id);
+            if (await this.fileExists(metadataPath)) {
+                continue;
+            }
+            const contentPath = this.contentPathForHash(entry.contentHash);
+            if (!(await this.fileExists(contentPath))) {
+                await this.writeFileAtomic(contentPath, entry.encryptedData);
+            }
+            const { encryptedData, ...metadata } = entry;
+            const serialized = JSON.stringify(serializeMetadata(metadata));
+            await this.writeFileAtomic(metadataPath, serialized);
+            hasMutation = true;
+            if (this.indexingEnabled) {
+                this.entries.set(entry.id, metadata);
+                if (!this.docIndex.has(entry.docId)) {
+                    this.docIndex.set(entry.docId, new Set());
+                }
+                this.docIndex.get(entry.docId).add(entry.id);
+                this.contentRefCount.set(entry.contentHash, (this.contentRefCount.get(entry.contentHash) || 0) + 1);
+                this.insertOrderedMetadata(metadata);
+            }
+        }
+        if (hasMutation && this.indexingEnabled) {
+            await this.persistMetadataIndex();
+        }
+    }
+    async getEntries(ids) {
+        await this.ensureInitialized();
+        const result = [];
+        for (const id of ids) {
+            const metadata = await this.readMetadataById(id);
+            if (!metadata) {
+                continue;
+            }
+            const contentPath = this.contentPathForHash(metadata.contentHash);
+            if (!(await this.fileExists(contentPath))) {
+                this.logger.warn(`Content ${metadata.contentHash} not found for entry ${id}`);
+                continue;
+            }
+            const encryptedData = new Uint8Array(await (0, promises_1.readFile)(contentPath));
+            result.push({ ...metadata, encryptedData });
+        }
+        return result;
+    }
+    async hasEntries(ids) {
+        await this.ensureInitialized();
+        const existing = [];
+        for (const id of ids) {
+            if (this.indexingEnabled) {
+                if (this.entries.has(id)) {
+                    existing.push(id);
+                }
+            }
+            else {
+                if (await this.fileExists(this.metadataPathForId(id))) {
+                    existing.push(id);
+                }
+            }
+        }
+        return existing;
+    }
+    async findNewEntries(knownIds) {
+        await this.ensureInitialized();
+        const known = new Set(knownIds);
+        const all = await this.listAllMetadata();
+        return all.filter((meta) => !known.has(meta.id));
+    }
+    async findNewEntriesForDoc(knownIds, docId) {
+        await this.ensureInitialized();
+        const known = new Set(knownIds);
+        if (this.indexingEnabled) {
+            const ids = this.docIndex.get(docId) || new Set();
+            const result = [];
+            for (const id of ids) {
+                if (known.has(id)) {
+                    continue;
+                }
+                const metadata = this.entries.get(id);
+                if (metadata) {
+                    result.push(metadata);
+                }
+            }
+            return result;
+        }
+        const all = await this.listAllMetadata();
+        return all.filter((meta) => meta.docId === docId && !known.has(meta.id));
+    }
+    async findEntries(type, creationDateFrom, creationDateUntil) {
+        await this.ensureInitialized();
+        const all = await this.listAllMetadata();
+        return all.filter((meta) => {
+            if (meta.entryType !== type) {
+                return false;
+            }
+            if (creationDateFrom !== null && meta.createdAt < creationDateFrom) {
+                return false;
+            }
+            if (creationDateUntil !== null && meta.createdAt >= creationDateUntil) {
+                return false;
+            }
+            return true;
+        });
+    }
+    async getAllIds() {
+        await this.ensureInitialized();
+        if (this.indexingEnabled) {
+            return this.orderedMetadata.map((m) => m.id);
+        }
+        const files = await this.listEntryFiles();
+        return files.map(fromEntryFileName);
+    }
+    /**
+     * Cursor-based metadata scan for high-volume synchronization.
+     * Ordering: createdAt ASC, id ASC.
+     */
+    async scanEntriesSince(cursor, limit = Number.MAX_SAFE_INTEGER, filters) {
+        await this.ensureInitialized();
+        if (this.indexingEnabled) {
+            const startIndex = this.lowerBoundForCursor(cursor);
+            if (!filters || Object.keys(filters).length === 0) {
+                const page = this.orderedMetadata.slice(startIndex, startIndex + limit);
+                const last = page.length > 0 ? page[page.length - 1] : null;
+                return {
+                    entries: page,
+                    nextCursor: last ? { createdAt: last.createdAt, id: last.id } : cursor,
+                    hasMore: startIndex + page.length < this.orderedMetadata.length,
+                };
+            }
+            const page = [];
+            let idx = startIndex;
+            while (idx < this.orderedMetadata.length && page.length < limit) {
+                const meta = this.orderedMetadata[idx];
+                if (this.metadataMatchesFilters(meta, filters)) {
+                    page.push(meta);
+                }
+                idx++;
+            }
+            let hasMore = false;
+            while (idx < this.orderedMetadata.length) {
+                if (this.metadataMatchesFilters(this.orderedMetadata[idx], filters)) {
+                    hasMore = true;
+                    break;
+                }
+                idx++;
+            }
+            const last = page.length > 0 ? page[page.length - 1] : null;
+            return {
+                entries: page,
+                nextCursor: last ? { createdAt: last.createdAt, id: last.id } : cursor,
+                hasMore,
+            };
+        }
+        const all = (await this.listAllMetadata())
+            .filter((meta) => {
+            if (filters?.docId && meta.docId !== filters.docId) {
+                return false;
+            }
+            if (filters?.entryTypes && filters.entryTypes.length > 0 && !filters.entryTypes.includes(meta.entryType)) {
+                return false;
+            }
+            if (filters?.creationDateFrom !== undefined && filters.creationDateFrom !== null && meta.createdAt < filters.creationDateFrom) {
+                return false;
+            }
+            if (filters?.creationDateUntil !== undefined && filters.creationDateUntil !== null && meta.createdAt >= filters.creationDateUntil) {
+                return false;
+            }
+            return true;
+        })
+            .sort((a, b) => (a.createdAt === b.createdAt ? a.id.localeCompare(b.id) : a.createdAt - b.createdAt));
+        const startIndex = cursor === null
+            ? 0
+            : all.findIndex((meta) => meta.createdAt > cursor.createdAt || (meta.createdAt === cursor.createdAt && meta.id > cursor.id));
+        if (startIndex === -1) {
+            return { entries: [], nextCursor: cursor, hasMore: false };
+        }
+        const page = all.slice(startIndex, startIndex + limit);
+        const last = page.length > 0 ? page[page.length - 1] : null;
+        return {
+            entries: page,
+            nextCursor: last ? { createdAt: last.createdAt, id: last.id } : cursor,
+            hasMore: startIndex + page.length < all.length,
+        };
+    }
+    async getIdBloomSummary() {
+        await this.ensureInitialized();
+        const ids = await this.getAllIds();
+        return (0, bloom_1.createIdBloomSummary)(ids);
+    }
+    async resolveDependencies(startId, options) {
+        await this.ensureInitialized();
+        const stopAtEntryType = options?.stopAtEntryType;
+        const maxDepth = options?.maxDepth;
+        const includeStart = options?.includeStart !== false;
+        const result = [];
+        const visited = new Set();
+        const queue = [{ id: startId, depth: 0 }];
+        while (queue.length > 0) {
+            const { id, depth } = queue.shift();
+            if (visited.has(id)) {
+                continue;
+            }
+            visited.add(id);
+            if (maxDepth !== undefined && depth > maxDepth) {
+                continue;
+            }
+            const entry = await this.readMetadataById(id);
+            if (!entry) {
+                continue;
+            }
+            if (stopAtEntryType && entry.entryType === stopAtEntryType && id !== startId) {
+                result.push(id);
+                continue;
+            }
+            if (id !== startId || includeStart) {
+                result.push(id);
+            }
+            for (const depId of entry.dependencyIds) {
+                if (!visited.has(depId)) {
+                    queue.push({ id: depId, depth: depth + 1 });
+                }
+            }
+        }
+        result.reverse();
+        return result;
+    }
+    async purgeDocHistory(docId) {
+        await this.ensureInitialized();
+        const all = await this.listAllMetadata();
+        const toDelete = all.filter((meta) => meta.docId === docId);
+        if (toDelete.length === 0) {
+            return;
+        }
+        const deletedHashes = new Set();
+        for (const meta of toDelete) {
+            deletedHashes.add(meta.contentHash);
+            const metadataPath = this.metadataPathForId(meta.id);
+            await (0, promises_1.unlink)(metadataPath).catch(() => {
+                // Ignore races or already-deleted files.
+            });
+            await this.syncParentDirectory(metadataPath);
+            if (this.indexingEnabled) {
+                this.entries.delete(meta.id);
+                this.docIndex.get(docId)?.delete(meta.id);
+                this.removeOrderedMetadataById(meta.id);
+                const nextRef = (this.contentRefCount.get(meta.contentHash) || 1) - 1;
+                if (nextRef <= 0) {
+                    this.contentRefCount.delete(meta.contentHash);
+                }
+                else {
+                    this.contentRefCount.set(meta.contentHash, nextRef);
+                }
+            }
+        }
+        if (this.indexingEnabled && (this.docIndex.get(docId)?.size || 0) === 0) {
+            this.docIndex.delete(docId);
+        }
+        const remaining = await this.listAllMetadata();
+        const remainingHashes = new Set(remaining.map((m) => m.contentHash));
+        for (const hash of deletedHashes) {
+            if (!remainingHashes.has(hash)) {
+                const contentPath = this.contentPathForHash(hash);
+                await (0, promises_1.unlink)(contentPath).catch(() => {
+                    // Ignore races or already-deleted files.
+                });
+                await this.syncParentDirectory(contentPath);
+            }
+        }
+        if (this.indexingEnabled) {
+            await this.persistMetadataIndex();
+        }
+    }
+    async clearAllLocalData() {
+        await this.ensureInitialized();
+        this.indexStatus = {
+            phase: this.indexingEnabled ? "building" : "ready",
+            indexingEnabled: this.indexingEnabled,
+            progress01: this.indexingEnabled ? 0 : 1,
+        };
+        await this.clearAllLocalDataInternal();
+        await (0, promises_1.mkdir)(this.entriesDir, { recursive: true });
+        await (0, promises_1.mkdir)(this.contentDir, { recursive: true });
+        if (this.indexingEnabled) {
+            this.indexStatus = { phase: "ready", indexingEnabled: true, progress01: 1 };
+        }
+    }
+    async awaitIndexReady(_options) {
+        await this.ensureInitialized();
+        return this.indexStatus;
+    }
+    getIndexBuildStatus() {
+        return this.indexStatus;
+    }
+}
+exports.BasicOnDiskContentAddressedStore = BasicOnDiskContentAddressedStore;
+class BasicOnDiskContentAddressedStoreFactory {
+    createStore(dbId, options) {
+        return {
+            docStore: new BasicOnDiskContentAddressedStore(dbId, undefined, options),
+        };
+    }
+}
+exports.BasicOnDiskContentAddressedStoreFactory = BasicOnDiskContentAddressedStoreFactory;
+//# sourceMappingURL=BasicOnDiskContentAddressedStore.js.map