@delma/fylo 2.0.0 → 2.1.0

package/src/engines/s3-files.ts

@@ -1,274 +1,256 @@
-import { mkdir, readFile, readdir, rm, stat, writeFile, open } from 'node:fs/promises'
+import { rename, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import { createHash } from 'node:crypto'
-import { Database } from 'bun:sqlite'
 import TTID from '@delma/ttid'
 import { Dir } from '../core/directory'
 import { validateCollectionName } from '../core/collection'
 import { Cipher } from '../adapters/cipher'
+import {
+    FyloSyncError,
+    resolveSyncMode,
+    type FyloDeleteSyncEvent,
+    type FyloSyncHooks,
+    type FyloSyncMode,
+    type FyloWriteSyncEvent
+} from '../sync'
 import type { EventBus, FyloStorageEngineKind, LockManager, StorageEngine } from './types'
+import {
+    type CollectionIndexCache,
+    type FyloRecord,
+    type StoredCollectionIndex,
+    type StoredIndexEntry
+} from './s3-files/types'
+import { FilesystemEventBus, FilesystemLockManager, FilesystemStorage } from './s3-files/filesystem'
+import { S3FilesDocuments } from './s3-files/documents'
+import { S3FilesQueryEngine } from './s3-files/query'
 
-type FyloRecord<T extends Record<string, any>> = Record<_ttid, T>
-
-type S3FilesQueryResult<T extends Record<string, any>> =
-    | _ttid
-    | FyloRecord<T>
-    | Record<string, _ttid[]>
-    | Record<string, Record<_ttid, Partial<T>>>
-    | Record<_ttid, Partial<T>>
-
-type S3FilesEvent<T extends Record<string, any>> = {
-    ts: number
-    action: 'insert' | 'delete'
-    id: _ttid
-    doc?: T
-}
+export class S3FilesEngine {
+    readonly kind: FyloStorageEngineKind = 's3-files'
 
-type StoredDoc<T extends Record<string, any>> = {
-    id: _ttid
-    createdAt: number
-    updatedAt: number
-    data: T
-}
+    private readonly indexes = new Map<string, CollectionIndexCache>()
+    private readonly writeLanes = new Map<string, Promise<void>>()
 
-class FilesystemStorage implements StorageEngine {
-    async read(target: string): Promise<string> {
-        return await readFile(target, 'utf8')
-    }
+    private readonly storage: StorageEngine
+    private readonly locks: LockManager
+    private readonly events: EventBus<Record<string, any>>
+    private readonly documents: S3FilesDocuments
+    private readonly queryEngine: S3FilesQueryEngine
+    private readonly sync?: FyloSyncHooks
+    private readonly syncMode: FyloSyncMode
 
-    async write(target: string, data: string): Promise<void> {
-        await mkdir(path.dirname(target), { recursive: true })
-        await writeFile(target, data, 'utf8')
+    constructor(
+        readonly root: string = process.env.FYLO_ROOT ??
+            process.env.FYLO_S3FILES_ROOT ??
+            path.join(process.cwd(), '.fylo-data'),
+        options: {
+            sync?: FyloSyncHooks
+            syncMode?: FyloSyncMode
+        } = {}
+    ) {
+        this.sync = options.sync
+        this.syncMode = resolveSyncMode(options.syncMode)
+        this.storage = new FilesystemStorage()
+        this.locks = new FilesystemLockManager(this.root, this.storage)
+        this.events = new FilesystemEventBus<Record<string, any>>(this.root, this.storage)
+        this.documents = new S3FilesDocuments(
+            this.storage,
+            this.docsRoot.bind(this),
+            this.docPath.bind(this),
+            this.ensureCollection.bind(this),
+            this.encodeEncrypted.bind(this),
+            this.decodeEncrypted.bind(this)
+        )
+        this.queryEngine = new S3FilesQueryEngine({
+            loadIndexCache: this.loadIndexCache.bind(this),
+            normalizeIndexValue: this.normalizeIndexValue.bind(this)
+        })
     }
 
-    async delete(target: string): Promise<void> {
-        await rm(target, { recursive: true, force: true })
+    private collectionRoot(collection: string) {
+        validateCollectionName(collection)
+        return path.join(this.root, collection)
     }
 
-    async list(target: string): Promise<string[]> {
-        const results: string[] = []
-
-        try {
-            const entries = await readdir(target, { withFileTypes: true })
-            for (const entry of entries) {
-                const child = path.join(target, entry.name)
-                if (entry.isDirectory()) {
-                    results.push(...(await this.list(child)))
-                } else {
-                    results.push(child)
-                }
-            }
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
-        }
-
-        return results
+    private docsRoot(collection: string) {
+        return path.join(this.collectionRoot(collection), '.fylo', 'docs')
     }
 
-    async mkdir(target: string): Promise<void> {
-        await mkdir(target, { recursive: true })
+    private metaRoot(collection: string) {
+        return path.join(this.collectionRoot(collection), '.fylo')
     }
 
-    async rmdir(target: string): Promise<void> {
-        await rm(target, { recursive: true, force: true })
+    private indexesRoot(collection: string) {
+        return path.join(this.metaRoot(collection), 'indexes')
     }
 
-    async exists(target: string): Promise<boolean> {
-        try {
-            await stat(target)
-            return true
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false
-            throw err
-        }
+    private indexFilePath(collection: string) {
+        return path.join(this.indexesRoot(collection), `${collection}.idx.json`)
     }
-}
 
-class FilesystemLockManager implements LockManager {
-    constructor(
-        private readonly root: string,
-        private readonly storage: StorageEngine
-    ) {}
-
-    private lockDir(collection: string, docId: _ttid) {
-        return path.join(this.root, collection, '.fylo', 'locks', `${docId}.lock`)
+    private docPath(collection: string, docId: _ttid) {
+        return path.join(this.docsRoot(collection), docId.slice(0, 2), `${docId}.json`)
     }
 
-    async acquire(
+    private async runSyncTask(
         collection: string,
         docId: _ttid,
-        owner: string,
-        ttlMs: number = 30_000
-    ): Promise<boolean> {
-        const dir = this.lockDir(collection, docId)
-        const metaPath = path.join(dir, 'meta.json')
-        await mkdir(path.dirname(dir), { recursive: true })
-
-        try {
-            await mkdir(dir, { recursive: false })
-            await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
-            return true
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err
+        operation: string,
+        targetPath: string,
+        task: () => Promise<void>
+    ) {
+        if (!this.sync?.onWrite && !this.sync?.onDelete) return
+
+        if (this.syncMode === 'fire-and-forget') {
+            void task().catch((cause) => {
+                console.error(
+                    new FyloSyncError({
+                        collection,
+                        docId,
+                        operation,
+                        path: targetPath,
+                        cause
+                    })
+                )
+            })
+            return
         }
 
         try {
-            const meta = JSON.parse(await this.storage.read(metaPath)) as { ts?: number }
-            if (meta.ts && Date.now() - meta.ts > ttlMs) {
-                await this.storage.rmdir(dir)
-                await mkdir(dir, { recursive: false })
-                await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
-                return true
-            }
-        } catch {
-            await this.storage.rmdir(dir)
-            await mkdir(dir, { recursive: false })
-            await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
-            return true
+            await task()
+        } catch (cause) {
+            throw new FyloSyncError({
+                collection,
+                docId,
+                operation,
+                path: targetPath,
+                cause
+            })
         }
-
-        return false
     }
 
-    async release(collection: string, docId: _ttid, owner: string): Promise<void> {
-        const dir = this.lockDir(collection, docId)
-        const metaPath = path.join(dir, 'meta.json')
-
-        try {
-            const meta = JSON.parse(await this.storage.read(metaPath)) as { owner?: string }
-            if (meta.owner === owner) await this.storage.rmdir(dir)
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
-        }
+    private async syncWrite<T extends Record<string, any>>(event: FyloWriteSyncEvent<T>) {
+        if (!this.sync?.onWrite) return
+        await this.sync.onWrite(event)
     }
-}
 
-class FilesystemEventBus<T extends Record<string, any>> implements EventBus<S3FilesEvent<T>> {
-    constructor(
-        private readonly root: string,
-        private readonly storage: StorageEngine
-    ) {}
-
-    private journalPath(collection: string) {
-        return path.join(this.root, collection, '.fylo', 'events', `${collection}.ndjson`)
+    private async syncDelete(event: FyloDeleteSyncEvent) {
+        if (!this.sync?.onDelete) return
+        await this.sync.onDelete(event)
     }
 
-    async publish(collection: string, event: S3FilesEvent<T>): Promise<void> {
-        const target = this.journalPath(collection)
-        await mkdir(path.dirname(target), { recursive: true })
-        const line = `${JSON.stringify(event)}\n`
-        const handle = await open(target, 'a')
-        try {
-            await handle.write(line)
-        } finally {
-            await handle.close()
-        }
+    private hash(value: string) {
+        return createHash('sha256').update(value).digest('hex')
     }
 
-    async *listen(collection: string): AsyncGenerator<S3FilesEvent<T>, void, unknown> {
-        const target = this.journalPath(collection)
-        let position = 0
-
-        while (true) {
-            try {
-                const fileStat = await stat(target)
-                if (fileStat.size > position) {
-                    const handle = await open(target, 'r')
-                    try {
-                        const size = fileStat.size - position
-                        const buffer = Buffer.alloc(size)
-                        await handle.read(buffer, 0, size, position)
-                        position = fileStat.size
-
-                        for (const line of buffer.toString('utf8').split('\n')) {
-                            if (line.trim().length === 0) continue
-                            yield JSON.parse(line) as S3FilesEvent<T>
-                        }
-                    } finally {
-                        await handle.close()
-                    }
-                }
-            } catch (err) {
-                if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
-            }
-
-            await Bun.sleep(100)
+    private createEmptyIndexCache(): CollectionIndexCache {
+        return {
+            docs: new Map(),
+            fieldHash: new Map(),
+            fieldNumeric: new Map(),
+            fieldString: new Map()
         }
     }
-}
 
-export class S3FilesEngine {
-    readonly kind: FyloStorageEngineKind = 's3-files'
+    private addEntryToCache(cache: CollectionIndexCache, docId: _ttid, entry: StoredIndexEntry) {
+        let valueHashBucket = cache.fieldHash.get(entry.fieldPath)
+        if (!valueHashBucket) {
+            valueHashBucket = new Map()
+            cache.fieldHash.set(entry.fieldPath, valueHashBucket)
+        }
 
-    private readonly databases = new Map<string, Database>()
+        let docsForValue = valueHashBucket.get(entry.valueHash)
+        if (!docsForValue) {
+            docsForValue = new Set()
+            valueHashBucket.set(entry.valueHash, docsForValue)
+        }
+        docsForValue.add(docId)
 
-    private readonly storage: StorageEngine
-    private readonly locks: LockManager
-    private readonly events: EventBus<Record<string, any>>
+        if (entry.numericValue !== null) {
+            const numericEntries = cache.fieldNumeric.get(entry.fieldPath) ?? []
+            numericEntries.push({ docId, numericValue: entry.numericValue })
+            cache.fieldNumeric.set(entry.fieldPath, numericEntries)
+        }
 
-    constructor(readonly root: string = process.env.FYLO_S3FILES_ROOT ?? '/mnt/fylo') {
-        this.storage = new FilesystemStorage()
-        this.locks = new FilesystemLockManager(this.root, this.storage)
-        this.events = new FilesystemEventBus<Record<string, any>>(this.root, this.storage)
+        if (entry.valueType === 'string') {
+            const stringEntries = cache.fieldString.get(entry.fieldPath) ?? []
+            stringEntries.push({ docId, rawValue: entry.rawValue })
+            cache.fieldString.set(entry.fieldPath, stringEntries)
+        }
     }
 
-    private collectionRoot(collection: string) {
-        validateCollectionName(collection)
-        return path.join(this.root, collection)
-    }
+    private deleteEntryFromCache(
+        cache: CollectionIndexCache,
+        docId: _ttid,
+        entry: StoredIndexEntry
+    ) {
+        const valueHashBucket = cache.fieldHash.get(entry.fieldPath)
+        const docsForValue = valueHashBucket?.get(entry.valueHash)
+        docsForValue?.delete(docId)
+        if (docsForValue?.size === 0) valueHashBucket?.delete(entry.valueHash)
+        if (valueHashBucket?.size === 0) cache.fieldHash.delete(entry.fieldPath)
+
+        if (entry.numericValue !== null) {
+            const numericEntries = cache.fieldNumeric
+                .get(entry.fieldPath)
+                ?.filter(
+                    (candidate) =>
+                        !(
+                            candidate.docId === docId &&
+                            candidate.numericValue === entry.numericValue
+                        )
+                )
+            if (!numericEntries?.length) cache.fieldNumeric.delete(entry.fieldPath)
+            else cache.fieldNumeric.set(entry.fieldPath, numericEntries)
+        }
 
-    private docsRoot(collection: string) {
-        return path.join(this.collectionRoot(collection), '.fylo', 'docs')
+        if (entry.valueType === 'string') {
+            const stringEntries = cache.fieldString
+                .get(entry.fieldPath)
+                ?.filter(
+                    (candidate) =>
+                        !(candidate.docId === docId && candidate.rawValue === entry.rawValue)
+                )
+            if (!stringEntries?.length) cache.fieldString.delete(entry.fieldPath)
+            else cache.fieldString.set(entry.fieldPath, stringEntries)
+        }
     }
 
-    private metaRoot(collection: string) {
-        return path.join(this.collectionRoot(collection), '.fylo')
-    }
+    private async writeIndexFile(collection: string, cache: CollectionIndexCache) {
+        await this.storage.mkdir(this.indexesRoot(collection))
+        const target = this.indexFilePath(collection)
+        const temp = `${target}.tmp`
 
-    private indexDbPath(collection: string) {
-        return path.join(this.metaRoot(collection), 'index.db')
-    }
+        const payload: StoredCollectionIndex = {
+            version: 1,
+            docs: Object.fromEntries(cache.docs)
+        }
 
-    private docPath(collection: string, docId: _ttid) {
-        return path.join(this.docsRoot(collection), docId.slice(0, 2), `${docId}.json`)
+        await writeFile(temp, JSON.stringify(payload), 'utf8')
+        await rename(temp, target)
     }
 
-    private hash(value: string) {
-        return createHash('sha256').update(value).digest('hex')
-    }
+    private async loadIndexCache(collection: string) {
+        const cache = this.createEmptyIndexCache()
 
-    private database(collection: string) {
-        const existing = this.databases.get(collection)
-        if (existing) return existing
-
-        const db = new Database(this.indexDbPath(collection))
-        db.exec(`
-            CREATE TABLE IF NOT EXISTS doc_index_entries (
-                doc_id TEXT NOT NULL,
-                field_path TEXT NOT NULL,
-                value_hash TEXT NOT NULL,
-                raw_value TEXT NOT NULL,
-                value_type TEXT NOT NULL,
-                numeric_value REAL,
-                PRIMARY KEY (doc_id, field_path, value_hash)
-            );
-
-            CREATE INDEX IF NOT EXISTS idx_doc_index_entries_field_hash
-            ON doc_index_entries (field_path, value_hash);
-
-            CREATE INDEX IF NOT EXISTS idx_doc_index_entries_field_numeric
-            ON doc_index_entries (field_path, numeric_value);
-        `)
-        this.databases.set(collection, db)
-        return db
-    }
-
-    private closeDatabase(collection: string) {
-        const db = this.databases.get(collection)
-        if (db) {
-            db.close()
-            this.databases.delete(collection)
+        try {
+            const raw = JSON.parse(await this.storage.read(this.indexFilePath(collection))) as
+                | StoredCollectionIndex
+                | undefined
+
+            if (raw?.version === 1 && raw.docs) {
+                for (const [docId, entries] of Object.entries(raw.docs) as Array<
+                    [_ttid, StoredIndexEntry[]]
+                >) {
+                    cache.docs.set(docId, entries)
+                    for (const entry of entries) this.addEntryToCache(cache, docId, entry)
+                }
+            }
+        } catch (err) {
+            if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
         }
+
+        this.indexes.set(collection, cache)
+        return cache
     }
 
     private normalizeIndexValue(rawValue: string) {
@@ -286,7 +268,30 @@ export class S3FilesEngine {
         await this.storage.mkdir(this.collectionRoot(collection))
         await this.storage.mkdir(this.metaRoot(collection))
         await this.storage.mkdir(this.docsRoot(collection))
-        this.database(collection)
+        await this.storage.mkdir(this.indexesRoot(collection))
+        await this.loadIndexCache(collection)
+    }
+
+    private async withCollectionWriteLock<T>(
+        collection: string,
+        action: () => Promise<T>
+    ): Promise<T> {
+        const previous = this.writeLanes.get(collection) ?? Promise.resolve()
+        let release!: () => void
+        const current = new Promise<void>((resolve) => {
+            release = resolve
+        })
+        const lane = previous.then(() => current)
+        this.writeLanes.set(collection, lane)
+
+        await previous
+
+        try {
+            return await action()
+        } finally {
+            release()
+            if (this.writeLanes.get(collection) === lane) this.writeLanes.delete(collection)
+        }
     }
 
     async createCollection(collection: string) {
@@ -294,7 +299,7 @@ export class S3FilesEngine {
     }
 
     async dropCollection(collection: string) {
-        this.closeDatabase(collection)
+        this.indexes.delete(collection)
         await this.storage.rmdir(this.collectionRoot(collection))
     }
 
@@ -392,198 +397,21 @@ export class S3FilesEngine {
         return value
     }
 
-    private async readStoredDoc<T extends Record<string, any>>(
-        collection: string,
-        docId: _ttid
-    ): Promise<StoredDoc<T> | null> {
-        const target = this.docPath(collection, docId)
-
-        try {
-            const raw = JSON.parse(await this.storage.read(target)) as StoredDoc<T>
-            raw.data = await this.decodeEncrypted(collection, raw.data)
-            return raw
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null
-            throw err
-        }
-    }
-
-    private async writeStoredDoc<T extends Record<string, any>>(
-        collection: string,
-        docId: _ttid,
-        data: T
-    ) {
-        await this.ensureCollection(collection)
-        const encoded = await this.encodeEncrypted(collection, data)
-        const { createdAt, updatedAt } = TTID.decodeTime(docId)
-        const target = this.docPath(collection, docId)
-        const record: StoredDoc<T> = {
-            id: docId,
-            createdAt,
-            updatedAt: updatedAt ?? createdAt,
-            data: encoded
-        }
-        await this.storage.write(target, JSON.stringify(record))
-    }
-
-    private async removeStoredDoc(collection: string, docId: _ttid) {
-        await this.storage.delete(this.docPath(collection, docId))
-    }
-
-    private async listDocIds(collection: string) {
-        const files = await this.storage.list(this.docsRoot(collection))
-        return files
-            .filter((file) => file.endsWith('.json'))
-            .map((file) => path.basename(file, '.json'))
-            .filter((key) => TTID.isTTID(key)) as _ttid[]
-    }
-
-    private getValueByPath(target: Record<string, any>, fieldPath: string) {
-        return fieldPath
-            .split('.')
-            .reduce<any>(
-                (acc, key) => (acc === undefined || acc === null ? undefined : acc[key]),
-                target
-            )
-    }
-
-    private matchesTimestamp(docId: _ttid, query?: _storeQuery<Record<string, any>>) {
-        if (!query?.$created && !query?.$updated) return true
-        const { createdAt, updatedAt } = TTID.decodeTime(docId)
-        const timestamps = { createdAt, updatedAt: updatedAt ?? createdAt }
-
-        const match = (value: number, range?: _timestamp) => {
-            if (!range) return true
-            if (range.$gt !== undefined && !(value > range.$gt)) return false
-            if (range.$gte !== undefined && !(value >= range.$gte)) return false
-            if (range.$lt !== undefined && !(value < range.$lt)) return false
-            if (range.$lte !== undefined && !(value <= range.$lte)) return false
-            return true
-        }
-
-        return (
-            match(timestamps.createdAt, query.$created) &&
-            match(timestamps.updatedAt, query.$updated)
-        )
-    }
-
-    private likeToRegex(pattern: string) {
-        const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replaceAll('%', '.*')
-        return new RegExp(`^${escaped}$`)
-    }
-
-    private matchesOperand(value: unknown, operand: _operand) {
-        if (operand.$eq !== undefined && value != operand.$eq) return false
-        if (operand.$ne !== undefined && value == operand.$ne) return false
-        if (operand.$gt !== undefined && !(Number(value) > operand.$gt)) return false
-        if (operand.$gte !== undefined && !(Number(value) >= operand.$gte)) return false
-        if (operand.$lt !== undefined && !(Number(value) < operand.$lt)) return false
-        if (operand.$lte !== undefined && !(Number(value) <= operand.$lte)) return false
-        if (
-            operand.$like !== undefined &&
-            (typeof value !== 'string' || !this.likeToRegex(operand.$like).test(value))
-        )
-            return false
-        if (operand.$contains !== undefined) {
-            if (!Array.isArray(value) || !value.some((item) => item == operand.$contains))
-                return false
-        }
-        return true
-    }
-
-    private matchesQuery<T extends Record<string, any>>(
-        docId: _ttid,
-        doc: T,
-        query?: _storeQuery<T>
-    ) {
-        if (!this.matchesTimestamp(docId, query as _storeQuery<Record<string, any>> | undefined))
-            return false
-        if (!query?.$ops || query.$ops.length === 0) return true
-
-        return query.$ops.some((operation) => {
-            for (const field in operation) {
-                const value = this.getValueByPath(doc, field)
-                if (!this.matchesOperand(value, operation[field as keyof T]!)) return false
-            }
-            return true
-        })
-    }
-
-    private selectValues<T extends Record<string, any>>(selection: Array<keyof T>, data: T) {
-        const copy = { ...data }
-        for (const field in copy) {
-            if (!selection.includes(field as keyof T)) delete copy[field]
-        }
-        return copy
-    }
-
-    private renameFields<T extends Record<string, any>>(
-        rename: Record<keyof Partial<T>, string>,
-        data: T
-    ) {
-        const copy = { ...data }
-        for (const field in copy) {
-            if (rename[field]) {
-                copy[rename[field]] = copy[field]
-                delete copy[field]
-            }
-        }
-        return copy
-    }
-
-    private processDoc<T extends Record<string, any>>(
-        doc: FyloRecord<T>,
-        query?: _storeQuery<T>
-    ): S3FilesQueryResult<T> | undefined {
-        if (Object.keys(doc).length === 0) return
-
-        const next = { ...doc }
-
-        for (let [_id, data] of Object.entries(next)) {
-            if (query?.$select?.length)
-                data = this.selectValues(query.$select as Array<keyof T>, data)
-            if (query?.$rename) data = this.renameFields(query.$rename, data)
-            next[_id as _ttid] = data as T
-        }
-
-        if (query?.$groupby) {
-            const docGroup: Record<string, Record<string, Partial<T>>> = {}
-            for (const [id, data] of Object.entries(next)) {
-                const groupValue = data[query.$groupby] as string
-                if (groupValue) {
-                    const groupData = { ...data }
-                    delete groupData[query.$groupby]
-                    docGroup[groupValue] = { [id]: groupData as Partial<T> }
-                }
-            }
-
-            if (query.$onlyIds) {
-                const groupedIds: Record<string, _ttid[]> = {}
-                for (const group in docGroup)
-                    groupedIds[group] = Object.keys(docGroup[group]) as _ttid[]
-                return groupedIds
-            }
-
-            return docGroup
-        }
-
-        if (query?.$onlyIds) return Object.keys(next).shift() as _ttid
-
-        return next
-    }
-
     private async docResults<T extends Record<string, any>>(
         collection: string,
         query?: _storeQuery<T>
     ) {
-        const ids = await this.listDocIds(collection)
+        const candidateIds = await this.queryEngine.candidateDocIdsForQuery(collection, query)
+        const ids = candidateIds
+            ? Array.from(candidateIds)
+            : await this.documents.listDocIds(collection)
         const limit = query?.$limit
         const results: Array<FyloRecord<T>> = []
 
         for (const id of ids) {
-            const stored = await this.readStoredDoc<T>(collection, id)
+            const stored = await this.documents.readStoredDoc<T>(collection, id)
             if (!stored) continue
-            if (!this.matchesQuery(id, stored.data, query)) continue
+            if (!this.queryEngine.matchesQuery(id, stored.data, query)) continue
             results.push({ [id]: stored.data } as FyloRecord<T>)
             if (limit && results.length >= limit) break
         }
@@ -597,74 +425,74 @@ export class S3FilesEngine {
         doc: T
     ) {
         const keys = await Dir.extractKeys(collection, docId, doc)
-        const db = this.database(collection)
-        const insert = db.query(`
-            INSERT OR REPLACE INTO doc_index_entries
-            (doc_id, field_path, value_hash, raw_value, value_type, numeric_value)
-            VALUES (?, ?, ?, ?, ?, ?)
-        `)
-
-        const transaction = db.transaction((logicalKeys: string[]) => {
-            for (const logicalKey of logicalKeys) {
-                const segments = logicalKey.split('/')
-                const fieldPath = segments.slice(0, -2).join('/')
-                const rawValue = segments.at(-2) ?? ''
-                const normalized = this.normalizeIndexValue(rawValue)
-                insert.run(
-                    docId,
-                    fieldPath,
-                    normalized.valueHash,
-                    normalized.rawValue,
-                    normalized.valueType,
-                    normalized.numericValue
-                )
-            }
+        const cache = await this.loadIndexCache(collection)
+        const entries = keys.indexes.map((logicalKey) => {
+            const segments = logicalKey.split('/')
+            const fieldPath = segments.slice(0, -2).join('/')
+            const rawValue = segments.at(-2) ?? ''
+            const normalized = this.normalizeIndexValue(rawValue)
+
+            return {
+                fieldPath,
+                rawValue: normalized.rawValue,
+                valueHash: normalized.valueHash,
+                valueType: normalized.valueType,
+                numericValue: normalized.numericValue
+            } satisfies StoredIndexEntry
         })
 
-        transaction(keys.indexes)
+        const existingEntries = cache.docs.get(docId)
+        if (existingEntries) {
+            for (const entry of existingEntries) this.deleteEntryFromCache(cache, docId, entry)
+        }
+
+        cache.docs.set(docId, entries)
+        for (const entry of entries) this.addEntryToCache(cache, docId, entry)
+        await this.writeIndexFile(collection, cache)
     }
 
     private async removeIndexes<T extends Record<string, any>>(
         collection: string,
         docId: _ttid,
-        doc: T
+        _doc: T
     ) {
-        const keys = await Dir.extractKeys(collection, docId, doc)
-        const db = this.database(collection)
-        const remove = db.query(`
-            DELETE FROM doc_index_entries
-            WHERE doc_id = ? AND field_path = ? AND value_hash = ?
-        `)
-
-        const transaction = db.transaction((logicalKeys: string[]) => {
-            for (const logicalKey of logicalKeys) {
-                const segments = logicalKey.split('/')
-                const fieldPath = segments.slice(0, -2).join('/')
-                const rawValue = segments.at(-2) ?? ''
-                remove.run(docId, fieldPath, this.hash(rawValue))
-            }
-        })
-
-        transaction(keys.indexes)
+        const cache = await this.loadIndexCache(collection)
+        const existingEntries = cache.docs.get(docId) ?? []
+        for (const entry of existingEntries) this.deleteEntryFromCache(cache, docId, entry)
+        cache.docs.delete(docId)
+        await this.writeIndexFile(collection, cache)
     }
 
     async putDocument<T extends Record<string, any>>(collection: string, docId: _ttid, doc: T) {
-        const owner = Bun.randomUUIDv7()
-        if (!(await this.locks.acquire(collection, docId, owner)))
-            throw new Error(`Unable to acquire filesystem lock for ${docId}`)
+        await this.withCollectionWriteLock(collection, async () => {
+            const owner = Bun.randomUUIDv7()
+            if (!(await this.locks.acquire(collection, docId, owner)))
+                throw new Error(`Unable to acquire filesystem lock for ${docId}`)
 
-        try {
-            await this.writeStoredDoc(collection, docId, doc)
-            await this.rebuildIndexes(collection, docId, doc)
-            await this.events.publish(collection, {
-                ts: Date.now(),
-                action: 'insert',
-                id: docId,
-                doc
-            })
-        } finally {
-            await this.locks.release(collection, docId, owner)
-        }
+            const targetPath = this.docPath(collection, docId)
+
+            try {
+                await this.documents.writeStoredDoc(collection, docId, doc)
+                await this.rebuildIndexes(collection, docId, doc)
+                await this.events.publish(collection, {
+                    ts: Date.now(),
+                    action: 'insert',
+                    id: docId,
+                    doc
+                })
+                await this.runSyncTask(collection, docId, 'put', targetPath, async () => {
+                    await this.syncWrite({
+                        operation: 'put',
+                        collection,
+                        docId,
+                        path: targetPath,
+                        data: doc
+                    })
+                })
+            } finally {
+                await this.locks.release(collection, docId, owner)
+            }
+        })
     }
 
     async patchDocument<T extends Record<string, any>>(
@@ -674,56 +502,90 @@ export class S3FilesEngine {
         patch: Partial<T>,
         oldDoc?: T
     ) {
-        const owner = Bun.randomUUIDv7()
-        if (!(await this.locks.acquire(collection, oldId, owner)))
-            throw new Error(`Unable to acquire filesystem lock for ${oldId}`)
+        return await this.withCollectionWriteLock(collection, async () => {
+            const owner = Bun.randomUUIDv7()
+            if (!(await this.locks.acquire(collection, oldId, owner)))
+                throw new Error(`Unable to acquire filesystem lock for ${oldId}`)
 
-        try {
-            const existing = oldDoc ?? (await this.readStoredDoc<T>(collection, oldId))?.data
-            if (!existing) return oldId
-
-            const nextDoc = { ...existing, ...patch } as T
-            await this.removeIndexes(collection, oldId, existing)
-            await this.removeStoredDoc(collection, oldId)
-            await this.events.publish(collection, {
-                ts: Date.now(),
-                action: 'delete',
-                id: oldId,
-                doc: existing
-            })
-            await this.writeStoredDoc(collection, newId, nextDoc)
-            await this.rebuildIndexes(collection, newId, nextDoc)
-            await this.events.publish(collection, {
-                ts: Date.now(),
-                action: 'insert',
-                id: newId,
-                doc: nextDoc
-            })
-            return newId
-        } finally {
-            await this.locks.release(collection, oldId, owner)
-        }
+            const oldPath = this.docPath(collection, oldId)
+
+            try {
+                const existing =
+                    oldDoc ?? (await this.documents.readStoredDoc<T>(collection, oldId))?.data
+                if (!existing) return oldId
+
+                const nextDoc = { ...existing, ...patch } as T
+                const newPath = this.docPath(collection, newId)
+                await this.removeIndexes(collection, oldId, existing)
+                await this.documents.removeStoredDoc(collection, oldId)
+                await this.events.publish(collection, {
+                    ts: Date.now(),
+                    action: 'delete',
+                    id: oldId,
+                    doc: existing
+                })
+                await this.documents.writeStoredDoc(collection, newId, nextDoc)
+                await this.rebuildIndexes(collection, newId, nextDoc)
+                await this.events.publish(collection, {
+                    ts: Date.now(),
+                    action: 'insert',
+                    id: newId,
+                    doc: nextDoc
+                })
+                await this.runSyncTask(collection, newId, 'patch', newPath, async () => {
+                    await this.syncDelete({
+                        operation: 'patch',
+                        collection,
+                        docId: oldId,
+                        path: oldPath
+                    })
+                    await this.syncWrite({
+                        operation: 'patch',
+                        collection,
+                        docId: newId,
+                        previousDocId: oldId,
+                        path: newPath,
+                        data: nextDoc
+                    })
+                })
+                return newId
+            } finally {
+                await this.locks.release(collection, oldId, owner)
+            }
+        })
     }
 
     async deleteDocument<T extends Record<string, any>>(collection: string, docId: _ttid) {
-        const owner = Bun.randomUUIDv7()
-        if (!(await this.locks.acquire(collection, docId, owner)))
-            throw new Error(`Unable to acquire filesystem lock for ${docId}`)
+        await this.withCollectionWriteLock(collection, async () => {
+            const owner = Bun.randomUUIDv7()
+            if (!(await this.locks.acquire(collection, docId, owner)))
+                throw new Error(`Unable to acquire filesystem lock for ${docId}`)
 
-        try {
-            const existing = await this.readStoredDoc<T>(collection, docId)
-            if (!existing) return
-            await this.removeIndexes(collection, docId, existing.data)
-            await this.removeStoredDoc(collection, docId)
-            await this.events.publish(collection, {
-                ts: Date.now(),
-                action: 'delete',
-                id: docId,
-                doc: existing.data
-            })
-        } finally {
-            await this.locks.release(collection, docId, owner)
-        }
+            const targetPath = this.docPath(collection, docId)
+
+            try {
+                const existing = await this.documents.readStoredDoc<T>(collection, docId)
+                if (!existing) return
+                await this.removeIndexes(collection, docId, existing.data)
+                await this.documents.removeStoredDoc(collection, docId)
+                await this.events.publish(collection, {
+                    ts: Date.now(),
+                    action: 'delete',
+                    id: docId,
+                    doc: existing.data
+                })
+                await this.runSyncTask(collection, docId, 'delete', targetPath, async () => {
+                    await this.syncDelete({
+                        operation: 'delete',
+                        collection,
+                        docId,
+                        path: targetPath
+                    })
+                })
+            } finally {
+                await this.locks.release(collection, docId, owner)
+            }
+        })
     }
 
     getDoc<T extends Record<string, any>>(
@@ -744,7 +606,7 @@ export class S3FilesEngine {
                 }
             },
             async once() {
-                const stored = await engine.readStoredDoc<T>(collection, docId)
+                const stored = await engine.documents.readStoredDoc<T>(collection, docId)
                 return stored ? ({ [docId]: stored.data } as FyloRecord<T>) : {}
             },
             async *onDelete() {
@@ -761,7 +623,7 @@ export class S3FilesEngine {
         const collectDocs = async function* () {
             const docs = await engine.docResults(collection, query)
             for (const doc of docs) {
-                const result = engine.processDoc(doc, query)
+                const result = engine.queryEngine.processDoc(doc, query)
                 if (result !== undefined) yield result
             }
         }
@@ -772,8 +634,8 @@ export class S3FilesEngine {
 
                 for await (const event of engine.events.listen(collection)) {
                     if (event.action !== 'insert' || !event.doc) continue
-                    if (!engine.matchesQuery(event.id, event.doc as T, query)) continue
-                    const processed = engine.processDoc(
+                    if (!engine.queryEngine.matchesQuery(event.id, event.doc as T, query)) continue
+                    const processed = engine.queryEngine.processDoc(
                         { [event.id]: event.doc as T } as FyloRecord<T>,
                         query
                     )
@@ -786,7 +648,7 @@ export class S3FilesEngine {
             async *onDelete() {
                 for await (const event of engine.events.listen(collection)) {
                     if (event.action !== 'delete' || !event.doc) continue
-                    if (!engine.matchesQuery(event.id, event.doc as T, query)) continue
+                    if (!engine.queryEngine.matchesQuery(event.id, event.doc as T, query)) continue
                     yield event.id
                 }
             }
@@ -794,9 +656,9 @@ export class S3FilesEngine {
     }
 
     async *exportBulkData<T extends Record<string, any>>(collection: string) {
-        const ids = await this.listDocIds(collection)
+        const ids = await this.documents.listDocIds(collection)
         for (const id of ids) {
-            const stored = await this.readStoredDoc<T>(collection, id)
+            const stored = await this.documents.readStoredDoc<T>(collection, id)
             if (stored) yield stored.data
         }
     }
@@ -829,11 +691,11 @@ export class S3FilesEngine {
                     for (const opKey of Object.keys(compareMap) as Array<keyof typeof compareMap>) {
                         const rightField = operand[opKey]
                         if (!rightField) continue
-                        const leftValue = this.getValueByPath(
+                        const leftValue = this.queryEngine.getValueByPath(
                             leftData as Record<string, any>,
                             String(field)
                         )
-                        const rightValue = this.getValueByPath(
+                        const rightValue = this.queryEngine.getValueByPath(
                             rightData as Record<string, any>,
                             String(rightField)
                         )
@@ -858,6 +720,25 @@ export class S3FilesEngine {
                         break
                 }
 
+                let projected = docs[`${leftId}, ${rightId}`] as Record<string, any>
+                if (join.$select?.length) {
+                    projected = this.queryEngine.selectValues(
+                        join.$select as Array<keyof typeof projected>,
+                        projected
+                    )
+                }
+                if (join.$rename) {
+                    projected = this.queryEngine.renameFields(
+                        join.$rename as Record<string, string>,
+                        projected
+                    )
+                }
+                docs[`${leftId}, ${rightId}`] = projected as
+                    | T
+                    | U
+                    | (T & U)
+                    | (Partial<T> & Partial<U>)
+
                 if (join.$limit && Object.keys(docs).length >= join.$limit) break
             }