@delma/fylo 1.1.1 → 2.0.0

package/src/index.ts

@@ -1,21 +1,22 @@
 /* eslint-disable @typescript-eslint/explicit-function-return-type */
 import { Query } from './core/query'
 import { Parser } from './core/parser'
-import { Dir } from "./core/directory";
-import TTID from '@delma/ttid';
-import Gen from "@delma/chex"
-import { Walker } from './core/walker';
-import { S3 } from "./adapters/s3"
-import { Cipher } from "./adapters/cipher"
-import { Redis } from "./adapters/redis"
+import { Dir } from './core/directory'
+import TTID from '@delma/ttid'
+import Gen from '@delma/chex'
+import { Walker } from './core/walker'
+import { S3 } from './adapters/s3'
+import { Cipher } from './adapters/cipher'
+import { Redis } from './adapters/redis'
 import { WriteQueue } from './core/write-queue'
+import { S3FilesEngine } from './engines/s3-files'
 import './core/format'
 import './core/extensions'
 import type { QueueStats, QueuedWriteResult, WriteJob } from './types/write-queue'
 import type { StreamJobEntry } from './types/write-queue'
+import type { FyloStorageEngineKind } from './engines/types'
 
 export default class Fylo {
-
     private static LOGGING = process.env.LOGGING
 
     private static MAX_CPUS = navigator.hardwareConcurrency
@@ -37,62 +38,98 @@ export default class Fylo {
 
     private static readonly WRITE_RETRY_BASE_MS = Number(process.env.FYLO_WRITE_RETRY_BASE_MS ?? 10)
 
-    private dir: Dir;
+    private readonly dir: Dir
+
+    private readonly engineKind: FyloStorageEngineKind
 
-    constructor() {
+    private readonly s3Files?: S3FilesEngine
+
+    constructor(options: { engine?: FyloStorageEngineKind; s3FilesRoot?: string } = {}) {
         this.dir = new Dir()
+        this.engineKind = options.engine ?? Fylo.defaultEngineKind()
+        if (this.engineKind === 's3-files') this.s3Files = new S3FilesEngine(options.s3FilesRoot)
     }
 
     private static get queueRedis(): Redis {
-        if(!Fylo._queueRedis) Fylo._queueRedis = new Redis()
+        if (!Fylo._queueRedis) Fylo._queueRedis = new Redis()
         return Fylo._queueRedis
     }
 
+    private static defaultEngineKind(): FyloStorageEngineKind {
+        return process.env.FYLO_STORAGE_ENGINE === 's3-files' ? 's3-files' : 'legacy-s3'
+    }
+
+    private static get defaultS3Files(): S3FilesEngine {
+        return new S3FilesEngine(process.env.FYLO_S3FILES_ROOT)
+    }
+
+    private assertS3FilesEngine(): S3FilesEngine {
+        if (!this.s3Files) throw new Error('S3 Files engine is not configured')
+        return this.s3Files
+    }
+
+    private ensureLegacyQueue(feature: string): Redis {
+        if (this.engineKind === 's3-files')
+            throw new Error(`${feature} is not supported in s3-files engine`)
+        return Fylo.queueRedis
+    }
+
     /**
      * Executes a SQL query and returns the results.
      * @param SQL The SQL query to execute.
      * @returns The results of the query.
      */
-    async executeSQL<T extends Record<string, any>, U extends Record<string, any> = Record<string, unknown>>(SQL: string) {
-        
+    async executeSQL<
+        T extends Record<string, any>,
+        U extends Record<string, any> = Record<string, unknown>
+    >(SQL: string) {
         const op = SQL.match(/^(SELECT|INSERT|UPDATE|DELETE|CREATE|DROP)/i)
 
-        if(!op) throw new Error("Missing SQL Operation")
+        if (!op) throw new Error('Missing SQL Operation')
 
-        switch(op.shift()) {
-            case "CREATE":
-                return await Fylo.createCollection((Parser.parse(SQL) as _storeDelete<T>).$collection!)
-            case "DROP":
-                return await Fylo.dropCollection((Parser.parse(SQL) as _storeDelete<T>).$collection!)
-            case "SELECT":
+        switch (op.shift()) {
+            case 'CREATE':
+                return await this.createCollection(
+                    (Parser.parse(SQL) as _storeDelete<T>).$collection!
+                )
+            case 'DROP':
+                return await this.dropCollection(
+                    (Parser.parse(SQL) as _storeDelete<T>).$collection!
+                )
+            case 'SELECT':
                 const query = Parser.parse<T>(SQL) as _storeQuery<T>
-                if(SQL.includes('JOIN')) return await Fylo.joinDocs(query as _join<T, U>)
+                if (SQL.includes('JOIN')) return await this.joinDocs(query as _join<T, U>)
                 const selCol = (query as _storeQuery<T>).$collection
                 delete (query as _storeQuery<T>).$collection
-                let docs: Record<string, unknown> | Array<_ttid> = query.$onlyIds ? new Array<_ttid> : {}
-                for await (const data of Fylo.findDocs(selCol! as string, query as _storeQuery<T>).collect()) {
-                    if(typeof data === 'object') {
+                let docs: Record<string, unknown> | Array<_ttid> = query.$onlyIds
+                    ? new Array<_ttid>()
+                    : {}
+                for await (const data of this.findDocs(
+                    selCol! as string,
+                    query as _storeQuery<T>
+                ).collect()) {
+                    if (typeof data === 'object') {
                         docs = Object.appendGroup(docs, data)
                     } else (docs as Array<_ttid>).push(data as _ttid)
                 }
                 return docs
-            case "INSERT":
+            case 'INSERT':
                 const insert = Parser.parse<T>(SQL) as _storeInsert<T>
                 const insCol = insert.$collection
                 delete insert.$collection
                 return await this.putData(insCol!, insert.$values)
-            case "UPDATE":
+            case 'UPDATE':
                 const update = Parser.parse<T>(SQL) as _storeUpdate<T>
                 const updateCol = update.$collection
                 delete update.$collection
                 return await this.patchDocs(updateCol!, update)
-            case "DELETE":
+            case 'DELETE':
                 const del = Parser.parse<T>(SQL) as _storeDelete<T>
                 const delCol = del.$collection
                 delete del.$collection
                 return await this.delDocs(delCol!, del)
             default:
-                throw new Error("Invalid Operation")
+                throw new Error('Invalid Operation')
         }
     }
 
@@ -101,7 +138,10 @@ export default class Fylo {
      * @param collection The name of the collection.
      */
     static async createCollection(collection: string) {
-
+        if (Fylo.defaultEngineKind() === 's3-files') {
+            await Fylo.defaultS3Files.createCollection(collection)
+            return
+        }
         await S3.createBucket(collection)
     }
 
@@ -110,10 +150,25 @@ export default class Fylo {
      * @param collection The name of the collection.
      */
     static async dropCollection(collection: string) {
-
+        if (Fylo.defaultEngineKind() === 's3-files') {
+            await Fylo.defaultS3Files.dropCollection(collection)
+            return
+        }
         await S3.deleteBucket(collection)
     }
 
+    async createCollection(collection: string) {
+        if (this.engineKind === 's3-files')
+            return await this.assertS3FilesEngine().createCollection(collection)
+        return await Fylo.createCollection(collection)
+    }
+
+    async dropCollection(collection: string) {
+        if (this.engineKind === 's3-files')
+            return await this.assertS3FilesEngine().dropCollection(collection)
+        return await Fylo.dropCollection(collection)
+    }
+
     /**
      * Loads encrypted field config from a collection's JSON schema if not already loaded.
      * Reads the `$encrypted` array from the schema and registers fields with Cipher.
@@ -133,8 +188,12 @@ export default class Fylo {
             if (Array.isArray(encrypted) && encrypted.length > 0) {
                 if (!Cipher.isConfigured()) {
                     const secret = process.env.ENCRYPTION_KEY
-                    if (!secret) throw new Error('Schema declares $encrypted fields but ENCRYPTION_KEY env var is not set')
-                    if (secret.length < 32) throw new Error('ENCRYPTION_KEY must be at least 32 characters long')
+                    if (!secret)
+                        throw new Error(
+                            'Schema declares $encrypted fields but ENCRYPTION_KEY env var is not set'
+                        )
+                    if (secret.length < 32)
+                        throw new Error('ENCRYPTION_KEY must be at least 32 characters long')
                     await Cipher.configure(secret)
                 }
                 Cipher.registerFields(collection, encrypted as string[])
@@ -144,47 +203,68 @@ export default class Fylo {
         }
     }
 
+    getDoc<T extends Record<string, any>>(collection: string, _id: _ttid, onlyId: boolean = false) {
+        if (this.engineKind === 's3-files')
+            return this.assertS3FilesEngine().getDoc<T>(collection, _id, onlyId)
+        return Fylo.getDoc<T>(collection, _id, onlyId)
+    }
+
+    findDocs<T extends Record<string, any>>(collection: string, query?: _storeQuery<T>) {
+        if (this.engineKind === 's3-files')
+            return this.assertS3FilesEngine().findDocs<T>(collection, query)
+        return Fylo.findDocs<T>(collection, query)
+    }
+
+    async joinDocs<T extends Record<string, any>, U extends Record<string, any>>(
+        join: _join<T, U>
+    ) {
+        if (this.engineKind === 's3-files') return await this.assertS3FilesEngine().joinDocs(join)
+        return await Fylo.joinDocs(join)
+    }
+
     /**
      * Rolls back all transcations in current instance
      * @deprecated Prefer queued write recovery, retries, dead letters, or compensating writes.
      */
     async rollback() {
-        if(!Fylo.rollbackWarningShown) {
+        if (this.engineKind === 's3-files') return
+        if (!Fylo.rollbackWarningShown) {
             Fylo.rollbackWarningShown = true
-            console.warn('[FYLO] rollback() is deprecated for queued-write flows. Prefer job recovery, dead letters, or compensating writes.')
+            console.warn(
+                '[FYLO] rollback() is deprecated for queued-write flows. Prefer job recovery, dead letters, or compensating writes.'
+            )
         }
         await this.dir.executeRollback()
     }
 
     async getJobStatus(jobId: string) {
-        return await Fylo.queueRedis.getJob(jobId)
+        return await this.ensureLegacyQueue('getJobStatus').getJob(jobId)
     }
 
     async getDocStatus(collection: string, docId: _ttid) {
-        return await Fylo.queueRedis.getDocStatus(collection, docId)
+        return await this.ensureLegacyQueue('getDocStatus').getDocStatus(collection, docId)
     }
 
     async getDeadLetters(count: number = 10) {
-        return await Fylo.queueRedis.readDeadLetters(count)
+        return await this.ensureLegacyQueue('getDeadLetters').readDeadLetters(count)
     }
 
     async getQueueStats(): Promise<QueueStats> {
-        return await Fylo.queueRedis.getQueueStats()
+        return await this.ensureLegacyQueue('getQueueStats').getQueueStats()
     }
 
     async replayDeadLetter(streamId: string) {
-        return await Fylo.queueRedis.replayDeadLetter(streamId)
+        return await this.ensureLegacyQueue('replayDeadLetter').replayDeadLetter(streamId)
     }
 
     private async waitForJob(jobId: string, timeoutMs: number = 5_000, intervalMs: number = 50) {
-
         const start = Date.now()
 
         do {
             const job = await this.getJobStatus(jobId)
-            if(job && (job.status === 'committed' || job.status === 'failed')) return job
+            if (job && (job.status === 'committed' || job.status === 'failed')) return job
             await Bun.sleep(intervalMs)
-        } while(Date.now() - start < timeoutMs)
+        } while (Date.now() - start < timeoutMs)
 
         throw new Error(`Timed out waiting for job ${jobId}`)
     }
@@ -199,16 +279,20 @@ export default class Fylo {
         } = {},
         resolveValue?: () => Promise<T> | T
     ): Promise<T | QueuedWriteResult> {
+        if (this.engineKind === 's3-files') {
+            if (!wait) return queued
+            return resolveValue ? await resolveValue() : queued
+        }
 
-        if(!wait) return queued
+        if (!wait) return queued
 
         const processed = await this.processQueuedWrites(1)
 
-        if(processed === 0) throw new Error(`No worker available to process job ${queued.jobId}`)
+        if (processed === 0) throw new Error(`No worker available to process job ${queued.jobId}`)
 
         const job = await this.getJobStatus(queued.jobId)
 
-        if(job?.status === 'failed') {
+        if (job?.status === 'failed') {
             throw new Error(job.error ?? `Queued job ${queued.jobId} failed`)
         }
 
@@ -216,24 +300,25 @@ export default class Fylo {
     }
 
     async processQueuedWrites(count: number = 1, recover: boolean = false) {
+        this.ensureLegacyQueue('processQueuedWrites')
         const jobs = recover
             ? await Fylo.queueRedis.claimPendingJobs(Bun.randomUUIDv7(), 30_000, count)
             : await Fylo.queueRedis.readWriteJobs(Bun.randomUUIDv7(), count)
 
         let processed = 0
 
-        for(const job of jobs) {
-            if(await this.processQueuedJob(job)) processed++
+        for (const job of jobs) {
+            if (await this.processQueuedJob(job)) processed++
         }
 
         return processed
     }
 
     private async processQueuedJob({ streamId, job }: StreamJobEntry) {
-        if(job.nextAttemptAt && job.nextAttemptAt > Date.now()) return false
+        if (job.nextAttemptAt && job.nextAttemptAt > Date.now()) return false
 
         const locked = await Fylo.queueRedis.acquireDocLock(job.collection, job.docId, job.jobId)
-        if(!locked) return false
+        if (!locked) return false
 
         try {
             await Fylo.queueRedis.setJobStatus(job.jobId, 'processing', {
@@ -247,27 +332,36 @@ export default class Fylo {
             await Fylo.queueRedis.setDocStatus(job.collection, job.docId, 'committed', job.jobId)
             await Fylo.queueRedis.ackWriteJob(streamId)
             return true
-
-        } catch(err) {
+        } catch (err) {
             const attempts = job.attempts + 1
             const message = err instanceof Error ? err.message : String(err)
 
-            if(attempts >= Fylo.MAX_WRITE_ATTEMPTS) {
+            if (attempts >= Fylo.MAX_WRITE_ATTEMPTS) {
                 await Fylo.queueRedis.setJobStatus(job.jobId, 'dead-letter', {
                     error: message,
                     attempts
                 })
-                await Fylo.queueRedis.setDocStatus(job.collection, job.docId, 'dead-letter', job.jobId)
-                await Fylo.queueRedis.deadLetterWriteJob(streamId, {
-                    ...job,
-                    attempts,
-                    status: 'dead-letter',
-                    error: message
-                }, message)
+                await Fylo.queueRedis.setDocStatus(
+                    job.collection,
+                    job.docId,
+                    'dead-letter',
+                    job.jobId
+                )
+                await Fylo.queueRedis.deadLetterWriteJob(
+                    streamId,
+                    {
+                        ...job,
+                        attempts,
+                        status: 'dead-letter',
+                        error: message
+                    },
+                    message
+                )
                 return false
             }
 
-            const nextAttemptAt = Date.now() + (Fylo.WRITE_RETRY_BASE_MS * Math.max(1, 2 ** (attempts - 1)))
+            const nextAttemptAt =
+                Date.now() + Fylo.WRITE_RETRY_BASE_MS * Math.max(1, 2 ** (attempts - 1))
 
             await Fylo.queueRedis.setJobStatus(job.jobId, 'failed', {
                 error: message,
@@ -276,7 +370,6 @@ export default class Fylo {
             })
             await Fylo.queueRedis.setDocStatus(job.collection, job.docId, 'failed', job.jobId)
             return false
-
         } finally {
             await Fylo.queueRedis.releaseDocLock(job.collection, job.docId, job.jobId)
         }
@@ -288,20 +381,24 @@ export default class Fylo {
      * @param url The URL of the data to import.
      * @param limit The maximum number of documents to import.
      */
-    async importBulkData<T extends Record<string, any>>(collection: string, url: URL, limit?: number) {
-
+    async importBulkData<T extends Record<string, any>>(
+        collection: string,
+        url: URL,
+        limit?: number
+    ) {
         const res = await fetch(url)
 
-        if(!res.headers.get('content-type')?.includes('application/json')) throw new Error('Response is not JSON')
+        if (!res.headers.get('content-type')?.includes('application/json'))
+            throw new Error('Response is not JSON')
 
         let count = 0
         let batchNum = 0
 
         const flush = async (batch: T[]) => {
+            if (!batch.length) return
 
-            if(!batch.length) return
-
-            const items = limit && count + batch.length > limit ? batch.slice(0, limit - count) : batch
+            const items =
+                limit && count + batch.length > limit ? batch.slice(0, limit - count) : batch
 
             batchNum++
 
@@ -309,13 +406,15 @@ export default class Fylo {
             await this.batchPutData(collection, items)
             count += items.length
 
-            if(count % 10000 === 0) console.log("Count:", count)
+            if (count % 10000 === 0) console.log('Count:', count)
 
-            if(Fylo.LOGGING) {
+            if (Fylo.LOGGING) {
                 const bytes = JSON.stringify(items).length
                 const elapsed = Date.now() - start
                 const bytesPerSec = (bytes / (elapsed / 1000)).toFixed(2)
-                console.log(`Batch ${batchNum} of ${bytes} bytes took ${elapsed === Infinity ? 'Infinity' : elapsed}ms (${bytesPerSec} bytes/sec)`)
+                console.log(
+                    `Batch ${batchNum} of ${bytes} bytes took ${elapsed === Infinity ? 'Infinity' : elapsed}ms (${bytesPerSec} bytes/sec)`
+                )
             }
         }
 
@@ -332,11 +431,10 @@ export default class Fylo {
         let pending = new Uint8Array(0)
         let batch: T[] = []
 
-        for await (const chunk of res.body as AsyncIterable<Uint8Array>) {
-
-            if(isJsonArray === null) isJsonArray = chunk[0] === 0x5b
+        for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
+            if (isJsonArray === null) isJsonArray = chunk[0] === 0x5b
 
-            if(isJsonArray) {
+            if (isJsonArray) {
                 jsonArrayChunks.push(chunk)
                 jsonArrayLength += chunk.length
                 continue
@@ -351,40 +449,40 @@ export default class Fylo {
             const { values, read } = Bun.JSONL.parseChunk(merged)
             pending = merged.subarray(read)
 
-            for(const item of values) {
+            for (const item of values) {
                 batch.push(item as T)
-                if(batch.length === Fylo.MAX_CPUS) {
+                if (batch.length === Fylo.MAX_CPUS) {
                     await flush(batch)
                     batch = []
-                    if(limit && count >= limit) return count
+                    if (limit && count >= limit) return count
                 }
             }
         }
 
-        if(isJsonArray) {
-
+        if (isJsonArray) {
             // Reassemble buffered chunks into a single Uint8Array and parse as JSON.
             const body = new Uint8Array(jsonArrayLength)
             let offset = 0
-            for(const c of jsonArrayChunks) { body.set(c, offset); offset += c.length }
+            for (const c of jsonArrayChunks) {
+                body.set(c, offset)
+                offset += c.length
+            }
 
             const data = JSON.parse(new TextDecoder().decode(body))
             const items: T[] = Array.isArray(data) ? data : [data]
 
-            for(let i = 0; i < items.length; i += Fylo.MAX_CPUS) {
-                if(limit && count >= limit) break
+            for (let i = 0; i < items.length; i += Fylo.MAX_CPUS) {
+                if (limit && count >= limit) break
                 await flush(items.slice(i, i + Fylo.MAX_CPUS))
             }
-
         } else {
-
             // Flush the in-progress batch and any final line that had no trailing newline.
-            if(pending.length > 0) {
+            if (pending.length > 0) {
                 const { values } = Bun.JSONL.parseChunk(pending)
-                for(const item of values) batch.push(item as T)
+                for (const item of values) batch.push(item as T)
             }
 
-            if(batch.length > 0) await flush(batch)
+            if (batch.length > 0) await flush(batch)
         }
 
         return count
@@ -396,31 +494,42 @@ export default class Fylo {
      * @returns The current data exported from the collection.
      */
     static async *exportBulkData<T extends Record<string, any>>(collection: string) {
+        if (Fylo.defaultEngineKind() === 's3-files') {
+            yield* Fylo.defaultS3Files.exportBulkData<T>(collection)
+            return
+        }
 
         // Kick off the first S3 list immediately so there is no idle time at the start.
-        let listPromise: Promise<Bun.S3ListObjectsResponse> | null = S3.list(collection, { delimiter: '/' })
-
-        while(listPromise !== null) {
+        let listPromise: Promise<Bun.S3ListObjectsResponse> | null = S3.list(collection, {
+            delimiter: '/'
+        })
 
+        while (listPromise !== null) {
             const data: Bun.S3ListObjectsResponse = await listPromise
 
-            if(!data.commonPrefixes?.length) break
+            if (!data.commonPrefixes?.length) break
 
             const ids = data.commonPrefixes
-                .map(item => item.prefix!.split('/')[0]!)
-                .filter(key => TTID.isTTID(key)) as _ttid[]
+                .map((item) => item.prefix!.split('/')[0]!)
+                .filter((key) => TTID.isTTID(key)) as _ttid[]
 
             // Start fetching the next page immediately — before awaiting doc reads —
             // so the S3 list round-trip overlaps with document reconstruction.
-            listPromise = data.isTruncated && data.nextContinuationToken
-                ? S3.list(collection, { delimiter: '/', continuationToken: data.nextContinuationToken })
-                : null
-
-            const results = await Promise.allSettled(ids.map(id => this.getDoc<T>(collection, id).once()))
-
-            for(const result of results) {
-                if(result.status === 'fulfilled') {
-                    for(const id in result.value) yield result.value[id]
+            listPromise =
+                data.isTruncated && data.nextContinuationToken
+                    ? S3.list(collection, {
+                          delimiter: '/',
+                          continuationToken: data.nextContinuationToken
+                      })
+                    : null
+
+            const results = await Promise.allSettled(
+                ids.map((id) => this.getDoc<T>(collection, id).once())
+            )
+
+            for (const result of results) {
+                if (result.status === 'fulfilled') {
+                    for (const id in result.value) yield result.value[id]
                 }
             }
         }
@@ -433,30 +542,39 @@ export default class Fylo {
      * @param onlyId Whether to only return the ID of the document.
      * @returns The document or the ID of the document.
      */
-    static getDoc<T extends Record<string, any>>(collection: string, _id: _ttid, onlyId: boolean = false) {
-        
-        return {
+    static getDoc<T extends Record<string, any>>(
+        collection: string,
+        _id: _ttid,
+        onlyId: boolean = false
+    ) {
+        if (Fylo.defaultEngineKind() === 's3-files') {
+            return Fylo.defaultS3Files.getDoc<T>(collection, _id, onlyId)
+        }
 
+        return {
             /**
              * Async iterator (listener) for the document.
              */
             async *[Symbol.asyncIterator]() {
-
                 const doc = await this.once()
 
-                if(Object.keys(doc).length > 0) yield doc
+                if (Object.keys(doc).length > 0) yield doc
 
                 let finished = false
 
-                const iter = Dir.searchDocs<T>(collection, `**/${_id.split('-')[0]}*`, {}, { listen: true, skip: true })
+                const iter = Dir.searchDocs<T>(
+                    collection,
+                    `**/${_id.split('-')[0]}*`,
+                    {},
+                    { listen: true, skip: true }
+                )
 
                 do {
-
                     const { value, done } = await iter.next({ count: 0 })
 
-                    if(value === undefined && !done) continue
+                    if (value === undefined && !done) continue
 
-                    if(done) {
+                    if (done) {
                         finished = true
                         break
                     }
@@ -465,26 +583,23 @@ export default class Fylo {
 
                     const keys = Object.keys(doc)
 
-                    if(onlyId && keys.length > 0) {
+                    if (onlyId && keys.length > 0) {
                         yield keys.shift()!
                         continue
-                    }
-                    else if(keys.length > 0) {
+                    } else if (keys.length > 0) {
                         yield doc
                         continue
                     }
-
-                } while(!finished)
+                } while (!finished)
             },
 
             /**
              * Gets the document once.
              */
             async once() {
-
                 const items = await Walker.getDocData(collection, _id)
 
-                if(items.length === 0) return {}
+                if (items.length === 0) return {}
 
                 const data = await Dir.reconstructData(collection, items)
 
@@ -495,25 +610,28 @@ export default class Fylo {
              * Async iterator (listener) for the document's deletion.
              */
             async *onDelete() {
-
                 let finished = false
 
-                const iter = Dir.searchDocs<T>(collection, `**/${_id.split('-')[0]}*`, {}, { listen: true, skip: true }, true)
+                const iter = Dir.searchDocs<T>(
+                    collection,
+                    `**/${_id.split('-')[0]}*`,
+                    {},
+                    { listen: true, skip: true },
+                    true
+                )
 
                 do {
-
                     const { value, done } = await iter.next({ count: 0 })
 
-                    if(value === undefined && !done) continue
+                    if (value === undefined && !done) continue
 
-                    if(done) {
+                    if (done) {
                         finished = true
                         break
                     }
 
                     yield value as _ttid
-
-                } while(!finished)
+                } while (!finished)
             }
         }
     }
@@ -525,23 +643,23 @@ export default class Fylo {
      * @returns The IDs of the documents.
      */
     async batchPutData<T extends Record<string, any>>(collection: string, batch: Array<T>) {
-
         const batches: Array<Array<T>> = []
         const ids: _ttid[] = []
 
-        if(batch.length > navigator.hardwareConcurrency) {
-
-            for(let i = 0; i < batch.length; i += navigator.hardwareConcurrency) {
+        if (batch.length > navigator.hardwareConcurrency) {
+            for (let i = 0; i < batch.length; i += navigator.hardwareConcurrency) {
                 batches.push(batch.slice(i, i + navigator.hardwareConcurrency))
             }
-
         } else batches.push(batch)
-        
-        for(const batch of batches) {
 
-            const res = await Promise.allSettled(batch.map(data => this.putData(collection, data)))
+        for (const batch of batches) {
+            const res = await Promise.allSettled(
+                batch.map((data) => this.putData(collection, data))
+            )
 
-            for(const _id of res.filter(item => item.status === 'fulfilled').map(item => item.value)) {
+            for (const _id of res
+                .filter((item) => item.status === 'fulfilled')
+                .map((item) => item.value)) {
                 ids.push(_id)
             }
         }
@@ -549,7 +667,12 @@ export default class Fylo {
         return ids
     }
 
-    async queuePutData<T extends Record<string, any>>(collection: string, data: Record<_ttid, T> | T): Promise<QueuedWriteResult> {
+    async queuePutData<T extends Record<string, any>>(
+        collection: string,
+        data: Record<_ttid, T> | T
+    ): Promise<QueuedWriteResult> {
+        if (this.engineKind === 's3-files')
+            throw new Error('queuePutData is not supported in s3-files engine')
 
         const { _id, doc } = await this.prepareInsert(collection, data)
         const job = WriteQueue.createInsertJob(collection, _id, doc)
@@ -568,10 +691,12 @@ export default class Fylo {
         newDoc: Record<_ttid, Partial<T>>,
         oldDoc: Record<_ttid, T> = {}
     ): Promise<QueuedWriteResult> {
+        if (this.engineKind === 's3-files')
+            throw new Error('queuePatchDoc is not supported in s3-files engine')
 
         const docId = Object.keys(newDoc).shift() as _ttid
 
-        if(!docId) throw new Error("this document does not contain an TTID")
+        if (!docId) throw new Error('this document does not contain an TTID')
 
         const job = WriteQueue.createUpdateJob(collection, docId, { newDoc, oldDoc })
 
@@ -585,6 +710,8 @@ export default class Fylo {
     }
 
     async queueDelDoc(collection: string, _id: _ttid): Promise<QueuedWriteResult> {
+        if (this.engineKind === 's3-files')
+            throw new Error('queueDelDoc is not supported in s3-files engine')
 
         const job = WriteQueue.createDeleteJob(collection, _id)
 
@@ -603,8 +730,10 @@ export default class Fylo {
      * @param data The document to put.
      * @returns The ID of the document.
      */
-    private static async uniqueTTID(existingId?: string): Promise<_ttid> {
-
+    private static async uniqueTTID(
+        existingId?: string,
+        claimInRedis: boolean = true
+    ): Promise<_ttid> {
         // Serialize TTID generation so concurrent callers (e.g. batchPutData)
         // never invoke TTID.generate() at the same sub-millisecond instant.
         let _id!: _ttid
@@ -612,118 +741,198 @@ export default class Fylo {
         Fylo.ttidLock = prev.then(async () => {
             _id = existingId ? TTID.generate(existingId) : TTID.generate()
             // Claim in Redis for cross-process uniqueness (no-op if Redis unavailable)
-            if(!(await Dir.claimTTID(_id))) throw new Error('TTID collision — retry')
+            if (claimInRedis && !(await Dir.claimTTID(_id)))
+                throw new Error('TTID collision — retry')
         })
         await Fylo.ttidLock
 
         return _id
     }
 
-    private async prepareInsert<T extends Record<string, any>>(collection: string, data: Record<_ttid, T> | T) {
-
+    private async prepareInsert<T extends Record<string, any>>(
+        collection: string,
+        data: Record<_ttid, T> | T
+    ) {
         await Fylo.loadEncryption(collection)
 
         const currId = Object.keys(data).shift()!
-        const _id = TTID.isTTID(currId) ? await Fylo.uniqueTTID(currId) : await Fylo.uniqueTTID()
+        const claimInRedis = this.engineKind !== 's3-files'
+        const _id = TTID.isTTID(currId)
+            ? await Fylo.uniqueTTID(currId, claimInRedis)
+            : await Fylo.uniqueTTID(undefined, claimInRedis)
 
-        let doc = TTID.isTTID(currId) ? Object.values(data).shift() as T : data as T
+        let doc = TTID.isTTID(currId) ? (Object.values(data).shift() as T) : (data as T)
 
-        if(Fylo.STRICT) doc = await Gen.validateData(collection, doc) as T
+        if (Fylo.STRICT) doc = (await Gen.validateData(collection, doc)) as T
 
         return { _id, doc }
     }
 
-    private async executePutDataDirect<T extends Record<string, any>>(collection: string, _id: _ttid, doc: T) {
+    private async executePutDataDirect<T extends Record<string, any>>(
+        collection: string,
+        _id: _ttid,
+        doc: T
+    ) {
+        if (this.engineKind === 's3-files') {
+            await this.assertS3FilesEngine().putDocument(collection, _id, doc)
+            return _id
+        }
 
         const keys = await Dir.extractKeys(collection, _id, doc)
 
-        const results = await Promise.allSettled(keys.data.map((item, i) => this.dir.putKeys(collection, { dataKey: item, indexKey: keys.indexes[i] })))
+        const results = await Promise.allSettled(
+            keys.data.map((item, i) =>
+                this.dir.putKeys(collection, { dataKey: item, indexKey: keys.indexes[i] })
+            )
+        )
 
-        if(results.some(res => res.status === "rejected")) {
+        if (results.some((res) => res.status === 'rejected')) {
             await this.dir.executeRollback()
             throw new Error(`Unable to write to ${collection} collection`)
         }
 
-        if(Fylo.LOGGING) console.log(`Finished Writing ${_id}`)
+        if (Fylo.LOGGING) console.log(`Finished Writing ${_id}`)
 
         return _id
     }
 
-    private async executePatchDocDirect<T extends Record<string, any>>(collection: string, newDoc: Record<_ttid, Partial<T>>, oldDoc: Record<_ttid, T> = {}) {
-
+    private async executePatchDocDirect<T extends Record<string, any>>(
+        collection: string,
+        newDoc: Record<_ttid, Partial<T>>,
+        oldDoc: Record<_ttid, T> = {}
+    ) {
         await Fylo.loadEncryption(collection)
 
         const _id = Object.keys(newDoc).shift() as _ttid
 
         let _newId = _id
 
-        if(!_id) throw new Error("this document does not contain an TTID")
+        if (!_id) throw new Error('this document does not contain an TTID')
 
-        const dataKeys = await Walker.getDocData(collection, _id)
+        if (this.engineKind === 's3-files') {
+            let existingDoc = oldDoc[_id]
+            if (!existingDoc) {
+                const existing = await this.assertS3FilesEngine().getDoc<T>(collection, _id).once()
+                existingDoc = existing[_id]
+            }
+            if (!existingDoc) return _id
+
+            const currData = { ...existingDoc, ...newDoc[_id] } as T
+            let docToWrite: T = currData
+            _newId = await Fylo.uniqueTTID(_id, false)
+            if (Fylo.STRICT) docToWrite = (await Gen.validateData(collection, currData)) as T
+            return await this.assertS3FilesEngine().patchDocument(
+                collection,
+                _id,
+                _newId,
+                docToWrite,
+                existingDoc
+            )
+        }
 
-        if(dataKeys.length === 0) return _newId
+        const dataKeys = await Walker.getDocData(collection, _id)
 
-        if(Object.keys(oldDoc).length === 0) {
+        if (dataKeys.length === 0) return _newId
 
+        if (Object.keys(oldDoc).length === 0) {
             const data = await Dir.reconstructData(collection, dataKeys)
 
             oldDoc = { [_id]: data } as Record<_ttid, T>
         }
 
-        if(Object.keys(oldDoc).length === 0) return _newId
+        if (Object.keys(oldDoc).length === 0) return _newId
 
         const currData = { ...oldDoc[_id] }
 
-        for(const field in newDoc[_id]) currData[field] = newDoc[_id][field]!
+        for (const field in newDoc[_id]) currData[field] = newDoc[_id][field]!
 
-        _newId = await Fylo.uniqueTTID(_id)
+        _newId = await Fylo.uniqueTTID(_id, this.engineKind === 'legacy-s3')
 
         let docToWrite: T = currData as T
 
-        if(Fylo.STRICT) docToWrite = await Gen.validateData(collection, currData) as T
+        if (Fylo.STRICT) docToWrite = (await Gen.validateData(collection, currData)) as T
 
         const newKeys = await Dir.extractKeys(collection, _newId, docToWrite)
 
         const [deleteResults, putResults] = await Promise.all([
-            Promise.allSettled(dataKeys.map(key => this.dir.deleteKeys(collection, key))),
-            Promise.allSettled(newKeys.data.map((item, i) => this.dir.putKeys(collection, { dataKey: item, indexKey: newKeys.indexes[i] })))
+            Promise.allSettled(dataKeys.map((key) => this.dir.deleteKeys(collection, key))),
+            Promise.allSettled(
+                newKeys.data.map((item, i) =>
+                    this.dir.putKeys(collection, { dataKey: item, indexKey: newKeys.indexes[i] })
+                )
+            )
         ])
 
-        if(deleteResults.some(r => r.status === 'rejected') || putResults.some(r => r.status === 'rejected')) {
+        if (
+            deleteResults.some((r) => r.status === 'rejected') ||
+            putResults.some((r) => r.status === 'rejected')
+        ) {
             await this.dir.executeRollback()
             throw new Error(`Unable to update ${collection} collection`)
         }
 
-        if(Fylo.LOGGING) console.log(`Finished Updating ${_id} to ${_newId}`)
+        if (Fylo.LOGGING) console.log(`Finished Updating ${_id} to ${_newId}`)
 
         return _newId
     }
 
     private async executeDelDocDirect(collection: string, _id: _ttid) {
+        if (this.engineKind === 's3-files') {
+            await this.assertS3FilesEngine().deleteDocument(collection, _id)
+            return
+        }
 
         const keys = await Walker.getDocData(collection, _id)
 
-        const results = await Promise.allSettled(keys.map(key => this.dir.deleteKeys(collection, key)))
+        const results = await Promise.allSettled(
+            keys.map((key) => this.dir.deleteKeys(collection, key))
+        )
 
-        if(results.some(res => res.status === "rejected")) {
+        if (results.some((res) => res.status === 'rejected')) {
             await this.dir.executeRollback()
             throw new Error(`Unable to delete from ${collection} collection`)
         }
 
-        if(Fylo.LOGGING) console.log(`Finished Deleting ${_id}`)
+        if (Fylo.LOGGING) console.log(`Finished Deleting ${_id}`)
     }
 
     async putData<T extends Record<string, any>>(collection: string, data: T): Promise<_ttid>
-    async putData<T extends Record<string, any>>(collection: string, data: Record<_ttid, T>): Promise<_ttid>
-    async putData<T extends Record<string, any>>(collection: string, data: T, options: { wait?: true, timeoutMs?: number }): Promise<_ttid>
-    async putData<T extends Record<string, any>>(collection: string, data: Record<_ttid, T>, options: { wait?: true, timeoutMs?: number }): Promise<_ttid>
-    async putData<T extends Record<string, any>>(collection: string, data: T, options: { wait: false, timeoutMs?: number }): Promise<QueuedWriteResult>
-    async putData<T extends Record<string, any>>(collection: string, data: Record<_ttid, T>, options: { wait: false, timeoutMs?: number }): Promise<QueuedWriteResult>
+    async putData<T extends Record<string, any>>(
+        collection: string,
+        data: Record<_ttid, T>
+    ): Promise<_ttid>
+    async putData<T extends Record<string, any>>(
+        collection: string,
+        data: T,
+        options: { wait?: true; timeoutMs?: number }
+    ): Promise<_ttid>
+    async putData<T extends Record<string, any>>(
+        collection: string,
+        data: Record<_ttid, T>,
+        options: { wait?: true; timeoutMs?: number }
+    ): Promise<_ttid>
+    async putData<T extends Record<string, any>>(
+        collection: string,
+        data: T,
+        options: { wait: false; timeoutMs?: number }
+    ): Promise<QueuedWriteResult>
+    async putData<T extends Record<string, any>>(
+        collection: string,
+        data: Record<_ttid, T>,
+        options: { wait: false; timeoutMs?: number }
+    ): Promise<QueuedWriteResult>
     async putData<T extends Record<string, any>>(
         collection: string,
         data: Record<_ttid, T> | T,
-        options: { wait?: boolean, timeoutMs?: number } = {}
+        options: { wait?: boolean; timeoutMs?: number } = {}
     ): Promise<_ttid | QueuedWriteResult> {
+        if (this.engineKind === 's3-files') {
+            if (options.wait === false)
+                throw new Error('wait:false is not supported in s3-files engine')
+            const { _id, doc } = await this.prepareInsert(collection, data)
+            await this.executePutDataDirect(collection, _id, doc)
+            return _id
+        }
 
         const queued = await this.queuePutData(collection, data)
 
@@ -731,8 +940,7 @@ export default class Fylo {
     }
 
     async executeQueuedWrite(job: WriteJob) {
-
-        switch(job.operation) {
+        switch (job.operation) {
             case 'insert':
                 await Fylo.loadEncryption(job.collection)
                 return await this.executePutDataDirect(job.collection, job.docId, job.payload)
@@ -756,15 +964,35 @@ export default class Fylo {
      * @param oldDoc The old document data.
      * @returns The number of documents patched.
      */
-    async patchDoc<T extends Record<string, any>>(collection: string, newDoc: Record<_ttid, Partial<T>>, oldDoc?: Record<_ttid, T>): Promise<_ttid>
-    async patchDoc<T extends Record<string, any>>(collection: string, newDoc: Record<_ttid, Partial<T>>, oldDoc: Record<_ttid, T> | undefined, options: { wait?: true, timeoutMs?: number }): Promise<_ttid>
-    async patchDoc<T extends Record<string, any>>(collection: string, newDoc: Record<_ttid, Partial<T>>, oldDoc: Record<_ttid, T> | undefined, options: { wait: false, timeoutMs?: number }): Promise<QueuedWriteResult>
+    async patchDoc<T extends Record<string, any>>(
+        collection: string,
+        newDoc: Record<_ttid, Partial<T>>,
+        oldDoc?: Record<_ttid, T>
+    ): Promise<_ttid>
+    async patchDoc<T extends Record<string, any>>(
+        collection: string,
+        newDoc: Record<_ttid, Partial<T>>,
+        oldDoc: Record<_ttid, T> | undefined,
+        options: { wait?: true; timeoutMs?: number }
+    ): Promise<_ttid>
+    async patchDoc<T extends Record<string, any>>(
+        collection: string,
+        newDoc: Record<_ttid, Partial<T>>,
+        oldDoc: Record<_ttid, T> | undefined,
+        options: { wait: false; timeoutMs?: number }
+    ): Promise<QueuedWriteResult>
     async patchDoc<T extends Record<string, any>>(
         collection: string,
         newDoc: Record<_ttid, Partial<T>>,
         oldDoc: Record<_ttid, T> = {},
-        options: { wait?: boolean, timeoutMs?: number } = {}
+        options: { wait?: boolean; timeoutMs?: number } = {}
     ): Promise<_ttid | QueuedWriteResult> {
+        if (this.engineKind === 's3-files') {
+            if (options.wait === false)
+                throw new Error('wait:false is not supported in s3-files engine')
+            const _id = await this.executePatchDocDirect(collection, newDoc, oldDoc)
+            return _id
+        }
         const queued = await this.queuePatchDoc(collection, newDoc, oldDoc)
 
         return await this.runQueuedJob(queued, options, async () => {
@@ -779,61 +1007,79 @@ export default class Fylo {
      * @param updateSchema The update schema.
      * @returns The number of documents patched.
      */
-    async patchDocs<T extends Record<string, any>>(collection: string, updateSchema: _storeUpdate<T>) {
-
+    async patchDocs<T extends Record<string, any>>(
+        collection: string,
+        updateSchema: _storeUpdate<T>
+    ) {
         await Fylo.loadEncryption(collection)
-        
-        const processDoc = (doc: Record<_ttid, T>, updateSchema: _storeUpdate<T>) => {
 
-            for(const _id in doc) 
+        const processDoc = (doc: Record<_ttid, T>, updateSchema: _storeUpdate<T>) => {
+            for (const _id in doc)
                 return this.patchDoc(collection, { [_id]: updateSchema.$set }, doc)
 
             return
         }
 
         let count = 0
-        
+
         const promises: Promise<_ttid>[] = []
 
-        let finished = false
+        if (this.engineKind === 's3-files') {
+            for await (const value of this.findDocs<T>(collection, updateSchema.$where).collect()) {
+                if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+                    const promise = processDoc(value as Record<_ttid, T>, updateSchema)
+                    if (promise) {
+                        promises.push(promise)
+                        count++
+                    }
+                }
+            }
 
-        const exprs = await Query.getExprs(collection, updateSchema.$where ?? {})
+            await Promise.all(promises)
+            return count
+        }
 
-        if(exprs.length === 1 && exprs[0] === `**/*`) {
+        let finished = false
 
-            for(const doc of await Fylo.allDocs<T>(collection, updateSchema.$where)) {
+        const exprs = await Query.getExprs(collection, updateSchema.$where ?? {})
 
+        if (exprs.length === 1 && exprs[0] === `**/*`) {
+            for (const doc of await Fylo.allDocs<T>(collection, updateSchema.$where)) {
                 const promise = processDoc(doc, updateSchema)
 
-                if(promise) {
+                if (promise) {
                     promises.push(promise)
                     count++
                 }
             }
-        
         } else {
-
-            const iter = Dir.searchDocs<T>(collection, exprs, { updated: updateSchema?.$where?.$updated, created: updateSchema?.$where?.$created }, { listen: false, skip: false })
+            const iter = Dir.searchDocs<T>(
+                collection,
+                exprs,
+                {
+                    updated: updateSchema?.$where?.$updated,
+                    created: updateSchema?.$where?.$created
+                },
+                { listen: false, skip: false }
+            )
 
             do {
-
                 const { value, done } = await iter.next({ count })
 
-                if(value === undefined && !done) continue
+                if (value === undefined && !done) continue
 
-                if(done) {
+                if (done) {
                     finished = true
                     break
                 }
 
                 const promise = processDoc(value as Record<_ttid, T>, updateSchema)
 
-                if(promise) {
+                if (promise) {
                     promises.push(promise)
                     count++
                 }
-
-            } while(!finished)
+            } while (!finished)
         }
 
         await Promise.all(promises)
@@ -848,13 +1094,27 @@ export default class Fylo {
      * @returns The number of documents deleted.
      */
     async delDoc(collection: string, _id: _ttid): Promise<void>
-    async delDoc(collection: string, _id: _ttid, options: { wait?: true, timeoutMs?: number }): Promise<void>
-    async delDoc(collection: string, _id: _ttid, options: { wait: false, timeoutMs?: number }): Promise<QueuedWriteResult>
     async delDoc(
         collection: string,
         _id: _ttid,
-        options: { wait?: boolean, timeoutMs?: number } = {}
+        options: { wait?: true; timeoutMs?: number }
+    ): Promise<void>
+    async delDoc(
+        collection: string,
+        _id: _ttid,
+        options: { wait: false; timeoutMs?: number }
+    ): Promise<QueuedWriteResult>
+    async delDoc(
+        collection: string,
+        _id: _ttid,
+        options: { wait?: boolean; timeoutMs?: number } = {}
     ): Promise<void | QueuedWriteResult> {
+        if (this.engineKind === 's3-files') {
+            if (options.wait === false)
+                throw new Error('wait:false is not supported in s3-files engine')
+            await this.executeDelDocDirect(collection, _id)
+            return
+        }
         const queued = await this.queueDelDoc(collection, _id)
 
         await this.runQueuedJob(queued, options, async () => undefined)
@@ -866,15 +1126,15 @@ export default class Fylo {
      * @param deleteSchema The delete schema.
      * @returns The number of documents deleted.
      */
-    async delDocs<T extends Record<string, any>>(collection: string, deleteSchema?: _storeDelete<T>) {
-
+    async delDocs<T extends Record<string, any>>(
+        collection: string,
+        deleteSchema?: _storeDelete<T>
+    ) {
         await Fylo.loadEncryption(collection)
-        
-        const processDoc = (doc: Record<_ttid, T>) => {
-
-            for(const _id in doc) {
 
-                if(TTID.isTTID(_id)) {
+        const processDoc = (doc: Record<_ttid, T>) => {
+            for (const _id in doc) {
+                if (TTID.isTTID(_id)) {
                     return this.delDoc(collection, _id)
                 }
             }
@@ -886,45 +1146,59 @@ export default class Fylo {
 
         const promises: Promise<void>[] = []
 
-        let finished = false
+        if (this.engineKind === 's3-files') {
+            for await (const value of this.findDocs<T>(collection, deleteSchema).collect()) {
+                if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+                    const promise = processDoc(value as Record<_ttid, T>)
+                    if (promise) {
+                        promises.push(promise)
+                        count++
+                    }
+                }
+            }
 
-        const exprs = await Query.getExprs(collection, deleteSchema ?? {})
+            await Promise.all(promises)
+            return count
+        }
 
-        if(exprs.length === 1 && exprs[0] === `**/*`) {
+        let finished = false
 
-            for(const doc of await Fylo.allDocs<T>(collection, deleteSchema)) {
+        const exprs = await Query.getExprs(collection, deleteSchema ?? {})
 
+        if (exprs.length === 1 && exprs[0] === `**/*`) {
+            for (const doc of await Fylo.allDocs<T>(collection, deleteSchema)) {
                 const promise = processDoc(doc)
 
-                if(promise) {
+                if (promise) {
                     promises.push(promise)
                     count++
                 }
             }
-
         } else {
-
-            const iter = Dir.searchDocs<T>(collection, exprs, { updated: deleteSchema?.$updated, created: deleteSchema?.$created }, { listen: false, skip: false })
+            const iter = Dir.searchDocs<T>(
+                collection,
+                exprs,
+                { updated: deleteSchema?.$updated, created: deleteSchema?.$created },
+                { listen: false, skip: false }
+            )
 
             do {
-
                 const { value, done } = await iter.next({ count })
 
-                if(value === undefined && !done) continue
+                if (value === undefined && !done) continue
 
-                if(done) {
+                if (done) {
                     finished = true
                     break
                 }
 
                 const promise = processDoc(value as Record<_ttid, T>)
 
-                if(promise) {
+                if (promise) {
                     promises.push(promise)
                     count++
                 }
-
-            } while(!finished)
+            } while (!finished)
         }
 
         await Promise.all(promises)
@@ -933,18 +1207,19 @@ export default class Fylo {
     }
 
     private static selectValues<T extends Record<string, any>>(selection: Array<keyof T>, data: T) {
-        
-        for(const field in data) {
-            if(!selection.includes(field as keyof T)) delete data[field]
+        for (const field in data) {
+            if (!selection.includes(field as keyof T)) delete data[field]
         }
 
         return data
     }
 
-    private static renameFields<T extends Record<string, any>>(rename: Record<keyof T, string>, data: T) {
-        
-        for(const field in data) {
-            if(rename[field]) {
+    private static renameFields<T extends Record<string, any>>(
+        rename: Record<keyof T, string>,
+        data: T
+    ) {
+        for (const field in data) {
+            if (rename[field]) {
                 data[rename[field]] = data[field]
                 delete data[field]
             }
@@ -958,50 +1233,54 @@ export default class Fylo {
      * @param join The join schema.
      * @returns The joined documents.
      */
-    static async joinDocs<T extends Record<string, any>, U extends Record<string, any>>(join: _join<T, U>) { 
-        
-        const docs: Record<`${_ttid}, ${_ttid}`, T | U | T & U | Partial<T> & Partial<U>> = {}
+    static async joinDocs<T extends Record<string, any>, U extends Record<string, any>>(
+        join: _join<T, U>
+    ) {
+        if (Fylo.defaultEngineKind() === 's3-files') {
+            return await Fylo.defaultS3Files.joinDocs(join)
+        }
+
+        const docs: Record<`${_ttid}, ${_ttid}`, T | U | (T & U) | (Partial<T> & Partial<U>)> = {}
 
-        const compareFields = async (leftField: keyof T, rightField: keyof U, compare: (leftVal: string, rightVal: string) => boolean) => {
+        const compareFields = async (
+            leftField: keyof T,
+            rightField: keyof U,
+            compare: (leftVal: string, rightVal: string) => boolean
+        ) => {
+            if (join.$leftCollection === join.$rightCollection)
+                throw new Error('Left and right collections cannot be the same')
 
-            if(join.$leftCollection === join.$rightCollection) throw new Error("Left and right collections cannot be the same")
-            
             let leftToken: string | undefined
             const leftFieldIndexes: string[] = []
 
             do {
-
                 const leftData = await S3.list(join.$leftCollection, {
                     prefix: String(leftField)
                 })
-                
-                if(!leftData.contents) break
 
-                leftFieldIndexes.push(...leftData.contents!.map(content => content.key!))
+                if (!leftData.contents) break
+
+                leftFieldIndexes.push(...leftData.contents!.map((content) => content.key!))
 
                 leftToken = leftData.nextContinuationToken
+            } while (leftToken !== undefined)
 
-            } while(leftToken !== undefined)
-            
             let rightToken: string | undefined
             const rightFieldIndexes: string[] = []
 
-            do { 
-
+            do {
                 const rightData = await S3.list(join.$rightCollection, {
                     prefix: String(rightField)
                 })
-                
-                if(!rightData.contents) break
 
-                rightFieldIndexes.push(...rightData.contents!.map(content => content.key!))
+                if (!rightData.contents) break
 
-                rightToken = rightData.nextContinuationToken
+                rightFieldIndexes.push(...rightData.contents!.map((content) => content.key!))
 
-            } while(rightToken !== undefined)
-
-            for(const leftIdx of leftFieldIndexes) {
+                rightToken = rightData.nextContinuationToken
+            } while (rightToken !== undefined)
 
+            for (const leftIdx of leftFieldIndexes) {
                 const leftSegs = leftIdx.split('/')
                 const left_id = leftSegs.pop()! as _ttid
                 const leftVal = leftSegs.pop()!
@@ -1010,140 +1289,219 @@ export default class Fylo {
 
                 const allVals = new Set<string>()
 
-                for(const rightIdx of rightFieldIndexes) {
-
+                for (const rightIdx of rightFieldIndexes) {
                     const rightSegs = rightIdx.split('/')
                     const right_id = rightSegs.pop()! as _ttid
                     const rightVal = rightSegs.pop()!
 
                     const rightCollection = join.$rightCollection
 
-                    if(compare(rightVal, leftVal) && !allVals.has(rightVal)) {
-
+                    if (compare(rightVal, leftVal) && !allVals.has(rightVal)) {
                         allVals.add(rightVal)
 
-                        switch(join.$mode) {
-                            case "inner":
-                                docs[`${left_id}, ${right_id}`] = { [leftField]: Dir.parseValue(leftVal), [rightField]: Dir.parseValue(rightVal) } as Partial<T> & Partial<U>
+                        switch (join.$mode) {
+                            case 'inner':
+                                docs[`${left_id}, ${right_id}`] = {
+                                    [leftField]: Dir.parseValue(leftVal),
+                                    [rightField]: Dir.parseValue(rightVal)
+                                } as Partial<T> & Partial<U>
                                 break
-                            case "left":
+                            case 'left':
                                 const leftDoc = await this.getDoc<T>(leftCollection, left_id).once()
-                                if(Object.keys(leftDoc).length > 0) {
+                                if (Object.keys(leftDoc).length > 0) {
                                     let leftData = leftDoc[left_id]
-                                    if(join.$select) leftData = this.selectValues<T>(join.$select as Array<keyof T>, leftData)
-                                    if(join.$rename) leftData = this.renameFields<T>(join.$rename, leftData)
+                                    if (join.$select)
+                                        leftData = this.selectValues<T>(
+                                            join.$select as Array<keyof T>,
+                                            leftData
+                                        )
+                                    if (join.$rename)
+                                        leftData = this.renameFields<T>(join.$rename, leftData)
                                     docs[`${left_id}, ${right_id}`] = leftData as T
                                 }
                                 break
-                            case "right":
-                                const rightDoc = await this.getDoc<U>(rightCollection, right_id).once()
-                                if(Object.keys(rightDoc).length > 0) {
+                            case 'right':
+                                const rightDoc = await this.getDoc<U>(
+                                    rightCollection,
+                                    right_id
+                                ).once()
+                                if (Object.keys(rightDoc).length > 0) {
                                     let rightData = rightDoc[right_id]
-                                    if(join.$select) rightData = this.selectValues<U>(join.$select as Array<keyof U>, rightData)
-                                    if(join.$rename) rightData = this.renameFields<U>(join.$rename, rightData)
+                                    if (join.$select)
+                                        rightData = this.selectValues<U>(
+                                            join.$select as Array<keyof U>,
+                                            rightData
+                                        )
+                                    if (join.$rename)
+                                        rightData = this.renameFields<U>(join.$rename, rightData)
                                     docs[`${left_id}, ${right_id}`] = rightData as U
                                 }
                                 break
-                            case "outer":
-
+                            case 'outer':
                                 let leftFullData: T = {} as T
                                 let rightFullData: U = {} as U
 
-                                const leftFullDoc = await this.getDoc<T>(leftCollection, left_id).once()
+                                const leftFullDoc = await this.getDoc<T>(
+                                    leftCollection,
+                                    left_id
+                                ).once()
 
-                                if(Object.keys(leftFullDoc).length > 0) {
+                                if (Object.keys(leftFullDoc).length > 0) {
                                     let leftData = leftFullDoc[left_id]
-                                    if(join.$select) leftData = this.selectValues<T>(join.$select as Array<keyof T>, leftData)
-                                    if(join.$rename) leftData = this.renameFields<T>(join.$rename, leftData)
+                                    if (join.$select)
+                                        leftData = this.selectValues<T>(
+                                            join.$select as Array<keyof T>,
+                                            leftData
+                                        )
+                                    if (join.$rename)
+                                        leftData = this.renameFields<T>(join.$rename, leftData)
                                     leftFullData = { ...leftData, ...leftFullData } as T
                                 }
 
-                                const rightFullDoc = await this.getDoc<U>(rightCollection, right_id).once()
+                                const rightFullDoc = await this.getDoc<U>(
+                                    rightCollection,
+                                    right_id
+                                ).once()
 
-                                if(Object.keys(rightFullDoc).length > 0) {
+                                if (Object.keys(rightFullDoc).length > 0) {
                                     let rightData = rightFullDoc[right_id]
-                                    if(join.$select) rightData = this.selectValues<U>(join.$select as Array<keyof U>, rightData)
-                                    if(join.$rename) rightData = this.renameFields<U>(join.$rename, rightData)
+                                    if (join.$select)
+                                        rightData = this.selectValues<U>(
+                                            join.$select as Array<keyof U>,
+                                            rightData
+                                        )
+                                    if (join.$rename)
+                                        rightData = this.renameFields<U>(join.$rename, rightData)
                                     rightFullData = { ...rightData, ...rightFullData } as U
                                 }
 
-                                docs[`${left_id}, ${right_id}`] = { ...leftFullData, ...rightFullData } as T & U
+                                docs[`${left_id}, ${right_id}`] = {
+                                    ...leftFullData,
+                                    ...rightFullData
+                                } as T & U
                                 break
                         }
 
-                        if(join.$limit && Object.keys(docs).length === join.$limit) break
+                        if (join.$limit && Object.keys(docs).length === join.$limit) break
                     }
                 }
 
-                if(join.$limit && Object.keys(docs).length === join.$limit) break
+                if (join.$limit && Object.keys(docs).length === join.$limit) break
             }
         }
 
-        for(const field in join.$on) {
+        for (const field in join.$on) {
+            if (join.$on[field]!.$eq)
+                await compareFields(
+                    field,
+                    join.$on[field]!.$eq,
+                    (leftVal, rightVal) => leftVal === rightVal
+                )
+
+            if (join.$on[field]!.$ne)
+                await compareFields(
+                    field,
+                    join.$on[field]!.$ne,
+                    (leftVal, rightVal) => leftVal !== rightVal
+                )
 
-            if(join.$on[field]!.$eq) await compareFields(field, join.$on[field]!.$eq, (leftVal, rightVal) => leftVal === rightVal)
+            if (join.$on[field]!.$gt)
+                await compareFields(
+                    field,
+                    join.$on[field]!.$gt,
+                    (leftVal, rightVal) => Number(leftVal) > Number(rightVal)
+                )
 
-            if(join.$on[field]!.$ne) await compareFields(field, join.$on[field]!.$ne, (leftVal, rightVal) => leftVal !== rightVal)
-            
-            if(join.$on[field]!.$gt) await compareFields(field, join.$on[field]!.$gt, (leftVal, rightVal) => Number(leftVal) > Number(rightVal))
-            
-            if(join.$on[field]!.$lt) await compareFields(field, join.$on[field]!.$lt, (leftVal, rightVal) => Number(leftVal) < Number(rightVal))
-            
-            if(join.$on[field]!.$gte) await compareFields(field, join.$on[field]!.$gte, (leftVal, rightVal) => Number(leftVal) >= Number(rightVal))
-            
-            if(join.$on[field]!.$lte) await compareFields(field, join.$on[field]!.$lte, (leftVal, rightVal) => Number(leftVal) <= Number(rightVal))
-        }
+            if (join.$on[field]!.$lt)
+                await compareFields(
+                    field,
+                    join.$on[field]!.$lt,
+                    (leftVal, rightVal) => Number(leftVal) < Number(rightVal)
+                )
 
-        if(join.$groupby) {
+            if (join.$on[field]!.$gte)
+                await compareFields(
+                    field,
+                    join.$on[field]!.$gte,
+                    (leftVal, rightVal) => Number(leftVal) >= Number(rightVal)
+                )
 
-            const groupedDocs: Record<string, Record<string, Partial<T | U>>> = {} as Record<string, Record<string, Partial<T | U>>>
+            if (join.$on[field]!.$lte)
+                await compareFields(
+                    field,
+                    join.$on[field]!.$lte,
+                    (leftVal, rightVal) => Number(leftVal) <= Number(rightVal)
+                )
+        }
 
-            for(const ids in docs) {
+        if (join.$groupby) {
+            const groupedDocs: Record<string, Record<string, Partial<T | U>>> = {} as Record<
+                string,
+                Record<string, Partial<T | U>>
+            >
 
+            for (const ids in docs) {
                 const data = docs[ids as `${_ttid}, ${_ttid}`]
 
                 // @ts-expect-error - Object.groupBy not yet in TS lib types
-                const grouping = Object.groupBy([data], elem => elem[join.$groupby!])
-
-                for(const group in grouping) {
+                const grouping = Object.groupBy([data], (elem) => elem[join.$groupby!])
 
-                    if(groupedDocs[group]) groupedDocs[group][ids] = data
+                for (const group in grouping) {
+                    if (groupedDocs[group]) groupedDocs[group][ids] = data
                     else groupedDocs[group] = { [ids]: data }
                 }
             }
 
-            if(join.$onlyIds) {
-
+            if (join.$onlyIds) {
                 const groupedIds: Record<string, _ttid[]> = {}
 
-                for(const group in groupedDocs) {
+                for (const group in groupedDocs) {
                     const doc = groupedDocs[group]
                     groupedIds[group] = Object.keys(doc).flat()
                 }
 
                 return groupedIds
             }
-            
+
             return groupedDocs
         }
 
-        if(join.$onlyIds) return Array.from(new Set(Object.keys(docs).flat()))    
+        if (join.$onlyIds) return Array.from(new Set(Object.keys(docs).flat()))
 
         return docs
     }
 
-    private static async allDocs<T extends Record<string, any>>(collection: string, query?: _storeQuery<T>) {
+    private static async allDocs<T extends Record<string, any>>(
+        collection: string,
+        query?: _storeQuery<T>
+    ) {
+        if (Fylo.defaultEngineKind() === 's3-files') {
+            const results: Array<Record<_ttid, T>> = []
+            for await (const data of Fylo.defaultS3Files.findDocs<T>(collection, query).collect()) {
+                if (typeof data === 'object' && !Array.isArray(data))
+                    results.push(data as Record<_ttid, T>)
+            }
+            return results
+        }
 
         const res = await S3.list(collection, {
             delimiter: '/',
             maxKeys: !query || !query.$limit ? undefined : query.$limit
         })
-        
-        const ids = res.commonPrefixes?.map(item => item.prefix!.split('/')[0]!).filter(key => TTID.isTTID(key)) as _ttid[] ?? [] as _ttid[]
-        
-        const docs = await Promise.allSettled(ids.map(id => Fylo.getDoc<T>(collection, id).once()))
-        
-        return docs.filter(item => item.status === 'fulfilled').map(item => item.value).filter(doc => Object.keys(doc).length > 0)
+
+        const ids =
+            (res.commonPrefixes
+                ?.map((item) => item.prefix!.split('/')[0]!)
+                .filter((key) => TTID.isTTID(key)) as _ttid[]) ?? ([] as _ttid[])
+
+        const docs = await Promise.allSettled(
+            ids.map((id) => Fylo.getDoc<T>(collection, id).once())
+        )
+
+        return docs
+            .filter((item) => item.status === 'fulfilled')
+            .map((item) => item.value)
+            .filter((doc) => Object.keys(doc).length > 0)
     }
 
     /**
@@ -1153,56 +1511,70 @@ export default class Fylo {
      * @returns The found documents.
      */
     static findDocs<T extends Record<string, any>>(collection: string, query?: _storeQuery<T>) {
-        
-        const processDoc = (doc: Record<_ttid, T>, query?: _storeQuery<T>) => {
-
-            if(Object.keys(doc).length > 0) {
+        if (Fylo.defaultEngineKind() === 's3-files') {
+            return Fylo.defaultS3Files.findDocs<T>(collection, query)
+        }
 
+        const processDoc = (doc: Record<_ttid, T>, query?: _storeQuery<T>) => {
+            if (Object.keys(doc).length > 0) {
                 // Post-filter for operators that cannot be expressed as globs ($ne, $gt, $gte, $lt, $lte).
                 // $ops use OR semantics: a document passes if it matches at least one op.
-                if(query?.$ops) {
-                    for(const [_id, data] of Object.entries(doc)) {
+                if (query?.$ops) {
+                    for (const [_id, data] of Object.entries(doc)) {
                         let matchesAny = false
-                        for(const op of query.$ops) {
+                        for (const op of query.$ops) {
                             let opMatches = true
-                            for(const col in op) {
+                            for (const col in op) {
                                 const val = (data as Record<string, unknown>)[col]
                                 const cond = op[col as keyof T]!
-                                if(cond.$ne !== undefined && val == cond.$ne) { opMatches = false; break }
-                                if(cond.$gt !== undefined && !(Number(val) > cond.$gt)) { opMatches = false; break }
-                                if(cond.$gte !== undefined && !(Number(val) >= cond.$gte)) { opMatches = false; break }
-                                if(cond.$lt !== undefined && !(Number(val) < cond.$lt)) { opMatches = false; break }
-                                if(cond.$lte !== undefined && !(Number(val) <= cond.$lte)) { opMatches = false; break }
+                                if (cond.$ne !== undefined && val == cond.$ne) {
+                                    opMatches = false
+                                    break
+                                }
+                                if (cond.$gt !== undefined && !(Number(val) > cond.$gt)) {
+                                    opMatches = false
+                                    break
+                                }
+                                if (cond.$gte !== undefined && !(Number(val) >= cond.$gte)) {
+                                    opMatches = false
+                                    break
+                                }
+                                if (cond.$lt !== undefined && !(Number(val) < cond.$lt)) {
+                                    opMatches = false
+                                    break
+                                }
+                                if (cond.$lte !== undefined && !(Number(val) <= cond.$lte)) {
+                                    opMatches = false
+                                    break
+                                }
+                            }
+                            if (opMatches) {
+                                matchesAny = true
+                                break
                             }
-                            if(opMatches) { matchesAny = true; break }
                         }
-                        if(!matchesAny) delete doc[_id as _ttid]
+                        if (!matchesAny) delete doc[_id as _ttid]
                     }
-                    if(Object.keys(doc).length === 0) return
+                    if (Object.keys(doc).length === 0) return
                 }
 
-                for(let [_id, data] of Object.entries(doc)) {
-
-                    if(query && query.$select && query.$select.length > 0) {
-
+                for (let [_id, data] of Object.entries(doc)) {
+                    if (query && query.$select && query.$select.length > 0) {
                         data = this.selectValues<T>(query.$select as Array<keyof T>, data)
                     }
 
-                    if(query && query.$rename) data = this.renameFields<T>(query.$rename, data)
+                    if (query && query.$rename) data = this.renameFields<T>(query.$rename, data)
 
                     doc[_id] = data
                 }
 
-                if(query && query.$groupby) {
-
+                if (query && query.$groupby) {
                     const docGroup: Record<string, Record<string, Partial<T>>> = {}
 
-                    for(const [id, data] of Object.entries(doc)) {
-
+                    for (const [id, data] of Object.entries(doc)) {
                         const groupValue = data[query.$groupby] as string
 
-                        if(groupValue) {
-
+                        if (groupValue) {
                             delete data[query.$groupby]
 
                             docGroup[groupValue] = {
@@ -1211,12 +1583,9 @@ export default class Fylo {
                         }
                     }
 
-                    if(query && query.$onlyIds) {
-
-                        for(const [groupValue, doc] of Object.entries(docGroup)) {
-
-                            for(const id in doc as Record<_ttid, T>) {
-
+                    if (query && query.$onlyIds) {
+                        for (const [groupValue, doc] of Object.entries(docGroup)) {
+                            for (const id in doc as Record<_ttid, T>) {
                                 // @ts-expect-error - dynamic key assignment on grouped object
                                 docGroup[groupValue][id] = null
                             }
@@ -1228,94 +1597,96 @@ export default class Fylo {
                     return docGroup
                 }
 
-                if(query && query.$onlyIds) {
+                if (query && query.$onlyIds) {
                     return Object.keys(doc).shift()
                 }
 
                 return doc
             }
 
-            return 
+            return
         }
 
         return {
-
             /**
              * Async iterator (listener) for the documents.
              */
             async *[Symbol.asyncIterator]() {
-
                 await Fylo.loadEncryption(collection)
 
                 const expression = await Query.getExprs(collection, query ?? {})
 
-                if(expression.length === 1 && expression[0] === `**/*`) {
-                    for(const doc of await Fylo.allDocs<T>(collection, query)) yield processDoc(doc, query)
-                } 
+                if (expression.length === 1 && expression[0] === `**/*`) {
+                    for (const doc of await Fylo.allDocs<T>(collection, query))
+                        yield processDoc(doc, query)
+                }
 
                 let count = 0
                 let finished = false
 
-                const iter = Dir.searchDocs<T>(collection, expression, { updated: query?.$updated, created: query?.$created }, { listen: true, skip: true })
+                const iter = Dir.searchDocs<T>(
+                    collection,
+                    expression,
+                    { updated: query?.$updated, created: query?.$created },
+                    { listen: true, skip: true }
+                )
 
                 do {
-
                     const { value, done } = await iter.next({ count, limit: query?.$limit })
 
-                    if(value === undefined && !done) continue
+                    if (value === undefined && !done) continue
 
-                    if(done) {
+                    if (done) {
                         finished = true
                         break
                     }
 
                     const result = processDoc(value as Record<_ttid, T>, query)
-                    if(result !== undefined) {
+                    if (result !== undefined) {
                         count++
                         yield result
                     }
-
-                } while(!finished)
+                } while (!finished)
             },
-            
+
             /**
              * Async iterator for the documents.
              */
             async *collect() {
-
                 await Fylo.loadEncryption(collection)
 
                 const expression = await Query.getExprs(collection, query ?? {})
 
-                if(expression.length === 1 && expression[0] === `**/*`) {
-
-                    for(const doc of await Fylo.allDocs<T>(collection, query)) yield processDoc(doc, query)
-                
+                if (expression.length === 1 && expression[0] === `**/*`) {
+                    for (const doc of await Fylo.allDocs<T>(collection, query))
+                        yield processDoc(doc, query)
                 } else {
-
                     let count = 0
                     let finished = false
 
-                    const iter = Dir.searchDocs<T>(collection, expression, { updated: query?.$updated, created: query?.$created }, { listen: false, skip: false })
+                    const iter = Dir.searchDocs<T>(
+                        collection,
+                        expression,
+                        { updated: query?.$updated, created: query?.$created },
+                        { listen: false, skip: false }
+                    )
 
                     do {
-
                         const { value, done } = await iter.next({ count, limit: query?.$limit })
 
-                        if(value === undefined && !done) continue
+                        if (value === undefined && !done) continue
 
-                        if(done) {
+                        if (done) {
                             finished = true
                             break
                         }
 
                         const result = processDoc(value as Record<_ttid, T>, query)
-                        if(result !== undefined) {
+                        if (result !== undefined) {
                             count++
                             yield result
                         }
-
-                    } while(!finished)
+                    } while (!finished)
                 }
             },
 
@@ -1323,29 +1694,34 @@ export default class Fylo {
              * Async iterator (listener) for the document's deletion.
              */
             async *onDelete() {
-
                 await Fylo.loadEncryption(collection)
 
                 let count = 0
                 let finished = false
 
-                const iter = Dir.searchDocs<T>(collection, await Query.getExprs(collection, query ?? {}), {}, { listen: true, skip: true }, true)
+                const iter = Dir.searchDocs<T>(
+                    collection,
+                    await Query.getExprs(collection, query ?? {}),
+                    {},
+                    { listen: true, skip: true },
+                    true
+                )
 
                 do {
-
                     const { value, done } = await iter.next({ count })
 
-                    if(value === undefined && !done) continue
+                    if (value === undefined && !done) continue
 
-                    if(done) {
+                    if (done) {
                         finished = true
                         break
                     }
 
-                    if(value) yield value as _ttid
-
-                } while(!finished)
+                    if (value) yield value as _ttid
+                } while (!finished)
             }
         }
     }
 }
+
+export { migrateLegacyS3ToS3Files } from './migrate'