@delma/fylo 2.0.1 → 2.1.0

package/tests/integration/operators.test.js

@@ -1,14 +1,13 @@
-import { test, expect, describe, beforeAll, afterAll, mock } from 'bun:test'
+import { test, expect, describe, beforeAll, afterAll } from 'bun:test'
+import { rm } from 'node:fs/promises'
 import Fylo from '../../src'
 import { albumURL } from '../data'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
+import { createTestRoot } from '../helpers/root'
 const ALBUMS = 'ops-album'
-const fylo = new Fylo()
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
+const root = await createTestRoot('fylo-operators-')
+const fylo = new Fylo({ root })
 beforeAll(async () => {
-    await Fylo.createCollection(ALBUMS)
+    await fylo.createCollection(ALBUMS)
     try {
         await fylo.importBulkData(ALBUMS, new URL(albumURL), 100)
     } catch {
@@ -16,14 +15,17 @@ beforeAll(async () => {
     }
 })
 afterAll(async () => {
-    await Fylo.dropCollection(ALBUMS)
+    await fylo.dropCollection(ALBUMS)
+    await rm(root, { recursive: true, force: true })
 })
 describe('NO-SQL', async () => {
     test('$ne — excludes matching value', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $ops: [{ userId: { $ne: 1 } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $ops: [{ userId: { $ne: 1 } }]
+            })
+            .collect()) {
             results = { ...results, ...data }
         }
         const albums = Object.values(results)
@@ -33,9 +35,11 @@ describe('NO-SQL', async () => {
     })
     test('$lt — returns documents where field is less than value', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $ops: [{ userId: { $lt: 5 } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $ops: [{ userId: { $lt: 5 } }]
+            })
+            .collect()) {
             results = { ...results, ...data }
         }
         const albums = Object.values(results)
@@ -45,9 +49,11 @@ describe('NO-SQL', async () => {
     })
     test('$lte — returns documents where field is less than or equal to value', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $ops: [{ userId: { $lte: 5 } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $ops: [{ userId: { $lte: 5 } }]
+            })
+            .collect()) {
             results = { ...results, ...data }
         }
         const albums = Object.values(results)
@@ -57,9 +63,11 @@ describe('NO-SQL', async () => {
     })
     test('$gt — returns documents where field is greater than value', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $ops: [{ userId: { $gt: 5 } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $ops: [{ userId: { $gt: 5 } }]
+            })
+            .collect()) {
             results = { ...results, ...data }
         }
         const albums = Object.values(results)
@@ -69,9 +77,11 @@ describe('NO-SQL', async () => {
     })
     test('$gte — returns documents where field is greater than or equal to value', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $ops: [{ userId: { $gte: 5 } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $ops: [{ userId: { $gte: 5 } }]
+            })
+            .collect()) {
             results = { ...results, ...data }
         }
         const albums = Object.values(results)
@@ -81,9 +91,11 @@ describe('NO-SQL', async () => {
     })
     test('$like — matches substring pattern', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $ops: [{ title: { $like: '%quidem%' } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $ops: [{ title: { $like: '%quidem%' } }]
+            })
+            .collect()) {
             results = { ...results, ...data }
         }
         const albums = Object.values(results)
@@ -93,9 +105,11 @@ describe('NO-SQL', async () => {
     })
     test('$like — prefix pattern', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $ops: [{ title: { $like: 'omnis%' } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $ops: [{ title: { $like: 'omnis%' } }]
+            })
+            .collect()) {
             results = { ...results, ...data }
         }
         const albums = Object.values(results)

package/tests/integration/read.test.js

@@ -1,16 +1,15 @@
-import { test, expect, describe, beforeAll, afterAll, mock } from 'bun:test'
+import { test, expect, describe, beforeAll, afterAll } from 'bun:test'
+import { rm } from 'node:fs/promises'
 import Fylo from '../../src'
 import { albumURL, postsURL } from '../data'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
+import { createTestRoot } from '../helpers/root'
 const POSTS = `post`
 const ALBUMS = `album`
 let count = 0
-const fylo = new Fylo()
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
+const root = await createTestRoot('fylo-read-')
+const fylo = new Fylo({ root })
 beforeAll(async () => {
-    await Promise.all([Fylo.createCollection(ALBUMS), fylo.executeSQL(`CREATE TABLE ${POSTS}`)])
+    await Promise.all([fylo.createCollection(ALBUMS), fylo.executeSQL(`CREATE TABLE ${POSTS}`)])
     try {
         count = await fylo.importBulkData(ALBUMS, new URL(albumURL), 100)
         await fylo.importBulkData(POSTS, new URL(postsURL), 100)
@@ -19,19 +18,20 @@ beforeAll(async () => {
     }
 })
 afterAll(async () => {
-    await Promise.all([Fylo.dropCollection(ALBUMS), Fylo.dropCollection(POSTS)])
+    await Promise.all([fylo.dropCollection(ALBUMS), fylo.dropCollection(POSTS)])
+    await rm(root, { recursive: true, force: true })
 })
 describe('NO-SQL', async () => {
     test('SELECT ALL', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS).collect()) {
+        for await (const data of fylo.findDocs(ALBUMS).collect()) {
             results = { ...results, ...data }
         }
         expect(Object.keys(results).length).toBe(count)
     })
     test('SELECT PARTIAL', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, { $select: ['title'] }).collect()) {
+        for await (const data of fylo.findDocs(ALBUMS, { $select: ['title'] }).collect()) {
             results = { ...results, ...data }
         }
         const allAlbums = Object.values(results)
@@ -40,18 +40,20 @@ describe('NO-SQL', async () => {
     })
     test('GET ONE', async () => {
         const ids = []
-        for await (const data of Fylo.findDocs(ALBUMS, { $limit: 1, $onlyIds: true }).collect()) {
+        for await (const data of fylo.findDocs(ALBUMS, { $limit: 1, $onlyIds: true }).collect()) {
             ids.push(data)
         }
-        const result = await Fylo.getDoc(ALBUMS, ids[0]).once()
+        const result = await fylo.getDoc(ALBUMS, ids[0]).once()
         const _id = Object.keys(result).shift()
         expect(ids[0]).toEqual(_id)
     })
     test('SELECT CLAUSE', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $ops: [{ userId: { $eq: 2 } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $ops: [{ userId: { $eq: 2 } }]
+            })
+            .collect()) {
             results = { ...results, ...data }
         }
         const allAlbums = Object.values(results)
@@ -60,23 +62,25 @@ describe('NO-SQL', async () => {
     })
     test('SELECT LIMIT', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, { $limit: 5 }).collect()) {
+        for await (const data of fylo.findDocs(ALBUMS, { $limit: 5 }).collect()) {
             results = { ...results, ...data }
         }
         expect(Object.keys(results).length).toBe(5)
     })
     test('SELECT GROUP BY', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(ALBUMS, {
-            $groupby: 'userId',
-            $onlyIds: true
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(ALBUMS, {
+                $groupby: 'userId',
+                $onlyIds: true
+            })
+            .collect()) {
             results = Object.appendGroup(results, data)
         }
         expect(Object.keys(results).length).toBeGreaterThan(0)
     })
     test('SELECT JOIN', async () => {
-        const results = await Fylo.joinDocs({
+        const results = await fylo.joinDocs({
             $leftCollection: ALBUMS,
             $rightCollection: POSTS,
             $mode: 'inner',

package/tests/integration/rollback.test.js

@@ -1,60 +1,30 @@
-import { test, expect, describe, beforeAll, afterAll, mock } from 'bun:test'
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
+import { mkdtemp, rm } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
 import Fylo from '../../src'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
+
+const root = await mkdtemp(path.join(os.tmpdir(), 'fylo-rollback-'))
 const POSTS = 'rb-post'
-const fylo = new Fylo()
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
+const fylo = new Fylo({ root })
+
 beforeAll(async () => {
-    await Fylo.createCollection(POSTS)
+    await fylo.createCollection(POSTS)
 })
+
 afterAll(async () => {
-    await Fylo.dropCollection(POSTS)
+    await rm(root, { recursive: true, force: true })
 })
-describe('NO-SQL', () => {
-    test('INSERT then rollback — document is not retrievable', async () => {
-        const _id = await fylo.putData(POSTS, {
-            userId: 99,
-            id: 9001,
-            title: 'Rollback Me',
-            body: 'This document should disappear after rollback'
+
+describe('rollback compatibility', () => {
+    test('rollback is a safe no-op in filesystem-first FYLO', async () => {
+        const id = await fylo.putData(POSTS, {
+            title: 'Still here'
         })
-        const before = await Fylo.getDoc(POSTS, _id).once()
-        expect(Object.keys(before).length).toBe(1)
-        await fylo.rollback()
-        const after = await Fylo.getDoc(POSTS, _id).once()
-        expect(Object.keys(after).length).toBe(0)
-    })
-    test('DELETE then rollback — document is restored', async () => {
-        const freshFylo = new Fylo()
-        const _id = await freshFylo.putData(POSTS, {
-            userId: 99,
-            id: 9002,
-            title: 'Restore Me',
-            body: 'This document should reappear after delete rollback'
-        })
-        const deleteInstance = new Fylo()
-        await deleteInstance.delDoc(POSTS, _id)
-        const after = await Fylo.getDoc(POSTS, _id).once()
-        expect(Object.keys(after).length).toBe(0)
-        await deleteInstance.rollback()
-        const restored = await Fylo.getDoc(POSTS, _id).once()
-        expect(Object.keys(restored).length).toBe(1)
-        expect(restored[_id].title).toBe('Restore Me')
-    })
-    test('batch INSERT then rollback — all documents are removed', async () => {
-        const batchFylo = new Fylo()
-        const batch = [
-            { userId: 98, id: 9003, title: 'Batch A', body: 'body a' },
-            { userId: 98, id: 9004, title: 'Batch B', body: 'body b' },
-            { userId: 98, id: 9005, title: 'Batch C', body: 'body c' }
-        ]
-        const ids = await batchFylo.batchPutData(POSTS, batch)
-        const beforeResults = await Promise.all(ids.map((id) => Fylo.getDoc(POSTS, id).once()))
-        expect(beforeResults.every((r) => Object.keys(r).length === 1)).toBe(true)
-        await batchFylo.rollback()
-        const afterResults = await Promise.all(ids.map((id) => Fylo.getDoc(POSTS, id).once()))
-        expect(afterResults.every((r) => Object.keys(r).length === 0)).toBe(true)
+
+        await expect(fylo.rollback()).resolves.toBeUndefined()
+
+        const after = await fylo.getDoc(POSTS, id).once()
+        expect(after[id].title).toBe('Still here')
     })
 })

package/tests/integration/s3-files.performance.test.js

@@ -0,0 +1,75 @@
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
+import { mkdtemp, rm, stat } from 'node:fs/promises'
+import { performance } from 'node:perf_hooks'
+import os from 'node:os'
+import path from 'node:path'
+import Fylo from '../../src'
+
+const runPerf = process.env.FYLO_RUN_PERF_TESTS === 'true'
+
+describe.skipIf(!runPerf)('s3-files engine performance', () => {
+    let root = path.join(os.tmpdir(), `fylo-s3files-perf-${Date.now()}`)
+    const collection = 's3files-perf'
+    let fylo = new Fylo({ root })
+
+    beforeAll(async () => {
+        root = await mkdtemp(root)
+        fylo = new Fylo({ root })
+        await fylo.createCollection(collection)
+    })
+
+    afterAll(async () => {
+        await rm(root, { recursive: true, force: true })
+    })
+
+    test('keeps a single durable index file while querying a large dataset', async () => {
+        const totalDocs = 2000
+        const insertStart = performance.now()
+
+        for (let index = 0; index < totalDocs; index++) {
+            await fylo.putData(collection, {
+                title: `doc-${index}`,
+                group: index % 10,
+                tags: [`tag-${index % 5}`, `batch-${Math.floor(index / 100)}`],
+                meta: { score: index }
+            })
+        }
+
+        const insertMs = performance.now() - insertStart
+
+        const exactStart = performance.now()
+        let exactResults = {}
+        for await (const data of fylo
+            .findDocs(collection, {
+                $ops: [{ title: { $eq: 'doc-1555' } }]
+            })
+            .collect()) {
+            exactResults = { ...exactResults, ...data }
+        }
+        const exactMs = performance.now() - exactStart
+
+        const rangeStart = performance.now()
+        let rangeCount = 0
+        for await (const data of fylo
+            .findDocs(collection, {
+                $ops: [{ ['meta.score']: { $gte: 1900 } }]
+            })
+            .collect()) {
+            rangeCount += Object.keys(data).length
+        }
+        const rangeMs = performance.now() - rangeStart
+
+        const indexFile = path.join(root, collection, '.fylo', 'indexes', `${collection}.idx.json`)
+        const indexStats = await stat(indexFile)
+
+        expect(Object.keys(exactResults)).toHaveLength(1)
+        expect(rangeCount).toBe(100)
+        expect(indexStats.isFile()).toBe(true)
+
+        console.log(
+            `[FYLO perf] docs=${totalDocs} insertMs=${insertMs.toFixed(1)} exactMs=${exactMs.toFixed(
+                1
+            )} rangeMs=${rangeMs.toFixed(1)} indexBytes=${indexStats.size}`
+        )
+    }, 120_000)
+})

package/tests/integration/s3-files.test.js

@@ -1,11 +1,10 @@
 import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
-import { mkdtemp, rm, stat } from 'node:fs/promises'
+import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'
 import os from 'node:os'
 import path from 'node:path'
-import { Database } from 'bun:sqlite'
 import Fylo from '../../src'
 const root = await mkdtemp(path.join(os.tmpdir(), 'fylo-s3files-'))
-const fylo = new Fylo({ engine: 's3-files', s3FilesRoot: root })
+const fylo = new Fylo({ root })
 const POSTS = 's3files-posts'
 const USERS = 's3files-users'
 describe('s3-files engine', () => {
@@ -60,12 +59,34 @@ describe('s3-files engine', () => {
         const result = await fylo.getDoc(POSTS, id).once()
         expect(result[id].body).toBe(longBody)
     })
-    test('stores indexes in a single SQLite database instead of per-entry files', async () => {
-        const dbStat = await stat(path.join(root, POSTS, '.fylo', 'index.db'))
-        expect(dbStat.isFile()).toBe(true)
-        await expect(stat(path.join(root, POSTS, '.fylo', 'indexes'))).rejects.toThrow()
+    test('stores only user document data in the file body', async () => {
+        const id = await fylo.putData(POSTS, {
+            title: 'Lean doc',
+            body: 'payload only'
+        })
+        const raw = JSON.parse(
+            await readFile(
+                path.join(root, POSTS, '.fylo', 'docs', id.slice(0, 2), `${id}.json`),
+                'utf8'
+            )
+        )
+
+        expect(raw).toEqual({
+            title: 'Lean doc',
+            body: 'payload only'
+        })
+        expect(raw.id).toBeUndefined()
+        expect(raw.createdAt).toBeUndefined()
+        expect(raw.updatedAt).toBeUndefined()
+    })
+    test('stores indexes in a single collection file instead of SQLite or per-entry files', async () => {
+        const indexStat = await stat(
+            path.join(root, POSTS, '.fylo', 'indexes', `${POSTS}.idx.json`)
+        )
+        expect(indexStat.isFile()).toBe(true)
+        await expect(stat(path.join(root, POSTS, '.fylo', 'index.db'))).rejects.toThrow()
     })
-    test('uses SQLite index rows to support exact, range, and contains queries', async () => {
+    test('uses the collection index file to support exact, range, and contains queries', async () => {
         const queryCollection = 's3files-query'
         await fylo.createCollection(queryCollection)
 
@@ -111,39 +132,33 @@ describe('s3-files engine', () => {
         expect(Object.keys(containsResults)).toEqual([bunId])
         expect(containsResults[nodeId]).toBeUndefined()
 
-        const db = new Database(path.join(root, queryCollection, '.fylo', 'index.db'))
-        const rows = db
-            .query(
-                `SELECT doc_id, field_path, raw_value, value_type, numeric_value
-                 FROM doc_index_entries
-                 WHERE doc_id = ?
-                 ORDER BY field_path, raw_value`
+        const index = JSON.parse(
+            await readFile(
+                path.join(root, queryCollection, '.fylo', 'indexes', `${queryCollection}.idx.json`),
+                'utf8'
             )
-            .all(bunId)
-        db.close()
+        )
+        const rows = index.docs[bunId]
 
         expect(rows).toEqual(
             expect.arrayContaining([
                 expect.objectContaining({
-                    doc_id: bunId,
-                    field_path: 'title',
-                    raw_value: 'Bun launch',
-                    value_type: 'string',
-                    numeric_value: null
+                    fieldPath: 'title',
+                    rawValue: 'Bun launch',
+                    valueType: 'string',
+                    numericValue: null
                 }),
                 expect.objectContaining({
-                    doc_id: bunId,
-                    field_path: 'meta/score',
-                    raw_value: '10',
-                    value_type: 'number',
-                    numeric_value: 10
+                    fieldPath: 'meta/score',
+                    rawValue: '10',
+                    valueType: 'number',
+                    numericValue: 10
                 }),
                 expect.objectContaining({
-                    doc_id: bunId,
-                    field_path: 'tags/1',
-                    raw_value: 'aws',
-                    value_type: 'string',
-                    numeric_value: null
+                    fieldPath: 'tags/1',
+                    rawValue: 'aws',
+                    valueType: 'string',
+                    numericValue: null
                 })
             ])
         )
@@ -163,30 +178,28 @@ describe('s3-files engine', () => {
     })
     test('queue APIs are explicitly unsupported', async () => {
         await expect(fylo.queuePutData(POSTS, { title: 'no queue' })).rejects.toThrow(
-            'queuePutData is not supported'
-        )
-        await expect(fylo.processQueuedWrites(1)).rejects.toThrow(
-            'processQueuedWrites is not supported'
+            'queuePutData was removed'
         )
+        await expect(fylo.processQueuedWrites(1)).rejects.toThrow('processQueuedWrites was removed')
     })
     test('rejects collection names that are unsafe for cross-platform filesystems', async () => {
         await expect(fylo.createCollection('bad/name')).rejects.toThrow('Invalid collection name')
         await expect(fylo.createCollection('bad\\name')).rejects.toThrow('Invalid collection name')
         await expect(fylo.createCollection('bad:name')).rejects.toThrow('Invalid collection name')
     })
-    test('static helpers can use s3-files through env defaults', async () => {
-        const prevEngine = process.env.FYLO_STORAGE_ENGINE
-        const prevRoot = process.env.FYLO_S3FILES_ROOT
-        process.env.FYLO_STORAGE_ENGINE = 's3-files'
+    test('static helpers can use filesystem root env defaults', async () => {
+        const prevFyloRoot = process.env.FYLO_ROOT
+        const prevS3FilesRoot = process.env.FYLO_S3FILES_ROOT
+        process.env.FYLO_ROOT = root
         process.env.FYLO_S3FILES_ROOT = root
         const collection = 's3files-static'
         await Fylo.createCollection(collection)
         const id = await fylo.putData(collection, { title: 'Static path' })
         const result = await Fylo.getDoc(collection, id).once()
         expect(result[id].title).toBe('Static path')
-        if (prevEngine === undefined) delete process.env.FYLO_STORAGE_ENGINE
-        else process.env.FYLO_STORAGE_ENGINE = prevEngine
-        if (prevRoot === undefined) delete process.env.FYLO_S3FILES_ROOT
-        else process.env.FYLO_S3FILES_ROOT = prevRoot
+        if (prevFyloRoot === undefined) delete process.env.FYLO_ROOT
+        else process.env.FYLO_ROOT = prevFyloRoot
+        if (prevS3FilesRoot === undefined) delete process.env.FYLO_S3FILES_ROOT
+        else process.env.FYLO_S3FILES_ROOT = prevS3FilesRoot
     })
 })