@delma/fylo 1.1.2 → 2.0.1

package/tests/integration/s3-files.test.js

@@ -0,0 +1,192 @@
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
+import { mkdtemp, rm, stat } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { Database } from 'bun:sqlite'
+import Fylo from '../../src'
+const root = await mkdtemp(path.join(os.tmpdir(), 'fylo-s3files-'))
+const fylo = new Fylo({ engine: 's3-files', s3FilesRoot: root })
+const POSTS = 's3files-posts'
+const USERS = 's3files-users'
+describe('s3-files engine', () => {
+    beforeAll(async () => {
+        await fylo.createCollection(POSTS)
+        await fylo.createCollection(USERS)
+    })
+    afterAll(async () => {
+        await rm(root, { recursive: true, force: true })
+    })
+    test('put/get/patch/delete works without Redis or S3 adapters', async () => {
+        const id = await fylo.putData(POSTS, {
+            title: 'Hello',
+            tags: ['bun', 'aws'],
+            meta: { score: 1 }
+        })
+        const created = await fylo.getDoc(POSTS, id).once()
+        expect(created[id].title).toBe('Hello')
+        expect(created[id].tags).toEqual(['bun', 'aws'])
+        const nextId = await fylo.patchDoc(POSTS, {
+            [id]: {
+                title: 'Hello 2',
+                meta: { score: 2 }
+            }
+        })
+        const updated = await fylo.getDoc(POSTS, nextId).once()
+        expect(updated[nextId].title).toBe('Hello 2')
+        expect(updated[nextId].meta.score).toBe(2)
+        expect(await fylo.getDoc(POSTS, id).once()).toEqual({})
+        await fylo.delDoc(POSTS, nextId)
+        expect(await fylo.getDoc(POSTS, nextId).once()).toEqual({})
+    })
+    test('findDocs listener is backed by the filesystem event journal', async () => {
+        const iter = fylo
+            .findDocs(POSTS, {
+                $ops: [{ title: { $eq: 'Live event' } }]
+            })
+            [Symbol.asyncIterator]()
+        const pending = iter.next()
+        await Bun.sleep(100)
+        const id = await fylo.putData(POSTS, { title: 'Live event' })
+        const { value } = await pending
+        expect(value).toEqual({ [id]: { title: 'Live event' } })
+        await iter.return?.()
+    })
+    test('supports long values without path-length issues', async () => {
+        const longBody = 'x'.repeat(5000)
+        const id = await fylo.putData(POSTS, {
+            title: 'Long payload',
+            body: longBody
+        })
+        const result = await fylo.getDoc(POSTS, id).once()
+        expect(result[id].body).toBe(longBody)
+    })
+    test('stores indexes in a single SQLite database instead of per-entry files', async () => {
+        const dbStat = await stat(path.join(root, POSTS, '.fylo', 'index.db'))
+        expect(dbStat.isFile()).toBe(true)
+        await expect(stat(path.join(root, POSTS, '.fylo', 'indexes'))).rejects.toThrow()
+    })
+    test('uses SQLite index rows to support exact, range, and contains queries', async () => {
+        const queryCollection = 's3files-query'
+        await fylo.createCollection(queryCollection)
+
+        const bunId = await fylo.putData(queryCollection, {
+            title: 'Bun launch',
+            tags: ['bun', 'aws'],
+            meta: { score: 10 }
+        })
+        const nodeId = await fylo.putData(queryCollection, {
+            title: 'Node launch',
+            tags: ['node'],
+            meta: { score: 2 }
+        })
+
+        let eqResults = {}
+        for await (const data of fylo
+            .findDocs(queryCollection, {
+                $ops: [{ title: { $eq: 'Bun launch' } }]
+            })
+            .collect()) {
+            eqResults = { ...eqResults, ...data }
+        }
+        expect(Object.keys(eqResults)).toEqual([bunId])
+
+        let rangeResults = {}
+        for await (const data of fylo
+            .findDocs(queryCollection, {
+                $ops: [{ ['meta.score']: { $gte: 5 } }]
+            })
+            .collect()) {
+            rangeResults = { ...rangeResults, ...data }
+        }
+        expect(Object.keys(rangeResults)).toEqual([bunId])
+
+        let containsResults = {}
+        for await (const data of fylo
+            .findDocs(queryCollection, {
+                $ops: [{ tags: { $contains: 'aws' } }]
+            })
+            .collect()) {
+            containsResults = { ...containsResults, ...data }
+        }
+        expect(Object.keys(containsResults)).toEqual([bunId])
+        expect(containsResults[nodeId]).toBeUndefined()
+
+        const db = new Database(path.join(root, queryCollection, '.fylo', 'index.db'))
+        const rows = db
+            .query(
+                `SELECT doc_id, field_path, raw_value, value_type, numeric_value
+                 FROM doc_index_entries
+                 WHERE doc_id = ?
+                 ORDER BY field_path, raw_value`
+            )
+            .all(bunId)
+        db.close()
+
+        expect(rows).toEqual(
+            expect.arrayContaining([
+                expect.objectContaining({
+                    doc_id: bunId,
+                    field_path: 'title',
+                    raw_value: 'Bun launch',
+                    value_type: 'string',
+                    numeric_value: null
+                }),
+                expect.objectContaining({
+                    doc_id: bunId,
+                    field_path: 'meta/score',
+                    raw_value: '10',
+                    value_type: 'number',
+                    numeric_value: 10
+                }),
+                expect.objectContaining({
+                    doc_id: bunId,
+                    field_path: 'tags/1',
+                    raw_value: 'aws',
+                    value_type: 'string',
+                    numeric_value: null
+                })
+            ])
+        )
+    })
+    test('joins work in s3-files mode', async () => {
+        const userId = await fylo.putData(USERS, { id: 42, name: 'Ada' })
+        const postId = await fylo.putData(POSTS, { id: 42, title: 'Shared', content: 'join me' })
+        const joined = await fylo.joinDocs({
+            $leftCollection: USERS,
+            $rightCollection: POSTS,
+            $mode: 'inner',
+            $on: {
+                id: { $eq: 'id' }
+            }
+        })
+        expect(joined[`${userId}, ${postId}`]).toBeDefined()
+    })
+    test('queue APIs are explicitly unsupported', async () => {
+        await expect(fylo.queuePutData(POSTS, { title: 'no queue' })).rejects.toThrow(
+            'queuePutData is not supported'
+        )
+        await expect(fylo.processQueuedWrites(1)).rejects.toThrow(
+            'processQueuedWrites is not supported'
+        )
+    })
+    test('rejects collection names that are unsafe for cross-platform filesystems', async () => {
+        await expect(fylo.createCollection('bad/name')).rejects.toThrow('Invalid collection name')
+        await expect(fylo.createCollection('bad\\name')).rejects.toThrow('Invalid collection name')
+        await expect(fylo.createCollection('bad:name')).rejects.toThrow('Invalid collection name')
+    })
+    test('static helpers can use s3-files through env defaults', async () => {
+        const prevEngine = process.env.FYLO_STORAGE_ENGINE
+        const prevRoot = process.env.FYLO_S3FILES_ROOT
+        process.env.FYLO_STORAGE_ENGINE = 's3-files'
+        process.env.FYLO_S3FILES_ROOT = root
+        const collection = 's3files-static'
+        await Fylo.createCollection(collection)
+        const id = await fylo.putData(collection, { title: 'Static path' })
+        const result = await Fylo.getDoc(collection, id).once()
+        expect(result[id].title).toBe('Static path')
+        if (prevEngine === undefined) delete process.env.FYLO_STORAGE_ENGINE
+        else process.env.FYLO_STORAGE_ENGINE = prevEngine
+        if (prevRoot === undefined) delete process.env.FYLO_S3FILES_ROOT
+        else process.env.FYLO_S3FILES_ROOT = prevRoot
+    })
+})

package/tests/integration/update.test.js

@@ -0,0 +1,99 @@
+import { test, expect, describe, beforeAll, afterAll, mock } from 'bun:test'
+import Fylo from '../../src'
+import { photosURL, todosURL } from '../data'
+import S3Mock from '../mocks/s3'
+import RedisMock from '../mocks/redis'
+const PHOTOS = `photo`
+const TODOS = `todo`
+const fylo = new Fylo()
+mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
+mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
+beforeAll(async () => {
+    await Promise.all([Fylo.createCollection(PHOTOS), fylo.executeSQL(`CREATE TABLE ${TODOS}`)])
+    try {
+        await fylo.importBulkData(PHOTOS, new URL(photosURL), 100)
+        await fylo.importBulkData(TODOS, new URL(todosURL), 100)
+    } catch {
+        await fylo.rollback()
+    }
+})
+afterAll(async () => {
+    await Promise.all([Fylo.dropCollection(PHOTOS), fylo.executeSQL(`DROP TABLE ${TODOS}`)])
+})
+describe('NO-SQL', async () => {
+    test('UPDATE ONE', async () => {
+        const ids = []
+        for await (const data of Fylo.findDocs(PHOTOS, { $limit: 1, $onlyIds: true }).collect()) {
+            ids.push(data)
+        }
+        try {
+            await fylo.patchDoc(PHOTOS, { [ids.shift()]: { title: 'All Mighty' } })
+        } catch {
+            await fylo.rollback()
+        }
+        let results = {}
+        for await (const data of Fylo.findDocs(PHOTOS, {
+            $ops: [{ title: { $eq: 'All Mighty' } }]
+        }).collect()) {
+            results = { ...results, ...data }
+        }
+        expect(Object.keys(results).length).toBe(1)
+    })
+    test('UPDATE CLAUSE', async () => {
+        let count = -1
+        try {
+            count = await fylo.patchDocs(PHOTOS, {
+                $set: { title: 'All Mighti' },
+                $where: { $ops: [{ title: { $like: '%est%' } }] }
+            })
+        } catch {
+            await fylo.rollback()
+        }
+        let results = {}
+        for await (const data of Fylo.findDocs(PHOTOS, {
+            $ops: [{ title: { $eq: 'All Mighti' } }]
+        }).collect()) {
+            results = { ...results, ...data }
+        }
+        expect(Object.keys(results).length).toBe(count)
+    })
+    test('UPDATE ALL', async () => {
+        let count = -1
+        try {
+            count = await fylo.patchDocs(PHOTOS, { $set: { title: 'All Mighter' } })
+        } catch {
+            await fylo.rollback()
+        }
+        let results = {}
+        for await (const data of Fylo.findDocs(PHOTOS, {
+            $ops: [{ title: { $eq: 'All Mighter' } }]
+        }).collect()) {
+            results = { ...results, ...data }
+        }
+        expect(Object.keys(results).length).toBe(count)
+    }, 20000)
+})
+describe('SQL', async () => {
+    test('UPDATE CLAUSE', async () => {
+        let count = -1
+        try {
+            count = await fylo.executeSQL(
+                `UPDATE ${TODOS} SET title = 'All Mighty' WHERE title LIKE '%est%'`
+            )
+        } catch {
+            await fylo.rollback()
+        }
+        const results = await fylo.executeSQL(`SELECT * FROM ${TODOS} WHERE title = 'All Mighty'`)
+        expect(Object.keys(results).length).toBe(count)
+    })
+    test('UPDATE ALL', async () => {
+        let count = -1
+        try {
+            count = await fylo.executeSQL(`UPDATE ${TODOS} SET title = 'All Mightier'`)
+        } catch {
+            await fylo.rollback()
+        }
+        const results = await fylo.executeSQL(`SELECT * FROM ${TODOS} WHERE title = 'All Mightier'`)
+        expect(Object.keys(results).length).toBe(count)
+    }, 20000)
+})

package/tests/mocks/{cipher.ts → cipher.js}

@@ -1,55 +1,40 @@
-/**
- * Pass-through Cipher mock for tests that don't need real encryption.
- * Returns values as-is (base64-encoded to match real adapter interface shape).
- */
 export class CipherMock {
-
-    private static _configured = false
-    private static collections: Map<string, Set<string>> = new Map()
-
-    static isConfigured(): boolean {
+    static _configured = false
+    static collections = new Map()
+    static isConfigured() {
         return CipherMock._configured
     }
-
-    static hasEncryptedFields(collection: string): boolean {
+    static hasEncryptedFields(collection) {
         const fields = CipherMock.collections.get(collection)
         return !!fields && fields.size > 0
     }
-
-    static isEncryptedField(collection: string, field: string): boolean {
+    static isEncryptedField(collection, field) {
         const fields = CipherMock.collections.get(collection)
         if (!fields || fields.size === 0) return false
-
         for (const pattern of fields) {
             if (field === pattern) return true
             if (field.startsWith(`${pattern}/`)) return true
         }
-
         return false
     }
-
-    static registerFields(collection: string, fields: string[]): void {
+    static registerFields(collection, fields) {
         if (fields.length > 0) {
             CipherMock.collections.set(collection, new Set(fields))
         }
     }
-
-    static async configure(_secret: string): Promise<void> {
+    static async configure(_secret) {
         CipherMock._configured = true
     }
-
-    static reset(): void {
+    static reset() {
         CipherMock._configured = false
         CipherMock.collections = new Map()
     }
-
-    static async encrypt(value: string): Promise<string> {
+    static async encrypt(value) {
         return btoa(value).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '')
     }
-
-    static async decrypt(encoded: string): Promise<string> {
+    static async decrypt(encoded) {
         const b64 = encoded.replace(/-/g, '+').replace(/_/g, '/')
-        const padded = b64 + '='.repeat((4 - b64.length % 4) % 4)
+        const padded = b64 + '='.repeat((4 - (b64.length % 4)) % 4)
         return atob(padded)
     }
 }

package/tests/mocks/redis.js

@@ -0,0 +1,123 @@
+export default class RedisMock {
+    static stream = []
+    static jobs = new Map()
+    static docs = new Map()
+    static locks = new Map()
+    static deadLetters = []
+    static nextId = 0
+    async publish(_collection, _action, _keyId) {}
+    async claimTTID(_id, _ttlSeconds = 10) {
+        return true
+    }
+    async enqueueWrite(job) {
+        RedisMock.jobs.set(job.jobId, { ...job, nextAttemptAt: job.nextAttemptAt ?? Date.now() })
+        RedisMock.docs.set(`fylo:doc:${job.collection}:${job.docId}`, {
+            status: 'queued',
+            lastJobId: job.jobId,
+            updatedAt: String(Date.now())
+        })
+        const streamId = String(++RedisMock.nextId)
+        RedisMock.stream.push({
+            streamId,
+            jobId: job.jobId,
+            collection: job.collection,
+            docId: job.docId,
+            operation: job.operation
+        })
+        return streamId
+    }
+    async readWriteJobs(workerId, count = 1) {
+        const available = RedisMock.stream.filter((entry) => !entry.claimedBy).slice(0, count)
+        for (const entry of available) entry.claimedBy = workerId
+        return available.map((entry) => ({
+            streamId: entry.streamId,
+            job: { ...RedisMock.jobs.get(entry.jobId) }
+        }))
+    }
+    async ackWriteJob(streamId) {
+        RedisMock.stream = RedisMock.stream.filter((item) => item.streamId !== streamId)
+    }
+    async deadLetterWriteJob(streamId, job, reason) {
+        RedisMock.deadLetters.push({
+            streamId: String(RedisMock.deadLetters.length + 1),
+            jobId: job.jobId,
+            reason,
+            failedAt: Date.now()
+        })
+        await this.ackWriteJob(streamId)
+    }
+    async claimPendingJobs(workerId, _minIdleMs = 30000, count = 10) {
+        const pending = RedisMock.stream.filter((entry) => entry.claimedBy).slice(0, count)
+        for (const entry of pending) entry.claimedBy = workerId
+        return pending.map((entry) => ({
+            streamId: entry.streamId,
+            job: { ...RedisMock.jobs.get(entry.jobId) }
+        }))
+    }
+    async setJobStatus(jobId, status, extra = {}) {
+        const job = RedisMock.jobs.get(jobId)
+        if (job) Object.assign(job, extra, { status, updatedAt: Date.now() })
+    }
+    async setDocStatus(collection, docId, status, jobId) {
+        const key = `fylo:doc:${collection}:${docId}`
+        const curr = RedisMock.docs.get(key) ?? {}
+        RedisMock.docs.set(key, {
+            ...curr,
+            status,
+            updatedAt: String(Date.now()),
+            ...(jobId ? { lastJobId: jobId } : {})
+        })
+    }
+    async getJob(jobId) {
+        const job = RedisMock.jobs.get(jobId)
+        return job ? { ...job } : null
+    }
+    async getDocStatus(collection, docId) {
+        return RedisMock.docs.get(`fylo:doc:${collection}:${docId}`) ?? null
+    }
+    async readDeadLetters(count = 10) {
+        return RedisMock.deadLetters.slice(0, count).map((item) => ({
+            streamId: item.streamId,
+            job: { ...RedisMock.jobs.get(item.jobId) },
+            reason: item.reason,
+            failedAt: item.failedAt
+        }))
+    }
+    async replayDeadLetter(streamId) {
+        const item = RedisMock.deadLetters.find((entry) => entry.streamId === streamId)
+        if (!item) return null
+        const job = RedisMock.jobs.get(item.jobId)
+        if (!job) return null
+        const replayed = {
+            ...job,
+            status: 'queued',
+            error: undefined,
+            workerId: undefined,
+            attempts: 0,
+            updatedAt: Date.now(),
+            nextAttemptAt: Date.now()
+        }
+        RedisMock.jobs.set(item.jobId, replayed)
+        await this.enqueueWrite(replayed)
+        RedisMock.deadLetters = RedisMock.deadLetters.filter((entry) => entry.streamId !== streamId)
+        return { ...replayed }
+    }
+    async getQueueStats() {
+        return {
+            queued: RedisMock.stream.length,
+            pending: RedisMock.stream.filter((entry) => entry.claimedBy).length,
+            deadLetters: RedisMock.deadLetters.length
+        }
+    }
+    async acquireDocLock(collection, docId, jobId) {
+        const key = `fylo:lock:${collection}:${docId}`
+        if (RedisMock.locks.has(key)) return false
+        RedisMock.locks.set(key, jobId)
+        return true
+    }
+    async releaseDocLock(collection, docId, jobId) {
+        const key = `fylo:lock:${collection}:${docId}`
+        if (RedisMock.locks.get(key) === jobId) RedisMock.locks.delete(key)
+    }
+    async *subscribe(_collection) {}
+}

package/tests/mocks/{s3.ts → s3.js}

@@ -1,66 +1,43 @@
-/**
- * In-memory S3 mock. Replaces src/adapters/s3 so tests never touch real S3
- * or the AWS CLI. Each test file gets a fresh store because mock.module is
- * hoisted before imports, and module-level state is isolated per test file
- * in Bun's test runner.
- *
- * createBucket / deleteBucket are no-ops (bucket creation/deletion is
- * handled implicitly by the in-memory store).
- */
-
-const store = new Map<string, Map<string, string>>()
-
-function getBucket(name: string): Map<string, string> {
+const store = new Map()
+function getBucket(name) {
     if (!store.has(name)) store.set(name, new Map())
-    return store.get(name)!
+    return store.get(name)
 }
-
 export default class S3Mock {
-
-    static readonly BUCKET_ENV = process.env.BUCKET_PREFIX
-
-    static readonly CREDS = {
+    static BUCKET_ENV = process.env.BUCKET_PREFIX
+    static CREDS = {
         accessKeyId: 'mock',
         secretAccessKey: 'mock',
         region: 'mock',
-        endpoint: undefined as string | undefined
+        endpoint: undefined
     }
-
-    static getBucketFormat(collection: string): string {
+    static getBucketFormat(collection) {
         return S3Mock.BUCKET_ENV ? `${S3Mock.BUCKET_ENV}-${collection}` : collection
     }
-
-    static file(collection: string, path: string) {
+    static file(collection, path) {
         const bucket = getBucket(S3Mock.getBucketFormat(collection))
         return {
             get size() {
                 const val = bucket.get(path)
                 return val !== undefined ? val.length : 0
             },
-            async text(): Promise<string> {
+            async text() {
                 return bucket.get(path) ?? ''
             }
         }
     }
-
-    static async list(collection: string, options: {
-        prefix?: string
-        delimiter?: string
-        maxKeys?: number
-        continuationToken?: string
-    } = {}) {
+    static async list(collection, options = {}) {
         const bucket = getBucket(S3Mock.getBucketFormat(collection))
         const prefix = options.prefix ?? ''
         const delimiter = options.delimiter
         const maxKeys = options.maxKeys ?? 1000
         const token = options.continuationToken
-
-        const allKeys = Array.from(bucket.keys()).filter(k => k.startsWith(prefix)).sort()
-
+        const allKeys = Array.from(bucket.keys())
+            .filter((k) => k.startsWith(prefix))
+            .sort()
         if (delimiter) {
-            const prefixSet = new Set<string>()
-            const contents: Array<{ key: string }> = []
-
+            const prefixSet = new Set()
+            const contents = []
             for (const key of allKeys) {
                 const rest = key.slice(prefix.length)
                 const idx = rest.indexOf(delimiter)
@@ -70,45 +47,34 @@ export default class S3Mock {
                     contents.push({ key })
                 }
             }
-
-            const allPrefixes = Array.from(prefixSet).map(p => ({ prefix: p }))
+            const allPrefixes = Array.from(prefixSet).map((p) => ({ prefix: p }))
             const limitedPrefixes = allPrefixes.slice(0, maxKeys)
-
             return {
                 contents: contents.length ? contents : undefined,
-                commonPrefixes: limitedPrefixes.length
-                    ? limitedPrefixes
-                    : undefined,
+                commonPrefixes: limitedPrefixes.length ? limitedPrefixes : undefined,
                 isTruncated: allPrefixes.length > maxKeys,
                 nextContinuationToken: undefined
             }
         }
-
         const startIdx = token ? parseInt(token) : 0
         const page = allKeys.slice(startIdx, startIdx + maxKeys)
-        const nextToken = startIdx + maxKeys < allKeys.length
-            ? String(startIdx + maxKeys)
-            : undefined
-
+        const nextToken =
+            startIdx + maxKeys < allKeys.length ? String(startIdx + maxKeys) : undefined
         return {
-            contents: page.length ? page.map(k => ({ key: k })) : undefined,
+            contents: page.length ? page.map((k) => ({ key: k })) : undefined,
             isTruncated: !!nextToken,
             nextContinuationToken: nextToken,
             commonPrefixes: undefined
         }
     }
-
-    static async put(collection: string, path: string, data: string): Promise<void> {
+    static async put(collection, path, data) {
         getBucket(S3Mock.getBucketFormat(collection)).set(path, data)
     }
-
-    static async delete(collection: string, path: string): Promise<void> {
+    static async delete(collection, path) {
         getBucket(S3Mock.getBucketFormat(collection)).delete(path)
     }
-
-    static async createBucket(_collection: string): Promise<void> {}
-
-    static async deleteBucket(collection: string): Promise<void> {
+    static async createBucket(_collection) {}
+    static async deleteBucket(collection) {
         store.delete(S3Mock.getBucketFormat(collection))
     }
 }

package/tests/schemas/album.json

@@ -2,4 +2,4 @@
     "id": -0,
     "userId": -0,
     "title": ""
-}
+}

package/tests/schemas/comment.json

@@ -4,4 +4,4 @@
     "email": "",
     "body": "",
     "postId": -0
-}
+}

package/tests/schemas/photo.json

@@ -4,4 +4,4 @@
     "title": "",
     "url": "",
     "thumbnailUrl": ""
-}
+}

package/tests/schemas/post.json

@@ -3,4 +3,4 @@
     "userId": -0,
     "title": "",
     "body": ""
-}
+}

package/tests/schemas/tip.json

@@ -4,4 +4,4 @@
     "text": "",
     "date": "",
     "compliment_count": -0
-}
+}

package/tests/schemas/todo.json

@@ -3,4 +3,4 @@
     "userId": -0,
     "title": "",
     "completed": false
-}
+}