@delma/fylo 2.0.0 → 2.0.1

package/README.md

@@ -2,7 +2,7 @@
 
 NoSQL document store with SQL parsing, real-time listeners, and Bun-first workflows.
 
-Fylo `2.0.0` supports two storage engines:
+Fylo `2.0.1` supports two storage engines:
 
 - `legacy-s3`: the existing S3 + Redis architecture with queued writes, bucket-per-collection storage, and Redis-backed pub/sub/locks.
 - `s3-files`: a new AWS S3 Files mode that stores canonical documents on a mounted S3 Files filesystem, keeps query indexes in a collection-level SQLite database under `.fylo/index.db`, and uses filesystem locks plus an append-only event journal instead of Redis.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@delma/fylo",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "main": "./dist/index.js",
   "types": "./dist/types/index.d.ts",
   "bin": {

package/src/engines/s3-files.ts

@@ -2,6 +2,7 @@ import { mkdir, readFile, readdir, rm, stat, writeFile, open } from 'node:fs/pro
 import path from 'node:path'
 import { createHash } from 'node:crypto'
 import { Database } from 'bun:sqlite'
+import type { SQLQueryBindings } from 'bun:sqlite'
 import TTID from '@delma/ttid'
 import { Dir } from '../core/directory'
 import { validateCollectionName } from '../core/collection'
@@ -440,6 +441,7 @@ export class S3FilesEngine {
 
     private getValueByPath(target: Record<string, any>, fieldPath: string) {
         return fieldPath
+            .replaceAll('/', '.')
             .split('.')
             .reduce<any>(
                 (acc, key) => (acc === undefined || acc === null ? undefined : acc[key]),
@@ -447,6 +449,10 @@ export class S3FilesEngine {
             )
     }
 
+    private normalizeFieldPath(fieldPath: string) {
+        return fieldPath.replaceAll('.', '/')
+    }
+
     private matchesTimestamp(docId: _ttid, query?: _storeQuery<Record<string, any>>) {
         if (!query?.$created && !query?.$updated) return true
         const { createdAt, updatedAt } = TTID.decodeTime(docId)
@@ -491,6 +497,179 @@ export class S3FilesEngine {
         return true
     }
 
+    private async normalizeQueryValue(collection: string, fieldPath: string, value: unknown) {
+        let rawValue = String(value).replaceAll('/', '%2F')
+        if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, fieldPath))
+            rawValue = await Cipher.encrypt(rawValue, true)
+        return this.normalizeIndexValue(rawValue)
+    }
+
+    private intersectDocIds(current: Set<_ttid> | null, next: Iterable<_ttid>) {
+        const nextSet = next instanceof Set ? next : new Set(next)
+        if (current === null) return new Set(nextSet)
+
+        const intersection = new Set<_ttid>()
+        for (const docId of current) {
+            if (nextSet.has(docId)) intersection.add(docId)
+        }
+        return intersection
+    }
+
+    private async queryDocIdsBySql(
+        collection: string,
+        sql: string,
+        ...params: SQLQueryBindings[]
+    ): Promise<Set<_ttid>> {
+        const db = this.database(collection)
+        const rows = db
+            .query(sql)
+            .all(...params)
+            .map((row) => (row as { doc_id: _ttid }).doc_id)
+
+        return new Set(rows)
+    }
+
+    private async candidateDocIdsForOperand(
+        collection: string,
+        fieldPath: string,
+        operand: _operand
+    ): Promise<Set<_ttid> | null> {
+        if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, fieldPath)) return null
+
+        let candidateIds: Set<_ttid> | null = null
+
+        if (operand.$eq !== undefined) {
+            const normalized = await this.normalizeQueryValue(collection, fieldPath, operand.$eq)
+            candidateIds = this.intersectDocIds(
+                candidateIds,
+                await this.queryDocIdsBySql(
+                    collection,
+                    `SELECT DISTINCT doc_id
+                     FROM doc_index_entries
+                     WHERE field_path = ? AND value_hash = ?`,
+                    fieldPath,
+                    normalized.valueHash
+                )
+            )
+        }
+
+        if (
+            operand.$gt !== undefined ||
+            operand.$gte !== undefined ||
+            operand.$lt !== undefined ||
+            operand.$lte !== undefined
+        ) {
+            const clauses = ['field_path = ?']
+            const params: SQLQueryBindings[] = [fieldPath]
+            if (operand.$gt !== undefined) {
+                clauses.push('numeric_value > ?')
+                params.push(operand.$gt)
+            }
+            if (operand.$gte !== undefined) {
+                clauses.push('numeric_value >= ?')
+                params.push(operand.$gte)
+            }
+            if (operand.$lt !== undefined) {
+                clauses.push('numeric_value < ?')
+                params.push(operand.$lt)
+            }
+            if (operand.$lte !== undefined) {
+                clauses.push('numeric_value <= ?')
+                params.push(operand.$lte)
+            }
+
+            candidateIds = this.intersectDocIds(
+                candidateIds,
+                await this.queryDocIdsBySql(
+                    collection,
+                    `SELECT DISTINCT doc_id
+                     FROM doc_index_entries
+                     WHERE ${clauses.join(' AND ')}`,
+                    ...params
+                )
+            )
+        }
+
+        if (operand.$like !== undefined) {
+            candidateIds = this.intersectDocIds(
+                candidateIds,
+                await this.queryDocIdsBySql(
+                    collection,
+                    `SELECT DISTINCT doc_id
+                     FROM doc_index_entries
+                     WHERE field_path = ? AND value_type = 'string' AND raw_value LIKE ?`,
+                    fieldPath,
+                    operand.$like.replaceAll('/', '%2F')
+                )
+            )
+        }
+
+        if (operand.$contains !== undefined) {
+            const normalized = await this.normalizeQueryValue(
+                collection,
+                fieldPath,
+                operand.$contains
+            )
+            candidateIds = this.intersectDocIds(
+                candidateIds,
+                await this.queryDocIdsBySql(
+                    collection,
+                    `SELECT DISTINCT doc_id
+                     FROM doc_index_entries
+                     WHERE (field_path = ? OR field_path LIKE ?)
+                       AND value_hash = ?`,
+                    fieldPath,
+                    `${fieldPath}/%`,
+                    normalized.valueHash
+                )
+            )
+        }
+
+        return candidateIds
+    }
+
+    private async candidateDocIdsForOperation<T extends Record<string, any>>(
+        collection: string,
+        operation: _op<T>
+    ): Promise<Set<_ttid> | null> {
+        let candidateIds: Set<_ttid> | null = null
+
+        for (const [field, operand] of Object.entries(operation) as Array<[keyof T, _operand]>) {
+            if (!operand) continue
+
+            const fieldPath = this.normalizeFieldPath(String(field))
+            const fieldCandidates = await this.candidateDocIdsForOperand(
+                collection,
+                fieldPath,
+                operand
+            )
+
+            if (fieldCandidates === null) continue
+            candidateIds = this.intersectDocIds(candidateIds, fieldCandidates)
+        }
+
+        return candidateIds
+    }
+
+    private async candidateDocIdsForQuery<T extends Record<string, any>>(
+        collection: string,
+        query?: _storeQuery<T>
+    ): Promise<Set<_ttid> | null> {
+        if (!query?.$ops || query.$ops.length === 0) return null
+
+        const union = new Set<_ttid>()
+        let usedIndex = false
+
+        for (const operation of query.$ops) {
+            const candidateIds = await this.candidateDocIdsForOperation(collection, operation)
+            if (candidateIds === null) return null
+            usedIndex = true
+            for (const docId of candidateIds) union.add(docId)
+        }
+
+        return usedIndex ? union : null
+    }
+
     private matchesQuery<T extends Record<string, any>>(
         docId: _ttid,
         doc: T,
@@ -576,7 +755,8 @@ export class S3FilesEngine {
         collection: string,
         query?: _storeQuery<T>
     ) {
-        const ids = await this.listDocIds(collection)
+        const candidateIds = await this.candidateDocIdsForQuery(collection, query)
+        const ids = candidateIds ? Array.from(candidateIds) : await this.listDocIds(collection)
         const limit = query?.$limit
         const results: Array<FyloRecord<T>> = []
 

package/tests/integration/s3-files.test.js

@@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'
 import { mkdtemp, rm, stat } from 'node:fs/promises'
 import os from 'node:os'
 import path from 'node:path'
+import { Database } from 'bun:sqlite'
 import Fylo from '../../src'
 const root = await mkdtemp(path.join(os.tmpdir(), 'fylo-s3files-'))
 const fylo = new Fylo({ engine: 's3-files', s3FilesRoot: root })
@@ -64,6 +65,89 @@ describe('s3-files engine', () => {
         expect(dbStat.isFile()).toBe(true)
         await expect(stat(path.join(root, POSTS, '.fylo', 'indexes'))).rejects.toThrow()
     })
+    test('uses SQLite index rows to support exact, range, and contains queries', async () => {
+        const queryCollection = 's3files-query'
+        await fylo.createCollection(queryCollection)
+
+        const bunId = await fylo.putData(queryCollection, {
+            title: 'Bun launch',
+            tags: ['bun', 'aws'],
+            meta: { score: 10 }
+        })
+        const nodeId = await fylo.putData(queryCollection, {
+            title: 'Node launch',
+            tags: ['node'],
+            meta: { score: 2 }
+        })
+
+        let eqResults = {}
+        for await (const data of fylo
+            .findDocs(queryCollection, {
+                $ops: [{ title: { $eq: 'Bun launch' } }]
+            })
+            .collect()) {
+            eqResults = { ...eqResults, ...data }
+        }
+        expect(Object.keys(eqResults)).toEqual([bunId])
+
+        let rangeResults = {}
+        for await (const data of fylo
+            .findDocs(queryCollection, {
+                $ops: [{ ['meta.score']: { $gte: 5 } }]
+            })
+            .collect()) {
+            rangeResults = { ...rangeResults, ...data }
+        }
+        expect(Object.keys(rangeResults)).toEqual([bunId])
+
+        let containsResults = {}
+        for await (const data of fylo
+            .findDocs(queryCollection, {
+                $ops: [{ tags: { $contains: 'aws' } }]
+            })
+            .collect()) {
+            containsResults = { ...containsResults, ...data }
+        }
+        expect(Object.keys(containsResults)).toEqual([bunId])
+        expect(containsResults[nodeId]).toBeUndefined()
+
+        const db = new Database(path.join(root, queryCollection, '.fylo', 'index.db'))
+        const rows = db
+            .query(
+                `SELECT doc_id, field_path, raw_value, value_type, numeric_value
+                 FROM doc_index_entries
+                 WHERE doc_id = ?
+                 ORDER BY field_path, raw_value`
+            )
+            .all(bunId)
+        db.close()
+
+        expect(rows).toEqual(
+            expect.arrayContaining([
+                expect.objectContaining({
+                    doc_id: bunId,
+                    field_path: 'title',
+                    raw_value: 'Bun launch',
+                    value_type: 'string',
+                    numeric_value: null
+                }),
+                expect.objectContaining({
+                    doc_id: bunId,
+                    field_path: 'meta/score',
+                    raw_value: '10',
+                    value_type: 'number',
+                    numeric_value: 10
+                }),
+                expect.objectContaining({
+                    doc_id: bunId,
+                    field_path: 'tags/1',
+                    raw_value: 'aws',
+                    value_type: 'string',
+                    numeric_value: null
+                })
+            ])
+        )
+    })
     test('joins work in s3-files mode', async () => {
         const userId = await fylo.putData(USERS, { id: 42, name: 'Ada' })
         const postId = await fylo.putData(POSTS, { id: 42, title: 'Shared', content: 'join me' })