@delma/fylo 2.1.0 → 2.1.1

package/src/core/query.ts

@@ -1,53 +0,0 @@
-import { Cipher } from '../adapters/cipher'
-
-const ENCRYPTED_FIELD_OPS = ['$ne', '$gt', '$gte', '$lt', '$lte', '$like', '$contains'] as const
-
-export class Query {
-    static async getExprs<T extends Record<string, any>>(
-        collection: string,
-        query: _storeQuery<T>
-    ) {
-        let exprs = new Set<string>()
-
-        if (query.$ops) {
-            for (const op of query.$ops) {
-                for (const column in op) {
-                    const col = op[column as keyof T]!
-
-                    const fieldPath = String(column).replaceAll('.', '/')
-                    const encrypted =
-                        Cipher.isConfigured() && Cipher.isEncryptedField(collection, fieldPath)
-
-                    if (encrypted) {
-                        for (const opKey of ENCRYPTED_FIELD_OPS) {
-                            if (col[opKey] !== undefined) {
-                                throw new Error(
-                                    `Operator ${opKey} is not supported on encrypted field "${String(column)}"`
-                                )
-                            }
-                        }
-                    }
-
-                    if (col.$eq) {
-                        const val = encrypted
-                            ? await Cipher.encrypt(String(col.$eq).replaceAll('/', '%2F'))
-                            : col.$eq
-                        exprs.add(`${column}/${val}/**/*`)
-                    }
-                    if (col.$ne) exprs.add(`${column}/**/*`)
-                    if (col.$gt) exprs.add(`${column}/**/*`)
-                    if (col.$gte) exprs.add(`${column}/**/*`)
-                    if (col.$lt) exprs.add(`${column}/**/*`)
-                    if (col.$lte) exprs.add(`${column}/**/*`)
-                    if (col.$like) exprs.add(`${column}/${col.$like.replaceAll('%', '*')}/**/*`)
-                    if (col.$contains !== undefined)
-                        exprs.add(
-                            `${column}/*/${String(col.$contains).split('/').join('%2F')}/**/*`
-                        )
-                }
-            }
-        } else exprs = new Set([`**/*`])
-
-        return Array.from(exprs)
-    }
-}

package/src/engines/s3-files/documents.ts

@@ -1,65 +0,0 @@
-import path from 'node:path'
-import TTID from '@delma/ttid'
-import type { StorageEngine } from '../types'
-import type { StoredDoc } from './types'
-
-export class S3FilesDocuments {
-    constructor(
-        private readonly storage: StorageEngine,
-        private readonly docsRoot: (collection: string) => string,
-        private readonly docPath: (collection: string, docId: _ttid) => string,
-        private readonly ensureCollection: (collection: string) => Promise<void>,
-        private readonly encodeEncrypted: <T extends Record<string, any>>(
-            collection: string,
-            value: T,
-            parentField?: string
-        ) => Promise<T>,
-        private readonly decodeEncrypted: <T extends Record<string, any>>(
-            collection: string,
-            value: T,
-            parentField?: string
-        ) => Promise<T>
-    ) {}
-
-    async readStoredDoc<T extends Record<string, any>>(
-        collection: string,
-        docId: _ttid
-    ): Promise<StoredDoc<T> | null> {
-        const target = this.docPath(collection, docId)
-
-        try {
-            const raw = JSON.parse(await this.storage.read(target)) as T
-            const decoded = await this.decodeEncrypted(collection, raw)
-            const { createdAt, updatedAt } = TTID.decodeTime(docId)
-
-            return {
-                id: docId,
-                createdAt,
-                updatedAt: updatedAt ?? createdAt,
-                data: decoded
-            }
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null
-            throw err
-        }
-    }
-
-    async writeStoredDoc<T extends Record<string, any>>(collection: string, docId: _ttid, data: T) {
-        await this.ensureCollection(collection)
-        const encoded = await this.encodeEncrypted(collection, data)
-        const target = this.docPath(collection, docId)
-        await this.storage.write(target, JSON.stringify(encoded))
-    }
-
-    async removeStoredDoc(collection: string, docId: _ttid) {
-        await this.storage.delete(this.docPath(collection, docId))
-    }
-
-    async listDocIds(collection: string) {
-        const files = await this.storage.list(this.docsRoot(collection))
-        return files
-            .filter((file) => file.endsWith('.json'))
-            .map((file) => path.basename(file, '.json'))
-            .filter((key) => TTID.isTTID(key)) as _ttid[]
-    }
-}

package/src/engines/s3-files/filesystem.ts

@@ -1,172 +0,0 @@
-import { mkdir, open, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'
-import path from 'node:path'
-import type { EventBus, LockManager, StorageEngine } from '../types'
-import type { S3FilesEvent } from './types'
-
-export class FilesystemStorage implements StorageEngine {
-    async read(target: string): Promise<string> {
-        return await readFile(target, 'utf8')
-    }
-
-    async write(target: string, data: string): Promise<void> {
-        await mkdir(path.dirname(target), { recursive: true })
-        await writeFile(target, data, 'utf8')
-    }
-
-    async delete(target: string): Promise<void> {
-        await rm(target, { recursive: true, force: true })
-    }
-
-    async list(target: string): Promise<string[]> {
-        const results: string[] = []
-
-        try {
-            const entries = await readdir(target, { withFileTypes: true })
-            for (const entry of entries) {
-                const child = path.join(target, entry.name)
-                if (entry.isDirectory()) {
-                    results.push(...(await this.list(child)))
-                } else {
-                    results.push(child)
-                }
-            }
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
-        }
-
-        return results
-    }
-
-    async mkdir(target: string): Promise<void> {
-        await mkdir(target, { recursive: true })
-    }
-
-    async rmdir(target: string): Promise<void> {
-        await rm(target, { recursive: true, force: true })
-    }
-
-    async exists(target: string): Promise<boolean> {
-        try {
-            await stat(target)
-            return true
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false
-            throw err
-        }
-    }
-}
-
-export class FilesystemLockManager implements LockManager {
-    constructor(
-        private readonly root: string,
-        private readonly storage: StorageEngine
-    ) {}
-
-    private lockDir(collection: string, docId: _ttid) {
-        return path.join(this.root, collection, '.fylo', 'locks', `${docId}.lock`)
-    }
-
-    async acquire(
-        collection: string,
-        docId: _ttid,
-        owner: string,
-        ttlMs: number = 30_000
-    ): Promise<boolean> {
-        const dir = this.lockDir(collection, docId)
-        const metaPath = path.join(dir, 'meta.json')
-        await mkdir(path.dirname(dir), { recursive: true })
-
-        try {
-            await mkdir(dir, { recursive: false })
-            await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
-            return true
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err
-        }
-
-        try {
-            const meta = JSON.parse(await this.storage.read(metaPath)) as { ts?: number }
-            if (meta.ts && Date.now() - meta.ts > ttlMs) {
-                await this.storage.rmdir(dir)
-                await mkdir(dir, { recursive: false })
-                await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
-                return true
-            }
-        } catch {
-            await this.storage.rmdir(dir)
-            await mkdir(dir, { recursive: false })
-            await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
-            return true
-        }
-
-        return false
-    }
-
-    async release(collection: string, docId: _ttid, owner: string): Promise<void> {
-        const dir = this.lockDir(collection, docId)
-        const metaPath = path.join(dir, 'meta.json')
-
-        try {
-            const meta = JSON.parse(await this.storage.read(metaPath)) as { owner?: string }
-            if (meta.owner === owner) await this.storage.rmdir(dir)
-        } catch (err) {
-            if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
-        }
-    }
-}
-
-export class FilesystemEventBus<T extends Record<string, any>> implements EventBus<
-    S3FilesEvent<T>
-> {
-    constructor(
-        private readonly root: string,
-        private readonly storage: StorageEngine
-    ) {}
-
-    private journalPath(collection: string) {
-        return path.join(this.root, collection, '.fylo', 'events', `${collection}.ndjson`)
-    }
-
-    async publish(collection: string, event: S3FilesEvent<T>): Promise<void> {
-        const target = this.journalPath(collection)
-        await mkdir(path.dirname(target), { recursive: true })
-        const line = `${JSON.stringify(event)}\n`
-        const handle = await open(target, 'a')
-        try {
-            await handle.write(line)
-        } finally {
-            await handle.close()
-        }
-    }
-
-    async *listen(collection: string): AsyncGenerator<S3FilesEvent<T>, void, unknown> {
-        const target = this.journalPath(collection)
-        let position = 0
-
-        while (true) {
-            try {
-                const fileStat = await stat(target)
-                if (fileStat.size > position) {
-                    const handle = await open(target, 'r')
-                    try {
-                        const size = fileStat.size - position
-                        const buffer = Buffer.alloc(size)
-                        await handle.read(buffer, 0, size, position)
-                        position = fileStat.size
-
-                        for (const line of buffer.toString('utf8').split('\n')) {
-                            if (line.trim().length === 0) continue
-                            yield JSON.parse(line) as S3FilesEvent<T>
-                        }
-                    } finally {
-                        await handle.close()
-                    }
-                }
-            } catch (err) {
-                if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
-            }
-
-            await Bun.sleep(100)
-        }
-    }
-}

package/src/engines/s3-files/query.ts

@@ -1,291 +0,0 @@
-import TTID from '@delma/ttid'
-import { Cipher } from '../../adapters/cipher'
-import type { CollectionIndexCache, FyloRecord, S3FilesQueryResult } from './types'
-
-type QueryContext = {
-    loadIndexCache: (collection: string) => Promise<CollectionIndexCache>
-    normalizeIndexValue: (rawValue: string) => {
-        rawValue: string
-        valueHash: string
-        valueType: string
-        numericValue: number | null
-    }
-}
-
-export class S3FilesQueryEngine {
-    constructor(private readonly context: QueryContext) {}
-
-    getValueByPath(target: Record<string, any>, fieldPath: string) {
-        return fieldPath
-            .replaceAll('/', '.')
-            .split('.')
-            .reduce<any>(
-                (acc, key) => (acc === undefined || acc === null ? undefined : acc[key]),
-                target
-            )
-    }
-
-    normalizeFieldPath(fieldPath: string) {
-        return fieldPath.replaceAll('.', '/')
-    }
-
-    matchesTimestamp(docId: _ttid, query?: _storeQuery<Record<string, any>>) {
-        if (!query?.$created && !query?.$updated) return true
-        const { createdAt, updatedAt } = TTID.decodeTime(docId)
-        const timestamps = { createdAt, updatedAt: updatedAt ?? createdAt }
-
-        const match = (value: number, range?: _timestamp) => {
-            if (!range) return true
-            if (range.$gt !== undefined && !(value > range.$gt)) return false
-            if (range.$gte !== undefined && !(value >= range.$gte)) return false
-            if (range.$lt !== undefined && !(value < range.$lt)) return false
-            if (range.$lte !== undefined && !(value <= range.$lte)) return false
-            return true
-        }
-
-        return (
-            match(timestamps.createdAt, query.$created) &&
-            match(timestamps.updatedAt, query.$updated)
-        )
-    }
-
-    likeToRegex(pattern: string) {
-        const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replaceAll('%', '.*')
-        return new RegExp(`^${escaped}$`)
-    }
-
-    matchesOperand(value: unknown, operand: _operand) {
-        if (operand.$eq !== undefined && value != operand.$eq) return false
-        if (operand.$ne !== undefined && value == operand.$ne) return false
-        if (operand.$gt !== undefined && !(Number(value) > operand.$gt)) return false
-        if (operand.$gte !== undefined && !(Number(value) >= operand.$gte)) return false
-        if (operand.$lt !== undefined && !(Number(value) < operand.$lt)) return false
-        if (operand.$lte !== undefined && !(Number(value) <= operand.$lte)) return false
-        if (
-            operand.$like !== undefined &&
-            (typeof value !== 'string' || !this.likeToRegex(operand.$like).test(value))
-        )
-            return false
-        if (operand.$contains !== undefined) {
-            if (!Array.isArray(value) || !value.some((item) => item == operand.$contains))
-                return false
-        }
-        return true
-    }
-
-    async normalizeQueryValue(collection: string, fieldPath: string, value: unknown) {
-        let rawValue = String(value).replaceAll('/', '%2F')
-        if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, fieldPath))
-            rawValue = await Cipher.encrypt(rawValue, true)
-        return this.context.normalizeIndexValue(rawValue)
-    }
-
-    intersectDocIds(current: Set<_ttid> | null, next: Iterable<_ttid>) {
-        const nextSet = next instanceof Set ? next : new Set(next)
-        if (current === null) return new Set(nextSet)
-
-        const intersection = new Set<_ttid>()
-        for (const docId of current) {
-            if (nextSet.has(docId)) intersection.add(docId)
-        }
-        return intersection
-    }
-
-    async candidateDocIdsForOperand(
-        collection: string,
-        fieldPath: string,
-        operand: _operand
-    ): Promise<Set<_ttid> | null> {
-        if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, fieldPath)) {
-            const unsupported =
-                operand.$ne !== undefined ||
-                operand.$gt !== undefined ||
-                operand.$gte !== undefined ||
-                operand.$lt !== undefined ||
-                operand.$lte !== undefined ||
-                operand.$like !== undefined ||
-                operand.$contains !== undefined
-
-            if (unsupported) {
-                throw new Error(`Operator is not supported on encrypted field: ${fieldPath}`)
-            }
-        }
-
-        const cache = await this.context.loadIndexCache(collection)
-        let candidateIds: Set<_ttid> | null = null
-
-        if (operand.$eq !== undefined) {
-            const normalized = await this.normalizeQueryValue(collection, fieldPath, operand.$eq)
-            candidateIds = this.intersectDocIds(
-                candidateIds,
-                cache.fieldHash.get(fieldPath)?.get(normalized.valueHash) ?? new Set<_ttid>()
-            )
-        }
-
-        if (
-            operand.$gt !== undefined ||
-            operand.$gte !== undefined ||
-            operand.$lt !== undefined ||
-            operand.$lte !== undefined
-        ) {
-            const numericMatches = new Set<_ttid>()
-            for (const entry of cache.fieldNumeric.get(fieldPath) ?? []) {
-                if (operand.$gt !== undefined && !(entry.numericValue > operand.$gt)) continue
-                if (operand.$gte !== undefined && !(entry.numericValue >= operand.$gte)) continue
-                if (operand.$lt !== undefined && !(entry.numericValue < operand.$lt)) continue
-                if (operand.$lte !== undefined && !(entry.numericValue <= operand.$lte)) continue
-                numericMatches.add(entry.docId)
-            }
-
-            candidateIds = this.intersectDocIds(candidateIds, numericMatches)
-        }
-
-        if (operand.$like !== undefined) {
-            const regex = this.likeToRegex(operand.$like.replaceAll('/', '%2F'))
-            const stringMatches = new Set<_ttid>()
-            for (const entry of cache.fieldString.get(fieldPath) ?? []) {
-                if (regex.test(entry.rawValue)) stringMatches.add(entry.docId)
-            }
-
-            candidateIds = this.intersectDocIds(candidateIds, stringMatches)
-        }
-
-        if (operand.$contains !== undefined) {
-            const normalized = await this.normalizeQueryValue(
-                collection,
-                fieldPath,
-                operand.$contains
-            )
-            const containsMatches = new Set<_ttid>()
-            for (const [candidateFieldPath, hashes] of cache.fieldHash.entries()) {
-                if (
-                    candidateFieldPath !== fieldPath &&
-                    !candidateFieldPath.startsWith(`${fieldPath}/`)
-                )
-                    continue
-                for (const docId of hashes.get(normalized.valueHash) ?? [])
-                    containsMatches.add(docId)
-            }
-
-            candidateIds = this.intersectDocIds(candidateIds, containsMatches)
-        }
-
-        return candidateIds
-    }
-
-    async candidateDocIdsForOperation<T extends Record<string, any>>(
-        collection: string,
-        operation: _op<T>
-    ): Promise<Set<_ttid> | null> {
-        let candidateIds: Set<_ttid> | null = null
-
-        for (const [field, operand] of Object.entries(operation) as Array<[keyof T, _operand]>) {
-            if (!operand) continue
-
-            const fieldPath = this.normalizeFieldPath(String(field))
-            const fieldCandidates = await this.candidateDocIdsForOperand(
-                collection,
-                fieldPath,
-                operand
-            )
-
-            if (fieldCandidates === null) continue
-            candidateIds = this.intersectDocIds(candidateIds, fieldCandidates)
-        }
-
-        return candidateIds
-    }
-
-    async candidateDocIdsForQuery<T extends Record<string, any>>(
-        collection: string,
-        query?: _storeQuery<T>
-    ): Promise<Set<_ttid> | null> {
-        if (!query?.$ops || query.$ops.length === 0) return null
-
-        const union = new Set<_ttid>()
-        let usedIndex = false
-
-        for (const operation of query.$ops) {
-            const candidateIds = await this.candidateDocIdsForOperation(collection, operation)
-            if (candidateIds === null) return null
-            usedIndex = true
-            for (const docId of candidateIds) union.add(docId)
-        }
-
-        return usedIndex ? union : null
-    }
-
-    matchesQuery<T extends Record<string, any>>(docId: _ttid, doc: T, query?: _storeQuery<T>) {
-        if (!this.matchesTimestamp(docId, query as _storeQuery<Record<string, any>> | undefined))
-            return false
-        if (!query?.$ops || query.$ops.length === 0) return true
-
-        return query.$ops.some((operation) => {
-            for (const field in operation) {
-                const value = this.getValueByPath(doc, field)
-                if (!this.matchesOperand(value, operation[field as keyof T]!)) return false
-            }
-            return true
-        })
-    }
-
-    selectValues<T extends Record<string, any>>(selection: Array<keyof T>, data: T) {
-        const copy = { ...data }
-        for (const field in copy) {
-            if (!selection.includes(field as keyof T)) delete copy[field]
-        }
-        return copy
-    }
-
-    renameFields<T extends Record<string, any>>(rename: Record<keyof Partial<T>, string>, data: T) {
-        const copy = { ...data }
-        for (const field in copy) {
-            if (rename[field]) {
-                copy[rename[field]] = copy[field]
-                delete copy[field]
-            }
-        }
-        return copy
-    }
-
-    processDoc<T extends Record<string, any>>(
-        doc: FyloRecord<T>,
-        query?: _storeQuery<T>
-    ): S3FilesQueryResult<T> | undefined {
-        if (Object.keys(doc).length === 0) return
-
-        const next = { ...doc }
-
-        for (let [_id, data] of Object.entries(next)) {
-            if (query?.$select?.length)
-                data = this.selectValues(query.$select as Array<keyof T>, data)
-            if (query?.$rename) data = this.renameFields(query.$rename, data)
-            next[_id as _ttid] = data as T
-        }
-
-        if (query?.$groupby) {
-            const docGroup: Record<string, Record<string, Partial<T>>> = {}
-            for (const [id, data] of Object.entries(next)) {
-                const groupValue = data[query.$groupby] as string
-                if (groupValue) {
-                    const groupData = { ...data }
-                    delete groupData[query.$groupby]
-                    docGroup[groupValue] = { [id]: groupData as Partial<T> }
-                }
-            }
-
-            if (query.$onlyIds) {
-                const groupedIds: Record<string, _ttid[]> = {}
-                for (const group in docGroup)
-                    groupedIds[group] = Object.keys(docGroup[group]) as _ttid[]
-                return groupedIds
-            }
-
-            return docGroup
-        }
-
-        if (query?.$onlyIds) return Object.keys(next).shift() as _ttid
-
-        return next
-    }
-}

package/src/engines/s3-files/types.ts

@@ -1,42 +0,0 @@
-export type FyloRecord<T extends Record<string, any>> = Record<_ttid, T>
-
-export type S3FilesQueryResult<T extends Record<string, any>> =
-    | _ttid
-    | FyloRecord<T>
-    | Record<string, _ttid[]>
-    | Record<string, Record<_ttid, Partial<T>>>
-    | Record<_ttid, Partial<T>>
-
-export type S3FilesEvent<T extends Record<string, any>> = {
-    ts: number
-    action: 'insert' | 'delete'
-    id: _ttid
-    doc?: T
-}
-
-export type StoredDoc<T extends Record<string, any>> = {
-    id: _ttid
-    createdAt: number
-    updatedAt: number
-    data: T
-}
-
-export type StoredIndexEntry = {
-    fieldPath: string
-    rawValue: string
-    valueHash: string
-    valueType: string
-    numericValue: number | null
-}
-
-export type StoredCollectionIndex = {
-    version: 1
-    docs: Record<_ttid, StoredIndexEntry[]>
-}
-
-export type CollectionIndexCache = {
-    docs: Map<_ttid, StoredIndexEntry[]>
-    fieldHash: Map<string, Map<string, Set<_ttid>>>
-    fieldNumeric: Map<string, Array<{ docId: _ttid; numericValue: number }>>
-    fieldString: Map<string, Array<{ docId: _ttid; rawValue: string }>>
-}