@delma/fylo 2.0.0 → 2.1.0

package/src/core/directory.ts

@@ -1,380 +1,48 @@
-import { Walker } from './walker'
-import TTID from '@delma/ttid'
-import { S3 } from '../adapters/s3'
-import { Redis } from '../adapters/redis'
 import { Cipher } from '../adapters/cipher'
 
 export class Dir {
-    private static readonly KEY_LIMIT = 1024
-
     private static readonly SLASH_ASCII = '%2F'
 
-    private readonly transactions: Array<{
-        action: (...args: string[]) => Promise<void>
-        args: string[]
-    }>
-
-    private static _redis: Redis | null = null
-
-    private static get redis(): Redis {
-        if (!Dir._redis) Dir._redis = new Redis()
-        return Dir._redis
-    }
-
-    constructor() {
-        this.transactions = []
-    }
-
-    static async claimTTID(_id: _ttid, ttlSeconds: number = 10): Promise<boolean> {
-        return await Dir.redis.claimTTID(_id, ttlSeconds)
-    }
-
-    static async reconstructData(collection: string, items: string[]) {
-        items = await this.readValues(collection, items)
-
-        let fieldVal: Record<string, string> = {}
-
-        for (const data of items) {
-            const segs = data.split('/')
-            const val = segs.pop()!
-            const fieldPath = segs.join('/')
-
-            // Decrypt value if field is encrypted — fieldPath starts with TTID segment
-            // so strip it to get the actual field name for the check
-            const fieldOnly = segs.slice(1).join('/')
-            if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, fieldOnly)) {
-                fieldVal[fieldPath] = await Cipher.decrypt(val)
-            } else {
-                fieldVal[fieldPath] = val
-            }
-        }
-
-        return this.constructData(fieldVal)
-    }
-
-    private static async readValues(collection: string, items: string[]) {
-        for (let i = 0; i < items.length; i++) {
-            const segments = items[i].split('/')
-
-            const filename = segments.pop()!
-
-            if (TTID.isUUID(filename)) {
-                const file = S3.file(collection, items[i])
-                const val = await file.text()
-
-                items[i] = `${segments.join('/')}/${val}`
-            }
-        }
-
-        return items
-    }
-
-    private static async filterByTimestamp(
-        _id: _ttid,
-        indexes: string[],
-        { updated, created }: { updated?: _timestamp; created?: _timestamp }
-    ) {
-        const { createdAt, updatedAt } = TTID.decodeTime(_id)
-
-        if (updated && updatedAt) {
-            if ((updated.$gt || updated.$gte) && (updated.$lt || updated.$lte)) {
-                if (updated.$gt && updated.$lt) {
-                    if (updated.$gt! > updated.$lt!) throw new Error('Invalid updated query')
-
-                    indexes = updatedAt > updated.$gt! && updatedAt < updated.$lt! ? indexes : []
-                } else if (updated.$gt && updated.$lte) {
-                    if (updated.$gt! > updated.$lte!) throw new Error('Invalid updated query')
-
-                    indexes = updatedAt > updated.$gt! && updatedAt <= updated.$lte! ? indexes : []
-                } else if (updated.$gte && updated.$lt) {
-                    if (updated.$gte! > updated.$lt!) throw new Error('Invalid updated query')
-
-                    indexes = updatedAt >= updated.$gte! && updatedAt < updated.$lt! ? indexes : []
-                } else if (updated.$gte && updated.$lte) {
-                    if (updated.$gte! > updated.$lte!) throw new Error('Invalid updated query')
-
-                    indexes =
-                        updatedAt >= updated.$gte! && updatedAt <= updated.$lte! ? indexes : []
-                }
-            } else if ((updated.$gt || updated.$gte) && !updated.$lt && !updated.$lte) {
-                indexes = updated.$gt
-                    ? updatedAt > updated.$gt!
-                        ? indexes
-                        : []
-                    : updatedAt >= updated.$gte!
-                      ? indexes
-                      : []
-            } else if (!updated.$gt && !updated.$gte && (updated.$lt || updated.$lte)) {
-                indexes = updated.$lt
-                    ? updatedAt < updated.$lt!
-                        ? indexes
-                        : []
-                    : updatedAt <= updated.$lte!
-                      ? indexes
-                      : []
-            }
-        }
-
-        if (created) {
-            if ((created.$gt || created.$gte) && (created.$lt || created.$lte)) {
-                if (created.$gt && created.$lt) {
-                    if (created.$gt! > created.$lt!) throw new Error('Invalid created query')
-
-                    indexes = createdAt > created.$gt! && createdAt < created.$lt! ? indexes : []
-                } else if (created.$gt && created.$lte) {
-                    if (created.$gt! > created.$lte!) throw new Error('Invalid updated query')
-
-                    indexes = createdAt > created.$gt! && createdAt <= created.$lte! ? indexes : []
-                } else if (created.$gte && created.$lt) {
-                    if (created.$gte! > created.$lt!) throw new Error('Invalid updated query')
-
-                    indexes = createdAt >= created.$gte! && createdAt < created.$lt! ? indexes : []
-                } else if (created.$gte && created.$lte) {
-                    if (created.$gte! > created.$lte!) throw new Error('Invalid updated query')
-
-                    indexes =
-                        createdAt >= created.$gte! && createdAt <= created.$lte! ? indexes : []
-                }
-            } else if ((created.$gt || created.$gte) && !created.$lt && !created.$lte) {
-                if (created.$gt) indexes = createdAt > created.$gt! ? indexes : []
-                else if (created.$gte) indexes = createdAt >= created.$gte! ? indexes : []
-            } else if (!created.$gt && !created.$gte && (created.$lt || created.$lte)) {
-                if (created.$lt) indexes = createdAt < created.$lt! ? indexes : []
-                else if (created.$lte) indexes = createdAt <= created.$lte! ? indexes : []
-            }
-        }
-
-        return indexes.length > 0
-    }
-
-    static async *searchDocs<T extends Record<string, any>>(
-        collection: string,
-        pattern: string | string[],
-        { updated, created }: { updated?: _timestamp; created?: _timestamp },
-        { listen = false, skip = false }: { listen: boolean; skip: boolean },
-        deleted: boolean = false
-    ): AsyncGenerator<Record<_ttid, T> | _ttid | void, void, { count: number; limit?: number }> {
-        const data = yield
-        let count = data.count
-        let limit = data.limit
-
-        const constructData = async (collection: string, _id: _ttid, items: string[]) => {
-            if (created || updated) {
-                if (await this.filterByTimestamp(_id, items, { created, updated })) {
-                    const data = await this.reconstructData(collection, items)
-
-                    return { [_id]: data } as Record<_ttid, T>
-                } else return {}
-            } else {
-                const data = await this.reconstructData(collection, items)
-
-                return { [_id]: data } as Record<_ttid, T>
-            }
-        }
-
-        const processQuery = async function* (
-            p: string
-        ): AsyncGenerator<
-            Record<_ttid, T> | _ttid | void,
-            void,
-            { count: number; limit?: number }
-        > {
-            let finished = false
-
-            if (listen && !deleted) {
-                const iter = Walker.search(collection, p, { listen, skip })
-
-                do {
-                    const { value, done } = await iter.next({ count, limit })
-
-                    if (done) finished = true
-
-                    if (value) {
-                        const data = yield await constructData(collection, value._id, value.data)
-                        count = data.count
-                        limit = data.limit
-                    }
-                } while (!finished)
-            } else if (listen && deleted) {
-                const iter = Walker.search(collection, p, { listen, skip }, 'delete')
-
-                do {
-                    const { value, done } = await iter.next({ count, limit })
-
-                    if (done) finished = true
-
-                    if (value) {
-                        const data = yield value._id
-                        count = data.count
-                        limit = data.limit
-                    }
-                } while (!finished)
-            } else {
-                const iter = Walker.search(collection, p, { listen, skip })
-
-                do {
-                    const { value, done } = await iter.next({ count, limit })
-
-                    if (done) finished = true
-
-                    if (value) {
-                        const data = yield await constructData(collection, value._id, value.data)
-                        count = data.count
-                        limit = data.limit
-                    }
-                } while (!finished)
-            }
-        }
-
-        if (Array.isArray(pattern)) {
-            for (const p of pattern) yield* processQuery(p)
-        } else yield* processQuery(pattern)
-    }
-
-    async putKeys(
-        collection: string,
-        { dataKey, indexKey }: { dataKey: string; indexKey: string }
-    ) {
-        let dataBody: string | undefined
-        let indexBody: string | undefined
-
-        if (dataKey.length > Dir.KEY_LIMIT) {
-            const dataSegs = dataKey.split('/')
-
-            dataBody = dataSegs.pop()!
-
-            indexKey = `${dataSegs.join('/')}/${Bun.randomUUIDv7()}`
-        }
-
-        if (indexKey.length > Dir.KEY_LIMIT) {
-            const indexSegs = indexKey.split('/')
-
-            const _id = indexSegs.pop()! as _ttid
-
-            indexBody = indexSegs.pop()!
-
-            dataKey = `${indexSegs.join('/')}/${_id}`
-        }
-
-        await Promise.all([
-            S3.put(collection, dataKey, dataBody ?? ''),
-            S3.put(collection, indexKey, indexBody ?? '')
-        ])
-
-        this.transactions.push({
-            action: S3.delete,
-            args: [collection, dataKey]
-        })
-
-        this.transactions.push({
-            action: S3.delete,
-            args: [collection, indexKey]
-        })
-
-        await Dir.redis.publish(collection, 'insert', indexKey)
-    }
-
-    async executeRollback() {
-        do {
-            const transaction = this.transactions.pop()
-
-            if (transaction) {
-                const { action, args } = transaction
-
-                await action(...args)
-            }
-        } while (this.transactions.length > 0)
-    }
-
-    async deleteKeys(collection: string, dataKey: string) {
-        const segments = dataKey.split('/')
-
-        const _id = segments.shift()!
-
-        const indexKey = `${segments.join('/')}/${_id}`
-
-        const dataFile = S3.file(collection, dataKey)
-        const indexFile = S3.file(collection, indexKey)
-
-        let dataBody: string | undefined
-        let indexBody: string | undefined
-
-        if (dataFile.size > 0) dataBody = await dataFile.text()
-        if (indexFile.size > 0) indexBody = await indexFile.text()
-
-        await Promise.all([S3.delete(collection, indexKey), S3.delete(collection, dataKey)])
-
-        this.transactions.push({
-            action: S3.put,
-            args: [collection, dataKey, dataBody ?? '']
-        })
-
-        this.transactions.push({
-            action: S3.put,
-            args: [collection, indexKey, indexBody ?? '']
-        })
-
-        await Dir.redis.publish(collection, 'delete', _id)
-    }
-
     static async extractKeys<T>(collection: string, _id: _ttid, data: T, parentField?: string) {
         const keys: { data: string[]; indexes: string[] } = { data: [], indexes: [] }
-
-        const obj = { ...data }
+        const obj = { ...data } as Record<string, any>
 
         for (const field in obj) {
             const newField = parentField ? `${parentField}/${field}` : field
+            const fieldValue = obj[field]
 
-            if (typeof obj[field] === 'object' && !Array.isArray(obj[field])) {
-                const items = await this.extractKeys(collection, _id, obj[field], newField)
+            if (fieldValue && typeof fieldValue === 'object' && !Array.isArray(fieldValue)) {
+                const items = await this.extractKeys(collection, _id, fieldValue, newField)
                 keys.data.push(...items.data)
                 keys.indexes.push(...items.indexes)
-            } else if (typeof obj[field] === 'object' && Array.isArray(obj[field])) {
-                const items: (string | number | boolean)[] = obj[field]
-                if (items.some((item) => typeof item === 'object'))
+                continue
+            }
+
+            if (Array.isArray(fieldValue)) {
+                if (fieldValue.some((item) => typeof item === 'object')) {
                     throw new Error(`Cannot have an array of objects`)
-                for (let i = 0; i < items.length; i++) {
-                    let val = String(items[i]).split('/').join(this.SLASH_ASCII)
-                    if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, newField))
+                }
+
+                for (let i = 0; i < fieldValue.length; i++) {
+                    let val = String(fieldValue[i]).replaceAll('/', this.SLASH_ASCII)
+                    if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, newField)) {
                         val = await Cipher.encrypt(val, true)
+                    }
                     keys.data.push(`${_id}/${newField}/${i}/${val}`)
                     keys.indexes.push(`${newField}/${i}/${val}/${_id}`)
                 }
-            } else {
-                let val = String(obj[field]).replaceAll('/', this.SLASH_ASCII)
-                if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, newField))
-                    val = await Cipher.encrypt(val, true)
-                keys.data.push(`${_id}/${newField}/${val}`)
-                keys.indexes.push(`${newField}/${val}/${_id}`)
+                continue
             }
-        }
-
-        return keys
-    }
-
-    static constructData(fieldVal: Record<string, string>) {
-        const data: Record<string, any> = {}
 
-        for (let fullField in fieldVal) {
-            const fields = fullField.split('/').slice(1)
-
-            let curr = data
-
-            while (fields.length > 1) {
-                const field = fields.shift()!
-
-                if (typeof curr[field] !== 'object' || curr[field] === null)
-                    curr[field] = isNaN(Number(fields[0])) ? {} : []
-
-                curr = curr[field]
+            let val = String(fieldValue).replaceAll('/', this.SLASH_ASCII)
+            if (Cipher.isConfigured() && Cipher.isEncryptedField(collection, newField)) {
+                val = await Cipher.encrypt(val, true)
             }
-
-            const lastKey = fields.shift()!
-
-            curr[lastKey] = this.parseValue(fieldVal[fullField].replaceAll(this.SLASH_ASCII, '/'))
+            keys.data.push(`${_id}/${newField}/${val}`)
+            keys.indexes.push(`${newField}/${val}/${_id}`)
         }
 
-        return data
+        return keys
     }
 
     static parseValue(value: string) {

package/src/engines/s3-files/documents.ts

@@ -0,0 +1,65 @@
+import path from 'node:path'
+import TTID from '@delma/ttid'
+import type { StorageEngine } from '../types'
+import type { StoredDoc } from './types'
+
+export class S3FilesDocuments {
+    constructor(
+        private readonly storage: StorageEngine,
+        private readonly docsRoot: (collection: string) => string,
+        private readonly docPath: (collection: string, docId: _ttid) => string,
+        private readonly ensureCollection: (collection: string) => Promise<void>,
+        private readonly encodeEncrypted: <T extends Record<string, any>>(
+            collection: string,
+            value: T,
+            parentField?: string
+        ) => Promise<T>,
+        private readonly decodeEncrypted: <T extends Record<string, any>>(
+            collection: string,
+            value: T,
+            parentField?: string
+        ) => Promise<T>
+    ) {}
+
+    async readStoredDoc<T extends Record<string, any>>(
+        collection: string,
+        docId: _ttid
+    ): Promise<StoredDoc<T> | null> {
+        const target = this.docPath(collection, docId)
+
+        try {
+            const raw = JSON.parse(await this.storage.read(target)) as T
+            const decoded = await this.decodeEncrypted(collection, raw)
+            const { createdAt, updatedAt } = TTID.decodeTime(docId)
+
+            return {
+                id: docId,
+                createdAt,
+                updatedAt: updatedAt ?? createdAt,
+                data: decoded
+            }
+        } catch (err) {
+            if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null
+            throw err
+        }
+    }
+
+    async writeStoredDoc<T extends Record<string, any>>(collection: string, docId: _ttid, data: T) {
+        await this.ensureCollection(collection)
+        const encoded = await this.encodeEncrypted(collection, data)
+        const target = this.docPath(collection, docId)
+        await this.storage.write(target, JSON.stringify(encoded))
+    }
+
+    async removeStoredDoc(collection: string, docId: _ttid) {
+        await this.storage.delete(this.docPath(collection, docId))
+    }
+
+    async listDocIds(collection: string) {
+        const files = await this.storage.list(this.docsRoot(collection))
+        return files
+            .filter((file) => file.endsWith('.json'))
+            .map((file) => path.basename(file, '.json'))
+            .filter((key) => TTID.isTTID(key)) as _ttid[]
+    }
+}

package/src/engines/s3-files/filesystem.ts

@@ -0,0 +1,172 @@
+import { mkdir, open, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import type { EventBus, LockManager, StorageEngine } from '../types'
+import type { S3FilesEvent } from './types'
+
+export class FilesystemStorage implements StorageEngine {
+    async read(target: string): Promise<string> {
+        return await readFile(target, 'utf8')
+    }
+
+    async write(target: string, data: string): Promise<void> {
+        await mkdir(path.dirname(target), { recursive: true })
+        await writeFile(target, data, 'utf8')
+    }
+
+    async delete(target: string): Promise<void> {
+        await rm(target, { recursive: true, force: true })
+    }
+
+    async list(target: string): Promise<string[]> {
+        const results: string[] = []
+
+        try {
+            const entries = await readdir(target, { withFileTypes: true })
+            for (const entry of entries) {
+                const child = path.join(target, entry.name)
+                if (entry.isDirectory()) {
+                    results.push(...(await this.list(child)))
+                } else {
+                    results.push(child)
+                }
+            }
+        } catch (err) {
+            if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
+        }
+
+        return results
+    }
+
+    async mkdir(target: string): Promise<void> {
+        await mkdir(target, { recursive: true })
+    }
+
+    async rmdir(target: string): Promise<void> {
+        await rm(target, { recursive: true, force: true })
+    }
+
+    async exists(target: string): Promise<boolean> {
+        try {
+            await stat(target)
+            return true
+        } catch (err) {
+            if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false
+            throw err
+        }
+    }
+}
+
+export class FilesystemLockManager implements LockManager {
+    constructor(
+        private readonly root: string,
+        private readonly storage: StorageEngine
+    ) {}
+
+    private lockDir(collection: string, docId: _ttid) {
+        return path.join(this.root, collection, '.fylo', 'locks', `${docId}.lock`)
+    }
+
+    async acquire(
+        collection: string,
+        docId: _ttid,
+        owner: string,
+        ttlMs: number = 30_000
+    ): Promise<boolean> {
+        const dir = this.lockDir(collection, docId)
+        const metaPath = path.join(dir, 'meta.json')
+        await mkdir(path.dirname(dir), { recursive: true })
+
+        try {
+            await mkdir(dir, { recursive: false })
+            await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
+            return true
+        } catch (err) {
+            if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err
+        }
+
+        try {
+            const meta = JSON.parse(await this.storage.read(metaPath)) as { ts?: number }
+            if (meta.ts && Date.now() - meta.ts > ttlMs) {
+                await this.storage.rmdir(dir)
+                await mkdir(dir, { recursive: false })
+                await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
+                return true
+            }
+        } catch {
+            await this.storage.rmdir(dir)
+            await mkdir(dir, { recursive: false })
+            await this.storage.write(metaPath, JSON.stringify({ owner, ts: Date.now() }))
+            return true
+        }
+
+        return false
+    }
+
+    async release(collection: string, docId: _ttid, owner: string): Promise<void> {
+        const dir = this.lockDir(collection, docId)
+        const metaPath = path.join(dir, 'meta.json')
+
+        try {
+            const meta = JSON.parse(await this.storage.read(metaPath)) as { owner?: string }
+            if (meta.owner === owner) await this.storage.rmdir(dir)
+        } catch (err) {
+            if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
+        }
+    }
+}
+
+export class FilesystemEventBus<T extends Record<string, any>> implements EventBus<
+    S3FilesEvent<T>
+> {
+    constructor(
+        private readonly root: string,
+        private readonly storage: StorageEngine
+    ) {}
+
+    private journalPath(collection: string) {
+        return path.join(this.root, collection, '.fylo', 'events', `${collection}.ndjson`)
+    }
+
+    async publish(collection: string, event: S3FilesEvent<T>): Promise<void> {
+        const target = this.journalPath(collection)
+        await mkdir(path.dirname(target), { recursive: true })
+        const line = `${JSON.stringify(event)}\n`
+        const handle = await open(target, 'a')
+        try {
+            await handle.write(line)
+        } finally {
+            await handle.close()
+        }
+    }
+
+    async *listen(collection: string): AsyncGenerator<S3FilesEvent<T>, void, unknown> {
+        const target = this.journalPath(collection)
+        let position = 0
+
+        while (true) {
+            try {
+                const fileStat = await stat(target)
+                if (fileStat.size > position) {
+                    const handle = await open(target, 'r')
+                    try {
+                        const size = fileStat.size - position
+                        const buffer = Buffer.alloc(size)
+                        await handle.read(buffer, 0, size, position)
+                        position = fileStat.size
+
+                        for (const line of buffer.toString('utf8').split('\n')) {
+                            if (line.trim().length === 0) continue
+                            yield JSON.parse(line) as S3FilesEvent<T>
+                        }
+                    } finally {
+                        await handle.close()
+                    }
+                }
+            } catch (err) {
+                if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
+            }
+
+            await Bun.sleep(100)
+        }
+    }
+}