@delma/fylo 2.0.0 → 2.1.0

package/src/core/walker.ts

@@ -1,174 +0,0 @@
-import { S3 } from '../adapters/s3'
-import TTID from '@delma/ttid'
-import { Redis } from '../adapters/redis'
-
-export class Walker {
-    private static readonly MAX_KEYS = 1000
-
-    private static _redis: Redis | null = null
-
-    private static get redis(): Redis {
-        if (!Walker._redis) Walker._redis = new Redis()
-        return Walker._redis
-    }
-
-    private static async *searchS3(
-        collection: string,
-        prefix: string,
-        pattern?: string
-    ): AsyncGenerator<
-        { _id: _ttid; data: string[] } | void,
-        void,
-        { count: number; limit?: number }
-    > {
-        const uniqueIds = new Set<string>()
-
-        let token: string | undefined
-
-        let filter = yield
-
-        let limit = filter ? filter.limit : this.MAX_KEYS
-
-        do {
-            const res = await S3.list(collection, {
-                prefix,
-                maxKeys: pattern ? limit : undefined,
-                continuationToken: token
-            })
-
-            if (res.contents === undefined) break
-
-            const keys = res.contents.map((item) => item.key!)
-
-            if (pattern) {
-                for (const key of keys) {
-                    const segements = key.split('/')
-
-                    const _id = segements.pop()! as _ttid
-
-                    if (
-                        TTID.isTTID(_id) &&
-                        !uniqueIds.has(_id) &&
-                        pattern.length <= 1024 &&
-                        new Bun.Glob(pattern).match(key)
-                    ) {
-                        filter = yield { _id, data: await this.getDocData(collection, _id) }
-
-                        limit = filter.limit ? filter.limit : this.MAX_KEYS
-
-                        uniqueIds.add(_id)
-
-                        if (filter.count === limit) break
-                    }
-                }
-            } else {
-                const _id = prefix.split('/').pop()! as _ttid
-
-                yield { _id, data: keys }
-
-                break
-            }
-
-            token = res.nextContinuationToken
-        } while (token !== undefined)
-    }
-
-    static async *search(
-        collection: string,
-        pattern: string,
-        { listen = false, skip = false }: { listen: boolean; skip: boolean },
-        action: 'insert' | 'delete' = 'insert'
-    ): AsyncGenerator<
-        { _id: _ttid; data: string[] } | void,
-        void,
-        { count: number; limit?: number }
-    > {
-        if (!skip) {
-            const segments = pattern.split('/')
-            const idx = segments.findIndex((seg) => seg.includes('*'))
-            const prefix = segments.slice(0, idx).join('/')
-
-            yield* this.searchS3(collection, prefix, pattern)
-        }
-
-        const eventIds = new Set<string>()
-
-        if (listen)
-            for await (const event of this.listen(collection, pattern)) {
-                if (event.action !== action && eventIds.has(event.id)) {
-                    eventIds.delete(event.id)
-                } else if (event.action === action && !eventIds.has(event.id)) {
-                    eventIds.add(event.id)
-                    yield { _id: event.id, data: event.data }
-                }
-            }
-    }
-
-    static async getDocData(collection: string, _id: _ttid) {
-        const prefix = _id.split('-')[0]
-
-        const data: string[] = []
-
-        let finished = false
-
-        const iter = this.searchS3(collection, prefix)
-
-        do {
-            const { value, done } = await iter.next()
-
-            if (done) {
-                finished = true
-                break
-            }
-
-            if (value) {
-                for (const key of value.data) {
-                    if (key.startsWith(_id + '/')) data.push(key)
-                }
-                finished = true
-                break
-            }
-        } while (!finished)
-
-        return data
-    }
-
-    private static async *processPattern(collection: string, pattern: string) {
-        const stackIds = new Set<string>()
-
-        for await (const { action, keyId } of Walker.redis.subscribe(collection)) {
-            if (
-                action === 'insert' &&
-                !TTID.isTTID(keyId) &&
-                pattern.length <= 1024 &&
-                new Bun.Glob(pattern).match(keyId)
-            ) {
-                const _id = keyId.split('/').pop()! as _ttid
-
-                if (!stackIds.has(_id)) {
-                    stackIds.add(_id)
-
-                    yield {
-                        id: _id,
-                        action: 'insert',
-                        data: await this.getDocData(collection, _id)
-                    }
-                }
-            } else if (action === 'delete' && TTID.isTTID(keyId)) {
-                yield { id: keyId as _ttid, action: 'delete', data: [] }
-            } else if (TTID.isTTID(keyId) && stackIds.has(keyId)) {
-                stackIds.delete(keyId)
-            }
-        }
-    }
-
-    static async *listen(collection: string, pattern: string | string[]) {
-        if (Array.isArray(pattern)) {
-            for (const p of pattern) {
-                for await (const event of this.processPattern(collection, p)) yield event
-            }
-        } else {
-            for await (const event of this.processPattern(collection, pattern)) yield event
-        }
-    }
-}

package/src/core/write-queue.ts

@@ -1,59 +0,0 @@
-import type { WriteJob } from '../types/write-queue'
-
-export class WriteQueue {
-    static createInsertJob<T extends Record<string, any>>(
-        collection: string,
-        docId: _ttid,
-        payload: T
-    ): WriteJob<T> {
-        const now = Date.now()
-
-        return {
-            jobId: Bun.randomUUIDv7(),
-            collection,
-            docId,
-            operation: 'insert',
-            payload,
-            status: 'queued',
-            attempts: 0,
-            createdAt: now,
-            updatedAt: now
-        }
-    }
-
-    static createUpdateJob<T extends Record<string, any>>(
-        collection: string,
-        docId: _ttid,
-        payload: { newDoc: Record<_ttid, Partial<T>>; oldDoc?: Record<_ttid, T> }
-    ): WriteJob<{ newDoc: Record<_ttid, Partial<T>>; oldDoc?: Record<_ttid, T> }> {
-        const now = Date.now()
-
-        return {
-            jobId: Bun.randomUUIDv7(),
-            collection,
-            docId,
-            operation: 'update',
-            payload,
-            status: 'queued',
-            attempts: 0,
-            createdAt: now,
-            updatedAt: now
-        }
-    }
-
-    static createDeleteJob(collection: string, docId: _ttid): WriteJob<{ _id: _ttid }> {
-        const now = Date.now()
-
-        return {
-            jobId: Bun.randomUUIDv7(),
-            collection,
-            docId,
-            operation: 'delete',
-            payload: { _id: docId },
-            status: 'queued',
-            attempts: 0,
-            createdAt: now,
-            updatedAt: now
-        }
-    }
-}

package/src/migrate-cli.ts

@@ -1,22 +0,0 @@
-#!/usr/bin/env bun
-import { migrateLegacyS3ToS3Files } from './migrate'
-
-const [, , ...args] = process.argv
-
-const collections = args.filter((arg) => !arg.startsWith('--'))
-const recreateCollections = !args.includes('--keep-existing')
-const verify = !args.includes('--no-verify')
-
-if (collections.length === 0) {
-    throw new Error(
-        'Usage: fylo.migrate <collection> [collection...] [--keep-existing] [--no-verify]'
-    )
-}
-
-const summary = await migrateLegacyS3ToS3Files({
-    collections,
-    recreateCollections,
-    verify
-})
-
-console.log(JSON.stringify(summary, null, 2))

package/src/migrate.ts

@@ -1,74 +0,0 @@
-import Fylo from './index'
-import { S3FilesEngine } from './engines/s3-files'
-
-type MigrateOptions = {
-    collections: string[]
-    s3FilesRoot?: string
-    recreateCollections?: boolean
-    verify?: boolean
-}
-
-function normalize<T>(value: T): T {
-    if (Array.isArray(value)) return value.map((item) => normalize(item)).sort() as T
-    if (value && typeof value === 'object') {
-        return Object.keys(value as Record<string, unknown>)
-            .sort()
-            .reduce(
-                (acc, key) => {
-                    acc[key] = normalize((value as Record<string, unknown>)[key])
-                    return acc
-                },
-                {} as Record<string, unknown>
-            ) as T
-    }
-    return value
-}
-
-export async function migrateLegacyS3ToS3Files({
-    collections,
-    s3FilesRoot = process.env.FYLO_S3FILES_ROOT,
-    recreateCollections = true,
-    verify = true
-}: MigrateOptions) {
-    if (!s3FilesRoot) throw new Error('s3FilesRoot is required')
-
-    const source = new Fylo({ engine: 'legacy-s3' })
-    const target = new S3FilesEngine(s3FilesRoot)
-
-    const summary: Record<string, { migrated: number; verified: boolean }> = {}
-
-    for (const collection of collections) {
-        if (recreateCollections) {
-            await target.dropCollection(collection)
-            await target.createCollection(collection)
-        } else if (!(await target.hasCollection(collection))) {
-            await target.createCollection(collection)
-        }
-
-        const docs = (await source.executeSQL<Record<string, any>>(
-            `SELECT * FROM ${collection}`
-        )) as Record<_ttid, Record<string, any>>
-
-        for (const [docId, doc] of Object.entries(docs) as Array<[_ttid, Record<string, any>]>) {
-            await target.putDocument(collection, docId, doc)
-        }
-
-        let verified = false
-
-        if (verify) {
-            const targetFylo = new Fylo({ engine: 's3-files', s3FilesRoot })
-            const migratedDocs = (await targetFylo.executeSQL<Record<string, any>>(
-                `SELECT * FROM ${collection}`
-            )) as Record<_ttid, Record<string, any>>
-            verified = JSON.stringify(normalize(docs)) === JSON.stringify(normalize(migratedDocs))
-            if (!verified) throw new Error(`Verification failed for ${collection}`)
-        }
-
-        summary[collection] = {
-            migrated: Object.keys(docs).length,
-            verified
-        }
-    }
-
-    return summary
-}

package/src/types/write-queue.ts

@@ -1,42 +0,0 @@
-export type WriteJobOperation = 'insert' | 'update' | 'delete'
-
-export type WriteJobStatus = 'queued' | 'processing' | 'committed' | 'failed' | 'dead-letter'
-
-export interface WriteJob<T extends Record<string, any> = Record<string, any>> {
-    jobId: string
-    collection: string
-    docId: _ttid
-    operation: WriteJobOperation
-    payload: T
-    status: WriteJobStatus
-    attempts: number
-    createdAt: number
-    updatedAt: number
-    nextAttemptAt?: number
-    workerId?: string
-    error?: string
-}
-
-export interface QueuedWriteResult {
-    jobId: string
-    docId: _ttid
-    status: WriteJobStatus
-}
-
-export interface StreamJobEntry<T extends Record<string, any> = Record<string, any>> {
-    streamId: string
-    job: WriteJob<T>
-}
-
-export interface DeadLetterJob<T extends Record<string, any> = Record<string, any>> {
-    streamId: string
-    job: WriteJob<T>
-    reason?: string
-    failedAt: number
-}
-
-export interface QueueStats {
-    queued: number
-    pending: number
-    deadLetters: number
-}

package/src/worker.ts

@@ -1,18 +0,0 @@
-#!/usr/bin/env bun
-import { WriteWorker } from './workers/write-worker'
-
-if (process.env.FYLO_STORAGE_ENGINE === 's3-files') {
-    throw new Error('fylo.worker is not supported in s3-files engine')
-}
-
-const worker = new WriteWorker(process.env.FYLO_WORKER_ID)
-
-await worker.run({
-    batchSize: process.env.FYLO_WORKER_BATCH_SIZE ? Number(process.env.FYLO_WORKER_BATCH_SIZE) : 1,
-    blockMs: process.env.FYLO_WORKER_BLOCK_MS ? Number(process.env.FYLO_WORKER_BLOCK_MS) : 1000,
-    recoverOnStart: process.env.FYLO_WORKER_RECOVER_ON_START !== 'false',
-    recoverIdleMs: process.env.FYLO_WORKER_RECOVER_IDLE_MS
-        ? Number(process.env.FYLO_WORKER_RECOVER_IDLE_MS)
-        : 30_000,
-    stopWhenIdle: process.env.FYLO_WORKER_STOP_WHEN_IDLE === 'true'
-})

package/src/workers/write-worker.ts

@@ -1,120 +0,0 @@
-import Fylo from '../index'
-import { Redis } from '../adapters/redis'
-import type { StreamJobEntry, WriteJob } from '../types/write-queue'
-
-export class WriteWorker {
-    private static readonly MAX_WRITE_ATTEMPTS = Number(process.env.FYLO_WRITE_MAX_ATTEMPTS ?? 3)
-
-    private static readonly WRITE_RETRY_BASE_MS = Number(process.env.FYLO_WRITE_RETRY_BASE_MS ?? 10)
-
-    private readonly fylo: Fylo
-
-    private readonly redis: Redis
-
-    readonly workerId: string
-
-    constructor(workerId: string = Bun.randomUUIDv7()) {
-        this.workerId = workerId
-        this.fylo = new Fylo()
-        this.redis = new Redis()
-    }
-
-    async recoverPending(minIdleMs: number = 30_000, count: number = 10) {
-        const jobs = await this.redis.claimPendingJobs(this.workerId, minIdleMs, count)
-        for (const job of jobs) await this.processJob(job)
-        return jobs.length
-    }
-
-    async processNext(count: number = 1, blockMs: number = 1000) {
-        const jobs = await this.redis.readWriteJobs(this.workerId, count, blockMs)
-        for (const job of jobs) await this.processJob(job)
-        return jobs.length
-    }
-
-    async processJob({ streamId, job }: StreamJobEntry) {
-        if (job.nextAttemptAt && job.nextAttemptAt > Date.now()) return false
-
-        const locked = await this.redis.acquireDocLock(job.collection, job.docId, job.jobId)
-        if (!locked) return false
-
-        try {
-            await this.redis.setJobStatus(job.jobId, 'processing', {
-                workerId: this.workerId,
-                attempts: job.attempts + 1
-            })
-            await this.redis.setDocStatus(job.collection, job.docId, 'processing', job.jobId)
-
-            await this.fylo.executeQueuedWrite(job)
-
-            await this.redis.setJobStatus(job.jobId, 'committed', { workerId: this.workerId })
-            await this.redis.setDocStatus(job.collection, job.docId, 'committed', job.jobId)
-            await this.redis.ackWriteJob(streamId)
-
-            return true
-        } catch (err) {
-            const attempts = job.attempts + 1
-            const message = err instanceof Error ? err.message : String(err)
-
-            if (attempts >= WriteWorker.MAX_WRITE_ATTEMPTS) {
-                await this.redis.setJobStatus(job.jobId, 'dead-letter', {
-                    workerId: this.workerId,
-                    attempts,
-                    error: message
-                })
-                await this.redis.setDocStatus(job.collection, job.docId, 'dead-letter', job.jobId)
-                await this.redis.deadLetterWriteJob(
-                    streamId,
-                    {
-                        ...job,
-                        attempts,
-                        status: 'dead-letter',
-                        workerId: this.workerId,
-                        error: message
-                    },
-                    message
-                )
-                return false
-            }
-
-            const nextAttemptAt =
-                Date.now() + WriteWorker.WRITE_RETRY_BASE_MS * Math.max(1, 2 ** (attempts - 1))
-
-            await this.redis.setJobStatus(job.jobId, 'failed', {
-                workerId: this.workerId,
-                attempts,
-                error: message,
-                nextAttemptAt
-            })
-            await this.redis.setDocStatus(job.collection, job.docId, 'failed', job.jobId)
-
-            return false
-        } finally {
-            await this.redis.releaseDocLock(job.collection, job.docId, job.jobId)
-        }
-    }
-
-    async processQueuedInsert(job: WriteJob) {
-        return await this.fylo.executeQueuedWrite(job)
-    }
-
-    async run({
-        batchSize = 1,
-        blockMs = 1000,
-        recoverOnStart = true,
-        recoverIdleMs = 30_000,
-        stopWhenIdle = false
-    }: {
-        batchSize?: number
-        blockMs?: number
-        recoverOnStart?: boolean
-        recoverIdleMs?: number
-        stopWhenIdle?: boolean
-    } = {}) {
-        if (recoverOnStart) await this.recoverPending(recoverIdleMs, batchSize)
-
-        while (true) {
-            const processed = await this.processNext(batchSize, blockMs)
-            if (stopWhenIdle && processed === 0) break
-        }
-    }
-}

package/tests/index.js

@@ -1,14 +0,0 @@
-import { Redis } from '../src/adapters/redis'
-import ttid from '@delma/ttid'
-const redisPub = new Redis()
-const redisSub = new Redis()
-setTimeout(async () => {
-    await redisPub.publish('bun', 'insert', ttid.generate())
-}, 2000)
-setTimeout(async () => {
-    await redisPub.publish('bun', 'insert', ttid.generate())
-}, 3000)
-await Bun.sleep(1000)
-for await (const data of redisSub.subscribe('bun')) {
-    console.log('Received:', data)
-}

package/tests/integration/migration.test.js

@@ -1,38 +0,0 @@
-import { afterAll, beforeAll, describe, expect, mock, test } from 'bun:test'
-import { mkdtemp, rm } from 'node:fs/promises'
-import os from 'node:os'
-import path from 'node:path'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
-const { default: Fylo, migrateLegacyS3ToS3Files } = await import('../../src')
-const root = await mkdtemp(path.join(os.tmpdir(), 'fylo-migrate-'))
-const legacyFylo = new Fylo({ engine: 'legacy-s3' })
-const s3FilesFylo = new Fylo({ engine: 's3-files', s3FilesRoot: root })
-const COLLECTION = 'migration-posts'
-describe('legacy-s3 to s3-files migration', () => {
-    beforeAll(async () => {
-        await Fylo.createCollection(COLLECTION)
-        await legacyFylo.putData(COLLECTION, { id: 1, title: 'Alpha' })
-        await legacyFylo.putData(COLLECTION, { id: 2, title: 'Beta' })
-    })
-    afterAll(async () => {
-        await rm(root, { recursive: true, force: true })
-    })
-    test('migrates legacy data and verifies parity', async () => {
-        const summary = await migrateLegacyS3ToS3Files({
-            collections: [COLLECTION],
-            s3FilesRoot: root,
-            verify: true
-        })
-        expect(summary[COLLECTION].migrated).toBe(2)
-        expect(summary[COLLECTION].verified).toBe(true)
-        const migrated = await s3FilesFylo.executeSQL(`SELECT * FROM ${COLLECTION}`)
-        expect(
-            Object.values(migrated)
-                .map((item) => item.title)
-                .sort()
-        ).toEqual(['Alpha', 'Beta'])
-    })
-})

package/tests/integration/queue.test.js

@@ -1,83 +0,0 @@
-import { test, expect, describe, beforeAll, afterAll, mock } from 'bun:test'
-import Fylo from '../../src'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
-const POSTS = 'queued-post'
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
-const fylo = new Fylo()
-beforeAll(async () => {
-    await Fylo.createCollection(POSTS)
-})
-afterAll(async () => {
-    await Fylo.dropCollection(POSTS)
-})
-describe('queue writes', () => {
-    test('queuePutData enqueues and worker commits insert', async () => {
-        const queued = await fylo.queuePutData(POSTS, {
-            title: 'Queued Title',
-            body: 'Queued Body'
-        })
-        const queuedStatus = await fylo.getJobStatus(queued.jobId)
-        expect(queuedStatus?.status).toBe('queued')
-        const processed = await fylo.processQueuedWrites(1)
-        expect(processed).toBe(1)
-        const doc = await Fylo.getDoc(POSTS, queued.docId, false).once()
-        expect(Object.keys(doc).length).toBe(1)
-    })
-    test('putData can return immediately when wait is false', async () => {
-        const queued = await fylo.putData(
-            POSTS,
-            {
-                title: 'Async Title',
-                body: 'Async Body'
-            },
-            { wait: false }
-        )
-        expect(typeof queued).toBe('object')
-        expect('jobId' in queued).toBe(true)
-        const before = await Fylo.getDoc(POSTS, queued.docId, false).once()
-        expect(Object.keys(before).length).toBe(0)
-        const processed = await fylo.processQueuedWrites(1)
-        expect(processed).toBe(1)
-        const stats = await fylo.getQueueStats()
-        expect(stats.deadLetters).toBe(0)
-        const after = await Fylo.getDoc(POSTS, queued.docId, false).once()
-        expect(Object.keys(after).length).toBe(1)
-    })
-    test('failed jobs can be recovered and eventually dead-lettered', async () => {
-        const originalExecute = fylo.executeQueuedWrite.bind(fylo)
-        fylo.executeQueuedWrite = async () => {
-            throw new Error('simulated write failure')
-        }
-        try {
-            const queued = await fylo.putData(
-                POSTS,
-                {
-                    title: 'Broken Title',
-                    body: 'Broken Body'
-                },
-                { wait: false }
-            )
-            expect(await fylo.processQueuedWrites(1)).toBe(0)
-            expect((await fylo.getJobStatus(queued.jobId))?.status).toBe('failed')
-            await Bun.sleep(15)
-            expect(await fylo.processQueuedWrites(1, true)).toBe(0)
-            expect((await fylo.getJobStatus(queued.jobId))?.status).toBe('failed')
-            await Bun.sleep(25)
-            expect(await fylo.processQueuedWrites(1, true)).toBe(0)
-            expect((await fylo.getJobStatus(queued.jobId))?.status).toBe('dead-letter')
-            const deadLetters = await fylo.getDeadLetters()
-            expect((await fylo.getQueueStats()).deadLetters).toBeGreaterThan(0)
-            expect(deadLetters.some((item) => item.job.jobId === queued.jobId)).toBe(true)
-            fylo.executeQueuedWrite = originalExecute
-            const replayed = await fylo.replayDeadLetter(deadLetters[0].streamId)
-            expect(replayed?.jobId).toBe(queued.jobId)
-            expect((await fylo.getQueueStats()).deadLetters).toBe(0)
-            expect(await fylo.processQueuedWrites(1)).toBe(1)
-            expect((await fylo.getJobStatus(queued.jobId))?.status).toBe('committed')
-        } finally {
-            fylo.executeQueuedWrite = originalExecute
-        }
-    })
-})