@delma/fylo 2.0.1 → 2.1.1

package/src/migrate-cli.ts

@@ -1,22 +0,0 @@
-#!/usr/bin/env bun
-import { migrateLegacyS3ToS3Files } from './migrate'
-
-const [, , ...args] = process.argv
-
-const collections = args.filter((arg) => !arg.startsWith('--'))
-const recreateCollections = !args.includes('--keep-existing')
-const verify = !args.includes('--no-verify')
-
-if (collections.length === 0) {
-    throw new Error(
-        'Usage: fylo.migrate <collection> [collection...] [--keep-existing] [--no-verify]'
-    )
-}
-
-const summary = await migrateLegacyS3ToS3Files({
-    collections,
-    recreateCollections,
-    verify
-})
-
-console.log(JSON.stringify(summary, null, 2))

package/src/migrate.ts

@@ -1,74 +0,0 @@
-import Fylo from './index'
-import { S3FilesEngine } from './engines/s3-files'
-
-type MigrateOptions = {
-    collections: string[]
-    s3FilesRoot?: string
-    recreateCollections?: boolean
-    verify?: boolean
-}
-
-function normalize<T>(value: T): T {
-    if (Array.isArray(value)) return value.map((item) => normalize(item)).sort() as T
-    if (value && typeof value === 'object') {
-        return Object.keys(value as Record<string, unknown>)
-            .sort()
-            .reduce(
-                (acc, key) => {
-                    acc[key] = normalize((value as Record<string, unknown>)[key])
-                    return acc
-                },
-                {} as Record<string, unknown>
-            ) as T
-    }
-    return value
-}
-
-export async function migrateLegacyS3ToS3Files({
-    collections,
-    s3FilesRoot = process.env.FYLO_S3FILES_ROOT,
-    recreateCollections = true,
-    verify = true
-}: MigrateOptions) {
-    if (!s3FilesRoot) throw new Error('s3FilesRoot is required')
-
-    const source = new Fylo({ engine: 'legacy-s3' })
-    const target = new S3FilesEngine(s3FilesRoot)
-
-    const summary: Record<string, { migrated: number; verified: boolean }> = {}
-
-    for (const collection of collections) {
-        if (recreateCollections) {
-            await target.dropCollection(collection)
-            await target.createCollection(collection)
-        } else if (!(await target.hasCollection(collection))) {
-            await target.createCollection(collection)
-        }
-
-        const docs = (await source.executeSQL<Record<string, any>>(
-            `SELECT * FROM ${collection}`
-        )) as Record<_ttid, Record<string, any>>
-
-        for (const [docId, doc] of Object.entries(docs) as Array<[_ttid, Record<string, any>]>) {
-            await target.putDocument(collection, docId, doc)
-        }
-
-        let verified = false
-
-        if (verify) {
-            const targetFylo = new Fylo({ engine: 's3-files', s3FilesRoot })
-            const migratedDocs = (await targetFylo.executeSQL<Record<string, any>>(
-                `SELECT * FROM ${collection}`
-            )) as Record<_ttid, Record<string, any>>
-            verified = JSON.stringify(normalize(docs)) === JSON.stringify(normalize(migratedDocs))
-            if (!verified) throw new Error(`Verification failed for ${collection}`)
-        }
-
-        summary[collection] = {
-            migrated: Object.keys(docs).length,
-            verified
-        }
-    }
-
-    return summary
-}

package/src/types/fylo.d.ts

@@ -1,261 +0,0 @@
-interface _getDoc {
-    [Symbol.asyncIterator]<T>(): AsyncGenerator<_ttid | Record<_ttid, T>, void, unknown>
-    once<T>(): Promise<Record<_ttid, T>>
-    onDelete(): AsyncGenerator<_ttid, void, unknown>
-}
-
-interface _findDocs {
-    [Symbol.asyncIterator]<T>(): AsyncGenerator<
-        _ttid | Record<_ttid, T> | Record<string, _ttid[]> | Record<_ttid, Partial<T>> | undefined,
-        void,
-        unknown
-    >
-    once<T>(): Promise<Record<_ttid, T>>
-    onDelete(): AsyncGenerator<_ttid, void, unknown>
-}
-
-interface _queuedWriteResult {
-    jobId: string
-    docId: _ttid
-    status: 'queued' | 'processing' | 'committed' | 'failed' | 'dead-letter'
-}
-
-interface ObjectConstructor {
-    appendGroup: (target: Record<string, any>, source: Record<string, any>) => Record<string, any>
-}
-
-interface Console {
-    format: (docs: Record<string, any>) => void
-}
-
-type _joinDocs<T, U> =
-    | _ttid[]
-    | Record<string, _ttid[]>
-    | Record<string, Record<_ttid, Partial<T | U>>>
-    | Record<`${_ttid}, ${_ttid}`, T | U | (T & U) | (Partial<T> & Partial<U>)>
-
-type _fyloEngineKind = 'legacy-s3' | 's3-files'
-
-interface _fyloOptions {
-    engine?: _fyloEngineKind
-    s3FilesRoot?: string
-}
-
-declare module '@delma/fylo' {
-    export default class {
-        constructor(options?: _fyloOptions)
-
-        /**
-         * Rolls back all transcations in current instance
-         * @deprecated Prefer queued write recovery, dead letters, or compensating writes.
-         */
-        rollback(): Promise<void>
-
-        /**
-         * Executes a SQL query and returns the results.
-         * @param SQL The SQL query to execute.
-         * @returns The results of the query.
-         */
-        executeSQL<T extends Record<string, any>, U extends Record<string, any> = {}>(
-            SQL: string
-        ): Promise<number | void | any[] | _ttid | Record<any, any>>
-
-        /**
-         * Creates a new schema for a collection.
-         * @param collection The name of the collection.
-         */
-        static createCollection(collection: string): Promise<void>
-
-        /**
-         * Drops an existing schema for a collection.
-         * @param collection The name of the collection.
-         */
-        static dropCollection(collection: string): Promise<void>
-
-        /**
-         * Imports data from a URL into a collection.
-         * @param collection The name of the collection.
-         * @param url The URL of the data to import.
-         * @param limit The maximum number of documents to import.
-         */
-        importBulkData(collection: string, url: URL, limit?: number): Promise<number>
-
-        /**
-         * Exports data from a collection to a URL.
-         * @param collection The name of the collection.
-         * @returns The current data exported from the collection.
-         */
-        exportBulkData<T extends Record<string, any>>(
-            collection: string
-        ): AsyncGenerator<T, void, unknown>
-
-        /**
-         * Gets a document from a collection.
-         * @param collection The name of the collection.
-         * @param _id The ID of the document.
-         * @param onlyId Whether to only return the ID of the document.
-         * @returns The document or the ID of the document.
-         */
-        static getDoc(collection: string, _id: _ttid, onlyId: boolean): _getDoc
-
-        /**
-         * Puts multiple documents into a collection.
-         * @param collection The name of the collection.
-         * @param batch The documents to put.
-         * @returns The IDs of the documents.
-         */
-        batchPutData<T extends Record<string, any>>(
-            collection: string,
-            batch: Array<T>
-        ): Promise<_ttid[]>
-
-        queuePutData<T extends Record<string, any>>(
-            collection: string,
-            data: Record<_ttid, T> | T
-        ): Promise<_queuedWriteResult>
-
-        queuePatchDoc<T extends Record<string, any>>(
-            collection: string,
-            newDoc: Record<_ttid, Partial<T>>,
-            oldDoc?: Record<_ttid, T>
-        ): Promise<_queuedWriteResult>
-
-        queueDelDoc(collection: string, _id: _ttid): Promise<_queuedWriteResult>
-
-        getJobStatus(jobId: string): Promise<Record<string, any> | null>
-
-        getDocStatus(collection: string, docId: _ttid): Promise<Record<string, any> | null>
-
-        getDeadLetters(count?: number): Promise<Array<Record<string, any>>>
-
-        getQueueStats(): Promise<{ queued: number; pending: number; deadLetters: number }>
-
-        replayDeadLetter(streamId: string): Promise<Record<string, any> | null>
-
-        processQueuedWrites(count?: number, recover?: boolean): Promise<number>
-
-        /**
-         * Puts a document into a collection.
-         * @param collection The name of the collection.
-         * @param data The document to put.
-         * @returns The ID of the document.
-         */
-        putData<T extends Record<string, any>>(collection: string, data: T): Promise<_ttid>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: Record<_ttid, T>
-        ): Promise<_ttid>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: T,
-            options: { wait?: true; timeoutMs?: number }
-        ): Promise<_ttid>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: Record<_ttid, T>,
-            options: { wait?: true; timeoutMs?: number }
-        ): Promise<_ttid>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: T,
-            options: { wait: false; timeoutMs?: number }
-        ): Promise<_queuedWriteResult>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: Record<_ttid, T>,
-            options: { wait: false; timeoutMs?: number }
-        ): Promise<_queuedWriteResult>
-
-        /**
-         * Patches a document in a collection.
-         * @param collection The name of the collection.
-         * @param newDoc The new document data.
-         * @param oldDoc The old document data.
-         * @returns The number of documents patched.
-         */
-        patchDoc<T extends Record<string, any>>(
-            collection: string,
-            newDoc: Record<_ttid, Partial<T>>,
-            oldDoc?: Record<_ttid, T>
-        ): Promise<_ttid>
-        patchDoc<T extends Record<string, any>>(
-            collection: string,
-            newDoc: Record<_ttid, Partial<T>>,
-            oldDoc: Record<_ttid, T> | undefined,
-            options: { wait?: true; timeoutMs?: number }
-        ): Promise<_ttid>
-        patchDoc<T extends Record<string, any>>(
-            collection: string,
-            newDoc: Record<_ttid, Partial<T>>,
-            oldDoc: Record<_ttid, T> | undefined,
-            options: { wait: false; timeoutMs?: number }
-        ): Promise<_queuedWriteResult>
-
-        /**
-         * Patches documents in a collection.
-         * @param collection The name of the collection.
-         * @param updateSchema The update schema.
-         * @returns The number of documents patched.
-         */
-        patchDocs<T extends Record<string, any>>(
-            collection: string,
-            updateSchema: _storeUpdate<T>
-        ): Promise<number>
-
-        /**
-         * Deletes a document from a collection.
-         * @param collection The name of the collection.
-         * @param _id The ID of the document.
-         * @returns The number of documents deleted.
-         */
-        delDoc(collection: string, _id: _ttid): Promise<void>
-        delDoc(
-            collection: string,
-            _id: _ttid,
-            options: { wait?: true; timeoutMs?: number }
-        ): Promise<void>
-        delDoc(
-            collection: string,
-            _id: _ttid,
-            options: { wait: false; timeoutMs?: number }
-        ): Promise<_queuedWriteResult>
-
-        /**
-         * Deletes documents from a collection.
-         * @param collection The name of the collection.
-         * @param deleteSchema The delete schema.
-         * @returns The number of documents deleted.
-         */
-        delDocs<T extends Record<string, any>>(
-            collection: string,
-            deleteSchema?: _storeDelete<T>
-        ): Promise<number>
-
-        /**
-         * Joins documents from two collections.
-         * @param join The join schema.
-         * @returns The joined documents.
-         */
-        static joinDocs<T extends Record<string, any>, U extends Record<string, any>>(
-            join: _join<T, U>
-        ): Promise<_joinDocs<T, U>>
-
-        /**
-         * Finds documents in a collection.
-         * @param collection The name of the collection.
-         * @param query The query schema.
-         * @returns The found documents.
-         */
-        static findDocs<T extends Record<string, any>>(
-            collection: string,
-            query?: _storeQuery<T>
-        ): _findDocs
-    }
-
-    export function migrateLegacyS3ToS3Files(options: {
-        collections: string[]
-        s3FilesRoot?: string
-        recreateCollections?: boolean
-        verify?: boolean
-    }): Promise<Record<string, { migrated: number; verified: boolean }>>
-}

package/src/types/write-queue.ts

@@ -1,42 +0,0 @@
-export type WriteJobOperation = 'insert' | 'update' | 'delete'
-
-export type WriteJobStatus = 'queued' | 'processing' | 'committed' | 'failed' | 'dead-letter'
-
-export interface WriteJob<T extends Record<string, any> = Record<string, any>> {
-    jobId: string
-    collection: string
-    docId: _ttid
-    operation: WriteJobOperation
-    payload: T
-    status: WriteJobStatus
-    attempts: number
-    createdAt: number
-    updatedAt: number
-    nextAttemptAt?: number
-    workerId?: string
-    error?: string
-}
-
-export interface QueuedWriteResult {
-    jobId: string
-    docId: _ttid
-    status: WriteJobStatus
-}
-
-export interface StreamJobEntry<T extends Record<string, any> = Record<string, any>> {
-    streamId: string
-    job: WriteJob<T>
-}
-
-export interface DeadLetterJob<T extends Record<string, any> = Record<string, any>> {
-    streamId: string
-    job: WriteJob<T>
-    reason?: string
-    failedAt: number
-}
-
-export interface QueueStats {
-    queued: number
-    pending: number
-    deadLetters: number
-}

package/src/worker.ts

@@ -1,18 +0,0 @@
-#!/usr/bin/env bun
-import { WriteWorker } from './workers/write-worker'
-
-if (process.env.FYLO_STORAGE_ENGINE === 's3-files') {
-    throw new Error('fylo.worker is not supported in s3-files engine')
-}
-
-const worker = new WriteWorker(process.env.FYLO_WORKER_ID)
-
-await worker.run({
-    batchSize: process.env.FYLO_WORKER_BATCH_SIZE ? Number(process.env.FYLO_WORKER_BATCH_SIZE) : 1,
-    blockMs: process.env.FYLO_WORKER_BLOCK_MS ? Number(process.env.FYLO_WORKER_BLOCK_MS) : 1000,
-    recoverOnStart: process.env.FYLO_WORKER_RECOVER_ON_START !== 'false',
-    recoverIdleMs: process.env.FYLO_WORKER_RECOVER_IDLE_MS
-        ? Number(process.env.FYLO_WORKER_RECOVER_IDLE_MS)
-        : 30_000,
-    stopWhenIdle: process.env.FYLO_WORKER_STOP_WHEN_IDLE === 'true'
-})

package/src/workers/write-worker.ts

@@ -1,120 +0,0 @@
-import Fylo from '../index'
-import { Redis } from '../adapters/redis'
-import type { StreamJobEntry, WriteJob } from '../types/write-queue'
-
-export class WriteWorker {
-    private static readonly MAX_WRITE_ATTEMPTS = Number(process.env.FYLO_WRITE_MAX_ATTEMPTS ?? 3)
-
-    private static readonly WRITE_RETRY_BASE_MS = Number(process.env.FYLO_WRITE_RETRY_BASE_MS ?? 10)
-
-    private readonly fylo: Fylo
-
-    private readonly redis: Redis
-
-    readonly workerId: string
-
-    constructor(workerId: string = Bun.randomUUIDv7()) {
-        this.workerId = workerId
-        this.fylo = new Fylo()
-        this.redis = new Redis()
-    }
-
-    async recoverPending(minIdleMs: number = 30_000, count: number = 10) {
-        const jobs = await this.redis.claimPendingJobs(this.workerId, minIdleMs, count)
-        for (const job of jobs) await this.processJob(job)
-        return jobs.length
-    }
-
-    async processNext(count: number = 1, blockMs: number = 1000) {
-        const jobs = await this.redis.readWriteJobs(this.workerId, count, blockMs)
-        for (const job of jobs) await this.processJob(job)
-        return jobs.length
-    }
-
-    async processJob({ streamId, job }: StreamJobEntry) {
-        if (job.nextAttemptAt && job.nextAttemptAt > Date.now()) return false
-
-        const locked = await this.redis.acquireDocLock(job.collection, job.docId, job.jobId)
-        if (!locked) return false
-
-        try {
-            await this.redis.setJobStatus(job.jobId, 'processing', {
-                workerId: this.workerId,
-                attempts: job.attempts + 1
-            })
-            await this.redis.setDocStatus(job.collection, job.docId, 'processing', job.jobId)
-
-            await this.fylo.executeQueuedWrite(job)
-
-            await this.redis.setJobStatus(job.jobId, 'committed', { workerId: this.workerId })
-            await this.redis.setDocStatus(job.collection, job.docId, 'committed', job.jobId)
-            await this.redis.ackWriteJob(streamId)
-
-            return true
-        } catch (err) {
-            const attempts = job.attempts + 1
-            const message = err instanceof Error ? err.message : String(err)
-
-            if (attempts >= WriteWorker.MAX_WRITE_ATTEMPTS) {
-                await this.redis.setJobStatus(job.jobId, 'dead-letter', {
-                    workerId: this.workerId,
-                    attempts,
-                    error: message
-                })
-                await this.redis.setDocStatus(job.collection, job.docId, 'dead-letter', job.jobId)
-                await this.redis.deadLetterWriteJob(
-                    streamId,
-                    {
-                        ...job,
-                        attempts,
-                        status: 'dead-letter',
-                        workerId: this.workerId,
-                        error: message
-                    },
-                    message
-                )
-                return false
-            }
-
-            const nextAttemptAt =
-                Date.now() + WriteWorker.WRITE_RETRY_BASE_MS * Math.max(1, 2 ** (attempts - 1))
-
-            await this.redis.setJobStatus(job.jobId, 'failed', {
-                workerId: this.workerId,
-                attempts,
-                error: message,
-                nextAttemptAt
-            })
-            await this.redis.setDocStatus(job.collection, job.docId, 'failed', job.jobId)
-
-            return false
-        } finally {
-            await this.redis.releaseDocLock(job.collection, job.docId, job.jobId)
-        }
-    }
-
-    async processQueuedInsert(job: WriteJob) {
-        return await this.fylo.executeQueuedWrite(job)
-    }
-
-    async run({
-        batchSize = 1,
-        blockMs = 1000,
-        recoverOnStart = true,
-        recoverIdleMs = 30_000,
-        stopWhenIdle = false
-    }: {
-        batchSize?: number
-        blockMs?: number
-        recoverOnStart?: boolean
-        recoverIdleMs?: number
-        stopWhenIdle?: boolean
-    } = {}) {
-        if (recoverOnStart) await this.recoverPending(recoverIdleMs, batchSize)
-
-        while (true) {
-            const processed = await this.processNext(batchSize, blockMs)
-            if (stopWhenIdle && processed === 0) break
-        }
-    }
-}

package/tests/collection/truncate.test.js

@@ -1,35 +0,0 @@
-import { test, expect, describe, afterAll, mock } from 'bun:test'
-import Fylo from '../../src'
-import { postsURL, albumURL } from '../data'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
-const POSTS = `post`
-const ALBUMS = `album`
-afterAll(async () => {
-    await Promise.all([Fylo.dropCollection(ALBUMS), Fylo.dropCollection(POSTS)])
-})
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
-describe('NO-SQL', () => {
-    test('TRUNCATE', async () => {
-        const fylo = new Fylo()
-        await Fylo.createCollection(POSTS)
-        await fylo.importBulkData(POSTS, new URL(postsURL))
-        await fylo.delDocs(POSTS)
-        const ids = []
-        for await (const data of Fylo.findDocs(POSTS, { $limit: 1, $onlyIds: true }).collect()) {
-            ids.push(data)
-        }
-        expect(ids.length).toBe(0)
-    })
-})
-describe('SQL', () => {
-    test('TRUNCATE', async () => {
-        const fylo = new Fylo()
-        await fylo.executeSQL(`CREATE TABLE ${ALBUMS}`)
-        await fylo.importBulkData(ALBUMS, new URL(albumURL))
-        await fylo.executeSQL(`DELETE FROM ${ALBUMS}`)
-        const ids = await fylo.executeSQL(`SELECT _id FROM ${ALBUMS} LIMIT 1`)
-        expect(ids.length).toBe(0)
-    })
-})

package/tests/data.js

@@ -1,97 +0,0 @@
-function makeDataUrl(data) {
-    return `data:application/json,${encodeURIComponent(JSON.stringify(data))}`
-}
-function generateAlbums() {
-    return Array.from({ length: 100 }, (_, index) => {
-        const id = index + 1
-        const userId = Math.ceil(id / 10)
-        const prefix = id <= 15 ? 'omnis' : id % 4 === 0 ? 'quidem' : 'album'
-        return {
-            id,
-            userId,
-            title: `${prefix} album ${id}`
-        }
-    })
-}
-function generatePosts() {
-    return Array.from({ length: 100 }, (_, index) => {
-        const id = index + 1
-        const userId = Math.ceil(id / 10)
-        return {
-            id,
-            userId,
-            title: `post title ${id}`,
-            body: `post body ${id} for user ${userId}`
-        }
-    })
-}
-function generateComments() {
-    return Array.from({ length: 100 }, (_, index) => {
-        const id = index + 1
-        return {
-            id,
-            postId: id,
-            name: `comment ${id}`,
-            email: `comment${id}@example.com`,
-            body: `comment body ${id}`
-        }
-    })
-}
-function generatePhotos() {
-    return Array.from({ length: 100 }, (_, index) => {
-        const id = index + 1
-        const title = id % 3 === 0 ? `test photo ${id}` : `photo ${id}`
-        return {
-            id,
-            albumId: Math.ceil(id / 10),
-            title,
-            url: `https://example.com/photos/${id}.jpg`,
-            thumbnailUrl: `https://example.com/photos/${id}-thumb.jpg`
-        }
-    })
-}
-function generateTodos() {
-    return Array.from({ length: 100 }, (_, index) => {
-        const id = index + 1
-        return {
-            id,
-            userId: Math.ceil(id / 10),
-            title: id % 4 === 0 ? `test todo ${id}` : `todo ${id}`,
-            completed: id % 2 === 0
-        }
-    })
-}
-function generateUsers() {
-    return Array.from({ length: 10 }, (_, index) => {
-        const id = index + 1
-        return {
-            id,
-            name: `User ${id}`,
-            username: `user${id}`,
-            email: `user${id}@example.com`,
-            address: {
-                street: `Main Street ${id}`,
-                suite: `Suite ${id}`,
-                city: id <= 5 ? 'South Christy' : 'North Christy',
-                zipcode: `0000${id}`,
-                geo: {
-                    lat: 10 + id,
-                    lng: -20 - id
-                }
-            },
-            phone: `555-000-${String(id).padStart(4, '0')}`,
-            website: `user${id}.example.com`,
-            company: {
-                name: id <= 5 ? 'Acme Labs' : 'Northwind Labs',
-                catchPhrase: `Catch phrase ${id}`,
-                bs: `business ${id}`
-            }
-        }
-    })
-}
-export const albumURL = makeDataUrl(generateAlbums())
-export const postsURL = makeDataUrl(generatePosts())
-export const commentsURL = makeDataUrl(generateComments())
-export const photosURL = makeDataUrl(generatePhotos())
-export const todosURL = makeDataUrl(generateTodos())
-export const usersURL = makeDataUrl(generateUsers())

package/tests/index.js

@@ -1,14 +0,0 @@
-import { Redis } from '../src/adapters/redis'
-import ttid from '@delma/ttid'
-const redisPub = new Redis()
-const redisSub = new Redis()
-setTimeout(async () => {
-    await redisPub.publish('bun', 'insert', ttid.generate())
-}, 2000)
-setTimeout(async () => {
-    await redisPub.publish('bun', 'insert', ttid.generate())
-}, 3000)
-await Bun.sleep(1000)
-for await (const data of redisSub.subscribe('bun')) {
-    console.log('Received:', data)
-}