@delma/fylo 2.0.1 → 2.1.0

package/src/sync.ts

@@ -0,0 +1,58 @@
+export type FyloSyncMode = 'await-sync' | 'fire-and-forget'
+
+export interface FyloWriteSyncEvent<T extends Record<string, any> = Record<string, any>> {
+    operation: 'put' | 'patch'
+    collection: string
+    docId: _ttid
+    previousDocId?: _ttid
+    path: string
+    data: T
+}
+
+export interface FyloDeleteSyncEvent {
+    operation: 'delete' | 'patch'
+    collection: string
+    docId: _ttid
+    path: string
+}
+
+export interface FyloSyncHooks<T extends Record<string, any> = Record<string, any>> {
+    onWrite?: (event: FyloWriteSyncEvent<T>) => Promise<void> | void
+    onDelete?: (event: FyloDeleteSyncEvent) => Promise<void> | void
+}
+
+export interface FyloOptions<T extends Record<string, any> = Record<string, any>> {
+    root?: string
+    s3FilesRoot?: string
+    sync?: FyloSyncHooks<T>
+    syncMode?: FyloSyncMode
+}
+
+export class FyloSyncError extends Error {
+    readonly collection: string
+    readonly docId: _ttid
+    readonly path: string
+    readonly operation: string
+
+    constructor(args: {
+        collection: string
+        docId: _ttid
+        path: string
+        operation: string
+        cause: unknown
+    }) {
+        super(
+            `FYLO sync failed after the local filesystem operation succeeded for ${args.operation} ${args.collection}/${args.docId}. Local state is already committed at ${args.path}.`,
+            { cause: args.cause }
+        )
+        this.name = 'FyloSyncError'
+        this.collection = args.collection
+        this.docId = args.docId
+        this.path = args.path
+        this.operation = args.operation
+    }
+}
+
+export function resolveSyncMode(syncMode?: FyloSyncMode): FyloSyncMode {
+    return syncMode ?? 'await-sync'
+}

package/src/types/fylo.d.ts

@@ -10,16 +10,14 @@ interface _findDocs {
         void,
         unknown
     >
-    once<T>(): Promise<Record<_ttid, T>>
+    collect<T>(): AsyncGenerator<
+        _ttid | Record<_ttid, T> | Record<string, _ttid[]> | Record<_ttid, Partial<T>> | undefined,
+        void,
+        unknown
+    >
     onDelete(): AsyncGenerator<_ttid, void, unknown>
 }
 
-interface _queuedWriteResult {
-    jobId: string
-    docId: _ttid
-    status: 'queued' | 'processing' | 'committed' | 'failed' | 'dead-letter'
-}
-
 interface ObjectConstructor {
     appendGroup: (target: Record<string, any>, source: Record<string, any>) => Record<string, any>
 }
@@ -34,20 +32,58 @@ type _joinDocs<T, U> =
     | Record<string, Record<_ttid, Partial<T | U>>>
     | Record<`${_ttid}, ${_ttid}`, T | U | (T & U) | (Partial<T> & Partial<U>)>
 
-type _fyloEngineKind = 'legacy-s3' | 's3-files'
+type _fyloSyncMode = 'await-sync' | 'fire-and-forget'
+
+interface _fyloWriteSyncEvent<T extends Record<string, any> = Record<string, any>> {
+    operation: 'put' | 'patch'
+    collection: string
+    docId: _ttid
+    previousDocId?: _ttid
+    path: string
+    data: T
+}
+
+interface _fyloDeleteSyncEvent {
+    operation: 'delete' | 'patch'
+    collection: string
+    docId: _ttid
+    path: string
+}
+
+interface _fyloSyncHooks<T extends Record<string, any> = Record<string, any>> {
+    onWrite?: (event: _fyloWriteSyncEvent<T>) => Promise<void> | void
+    onDelete?: (event: _fyloDeleteSyncEvent) => Promise<void> | void
+}
 
 interface _fyloOptions {
-    engine?: _fyloEngineKind
+    root?: string
     s3FilesRoot?: string
+    sync?: _fyloSyncHooks
+    syncMode?: _fyloSyncMode
 }
 
 declare module '@delma/fylo' {
+    export class FyloSyncError extends Error {
+        readonly collection: string
+        readonly docId: _ttid
+        readonly path: string
+        readonly operation: string
+    }
+
+    export type FyloSyncMode = _fyloSyncMode
+    export type FyloWriteSyncEvent<T extends Record<string, any> = Record<string, any>> =
+        _fyloWriteSyncEvent<T>
+    export type FyloDeleteSyncEvent = _fyloDeleteSyncEvent
+    export type FyloSyncHooks<T extends Record<string, any> = Record<string, any>> =
+        _fyloSyncHooks<T>
+    export type FyloOptions = _fyloOptions
+
     export default class {
         constructor(options?: _fyloOptions)
 
         /**
-         * Rolls back all transcations in current instance
-         * @deprecated Prefer queued write recovery, dead letters, or compensating writes.
+         * Compatibility helper. FYLO now writes synchronously to the filesystem,
+         * so rollback is a no-op.
          */
         rollback(): Promise<void>
 
@@ -60,202 +96,71 @@ declare module '@delma/fylo' {
             SQL: string
         ): Promise<number | void | any[] | _ttid | Record<any, any>>
 
-        /**
-         * Creates a new schema for a collection.
-         * @param collection The name of the collection.
-         */
         static createCollection(collection: string): Promise<void>
-
-        /**
-         * Drops an existing schema for a collection.
-         * @param collection The name of the collection.
-         */
         static dropCollection(collection: string): Promise<void>
 
-        /**
-         * Imports data from a URL into a collection.
-         * @param collection The name of the collection.
-         * @param url The URL of the data to import.
-         * @param limit The maximum number of documents to import.
-         */
+        createCollection(collection: string): Promise<void>
+        dropCollection(collection: string): Promise<void>
+
         importBulkData(collection: string, url: URL, limit?: number): Promise<number>
 
-        /**
-         * Exports data from a collection to a URL.
-         * @param collection The name of the collection.
-         * @returns The current data exported from the collection.
-         */
         exportBulkData<T extends Record<string, any>>(
             collection: string
         ): AsyncGenerator<T, void, unknown>
 
-        /**
-         * Gets a document from a collection.
-         * @param collection The name of the collection.
-         * @param _id The ID of the document.
-         * @param onlyId Whether to only return the ID of the document.
-         * @returns The document or the ID of the document.
-         */
-        static getDoc(collection: string, _id: _ttid, onlyId: boolean): _getDoc
+        static exportBulkData<T extends Record<string, any>>(
+            collection: string
+        ): AsyncGenerator<T, void, unknown>
+
+        static getDoc(collection: string, _id: _ttid, onlyId?: boolean): _getDoc
+
+        getDoc(collection: string, _id: _ttid, onlyId?: boolean): _getDoc
 
-        /**
-         * Puts multiple documents into a collection.
-         * @param collection The name of the collection.
-         * @param batch The documents to put.
-         * @returns The IDs of the documents.
-         */
         batchPutData<T extends Record<string, any>>(
             collection: string,
             batch: Array<T>
         ): Promise<_ttid[]>
 
-        queuePutData<T extends Record<string, any>>(
-            collection: string,
-            data: Record<_ttid, T> | T
-        ): Promise<_queuedWriteResult>
-
-        queuePatchDoc<T extends Record<string, any>>(
-            collection: string,
-            newDoc: Record<_ttid, Partial<T>>,
-            oldDoc?: Record<_ttid, T>
-        ): Promise<_queuedWriteResult>
-
-        queueDelDoc(collection: string, _id: _ttid): Promise<_queuedWriteResult>
-
-        getJobStatus(jobId: string): Promise<Record<string, any> | null>
-
-        getDocStatus(collection: string, docId: _ttid): Promise<Record<string, any> | null>
-
-        getDeadLetters(count?: number): Promise<Array<Record<string, any>>>
-
-        getQueueStats(): Promise<{ queued: number; pending: number; deadLetters: number }>
-
-        replayDeadLetter(streamId: string): Promise<Record<string, any> | null>
-
-        processQueuedWrites(count?: number, recover?: boolean): Promise<number>
-
-        /**
-         * Puts a document into a collection.
-         * @param collection The name of the collection.
-         * @param data The document to put.
-         * @returns The ID of the document.
-         */
         putData<T extends Record<string, any>>(collection: string, data: T): Promise<_ttid>
         putData<T extends Record<string, any>>(
             collection: string,
             data: Record<_ttid, T>
         ): Promise<_ttid>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: T,
-            options: { wait?: true; timeoutMs?: number }
-        ): Promise<_ttid>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: Record<_ttid, T>,
-            options: { wait?: true; timeoutMs?: number }
-        ): Promise<_ttid>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: T,
-            options: { wait: false; timeoutMs?: number }
-        ): Promise<_queuedWriteResult>
-        putData<T extends Record<string, any>>(
-            collection: string,
-            data: Record<_ttid, T>,
-            options: { wait: false; timeoutMs?: number }
-        ): Promise<_queuedWriteResult>
 
-        /**
-         * Patches a document in a collection.
-         * @param collection The name of the collection.
-         * @param newDoc The new document data.
-         * @param oldDoc The old document data.
-         * @returns The number of documents patched.
-         */
         patchDoc<T extends Record<string, any>>(
             collection: string,
             newDoc: Record<_ttid, Partial<T>>,
             oldDoc?: Record<_ttid, T>
         ): Promise<_ttid>
-        patchDoc<T extends Record<string, any>>(
-            collection: string,
-            newDoc: Record<_ttid, Partial<T>>,
-            oldDoc: Record<_ttid, T> | undefined,
-            options: { wait?: true; timeoutMs?: number }
-        ): Promise<_ttid>
-        patchDoc<T extends Record<string, any>>(
-            collection: string,
-            newDoc: Record<_ttid, Partial<T>>,
-            oldDoc: Record<_ttid, T> | undefined,
-            options: { wait: false; timeoutMs?: number }
-        ): Promise<_queuedWriteResult>
 
-        /**
-         * Patches documents in a collection.
-         * @param collection The name of the collection.
-         * @param updateSchema The update schema.
-         * @returns The number of documents patched.
-         */
         patchDocs<T extends Record<string, any>>(
             collection: string,
             updateSchema: _storeUpdate<T>
         ): Promise<number>
 
-        /**
-         * Deletes a document from a collection.
-         * @param collection The name of the collection.
-         * @param _id The ID of the document.
-         * @returns The number of documents deleted.
-         */
         delDoc(collection: string, _id: _ttid): Promise<void>
-        delDoc(
-            collection: string,
-            _id: _ttid,
-            options: { wait?: true; timeoutMs?: number }
-        ): Promise<void>
-        delDoc(
-            collection: string,
-            _id: _ttid,
-            options: { wait: false; timeoutMs?: number }
-        ): Promise<_queuedWriteResult>
 
-        /**
-         * Deletes documents from a collection.
-         * @param collection The name of the collection.
-         * @param deleteSchema The delete schema.
-         * @returns The number of documents deleted.
-         */
         delDocs<T extends Record<string, any>>(
             collection: string,
             deleteSchema?: _storeDelete<T>
         ): Promise<number>
 
-        /**
-         * Joins documents from two collections.
-         * @param join The join schema.
-         * @returns The joined documents.
-         */
         static joinDocs<T extends Record<string, any>, U extends Record<string, any>>(
             join: _join<T, U>
         ): Promise<_joinDocs<T, U>>
 
-        /**
-         * Finds documents in a collection.
-         * @param collection The name of the collection.
-         * @param query The query schema.
-         * @returns The found documents.
-         */
+        joinDocs<T extends Record<string, any>, U extends Record<string, any>>(
+            join: _join<T, U>
+        ): Promise<_joinDocs<T, U>>
+
         static findDocs<T extends Record<string, any>>(
             collection: string,
             query?: _storeQuery<T>
         ): _findDocs
-    }
 
-    export function migrateLegacyS3ToS3Files(options: {
-        collections: string[]
-        s3FilesRoot?: string
-        recreateCollections?: boolean
-        verify?: boolean
-    }): Promise<Record<string, { migrated: number; verified: boolean }>>
+        findDocs<T extends Record<string, any>>(
+            collection: string,
+            query?: _storeQuery<T>
+        ): _findDocs
+    }
 }

package/src/types/node-runtime.d.ts

@@ -1,5 +1,6 @@
 declare const process: {
     env: Record<string, string | undefined>
+    cwd(): string
 }
 
 declare const Buffer: {

package/tests/collection/truncate.test.js

@@ -1,23 +1,24 @@
-import { test, expect, describe, afterAll, mock } from 'bun:test'
+import { test, expect, describe, afterAll } from 'bun:test'
+import { rm } from 'node:fs/promises'
 import Fylo from '../../src'
 import { postsURL, albumURL } from '../data'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
+import { createTestRoot } from '../helpers/root'
 const POSTS = `post`
 const ALBUMS = `album`
+const root = await createTestRoot('fylo-truncate-')
 afterAll(async () => {
-    await Promise.all([Fylo.dropCollection(ALBUMS), Fylo.dropCollection(POSTS)])
+    const fylo = new Fylo({ root })
+    await Promise.all([fylo.dropCollection(ALBUMS), fylo.dropCollection(POSTS)])
+    await rm(root, { recursive: true, force: true })
 })
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
 describe('NO-SQL', () => {
     test('TRUNCATE', async () => {
-        const fylo = new Fylo()
-        await Fylo.createCollection(POSTS)
+        const fylo = new Fylo({ root })
+        await fylo.createCollection(POSTS)
         await fylo.importBulkData(POSTS, new URL(postsURL))
         await fylo.delDocs(POSTS)
         const ids = []
-        for await (const data of Fylo.findDocs(POSTS, { $limit: 1, $onlyIds: true }).collect()) {
+        for await (const data of fylo.findDocs(POSTS, { $limit: 1, $onlyIds: true }).collect()) {
             ids.push(data)
         }
         expect(ids.length).toBe(0)
@@ -25,7 +26,7 @@ describe('NO-SQL', () => {
 })
 describe('SQL', () => {
     test('TRUNCATE', async () => {
-        const fylo = new Fylo()
+        const fylo = new Fylo({ root })
         await fylo.executeSQL(`CREATE TABLE ${ALBUMS}`)
         await fylo.importBulkData(ALBUMS, new URL(albumURL))
         await fylo.executeSQL(`DELETE FROM ${ALBUMS}`)

package/tests/helpers/root.js

@@ -0,0 +1,7 @@
+import { mkdtemp } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+
+export async function createTestRoot(prefix = 'fylo-test-') {
+    return await mkdtemp(path.join(os.tmpdir(), prefix))
+}

package/tests/integration/create.test.js

@@ -1,17 +1,16 @@
-import { test, expect, describe, beforeAll, afterAll, mock } from 'bun:test'
+import { test, expect, describe, beforeAll, afterAll } from 'bun:test'
+import { rm } from 'node:fs/promises'
 import Fylo from '../../src'
 import { albumURL, postsURL } from '../data'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
+import { createTestRoot } from '../helpers/root'
 const POSTS = `post`
 const ALBUMS = `album`
 let postsCount = 0
 let albumsCount = 0
-const fylo = new Fylo()
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
+const root = await createTestRoot('fylo-create-')
+const fylo = new Fylo({ root })
 beforeAll(async () => {
-    await Promise.all([Fylo.createCollection(POSTS), fylo.executeSQL(`CREATE TABLE ${ALBUMS}`)])
+    await Promise.all([fylo.createCollection(POSTS), fylo.executeSQL(`CREATE TABLE ${ALBUMS}`)])
     try {
         albumsCount = await fylo.importBulkData(ALBUMS, new URL(albumURL), 100)
         postsCount = await fylo.importBulkData(POSTS, new URL(postsURL), 100)
@@ -20,12 +19,13 @@ beforeAll(async () => {
     }
 })
 afterAll(async () => {
-    await Promise.all([Fylo.dropCollection(POSTS), fylo.executeSQL(`DROP TABLE ${ALBUMS}`)])
+    await Promise.all([fylo.dropCollection(POSTS), fylo.executeSQL(`DROP TABLE ${ALBUMS}`)])
+    await rm(root, { recursive: true, force: true })
 })
 describe('NO-SQL', async () => {
     test('PUT', async () => {
         let results = {}
-        for await (const data of Fylo.findDocs(POSTS).collect()) {
+        for await (const data of fylo.findDocs(POSTS).collect()) {
             results = { ...results, ...data }
         }
         expect(Object.keys(results).length).toEqual(postsCount)

package/tests/integration/delete.test.js

@@ -1,17 +1,16 @@
-import { test, expect, describe, beforeAll, afterAll, mock } from 'bun:test'
+import { test, expect, describe, beforeAll, afterAll } from 'bun:test'
+import { rm } from 'node:fs/promises'
 import Fylo from '../../src'
 import { commentsURL, usersURL } from '../data'
-import S3Mock from '../mocks/s3'
-import RedisMock from '../mocks/redis'
+import { createTestRoot } from '../helpers/root'
 const COMMENTS = `comment`
 const USERS = `user`
 let commentsResults = {}
 let usersResults = {}
-const fylo = new Fylo()
-mock.module('../../src/adapters/s3', () => ({ S3: S3Mock }))
-mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
+const root = await createTestRoot('fylo-delete-')
+const fylo = new Fylo({ root })
 beforeAll(async () => {
-    await Promise.all([Fylo.createCollection(COMMENTS), fylo.executeSQL(`CREATE TABLE ${USERS}`)])
+    await Promise.all([fylo.createCollection(COMMENTS), fylo.executeSQL(`CREATE TABLE ${USERS}`)])
     try {
         await Promise.all([
             fylo.importBulkData(COMMENTS, new URL(commentsURL), 100),
@@ -20,13 +19,14 @@ beforeAll(async () => {
     } catch {
         await fylo.rollback()
     }
-    for await (const data of Fylo.findDocs(COMMENTS, { $limit: 1 }).collect()) {
+    for await (const data of fylo.findDocs(COMMENTS, { $limit: 1 }).collect()) {
         commentsResults = { ...commentsResults, ...data }
     }
     usersResults = await fylo.executeSQL(`SELECT * FROM ${USERS} LIMIT 1`)
 })
 afterAll(async () => {
-    await Promise.all([Fylo.dropCollection(COMMENTS), fylo.executeSQL(`DROP TABLE ${USERS}`)])
+    await Promise.all([fylo.dropCollection(COMMENTS), fylo.executeSQL(`DROP TABLE ${USERS}`)])
+    await rm(root, { recursive: true, force: true })
 })
 describe('NO-SQL', async () => {
     test('DELETE ONE', async () => {
@@ -37,7 +37,7 @@ describe('NO-SQL', async () => {
             await fylo.rollback()
         }
         commentsResults = {}
-        for await (const data of Fylo.findDocs(COMMENTS).collect()) {
+        for await (const data of fylo.findDocs(COMMENTS).collect()) {
             commentsResults = { ...commentsResults, ...data }
         }
         const idx = Object.keys(commentsResults).findIndex((_id) => _id === id)
@@ -51,9 +51,11 @@ describe('NO-SQL', async () => {
             await fylo.rollback()
         }
         commentsResults = {}
-        for await (const data of Fylo.findDocs(COMMENTS, {
-            $ops: [{ name: { $like: '%et%' } }]
-        }).collect()) {
+        for await (const data of fylo
+            .findDocs(COMMENTS, {
+                $ops: [{ name: { $like: '%et%' } }]
+            })
+            .collect()) {
             commentsResults = { ...commentsResults, ...data }
         }
         expect(Object.keys(commentsResults).length).toEqual(0)
@@ -65,7 +67,7 @@ describe('NO-SQL', async () => {
             await fylo.rollback()
         }
         commentsResults = {}
-        for await (const data of Fylo.findDocs(COMMENTS).collect()) {
+        for await (const data of fylo.findDocs(COMMENTS).collect()) {
             commentsResults = { ...commentsResults, ...data }
         }
         expect(Object.keys(commentsResults).length).toEqual(0)