@delma/fylo 1.0.1 → 1.1.0

package/README.md

@@ -1,11 +1,13 @@
 # Fylo
 
-S3-backed NoSQL document store with SQL parsing, Redis pub/sub for real-time events, and a CLI.
+S3-backed NoSQL document store with SQL parsing, Redis-backed write coordination and pub/sub for real-time events, and a CLI.
 
 Documents are stored as **S3 key paths** — not file contents. Each document produces two keys per field: a **data key** (`{ttid}/{field}/{value}`) for full-doc retrieval and an **index key** (`{field}/{value}/{ttid}`) for query lookups. This enables fast reads and filtered queries without a traditional database engine.
 
 Built for **serverless** runtimes (AWS Lambda, Cloudflare Workers) — no persistent in-memory state, lazy connections, minimal cold-start overhead.
 
+Writes are coordinated through Redis before they are flushed to S3. By default the high-level CRUD methods wait for the queued write to be processed so existing code can continue to behave synchronously. If you want fire-and-forget semantics, pass `{ wait: false }` and process queued jobs with a worker or `processQueuedWrites()`.
+
 ## Install
 
 ```bash
@@ -21,7 +23,15 @@ bun add @delma/fylo
 | `S3_SECRET_ACCESS_KEY` / `AWS_SECRET_ACCESS_KEY` | S3 credentials |
 | `S3_REGION` / `AWS_REGION` | S3 region |
 | `S3_ENDPOINT` / `AWS_ENDPOINT` | S3 endpoint (for LocalStack, MinIO, etc.) |
-| `REDIS_URL` | Redis connection URL (default: `redis://localhost:6379`) |
+| `REDIS_URL` | Redis connection URL used for pub/sub, document locks, and queued write coordination |
+| `FYLO_WRITE_MAX_ATTEMPTS` | Maximum retry attempts before a queued job is dead-lettered |
+| `FYLO_WRITE_RETRY_BASE_MS` | Base retry delay used for exponential backoff between recovery attempts |
+| `FYLO_WORKER_ID` | Optional stable identifier for a write worker process |
+| `FYLO_WORKER_BATCH_SIZE` | Number of queued jobs a worker pulls per read loop |
+| `FYLO_WORKER_BLOCK_MS` | Redis stream block time for waiting on new jobs |
+| `FYLO_WORKER_RECOVER_ON_START` | Whether the worker reclaims stale pending jobs on startup |
+| `FYLO_WORKER_RECOVER_IDLE_MS` | Minimum idle time before a pending job is reclaimed |
+| `FYLO_WORKER_STOP_WHEN_IDLE` | Exit the worker loop when no jobs are available |
 | `LOGGING` | Enable debug logging |
 | `STRICT` | Enable schema validation via CHEX |
 
@@ -69,6 +79,44 @@ const deleted = await fylo.delDocs<_user>("users", {
 await Fylo.dropCollection("users")
 ```
 
+### Queued Writes
+
+```typescript
+const fylo = new Fylo()
+
+// Default behavior waits for the queued write to finish.
+const _id = await fylo.putData("users", { name: "John Doe" })
+
+// Async mode returns the queued job immediately.
+const queued = await fylo.putData("users", { name: "Jane Doe" }, { wait: false })
+
+// Poll status if you need to track progress.
+const status = await fylo.getJobStatus(queued.jobId)
+
+// Process pending writes in-process when you are not running a separate worker.
+await fylo.processQueuedWrites()
+```
+
+When `wait: false` is used, the job is durable in Redis but the document is not visible in S3 until a worker commits it.
+
+Queued jobs that fail are left pending for recovery. Recovered jobs retry up to `FYLO_WRITE_MAX_ATTEMPTS` times before being moved to a dead-letter stream. You can inspect dead letters with `getDeadLetters()` and reclaim stale pending jobs with `processQueuedWrites(count, true)`.
+
+Operational helpers:
+
+- `getQueueStats()` returns current queue, pending, and dead-letter counts
+- `getDeadLetters()` lists exhausted jobs
+- `replayDeadLetter(streamId)` moves a dead-lettered job back into the main queue
+
+### Worker
+
+Run a dedicated write worker when you want queued writes to be flushed outside the request path:
+
+```bash
+bun run worker
+```
+
+The worker entrypoint lives at [worker.ts](/Users/iyor/Library/CloudStorage/Dropbox/myProjects/FYLO/src/worker.ts) and continuously drains the Redis stream, recovers stale pending jobs on startup, and respects the retry/dead-letter settings above.
+
 ### CRUD — SQL API
 
 ```typescript
@@ -160,7 +208,9 @@ for await (const doc of Fylo.exportBulkData<_user>("users")) {
 
 ### Rollback
 
-Every write is tracked as a transaction. If a batch write partially fails, Fylo automatically rolls back. You can also trigger it manually:
+`rollback()` is now a legacy escape hatch.
+
+Fylo still keeps best-effort rollback data for writes performed by the current instance. This is mainly useful for in-process failures and test workflows:
 
 ```typescript
 const fylo = new Fylo()
@@ -168,6 +218,15 @@ await fylo.putData("users", { name: "test" })
 await fylo.rollback() // undoes all writes in this instance
 ```
 
+For queued writes, prefer:
+
+- `getJobStatus()` to inspect an individual write
+- `processQueuedWrites(count, true)` to recover stale pending jobs
+- `getDeadLetters()` to inspect exhausted jobs
+- compensating writes instead of `rollback()` after a commit
+
+`rollback()` may be removed from the main queued-write path in a future major release.
+
 ### CLI
 
 ```bash

package/package.json

@@ -1,15 +1,17 @@
 {
   "name": "@delma/fylo",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "main": "./dist/index.js",
   "types": "./dist/types/index.d.ts",
   "bin": {
-    "fylo.query": "./dist/cli/index.js"
+    "fylo.query": "./dist/cli/index.js",
+    "fylo.worker": "./dist/worker.js"
   },
   "scripts": {
     "build": "tsc",
     "test": "bun test",
     "typecheck": "tsc --noEmit",
+    "worker": "bun run ./src/worker.ts",
     "lint": "eslint src tests",
     "format": "prettier --write src tests"
   },

package/src/adapters/redis.ts

@@ -1,8 +1,15 @@
 import { RedisClient } from "bun";
 import { S3 } from "./s3";
+import type { DeadLetterJob, QueueStats, StreamJobEntry, WriteJob, WriteJobStatus } from "../types/write-queue";
 
 export class Redis {
 
+    static readonly WRITE_STREAM = 'fylo:writes'
+
+    static readonly WRITE_GROUP = 'fylo-workers'
+
+    static readonly DEAD_LETTER_STREAM = 'fylo:writes:dead'
+
     private client: RedisClient
 
     private static LOGGING = process.env.LOGGING
@@ -31,6 +38,41 @@ export class Redis {
         this.client.connect()
     }
 
+    private async ensureWriteGroup() {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        try {
+            await this.client.send('XGROUP', ['CREATE', Redis.WRITE_STREAM, Redis.WRITE_GROUP, '$', 'MKSTREAM'])
+        } catch(err) {
+            if(!(err instanceof Error) || !err.message.includes('BUSYGROUP')) throw err
+        }
+    }
+
+    private static hashKey(jobId: string) {
+        return `fylo:job:${jobId}`
+    }
+
+    private static docKey(collection: string, docId: _ttid) {
+        return `fylo:doc:${collection}:${docId}`
+    }
+
+    private static lockKey(collection: string, docId: _ttid) {
+        return `fylo:lock:${collection}:${docId}`
+    }
+
+    private static parseHash(values: unknown): Record<string, string> {
+        if(!Array.isArray(values)) return {}
+
+        const parsed: Record<string, string> = {}
+
+        for(let i = 0; i < values.length; i += 2) {
+            parsed[String(values[i])] = String(values[i + 1] ?? '')
+        }
+
+        return parsed
+    }
+
     async publish(collection: string, action: 'insert' | 'delete', keyId: string | _ttid) {
 
         if(this.client.connected) {
@@ -48,6 +90,310 @@ export class Redis {
         return result === 'OK'
     }
 
+    async enqueueWrite<T extends Record<string, any>>(job: WriteJob<T>) {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        await this.ensureWriteGroup()
+
+        const now = Date.now()
+        const payload = JSON.stringify(job.payload)
+
+        await this.client.send('HSET', [
+            Redis.hashKey(job.jobId),
+            'jobId', job.jobId,
+            'collection', job.collection,
+            'docId', job.docId,
+            'operation', job.operation,
+            'payload', payload,
+            'status', job.status,
+            'attempts', String(job.attempts),
+            'createdAt', String(job.createdAt),
+            'updatedAt', String(now),
+            'nextAttemptAt', String(job.nextAttemptAt ?? now)
+        ])
+
+        await this.client.send('HSET', [
+            Redis.docKey(job.collection, job.docId),
+            'status', 'queued',
+            'lastJobId', job.jobId,
+            'updatedAt', String(now)
+        ])
+
+        return await this.client.send('XADD', [
+            Redis.WRITE_STREAM,
+            '*',
+            'jobId', job.jobId,
+            'collection', job.collection,
+            'docId', job.docId,
+            'operation', job.operation
+        ])
+    }
+
+    async readWriteJobs(workerId: string, count: number = 1, blockMs: number = 1000): Promise<Array<StreamJobEntry>> {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        await this.ensureWriteGroup()
+
+        const rows = await this.client.send('XREADGROUP', [
+            'GROUP', Redis.WRITE_GROUP, workerId,
+            'COUNT', String(count),
+            'BLOCK', String(blockMs),
+            'STREAMS', Redis.WRITE_STREAM, '>'
+        ])
+
+        if(!Array.isArray(rows) || rows.length === 0) return []
+
+        const items: Array<StreamJobEntry> = []
+
+        for(const streamRow of rows as unknown[]) {
+            if(!Array.isArray(streamRow) || streamRow.length < 2) continue
+            const entries = streamRow[1]
+            if(!Array.isArray(entries)) continue
+
+            for(const entry of entries as unknown[]) {
+                if(!Array.isArray(entry) || entry.length < 2) continue
+                const streamId = String(entry[0])
+                const fields = Redis.parseHash(entry[1])
+                const job = await this.getJob(fields.jobId)
+                if(job) items.push({ streamId, job })
+            }
+        }
+
+        return items
+    }
+
+    async ackWriteJob(streamId: string) {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        await this.client.send('XACK', [Redis.WRITE_STREAM, Redis.WRITE_GROUP, streamId])
+    }
+
+    async deadLetterWriteJob(streamId: string, job: WriteJob, reason?: string) {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const failedAt = Date.now()
+
+        await this.client.send('XADD', [
+            Redis.DEAD_LETTER_STREAM,
+            '*',
+            'jobId', job.jobId,
+            'collection', job.collection,
+            'docId', job.docId,
+            'operation', job.operation,
+            'reason', reason ?? '',
+            'failedAt', String(failedAt)
+        ])
+
+        await this.ackWriteJob(streamId)
+    }
+
+    async claimPendingJobs(workerId: string, minIdleMs: number = 30_000, count: number = 10): Promise<Array<StreamJobEntry>> {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        await this.ensureWriteGroup()
+
+        const result = await this.client.send('XAUTOCLAIM', [
+            Redis.WRITE_STREAM,
+            Redis.WRITE_GROUP,
+            workerId,
+            String(minIdleMs),
+            '0-0',
+            'COUNT',
+            String(count)
+        ])
+
+        if(!Array.isArray(result) || result.length < 2 || !Array.isArray(result[1])) return []
+
+        const items: Array<StreamJobEntry> = []
+
+        for(const entry of result[1] as unknown[]) {
+            if(!Array.isArray(entry) || entry.length < 2) continue
+            const streamId = String(entry[0])
+            const fields = Redis.parseHash(entry[1])
+            const job = await this.getJob(fields.jobId)
+            if(job) items.push({ streamId, job })
+        }
+
+        return items
+    }
+
+    async setJobStatus(jobId: string, status: WriteJobStatus, extra: Partial<Pick<WriteJob, 'workerId' | 'error' | 'attempts' | 'nextAttemptAt'>> = {}) {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const args = [
+            Redis.hashKey(jobId),
+            'status', status,
+            'updatedAt', String(Date.now())
+        ]
+
+        if(extra.workerId) args.push('workerId', extra.workerId)
+        if(extra.error) args.push('error', extra.error)
+        if(typeof extra.attempts === 'number') args.push('attempts', String(extra.attempts))
+        if(typeof extra.nextAttemptAt === 'number') args.push('nextAttemptAt', String(extra.nextAttemptAt))
+
+        await this.client.send('HSET', args)
+    }
+
+    async setDocStatus(collection: string, docId: _ttid, status: WriteJobStatus, jobId?: string) {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const args = [
+            Redis.docKey(collection, docId),
+            'status', status,
+            'updatedAt', String(Date.now())
+        ]
+
+        if(jobId) args.push('lastJobId', jobId)
+
+        await this.client.send('HSET', args)
+    }
+
+    async getJob(jobId: string): Promise<WriteJob | null> {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const hash = Redis.parseHash(await this.client.send('HGETALL', [Redis.hashKey(jobId)]))
+
+        if(Object.keys(hash).length === 0) return null
+
+        return {
+            jobId: hash.jobId,
+            collection: hash.collection,
+            docId: hash.docId as _ttid,
+            operation: hash.operation as WriteJob['operation'],
+            payload: JSON.parse(hash.payload),
+            status: hash.status as WriteJobStatus,
+            attempts: Number(hash.attempts ?? 0),
+            createdAt: Number(hash.createdAt ?? 0),
+            updatedAt: Number(hash.updatedAt ?? 0),
+            nextAttemptAt: Number(hash.nextAttemptAt ?? 0) || undefined,
+            workerId: hash.workerId || undefined,
+            error: hash.error || undefined
+        }
+    }
+
+    async getDocStatus(collection: string, docId: _ttid) {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const hash = Redis.parseHash(await this.client.send('HGETALL', [Redis.docKey(collection, docId)]))
+
+        return Object.keys(hash).length > 0 ? hash : null
+    }
+
+    async readDeadLetters(count: number = 10): Promise<Array<DeadLetterJob>> {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const rows = await this.client.send('XRANGE', [Redis.DEAD_LETTER_STREAM, '-', '+', 'COUNT', String(count)])
+
+        if(!Array.isArray(rows)) return []
+
+        const items: Array<DeadLetterJob> = []
+
+        for(const row of rows as unknown[]) {
+            if(!Array.isArray(row) || row.length < 2) continue
+            const streamId = String(row[0])
+            const fields = Redis.parseHash(row[1])
+            const job = await this.getJob(fields.jobId)
+
+            if(job) {
+                items.push({
+                    streamId,
+                    job,
+                    reason: fields.reason || undefined,
+                    failedAt: Number(fields.failedAt ?? 0)
+                })
+            }
+        }
+
+        return items
+    }
+
+    async replayDeadLetter(streamId: string): Promise<WriteJob | null> {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const rows = await this.client.send('XRANGE', [Redis.DEAD_LETTER_STREAM, streamId, streamId, 'COUNT', '1'])
+
+        if(!Array.isArray(rows) || rows.length === 0) return null
+
+        const row = rows[0]
+        if(!Array.isArray(row) || row.length < 2) return null
+
+        const fields = Redis.parseHash(row[1])
+        const job = await this.getJob(fields.jobId)
+
+        if(!job) return null
+
+        const replayed: WriteJob = {
+            ...job,
+            status: 'queued',
+            error: undefined,
+            workerId: undefined,
+            attempts: 0,
+            updatedAt: Date.now(),
+            nextAttemptAt: Date.now()
+        }
+
+        await this.enqueueWrite(replayed)
+        await this.client.send('XDEL', [Redis.DEAD_LETTER_STREAM, streamId])
+
+        return replayed
+    }
+
+    async getQueueStats(): Promise<QueueStats> {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        await this.ensureWriteGroup()
+
+        const [queuedRaw, deadRaw, pendingRaw] = await Promise.all([
+            this.client.send('XLEN', [Redis.WRITE_STREAM]),
+            this.client.send('XLEN', [Redis.DEAD_LETTER_STREAM]),
+            this.client.send('XPENDING', [Redis.WRITE_STREAM, Redis.WRITE_GROUP])
+        ])
+
+        const pending = Array.isArray(pendingRaw) ? Number(pendingRaw[0] ?? 0) : 0
+
+        return {
+            queued: Number(queuedRaw ?? 0),
+            pending,
+            deadLetters: Number(deadRaw ?? 0)
+        }
+    }
+
+    async acquireDocLock(collection: string, docId: _ttid, jobId: string, ttlSeconds: number = 60) {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const result = await this.client.send('SET', [
+            Redis.lockKey(collection, docId),
+            jobId,
+            'NX',
+            'EX',
+            String(ttlSeconds)
+        ])
+
+        return result === 'OK'
+    }
+
+    async releaseDocLock(collection: string, docId: _ttid, jobId: string) {
+
+        if(!this.client.connected) throw new Error('Redis not connected!')
+
+        const key = Redis.lockKey(collection, docId)
+        const current = await this.client.send('GET', [key])
+        if(current === jobId) await this.client.send('DEL', [key])
+    }
+
     async *subscribe(collection: string) {
 
         if(!this.client.connected) throw new Error('Redis not connected!')

package/src/core/write-queue.ts

@@ -0,0 +1,56 @@
+import type { WriteJob } from '../types/write-queue'
+
+export class WriteQueue {
+
+    static createInsertJob<T extends Record<string, any>>(collection: string, docId: _ttid, payload: T): WriteJob<T> {
+        const now = Date.now()
+
+        return {
+            jobId: Bun.randomUUIDv7(),
+            collection,
+            docId,
+            operation: 'insert',
+            payload,
+            status: 'queued',
+            attempts: 0,
+            createdAt: now,
+            updatedAt: now
+        }
+    }
+
+    static createUpdateJob<T extends Record<string, any>>(
+        collection: string,
+        docId: _ttid,
+        payload: { newDoc: Record<_ttid, Partial<T>>, oldDoc?: Record<_ttid, T> }
+    ): WriteJob<{ newDoc: Record<_ttid, Partial<T>>, oldDoc?: Record<_ttid, T> }> {
+        const now = Date.now()
+
+        return {
+            jobId: Bun.randomUUIDv7(),
+            collection,
+            docId,
+            operation: 'update',
+            payload,
+            status: 'queued',
+            attempts: 0,
+            createdAt: now,
+            updatedAt: now
+        }
+    }
+
+    static createDeleteJob(collection: string, docId: _ttid): WriteJob<{ _id: _ttid }> {
+        const now = Date.now()
+
+        return {
+            jobId: Bun.randomUUIDv7(),
+            collection,
+            docId,
+            operation: 'delete',
+            payload: { _id: docId },
+            status: 'queued',
+            attempts: 0,
+            createdAt: now,
+            updatedAt: now
+        }
+    }
+}