@delma/fylo 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +185 -267
- package/package.json +2 -5
- package/src/core/directory.ts +22 -354
- package/src/engines/s3-files/documents.ts +65 -0
- package/src/engines/s3-files/filesystem.ts +172 -0
- package/src/engines/s3-files/query.ts +291 -0
- package/src/engines/s3-files/types.ts +42 -0
- package/src/engines/s3-files.ts +391 -510
- package/src/engines/types.ts +1 -1
- package/src/index.ts +142 -1237
- package/src/sync.ts +58 -0
- package/src/types/fylo.d.ts +66 -161
- package/src/types/node-runtime.d.ts +1 -0
- package/tests/collection/truncate.test.js +11 -10
- package/tests/helpers/root.js +7 -0
- package/tests/integration/create.test.js +9 -9
- package/tests/integration/delete.test.js +16 -14
- package/tests/integration/edge-cases.test.js +29 -25
- package/tests/integration/encryption.test.js +47 -30
- package/tests/integration/export.test.js +11 -11
- package/tests/integration/join-modes.test.js +16 -16
- package/tests/integration/nested.test.js +26 -24
- package/tests/integration/operators.test.js +43 -29
- package/tests/integration/read.test.js +25 -21
- package/tests/integration/rollback.test.js +21 -51
- package/tests/integration/s3-files.performance.test.js +75 -0
- package/tests/integration/s3-files.test.js +115 -18
- package/tests/integration/sync.test.js +154 -0
- package/tests/integration/update.test.js +24 -18
- package/src/adapters/redis.ts +0 -487
- package/src/adapters/s3.ts +0 -61
- package/src/core/walker.ts +0 -174
- package/src/core/write-queue.ts +0 -59
- package/src/migrate-cli.ts +0 -22
- package/src/migrate.ts +0 -74
- package/src/types/write-queue.ts +0 -42
- package/src/worker.ts +0 -18
- package/src/workers/write-worker.ts +0 -120
- package/tests/index.js +0 -14
- package/tests/integration/migration.test.js +0 -38
- package/tests/integration/queue.test.js +0 -83
- package/tests/mocks/redis.js +0 -123
- package/tests/mocks/s3.js +0 -80
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { afterAll, describe, expect, test } from 'bun:test'
|
|
2
|
+
import { mkdtemp, rm } from 'node:fs/promises'
|
|
3
|
+
import os from 'node:os'
|
|
4
|
+
import path from 'node:path'
|
|
5
|
+
import Fylo, { FyloSyncError } from '../../src'
|
|
6
|
+
|
|
7
|
+
const roots = []
|
|
8
|
+
|
|
9
|
+
async function createRoot(prefix) {
|
|
10
|
+
const root = await mkdtemp(path.join(os.tmpdir(), prefix))
|
|
11
|
+
roots.push(root)
|
|
12
|
+
return root
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
afterAll(async () => {
|
|
16
|
+
await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true })))
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
describe('sync hooks', () => {
|
|
20
|
+
test('await-sync emits write, patch, and delete events with filesystem paths', async () => {
|
|
21
|
+
const root = await createRoot('fylo-sync-await-')
|
|
22
|
+
const calls = []
|
|
23
|
+
const fylo = new Fylo({
|
|
24
|
+
root,
|
|
25
|
+
sync: {
|
|
26
|
+
onWrite: async (event) => {
|
|
27
|
+
calls.push({ hook: 'write', ...event })
|
|
28
|
+
},
|
|
29
|
+
onDelete: async (event) => {
|
|
30
|
+
calls.push({ hook: 'delete', ...event })
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
const collection = 'sync-posts'
|
|
36
|
+
await fylo.createCollection(collection)
|
|
37
|
+
|
|
38
|
+
const id = await fylo.putData(collection, { title: 'Hello sync' })
|
|
39
|
+
const nextId = await fylo.patchDoc(collection, {
|
|
40
|
+
[id]: { title: 'Hello sync 2' }
|
|
41
|
+
})
|
|
42
|
+
await fylo.delDoc(collection, nextId)
|
|
43
|
+
|
|
44
|
+
expect(calls).toEqual([
|
|
45
|
+
{
|
|
46
|
+
hook: 'write',
|
|
47
|
+
operation: 'put',
|
|
48
|
+
collection,
|
|
49
|
+
docId: id,
|
|
50
|
+
path: path.join(root, collection, '.fylo', 'docs', id.slice(0, 2), `${id}.json`),
|
|
51
|
+
data: { title: 'Hello sync' }
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
hook: 'delete',
|
|
55
|
+
operation: 'patch',
|
|
56
|
+
collection,
|
|
57
|
+
docId: id,
|
|
58
|
+
path: path.join(root, collection, '.fylo', 'docs', id.slice(0, 2), `${id}.json`)
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
hook: 'write',
|
|
62
|
+
operation: 'patch',
|
|
63
|
+
collection,
|
|
64
|
+
docId: nextId,
|
|
65
|
+
previousDocId: id,
|
|
66
|
+
path: path.join(
|
|
67
|
+
root,
|
|
68
|
+
collection,
|
|
69
|
+
'.fylo',
|
|
70
|
+
'docs',
|
|
71
|
+
nextId.slice(0, 2),
|
|
72
|
+
`${nextId}.json`
|
|
73
|
+
),
|
|
74
|
+
data: { title: 'Hello sync 2' }
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
hook: 'delete',
|
|
78
|
+
operation: 'delete',
|
|
79
|
+
collection,
|
|
80
|
+
docId: nextId,
|
|
81
|
+
path: path.join(
|
|
82
|
+
root,
|
|
83
|
+
collection,
|
|
84
|
+
'.fylo',
|
|
85
|
+
'docs',
|
|
86
|
+
nextId.slice(0, 2),
|
|
87
|
+
`${nextId}.json`
|
|
88
|
+
)
|
|
89
|
+
}
|
|
90
|
+
])
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
test('fire-and-forget does not block the local write', async () => {
|
|
94
|
+
const root = await createRoot('fylo-sync-fire-')
|
|
95
|
+
let releaseHook
|
|
96
|
+
let writeStarted = false
|
|
97
|
+
const started = Promise.withResolvers()
|
|
98
|
+
const hookBlocker = new Promise((resolve) => {
|
|
99
|
+
releaseHook = resolve
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
const fylo = new Fylo({
|
|
103
|
+
root,
|
|
104
|
+
syncMode: 'fire-and-forget',
|
|
105
|
+
sync: {
|
|
106
|
+
onWrite: async () => {
|
|
107
|
+
writeStarted = true
|
|
108
|
+
started.resolve()
|
|
109
|
+
await hookBlocker
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
})
|
|
113
|
+
|
|
114
|
+
await fylo.createCollection('fire-posts')
|
|
115
|
+
|
|
116
|
+
const putPromise = fylo.putData('fire-posts', { title: 'Fast local write' })
|
|
117
|
+
await started.promise
|
|
118
|
+
|
|
119
|
+
const state = await Promise.race([
|
|
120
|
+
putPromise.then(() => 'resolved'),
|
|
121
|
+
Bun.sleep(25).then(() => 'pending')
|
|
122
|
+
])
|
|
123
|
+
|
|
124
|
+
expect(writeStarted).toBe(true)
|
|
125
|
+
expect(state).toBe('resolved')
|
|
126
|
+
|
|
127
|
+
releaseHook()
|
|
128
|
+
await putPromise
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
test('await-sync surfaces sync failures as FyloSyncError after the local write', async () => {
|
|
132
|
+
const root = await createRoot('fylo-sync-error-')
|
|
133
|
+
let failedDocId
|
|
134
|
+
const fylo = new Fylo({
|
|
135
|
+
root,
|
|
136
|
+
sync: {
|
|
137
|
+
onWrite: async (event) => {
|
|
138
|
+
failedDocId = event.docId
|
|
139
|
+
throw new Error('remote unavailable')
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
})
|
|
143
|
+
|
|
144
|
+
await fylo.createCollection('error-posts')
|
|
145
|
+
|
|
146
|
+
await expect(
|
|
147
|
+
fylo.putData('error-posts', { title: 'Still written locally' })
|
|
148
|
+
).rejects.toBeInstanceOf(FyloSyncError)
|
|
149
|
+
|
|
150
|
+
expect(failedDocId).toBeDefined()
|
|
151
|
+
const stored = await fylo.getDoc('error-posts', failedDocId).once()
|
|
152
|
+
expect(stored[failedDocId]).toEqual({ title: 'Still written locally' })
|
|
153
|
+
})
|
|
154
|
+
})
|
|
@@ -1,15 +1,14 @@
|
|
|
1
|
-
import { test, expect, describe, beforeAll, afterAll
|
|
1
|
+
import { test, expect, describe, beforeAll, afterAll } from 'bun:test'
|
|
2
|
+
import { rm } from 'node:fs/promises'
|
|
2
3
|
import Fylo from '../../src'
|
|
3
4
|
import { photosURL, todosURL } from '../data'
|
|
4
|
-
import
|
|
5
|
-
import RedisMock from '../mocks/redis'
|
|
5
|
+
import { createTestRoot } from '../helpers/root'
|
|
6
6
|
const PHOTOS = `photo`
|
|
7
7
|
const TODOS = `todo`
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
mock.module('../../src/adapters/redis', () => ({ Redis: RedisMock }))
|
|
8
|
+
const root = await createTestRoot('fylo-update-')
|
|
9
|
+
const fylo = new Fylo({ root })
|
|
11
10
|
beforeAll(async () => {
|
|
12
|
-
await Promise.all([
|
|
11
|
+
await Promise.all([fylo.createCollection(PHOTOS), fylo.executeSQL(`CREATE TABLE ${TODOS}`)])
|
|
13
12
|
try {
|
|
14
13
|
await fylo.importBulkData(PHOTOS, new URL(photosURL), 100)
|
|
15
14
|
await fylo.importBulkData(TODOS, new URL(todosURL), 100)
|
|
@@ -18,12 +17,13 @@ beforeAll(async () => {
|
|
|
18
17
|
}
|
|
19
18
|
})
|
|
20
19
|
afterAll(async () => {
|
|
21
|
-
await Promise.all([
|
|
20
|
+
await Promise.all([fylo.dropCollection(PHOTOS), fylo.executeSQL(`DROP TABLE ${TODOS}`)])
|
|
21
|
+
await rm(root, { recursive: true, force: true })
|
|
22
22
|
})
|
|
23
23
|
describe('NO-SQL', async () => {
|
|
24
24
|
test('UPDATE ONE', async () => {
|
|
25
25
|
const ids = []
|
|
26
|
-
for await (const data of
|
|
26
|
+
for await (const data of fylo.findDocs(PHOTOS, { $limit: 1, $onlyIds: true }).collect()) {
|
|
27
27
|
ids.push(data)
|
|
28
28
|
}
|
|
29
29
|
try {
|
|
@@ -32,9 +32,11 @@ describe('NO-SQL', async () => {
|
|
|
32
32
|
await fylo.rollback()
|
|
33
33
|
}
|
|
34
34
|
let results = {}
|
|
35
|
-
for await (const data of
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
for await (const data of fylo
|
|
36
|
+
.findDocs(PHOTOS, {
|
|
37
|
+
$ops: [{ title: { $eq: 'All Mighty' } }]
|
|
38
|
+
})
|
|
39
|
+
.collect()) {
|
|
38
40
|
results = { ...results, ...data }
|
|
39
41
|
}
|
|
40
42
|
expect(Object.keys(results).length).toBe(1)
|
|
@@ -50,9 +52,11 @@ describe('NO-SQL', async () => {
|
|
|
50
52
|
await fylo.rollback()
|
|
51
53
|
}
|
|
52
54
|
let results = {}
|
|
53
|
-
for await (const data of
|
|
54
|
-
|
|
55
|
-
|
|
55
|
+
for await (const data of fylo
|
|
56
|
+
.findDocs(PHOTOS, {
|
|
57
|
+
$ops: [{ title: { $eq: 'All Mighti' } }]
|
|
58
|
+
})
|
|
59
|
+
.collect()) {
|
|
56
60
|
results = { ...results, ...data }
|
|
57
61
|
}
|
|
58
62
|
expect(Object.keys(results).length).toBe(count)
|
|
@@ -65,9 +69,11 @@ describe('NO-SQL', async () => {
|
|
|
65
69
|
await fylo.rollback()
|
|
66
70
|
}
|
|
67
71
|
let results = {}
|
|
68
|
-
for await (const data of
|
|
69
|
-
|
|
70
|
-
|
|
72
|
+
for await (const data of fylo
|
|
73
|
+
.findDocs(PHOTOS, {
|
|
74
|
+
$ops: [{ title: { $eq: 'All Mighter' } }]
|
|
75
|
+
})
|
|
76
|
+
.collect()) {
|
|
71
77
|
results = { ...results, ...data }
|
|
72
78
|
}
|
|
73
79
|
expect(Object.keys(results).length).toBe(count)
|