@uploadista/kv-store-filesystem 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/.turbo/turbo-check.log +5 -0
- package/LICENSE +21 -0
- package/README.md +707 -0
- package/dist/file-kv-store.d.ts +8 -0
- package/dist/file-kv-store.d.ts.map +1 -0
- package/dist/file-kv-store.js +36 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1 -0
- package/package.json +29 -0
- package/src/file-kv-store.ts +58 -0
- package/src/index.ts +1 -0
- package/tsconfig.json +13 -0
- package/tsconfig.tsbuildinfo +1 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 uploadista
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,707 @@
|
|
|
1
|
+
# @uploadista/kv-store-filesystem
|
|
2
|
+
|
|
3
|
+
Filesystem-backed key-value store for Uploadista. Provides persistent storage without external dependencies, perfect for development and self-hosted deployments.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
The filesystem KV store stores data as JSON files on disk. It's designed for:
|
|
8
|
+
|
|
9
|
+
- **Development & Testing**: No external services needed
|
|
10
|
+
- **Self-Hosted Deployments**: Full control over data storage
|
|
11
|
+
- **Small to Medium Deployments**: Suitable for 1-100 GB of data
|
|
12
|
+
- **Docker & VPS Hosting**: Persistent storage in mounted volumes
|
|
13
|
+
- **Backup-Friendly**: Easy filesystem snapshots and backups
|
|
14
|
+
|
|
15
|
+
Data is persisted to disk and survives process restarts. Performance depends on disk I/O speed.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
npm install @uploadista/kv-store-filesystem
|
|
21
|
+
# or
|
|
22
|
+
pnpm add @uploadista/kv-store-filesystem
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
### Prerequisites
|
|
26
|
+
|
|
27
|
+
- Node.js 18+
|
|
28
|
+
- Writable filesystem (local disk, mounted volume, or network storage)
|
|
29
|
+
- For concurrent access: Avoid multiple processes writing to same directory
|
|
30
|
+
|
|
31
|
+
## Quick Start
|
|
32
|
+
|
|
33
|
+
```typescript
|
|
34
|
+
import { fileKvStore } from "@uploadista/kv-store-filesystem";
|
|
35
|
+
import { Effect } from "effect";
|
|
36
|
+
|
|
37
|
+
// Create store backed by filesystem
|
|
38
|
+
const layer = fileKvStore({
|
|
39
|
+
directory: "./data/kv-store",
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
const program = Effect.gen(function* () {
|
|
43
|
+
// The filesystem store is automatically available
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
Effect.runSync(
|
|
47
|
+
program.pipe(
|
|
48
|
+
Effect.provide(layer),
|
|
49
|
+
// ... other layers
|
|
50
|
+
)
|
|
51
|
+
);
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Features
|
|
55
|
+
|
|
56
|
+
- ✅ **Persistent Storage**: Data survives process restarts and crashes
|
|
57
|
+
- ✅ **No Dependencies**: No external services required
|
|
58
|
+
- ✅ **Easy Backups**: Standard filesystem backup tools work
|
|
59
|
+
- ✅ **Controlled Performance**: Tune with SSD vs HDD, caching strategies
|
|
60
|
+
- ✅ **Volume-Mounted**: Works perfectly in Docker/Kubernetes
|
|
61
|
+
- ✅ **Type Safe**: Full TypeScript support
|
|
62
|
+
- ✅ **Simple Debugging**: Data stored as readable JSON files
|
|
63
|
+
|
|
64
|
+
## API Reference
|
|
65
|
+
|
|
66
|
+
### Main Exports
|
|
67
|
+
|
|
68
|
+
#### `fileKvStore(config: FileKvStoreOptions): Layer<BaseKvStoreService>`
|
|
69
|
+
|
|
70
|
+
Creates an Effect layer providing the `BaseKvStoreService` backed by filesystem.
|
|
71
|
+
|
|
72
|
+
```typescript
|
|
73
|
+
import { fileKvStore } from "@uploadista/kv-store-filesystem";
|
|
74
|
+
|
|
75
|
+
const layer = fileKvStore({
|
|
76
|
+
directory: "./data/uploads",
|
|
77
|
+
});
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
**Configuration**:
|
|
81
|
+
|
|
82
|
+
```typescript
|
|
83
|
+
type FileKvStoreOptions = {
|
|
84
|
+
directory: string; // Directory path for storing files
|
|
85
|
+
};
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
#### `makeFileBaseKvStore(config: FileKvStoreOptions): BaseKvStore`
|
|
89
|
+
|
|
90
|
+
Factory function for creating a filesystem KV store.
|
|
91
|
+
|
|
92
|
+
```typescript
|
|
93
|
+
import { makeFileBaseKvStore } from "@uploadista/kv-store-filesystem";
|
|
94
|
+
|
|
95
|
+
const store = makeFileBaseKvStore({
|
|
96
|
+
directory: "./data/kv-store",
|
|
97
|
+
});
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### Available Operations
|
|
101
|
+
|
|
102
|
+
The filesystem store implements the `BaseKvStore` interface:
|
|
103
|
+
|
|
104
|
+
#### `get(key: string): Effect<string | null>`
|
|
105
|
+
|
|
106
|
+
Retrieve a value by key. Returns `null` if key doesn't exist.
|
|
107
|
+
|
|
108
|
+
```typescript
|
|
109
|
+
const program = Effect.gen(function* () {
|
|
110
|
+
const value = yield* store.get("user:123");
|
|
111
|
+
// Reads from ./data/kv-store/user:123.json
|
|
112
|
+
});
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
#### `set(key: string, value: string): Effect<void>`
|
|
116
|
+
|
|
117
|
+
Store a string value. Creates file if doesn't exist, overwrites if exists.
|
|
118
|
+
|
|
119
|
+
```typescript
|
|
120
|
+
const program = Effect.gen(function* () {
|
|
121
|
+
yield* store.set("user:123", JSON.stringify({ name: "Alice" }));
|
|
122
|
+
// Writes to ./data/kv-store/user:123.json
|
|
123
|
+
});
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
#### `delete(key: string): Effect<void>`
|
|
127
|
+
|
|
128
|
+
Remove a key from storage. Safe to call on non-existent keys.
|
|
129
|
+
|
|
130
|
+
```typescript
|
|
131
|
+
const program = Effect.gen(function* () {
|
|
132
|
+
yield* store.delete("user:123");
|
|
133
|
+
// Deletes ./data/kv-store/user:123.json
|
|
134
|
+
});
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
#### `list(keyPrefix: string): Effect<string[]>`
|
|
138
|
+
|
|
139
|
+
List all keys matching a prefix.
|
|
140
|
+
|
|
141
|
+
```typescript
|
|
142
|
+
const program = Effect.gen(function* () {
|
|
143
|
+
const keys = yield* store.list("user:");
|
|
144
|
+
// Returns: ["123", "456"] for files ["user:123.json", "user:456.json"]
|
|
145
|
+
});
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
## Configuration
|
|
149
|
+
|
|
150
|
+
### Basic Setup
|
|
151
|
+
|
|
152
|
+
```typescript
|
|
153
|
+
import { fileKvStore } from "@uploadista/kv-store-filesystem";
|
|
154
|
+
|
|
155
|
+
const layer = fileKvStore({
|
|
156
|
+
directory: "./data/kv-store",
|
|
157
|
+
});
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Environment-Based Configuration
|
|
161
|
+
|
|
162
|
+
```typescript
|
|
163
|
+
import { fileKvStore } from "@uploadista/kv-store-filesystem";
|
|
164
|
+
import path from "path";
|
|
165
|
+
|
|
166
|
+
const dataDir = process.env.DATA_DIR || path.join(process.cwd(), "data");
|
|
167
|
+
|
|
168
|
+
const layer = fileKvStore({
|
|
169
|
+
directory: path.join(dataDir, "kv-store"),
|
|
170
|
+
});
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
### Production Configuration with Volume Mounts
|
|
174
|
+
|
|
175
|
+
```typescript
|
|
176
|
+
import { fileKvStore } from "@uploadista/kv-store-filesystem";
|
|
177
|
+
|
|
178
|
+
// Use mount point provided by orchestration system
|
|
179
|
+
const layer = fileKvStore({
|
|
180
|
+
directory: process.env.KV_STORE_PATH || "/mnt/persistent-data/kv-store",
|
|
181
|
+
});
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
## Examples
|
|
185
|
+
|
|
186
|
+
### Example 1: Local Development Server
|
|
187
|
+
|
|
188
|
+
```typescript
|
|
189
|
+
import { fileKvStore } from "@uploadista/kv-store-filesystem";
|
|
190
|
+
import { uploadServer } from "@uploadista/server";
|
|
191
|
+
import { Effect } from "effect";
|
|
192
|
+
import path from "path";
|
|
193
|
+
|
|
194
|
+
const developmentLayer = fileKvStore({
|
|
195
|
+
directory: path.join(process.cwd(), "./dev-data"),
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
const program = Effect.gen(function* () {
|
|
199
|
+
const server = yield* uploadServer;
|
|
200
|
+
|
|
201
|
+
// Use filesystem store for development
|
|
202
|
+
const upload = yield* server.createUpload(
|
|
203
|
+
{
|
|
204
|
+
filename: "test-file.pdf",
|
|
205
|
+
size: 2097152,
|
|
206
|
+
mimeType: "application/pdf",
|
|
207
|
+
},
|
|
208
|
+
"client-dev"
|
|
209
|
+
);
|
|
210
|
+
|
|
211
|
+
console.log(`Upload created: ${upload.id}`);
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
Effect.runSync(
|
|
215
|
+
program.pipe(
|
|
216
|
+
Effect.provide(developmentLayer),
|
|
217
|
+
// ... other layers
|
|
218
|
+
)
|
|
219
|
+
);
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
### Example 2: Session Storage
|
|
223
|
+
|
|
224
|
+
```typescript
|
|
225
|
+
import { makeFileBaseKvStore } from "@uploadista/kv-store-filesystem";
|
|
226
|
+
import { Effect } from "effect";
|
|
227
|
+
|
|
228
|
+
const store = makeFileBaseKvStore({
|
|
229
|
+
directory: "./data/sessions",
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
interface Session {
|
|
233
|
+
userId: string;
|
|
234
|
+
loginTime: number;
|
|
235
|
+
lastActivity: number;
|
|
236
|
+
permissions: string[];
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const createSession = (sessionId: string, userId: string) =>
|
|
240
|
+
Effect.gen(function* () {
|
|
241
|
+
const session: Session = {
|
|
242
|
+
userId,
|
|
243
|
+
loginTime: Date.now(),
|
|
244
|
+
lastActivity: Date.now(),
|
|
245
|
+
permissions: ["upload", "download"],
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
yield* store.set(`session:${sessionId}`, JSON.stringify(session));
|
|
249
|
+
console.log(`Session created: ${sessionId}`);
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
const getSession = (sessionId: string) =>
|
|
253
|
+
Effect.gen(function* () {
|
|
254
|
+
const data = yield* store.get(`session:${sessionId}`);
|
|
255
|
+
return data ? JSON.parse(data) : null;
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
// Usage
|
|
259
|
+
const program = Effect.gen(function* () {
|
|
260
|
+
yield* createSession("sess_abc123", "user_xyz");
|
|
261
|
+
const session = yield* getSession("sess_abc123");
|
|
262
|
+
console.log(session);
|
|
263
|
+
// {
|
|
264
|
+
// userId: "user_xyz",
|
|
265
|
+
// loginTime: 1729516800000,
|
|
266
|
+
// lastActivity: 1729516800000,
|
|
267
|
+
// permissions: ["upload", "download"]
|
|
268
|
+
// }
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
Effect.runSync(program);
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
### Example 3: Upload Metadata Tracking
|
|
275
|
+
|
|
276
|
+
```typescript
|
|
277
|
+
import { makeFileBaseKvStore } from "@uploadista/kv-store-filesystem";
|
|
278
|
+
import { Effect } from "effect";
|
|
279
|
+
|
|
280
|
+
const store = makeFileBaseKvStore({
|
|
281
|
+
directory: "./data/uploads",
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
interface UploadMetadata {
|
|
285
|
+
id: string;
|
|
286
|
+
filename: string;
|
|
287
|
+
size: number;
|
|
288
|
+
uploadedAt: string;
|
|
289
|
+
completedAt?: string;
|
|
290
|
+
status: "in-progress" | "completed" | "failed";
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const trackUpload = (metadata: UploadMetadata) =>
|
|
294
|
+
Effect.gen(function* () {
|
|
295
|
+
const key = `upload:${metadata.id}`;
|
|
296
|
+
yield* store.set(key, JSON.stringify(metadata));
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
const completeUpload = (uploadId: string) =>
|
|
300
|
+
Effect.gen(function* () {
|
|
301
|
+
const key = `upload:${uploadId}`;
|
|
302
|
+
const dataStr = yield* store.get(key);
|
|
303
|
+
|
|
304
|
+
if (!dataStr) {
|
|
305
|
+
return; // Upload not found
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
const metadata: UploadMetadata = JSON.parse(dataStr);
|
|
309
|
+
metadata.status = "completed";
|
|
310
|
+
metadata.completedAt = new Date().toISOString();
|
|
311
|
+
|
|
312
|
+
yield* store.set(key, JSON.stringify(metadata));
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
const listAllUploads = () =>
|
|
316
|
+
Effect.gen(function* () {
|
|
317
|
+
const keys = yield* store.list("upload:");
|
|
318
|
+
|
|
319
|
+
const uploads = yield* Effect.all(
|
|
320
|
+
keys.map((key) =>
|
|
321
|
+
Effect.gen(function* () {
|
|
322
|
+
const data = yield* store.get(`upload:${key}`);
|
|
323
|
+
return data ? JSON.parse(data) : null;
|
|
324
|
+
})
|
|
325
|
+
)
|
|
326
|
+
);
|
|
327
|
+
|
|
328
|
+
return uploads.filter((u) => u !== null);
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
// Usage
|
|
332
|
+
const program = Effect.gen(function* () {
|
|
333
|
+
// Track new upload
|
|
334
|
+
yield* trackUpload({
|
|
335
|
+
id: "upl_123",
|
|
336
|
+
filename: "document.pdf",
|
|
337
|
+
size: 1048576,
|
|
338
|
+
uploadedAt: new Date().toISOString(),
|
|
339
|
+
status: "in-progress",
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
// Complete upload
|
|
343
|
+
yield* completeUpload("upl_123");
|
|
344
|
+
|
|
345
|
+
// List all uploads
|
|
346
|
+
const uploads = yield* listAllUploads();
|
|
347
|
+
console.log(uploads);
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
Effect.runSync(program);
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
## Performance Characteristics
|
|
354
|
+
|
|
355
|
+
| Operation | Latency | Scaling |
|
|
356
|
+
|-----------|---------|---------|
|
|
357
|
+
| get() | 1-5ms | O(1) + disk I/O |
|
|
358
|
+
| set() | 2-10ms | O(1) + disk write |
|
|
359
|
+
| delete() | 1-5ms | O(1) + disk delete |
|
|
360
|
+
| list() | 5-50ms | O(n) where n = files |
|
|
361
|
+
|
|
362
|
+
Performance depends heavily on:
|
|
363
|
+
- **Disk Type**: SSD (1-5ms) vs HDD (10-50ms)
|
|
364
|
+
- **I/O System**: NVMe >> SSD >> HDD
|
|
365
|
+
- **File Count**: More files in directory = slower list operations
|
|
366
|
+
|
|
367
|
+
## Deployment
|
|
368
|
+
|
|
369
|
+
### Docker with Volume Mount
|
|
370
|
+
|
|
371
|
+
```dockerfile
|
|
372
|
+
FROM node:18-alpine
|
|
373
|
+
|
|
374
|
+
WORKDIR /app
|
|
375
|
+
COPY package*.json ./
|
|
376
|
+
RUN npm ci --omit=dev
|
|
377
|
+
|
|
378
|
+
COPY . .
|
|
379
|
+
RUN npm run build
|
|
380
|
+
|
|
381
|
+
ENV KV_STORE_PATH=/data/kv-store
|
|
382
|
+
EXPOSE 3000
|
|
383
|
+
|
|
384
|
+
CMD ["npm", "start"]
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
```yaml
|
|
388
|
+
version: "3"
|
|
389
|
+
services:
|
|
390
|
+
app:
|
|
391
|
+
build: .
|
|
392
|
+
environment:
|
|
393
|
+
KV_STORE_PATH: /data/kv-store
|
|
394
|
+
volumes:
|
|
395
|
+
- kv_data:/data/kv-store
|
|
396
|
+
ports:
|
|
397
|
+
- "3000:3000"
|
|
398
|
+
|
|
399
|
+
volumes:
|
|
400
|
+
kv_data:
|
|
401
|
+
driver: local
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
### Kubernetes with PersistentVolume
|
|
405
|
+
|
|
406
|
+
```yaml
|
|
407
|
+
apiVersion: v1
|
|
408
|
+
kind: PersistentVolumeClaim
|
|
409
|
+
metadata:
|
|
410
|
+
name: kv-store-pvc
|
|
411
|
+
spec:
|
|
412
|
+
accessModes:
|
|
413
|
+
- ReadWriteOnce
|
|
414
|
+
resources:
|
|
415
|
+
requests:
|
|
416
|
+
storage: 10Gi
|
|
417
|
+
|
|
418
|
+
---
|
|
419
|
+
apiVersion: apps/v1
|
|
420
|
+
kind: Deployment
|
|
421
|
+
metadata:
|
|
422
|
+
name: uploadista-app
|
|
423
|
+
spec:
|
|
424
|
+
replicas: 1
|
|
425
|
+
template:
|
|
426
|
+
spec:
|
|
427
|
+
containers:
|
|
428
|
+
- name: app
|
|
429
|
+
image: uploadista:latest
|
|
430
|
+
env:
|
|
431
|
+
- name: KV_STORE_PATH
|
|
432
|
+
value: /data/kv-store
|
|
433
|
+
volumeMounts:
|
|
434
|
+
- name: kv-storage
|
|
435
|
+
mountPath: /data/kv-store
|
|
436
|
+
volumes:
|
|
437
|
+
- name: kv-storage
|
|
438
|
+
persistentVolumeClaim:
|
|
439
|
+
claimName: kv-store-pvc
|
|
440
|
+
```
|
|
441
|
+
|
|
442
|
+
### Manual Backup
|
|
443
|
+
|
|
444
|
+
```bash
|
|
445
|
+
# Backup using tar
|
|
446
|
+
tar -czf kv-backup-$(date +%Y%m%d).tar.gz ./data/kv-store
|
|
447
|
+
|
|
448
|
+
# Backup using rsync
|
|
449
|
+
rsync -avz ./data/kv-store /backup/location/
|
|
450
|
+
|
|
451
|
+
# Restore from backup
|
|
452
|
+
tar -xzf kv-backup-20251021.tar.gz -C ./
|
|
453
|
+
```
|
|
454
|
+
|
|
455
|
+
## Best Practices
|
|
456
|
+
|
|
457
|
+
### 1. Use Hierarchical Key Naming
|
|
458
|
+
|
|
459
|
+
```typescript
|
|
460
|
+
// Good: Organized by type and owner
|
|
461
|
+
"upload:user:123:abc"
|
|
462
|
+
"session:user:456:xyz"
|
|
463
|
+
"metadata:upload:abc"
|
|
464
|
+
|
|
465
|
+
// Avoid: Flat, unclear naming
|
|
466
|
+
"data1", "x", "tmp123"
|
|
467
|
+
```
|
|
468
|
+
|
|
469
|
+
### 2. Implement Cleanup for Old Data
|
|
470
|
+
|
|
471
|
+
```typescript
|
|
472
|
+
import { makeFileBaseKvStore } from "@uploadista/kv-store-filesystem";
|
|
473
|
+
import { Effect } from "effect";
|
|
474
|
+
import fs from "fs/promises";
|
|
475
|
+
import path from "path";
|
|
476
|
+
|
|
477
|
+
const cleanupOldSessions = (storePath: string, maxAge: number) =>
|
|
478
|
+
Effect.gen(function* () {
|
|
479
|
+
const now = Date.now();
|
|
480
|
+
const files = yield* Effect.tryPromise({
|
|
481
|
+
try: () => fs.readdir(storePath),
|
|
482
|
+
catch: (e) => e as Error,
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
for (const file of files) {
|
|
486
|
+
if (!file.startsWith("session:")) continue;
|
|
487
|
+
|
|
488
|
+
const filePath = path.join(storePath, file);
|
|
489
|
+
const stats = yield* Effect.tryPromise({
|
|
490
|
+
try: () => fs.stat(filePath),
|
|
491
|
+
catch: (e) => e as Error,
|
|
492
|
+
});
|
|
493
|
+
|
|
494
|
+
if (now - stats.mtimeMs > maxAge) {
|
|
495
|
+
yield* Effect.tryPromise({
|
|
496
|
+
try: () => fs.unlink(filePath),
|
|
497
|
+
catch: (e) => e as Error,
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
// Run cleanup daily
|
|
504
|
+
setInterval(() => {
|
|
505
|
+
Effect.runSync(
|
|
506
|
+
cleanupOldSessions("./data/kv-store", 24 * 60 * 60 * 1000) // 24 hours
|
|
507
|
+
);
|
|
508
|
+
}, 60 * 60 * 1000); // Every hour
|
|
509
|
+
```
|
|
510
|
+
|
|
511
|
+
### 3. Handle Directory Creation
|
|
512
|
+
|
|
513
|
+
```typescript
|
|
514
|
+
import { fileKvStore } from "@uploadista/kv-store-filesystem";
|
|
515
|
+
import fs from "fs/promises";
|
|
516
|
+
import path from "path";
|
|
517
|
+
|
|
518
|
+
const ensureDirectory = async (dir: string) => {
|
|
519
|
+
try {
|
|
520
|
+
await fs.mkdir(dir, { recursive: true });
|
|
521
|
+
} catch (e) {
|
|
522
|
+
if ((e as any).code !== "EEXIST") {
|
|
523
|
+
throw e;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
};
|
|
527
|
+
|
|
528
|
+
// In initialization
|
|
529
|
+
await ensureDirectory("./data/kv-store");
|
|
530
|
+
|
|
531
|
+
const layer = fileKvStore({
|
|
532
|
+
directory: "./data/kv-store",
|
|
533
|
+
});
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
### 4. Monitor Disk Space
|
|
537
|
+
|
|
538
|
+
```typescript
|
|
539
|
+
import { Effect } from "effect";
|
|
540
|
+
import { exec } from "child_process";
|
|
541
|
+
import { promisify } from "util";
|
|
542
|
+
|
|
543
|
+
const checkDiskSpace = (dir: string) =>
|
|
544
|
+
Effect.gen(function* () {
|
|
545
|
+
const execPromise = promisify(exec);
|
|
546
|
+
const { stdout } = yield* Effect.tryPromise({
|
|
547
|
+
try: () => execPromise(`df -B1 "${dir}" | tail -1`),
|
|
548
|
+
catch: (e) => e as Error,
|
|
549
|
+
});
|
|
550
|
+
|
|
551
|
+
const [, , available] = stdout.trim().split(/\s+/);
|
|
552
|
+
const availableGB = parseInt(available) / 1024 / 1024 / 1024;
|
|
553
|
+
|
|
554
|
+
if (availableGB < 1) {
|
|
555
|
+
console.warn("Less than 1GB disk space remaining!");
|
|
556
|
+
}
|
|
557
|
+
});
|
|
558
|
+
```
|
|
559
|
+
|
|
560
|
+
## Scaling Limitations
|
|
561
|
+
|
|
562
|
+
The filesystem store is suitable for:
|
|
563
|
+
|
|
564
|
+
| Data Size | Deployment | Recommendation |
|
|
565
|
+
|-----------|-----------|-----------------|
|
|
566
|
+
| < 1 GB | Single Server | ✅ Perfect |
|
|
567
|
+
| 1-10 GB | Single Server | ✅ Good |
|
|
568
|
+
| 10-100 GB | Single Server with fast disk | ✅ Acceptable |
|
|
569
|
+
| > 100 GB | Distributed | ❌ Use Redis or Database |
|
|
570
|
+
|
|
571
|
+
For larger scale or distributed systems, migrate to [Redis](#see-also) or a database.
|
|
572
|
+
|
|
573
|
+
## Troubleshooting
|
|
574
|
+
|
|
575
|
+
### "ENOENT: no such file or directory"
|
|
576
|
+
|
|
577
|
+
Directory doesn't exist. Solutions:
|
|
578
|
+
|
|
579
|
+
```typescript
|
|
580
|
+
import { mkdirSync } from "fs";
|
|
581
|
+
import path from "path";
|
|
582
|
+
|
|
583
|
+
const dir = "./data/kv-store";
|
|
584
|
+
mkdirSync(dir, { recursive: true });
|
|
585
|
+
|
|
586
|
+
const layer = fileKvStore({ directory: dir });
|
|
587
|
+
```
|
|
588
|
+
|
|
589
|
+
### "EACCES: permission denied"
|
|
590
|
+
|
|
591
|
+
No write permissions to directory:
|
|
592
|
+
|
|
593
|
+
```bash
|
|
594
|
+
# Check permissions
|
|
595
|
+
ls -la ./data/
|
|
596
|
+
|
|
597
|
+
# Fix permissions
|
|
598
|
+
chmod 755 ./data/
|
|
599
|
+
chmod 755 ./data/kv-store
|
|
600
|
+
```
|
|
601
|
+
|
|
602
|
+
### "Disk quota exceeded" or "No space left on device"
|
|
603
|
+
|
|
604
|
+
Disk is full:
|
|
605
|
+
|
|
606
|
+
```bash
|
|
607
|
+
# Check disk usage
|
|
608
|
+
df -h
|
|
609
|
+
|
|
610
|
+
# Find large files
|
|
611
|
+
du -sh ./data/kv-store/*
|
|
612
|
+
|
|
613
|
+
# Clean up old files
|
|
614
|
+
find ./data/kv-store -mtime +30 -delete
|
|
615
|
+
```
|
|
616
|
+
|
|
617
|
+
### Slow Performance on list() Operations
|
|
618
|
+
|
|
619
|
+
Too many files in directory:
|
|
620
|
+
|
|
621
|
+
1. **Implement archiving**: Move old files to separate directory
|
|
622
|
+
2. **Partition by date**: Use `./data/2025-10/uploads` structure
|
|
623
|
+
3. **Switch backends**: Migrate to Redis for frequent queries
|
|
624
|
+
|
|
625
|
+
```typescript
|
|
626
|
+
// Better structure
|
|
627
|
+
"./data/kv-store/2025-10/upload:abc123.json"
|
|
628
|
+
"./data/kv-store/2025-11/upload:def456.json"
|
|
629
|
+
```
|
|
630
|
+
|
|
631
|
+
### Multi-Process Conflicts
|
|
632
|
+
|
|
633
|
+
Multiple processes writing to same directory:
|
|
634
|
+
|
|
635
|
+
```typescript
|
|
636
|
+
// Use process-level locking
|
|
637
|
+
import lockfile from "proper-lockfile";
|
|
638
|
+
|
|
639
|
+
const store = makeFileBaseKvStore({
|
|
640
|
+
directory: "./data/kv-store",
|
|
641
|
+
});
|
|
642
|
+
|
|
643
|
+
// Wrap operations with locks if needed
|
|
644
|
+
const safeSet = (key: string, value: string) =>
|
|
645
|
+
Effect.gen(function* () {
|
|
646
|
+
const lock = yield* Effect.tryPromise({
|
|
647
|
+
try: () => lockfile.lock(`${key}.lock`),
|
|
648
|
+
catch: (e) => e as Error,
|
|
649
|
+
});
|
|
650
|
+
|
|
651
|
+
try {
|
|
652
|
+
yield* store.set(key, value);
|
|
653
|
+
} finally {
|
|
654
|
+
yield* Effect.tryPromise({
|
|
655
|
+
try: () => lockfile.unlock(lock),
|
|
656
|
+
catch: (e) => e as Error,
|
|
657
|
+
});
|
|
658
|
+
}
|
|
659
|
+
});
|
|
660
|
+
```
|
|
661
|
+
|
|
662
|
+
## Migration Paths
|
|
663
|
+
|
|
664
|
+
### From Memory Store
|
|
665
|
+
|
|
666
|
+
```typescript
|
|
667
|
+
// Replace
|
|
668
|
+
import { memoryKvStore } from "@uploadista/kv-store-memory";
|
|
669
|
+
// With
|
|
670
|
+
import { fileKvStore } from "@uploadista/kv-store-filesystem";
|
|
671
|
+
|
|
672
|
+
// Data is not automatically migrated - applications must handle data transfer
|
|
673
|
+
```
|
|
674
|
+
|
|
675
|
+
### To Redis
|
|
676
|
+
|
|
677
|
+
When your data grows beyond filesystem capacity:
|
|
678
|
+
|
|
679
|
+
```bash
|
|
680
|
+
# Export filesystem data
|
|
681
|
+
node scripts/export-to-redis.js ./data/kv-store
|
|
682
|
+
|
|
683
|
+
# Verify Redis has all data
|
|
684
|
+
redis-cli KEYS "upload:*"
|
|
685
|
+
|
|
686
|
+
# Switch connection and test
|
|
687
|
+
# Then deploy new code with Redis store
|
|
688
|
+
```
|
|
689
|
+
|
|
690
|
+
## Related Packages
|
|
691
|
+
|
|
692
|
+
- [@uploadista/core](../../core) - Core types
|
|
693
|
+
- [@uploadista/kv-store-memory](../memory) - For development/testing
|
|
694
|
+
- [@uploadista/kv-store-redis](../redis) - For distributed systems
|
|
695
|
+
- [@uploadista/server](../../servers/server) - Upload server
|
|
696
|
+
- [@uploadista/data-store-s3](../../data-stores/s3) - For file content storage
|
|
697
|
+
|
|
698
|
+
## License
|
|
699
|
+
|
|
700
|
+
See [LICENSE](../../../LICENSE) in the main repository.
|
|
701
|
+
|
|
702
|
+
## See Also
|
|
703
|
+
|
|
704
|
+
- [KV Stores Comparison Guide](../KV_STORES_COMPARISON.md) - Compare storage options
|
|
705
|
+
- [Server Setup Guide](../../../SERVER_SETUP.md) - Filesystem in production
|
|
706
|
+
- [Docker Documentation](https://docs.docker.com/storage/) - Volume mounting
|
|
707
|
+
- [Kubernetes PersistentVolumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) - Persistent storage
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type BaseKvStore, BaseKvStoreService } from "@uploadista/core/types";
|
|
2
|
+
import { Layer } from "effect";
|
|
3
|
+
export type FileKvStoreOptions = {
|
|
4
|
+
directory: string;
|
|
5
|
+
};
|
|
6
|
+
export declare function makeFileBaseKvStore({ directory, }: FileKvStoreOptions): BaseKvStore;
|
|
7
|
+
export declare const fileKvStore: (config: FileKvStoreOptions) => Layer.Layer<BaseKvStoreService, never, never>;
|
|
8
|
+
//# sourceMappingURL=file-kv-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-kv-store.d.ts","sourceRoot":"","sources":["../src/file-kv-store.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,WAAW,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC9E,OAAO,EAAU,KAAK,EAAE,MAAM,QAAQ,CAAC;AAEvC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAGF,wBAAgB,mBAAmB,CAAC,EAClC,SAAS,GACV,EAAE,kBAAkB,GAAG,WAAW,CAwClC;AAGD,eAAO,MAAM,WAAW,GAAI,QAAQ,kBAAkB,kDACU,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { UploadistaError } from "@uploadista/core/errors";
|
|
4
|
+
import { BaseKvStoreService } from "@uploadista/core/types";
|
|
5
|
+
import { Effect, Layer } from "effect";
|
|
6
|
+
// Base Filesystem KV store that stores raw strings
|
|
7
|
+
export function makeFileBaseKvStore({ directory, }) {
|
|
8
|
+
const resolve = (key) => {
|
|
9
|
+
return path.resolve(directory, `${key}.json`);
|
|
10
|
+
};
|
|
11
|
+
return {
|
|
12
|
+
get: (key) => Effect.tryPromise({
|
|
13
|
+
try: () => fs.readFile(resolve(key), "utf8"),
|
|
14
|
+
catch: (cause) => UploadistaError.fromCode("FILE_NOT_FOUND", { cause }),
|
|
15
|
+
}),
|
|
16
|
+
set: (key, value) => Effect.tryPromise({
|
|
17
|
+
try: () => fs.writeFile(resolve(key), value),
|
|
18
|
+
catch: (cause) => UploadistaError.fromCode("FILE_WRITE_ERROR", { cause }),
|
|
19
|
+
}),
|
|
20
|
+
delete: (key) => Effect.tryPromise({
|
|
21
|
+
try: () => fs.rm(resolve(key)),
|
|
22
|
+
catch: (cause) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause }),
|
|
23
|
+
}),
|
|
24
|
+
list: (keyPrefix) => Effect.tryPromise({
|
|
25
|
+
try: () => fs.readdir(directory),
|
|
26
|
+
catch: (cause) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause }),
|
|
27
|
+
}).pipe(Effect.map((files) => {
|
|
28
|
+
return files
|
|
29
|
+
.filter((file) => file.endsWith(".json") && file.startsWith(keyPrefix))
|
|
30
|
+
.map((file) => path.basename(file, ".json"))
|
|
31
|
+
.sort((a, b) => a.localeCompare(b));
|
|
32
|
+
})),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
// Base store layer
|
|
36
|
+
export const fileKvStore = (config) => Layer.succeed(BaseKvStoreService, makeFileBaseKvStore(config));
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./file-kv-store";
|
package/package.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@uploadista/kv-store-filesystem",
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "0.0.3",
|
|
5
|
+
"description": "File system KV store for Uploadista",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "Uploadista",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"default": "./dist/index.js"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dependencies": {
|
|
16
|
+
"effect": "3.18.4",
|
|
17
|
+
"@uploadista/core": "0.0.3"
|
|
18
|
+
},
|
|
19
|
+
"devDependencies": {
|
|
20
|
+
"@types/node": "24.8.1",
|
|
21
|
+
"@uploadista/typescript-config": "0.0.3"
|
|
22
|
+
},
|
|
23
|
+
"scripts": {
|
|
24
|
+
"build": "tsc -b",
|
|
25
|
+
"format": "biome format --write ./src",
|
|
26
|
+
"lint": "biome lint --write ./src",
|
|
27
|
+
"check": "biome check --write ./src"
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { UploadistaError } from "@uploadista/core/errors";
|
|
4
|
+
import { type BaseKvStore, BaseKvStoreService } from "@uploadista/core/types";
|
|
5
|
+
import { Effect, Layer } from "effect";
|
|
6
|
+
|
|
7
|
+
export type FileKvStoreOptions = {
|
|
8
|
+
directory: string;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
// Base Filesystem KV store that stores raw strings
|
|
12
|
+
export function makeFileBaseKvStore({
|
|
13
|
+
directory,
|
|
14
|
+
}: FileKvStoreOptions): BaseKvStore {
|
|
15
|
+
const resolve = (key: string): string => {
|
|
16
|
+
return path.resolve(directory, `${key}.json`);
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
return {
|
|
20
|
+
get: (key: string) =>
|
|
21
|
+
Effect.tryPromise({
|
|
22
|
+
try: () => fs.readFile(resolve(key), "utf8"),
|
|
23
|
+
catch: (cause) => UploadistaError.fromCode("FILE_NOT_FOUND", { cause }),
|
|
24
|
+
}),
|
|
25
|
+
|
|
26
|
+
set: (key: string, value: string) =>
|
|
27
|
+
Effect.tryPromise({
|
|
28
|
+
try: () => fs.writeFile(resolve(key), value),
|
|
29
|
+
catch: (cause) =>
|
|
30
|
+
UploadistaError.fromCode("FILE_WRITE_ERROR", { cause }),
|
|
31
|
+
}),
|
|
32
|
+
|
|
33
|
+
delete: (key: string) =>
|
|
34
|
+
Effect.tryPromise({
|
|
35
|
+
try: () => fs.rm(resolve(key)),
|
|
36
|
+
catch: (cause) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause }),
|
|
37
|
+
}),
|
|
38
|
+
|
|
39
|
+
list: (keyPrefix: string) =>
|
|
40
|
+
Effect.tryPromise({
|
|
41
|
+
try: () => fs.readdir(directory),
|
|
42
|
+
catch: (cause) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause }),
|
|
43
|
+
}).pipe(
|
|
44
|
+
Effect.map((files) => {
|
|
45
|
+
return files
|
|
46
|
+
.filter(
|
|
47
|
+
(file) => file.endsWith(".json") && file.startsWith(keyPrefix),
|
|
48
|
+
)
|
|
49
|
+
.map((file) => path.basename(file, ".json"))
|
|
50
|
+
.sort((a, b) => a.localeCompare(b));
|
|
51
|
+
}),
|
|
52
|
+
),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Base store layer
|
|
57
|
+
export const fileKvStore = (config: FileKvStoreOptions) =>
|
|
58
|
+
Layer.succeed(BaseKvStoreService, makeFileBaseKvStore(config));
|
package/src/index.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./file-kv-store";
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"root":["./src/file-kv-store.ts","./src/index.ts"],"version":"5.9.3"}
|