@valentinkolb/filegate 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +275 -311
- package/package.json +2 -2
- package/src/handlers/upload.ts +29 -20
- package/src/index.ts +6 -1
package/README.md
CHANGED
|
@@ -1,429 +1,393 @@
|
|
|
1
1
|
# Filegate
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Secure file proxy for building custom file management systems. Streaming uploads, chunked transfers, Unix permissions.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
```
|
|
6
|
+
Browser/App Your Backend Filegate Filesystem
|
|
7
|
+
| | | |
|
|
8
|
+
| upload request | | |
|
|
9
|
+
|------------------->| | |
|
|
10
|
+
| | proxy to filegate | |
|
|
11
|
+
| |-------------------->| |
|
|
12
|
+
| | | write file |
|
|
13
|
+
| | |------------------->|
|
|
14
|
+
| | | |
|
|
15
|
+
| |<--------------------|<-------------------|
|
|
16
|
+
|<-------------------| | |
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Filegate runs behind your backend, not as a public-facing service. Your backend handles authentication and authorization, then proxies requests to Filegate. You control access logic - Filegate handles file operations.
|
|
20
|
+
|
|
21
|
+
## Features
|
|
22
|
+
|
|
23
|
+
- Streaming uploads and downloads (no memory buffering)
|
|
8
24
|
- Resumable chunked uploads with SHA-256 verification
|
|
25
|
+
- Directory downloads as TAR archives
|
|
26
|
+
- Unix file permissions (uid, gid, mode)
|
|
9
27
|
- Glob-based file search
|
|
10
|
-
- Type-safe client
|
|
11
|
-
-
|
|
12
|
-
-
|
|
13
|
-
- LLM-friendly markdown docs at `/llms.txt`
|
|
28
|
+
- Type-safe client with full TypeScript support
|
|
29
|
+
- OpenAPI documentation
|
|
30
|
+
- Minimal dependencies (Hono, Zod - no database required)
|
|
14
31
|
|
|
15
32
|
## Quick Start
|
|
16
33
|
|
|
17
|
-
|
|
18
|
-
export FILE_PROXY_TOKEN=your-secret-token
|
|
19
|
-
export ALLOWED_BASE_PATHS=/export/homes,/export/groups
|
|
34
|
+
### 1. Start Filegate with Docker
|
|
20
35
|
|
|
21
|
-
|
|
36
|
+
```bash
|
|
37
|
+
docker run -d \
|
|
38
|
+
--name filegate \
|
|
39
|
+
-p 4000:4000 \
|
|
40
|
+
-e FILE_PROXY_TOKEN=your-secret-token \
|
|
41
|
+
-e ALLOWED_BASE_PATHS=/data \
|
|
42
|
+
-v /path/to/your/files:/data \
|
|
43
|
+
ghcr.io/valentinkolb/filegate:latest
|
|
22
44
|
```
|
|
23
45
|
|
|
24
|
-
|
|
46
|
+
### 2. Install the Client
|
|
25
47
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
48
|
+
```bash
|
|
49
|
+
npm install @valentinkolb/filegate
|
|
50
|
+
```
|
|
29
51
|
|
|
30
|
-
|
|
52
|
+
### 3. Configure Environment
|
|
31
53
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
| `REDIS_URL` | No | localhost:6379 | Redis connection URL (for chunked uploads) |
|
|
37
|
-
| `PORT` | No | 4000 | Server port |
|
|
54
|
+
```bash
|
|
55
|
+
export FILEGATE_URL=http://localhost:4000
|
|
56
|
+
export FILEGATE_TOKEN=your-secret-token
|
|
57
|
+
```
|
|
38
58
|
|
|
39
|
-
|
|
59
|
+
### 4. Upload a File
|
|
40
60
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
| `MAX_UPLOAD_MB` | 500 | Maximum file size for uploads (simple + chunked) |
|
|
44
|
-
| `MAX_DOWNLOAD_MB` | 5000 | Maximum file/directory size for downloads |
|
|
45
|
-
| `MAX_CHUNK_SIZE_MB` | 50 | Maximum chunk size (server rejects larger chunks) |
|
|
61
|
+
```typescript
|
|
62
|
+
import { filegate } from "@valentinkolb/filegate/client";
|
|
46
63
|
|
|
47
|
-
|
|
64
|
+
const result = await filegate.upload.single({
|
|
65
|
+
path: "/data/uploads",
|
|
66
|
+
filename: "document.pdf",
|
|
67
|
+
data: fileBuffer,
|
|
68
|
+
});
|
|
48
69
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
70
|
+
if (result.ok) {
|
|
71
|
+
console.log("Uploaded:", result.data.path);
|
|
72
|
+
}
|
|
73
|
+
```
|
|
53
74
|
|
|
54
|
-
|
|
75
|
+
### 5. Download a File
|
|
55
76
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
| `UPLOAD_EXPIRY_HOURS` | 24 | Chunked upload expiry (resets on each chunk) |
|
|
59
|
-
| `DISK_CLEANUP_INTERVAL_HOURS` | 6 | Interval to clean orphaned chunk files |
|
|
60
|
-
| `DEV_UID_OVERRIDE` | - | Override file ownership UID (dev mode only) |
|
|
61
|
-
| `DEV_GID_OVERRIDE` | - | Override file ownership GID (dev mode only) |
|
|
77
|
+
```typescript
|
|
78
|
+
import { filegate } from "@valentinkolb/filegate/client";
|
|
62
79
|
|
|
63
|
-
|
|
80
|
+
const result = await filegate.download({ path: "/data/uploads/document.pdf" });
|
|
64
81
|
|
|
65
|
-
|
|
82
|
+
if (result.ok) {
|
|
83
|
+
const blob = await result.data.blob();
|
|
84
|
+
}
|
|
85
|
+
```
|
|
66
86
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
| GET | `/docs` | Scalar API docs (public) |
|
|
71
|
-
| GET | `/openapi.json` | OpenAPI spec (public) |
|
|
72
|
-
| GET | `/llms.txt` | LLM-friendly docs (public) |
|
|
73
|
-
| GET | `/files/info` | File/directory info |
|
|
74
|
-
| GET | `/files/content` | Download file or directory (ZIP) |
|
|
75
|
-
| PUT | `/files/content` | Upload file |
|
|
76
|
-
| POST | `/files/mkdir` | Create directory |
|
|
77
|
-
| DELETE | `/files/delete` | Delete file/directory |
|
|
78
|
-
| POST | `/files/move` | Move (same basepath) |
|
|
79
|
-
| POST | `/files/copy` | Copy (same basepath) |
|
|
80
|
-
| GET | `/files/search` | Glob search |
|
|
81
|
-
| POST | `/files/upload/start` | Start/resume chunked upload |
|
|
82
|
-
| POST | `/files/upload/chunk` | Upload chunk |
|
|
87
|
+
## Core Concepts
|
|
88
|
+
|
|
89
|
+
### Base Paths
|
|
83
90
|
|
|
84
|
-
|
|
91
|
+
Filegate restricts all file operations to explicitly allowed directories called "base paths". This is a security boundary - files outside these paths cannot be accessed.
|
|
85
92
|
|
|
86
|
-
### Get Directory Info
|
|
87
93
|
```bash
|
|
88
|
-
|
|
89
|
-
"http://localhost:4000/files/info?path=/export/homes/alice&showHidden=false"
|
|
94
|
+
ALLOWED_BASE_PATHS=/data/uploads,/data/exports
|
|
90
95
|
```
|
|
91
96
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
97
|
+
With this configuration:
|
|
98
|
+
- `/data/uploads/file.txt` - allowed
|
|
99
|
+
- `/data/exports/report.pdf` - allowed
|
|
100
|
+
- `/home/user/file.txt` - forbidden
|
|
101
|
+
- `/data/../etc/passwd` - forbidden (path traversal blocked)
|
|
102
|
+
|
|
103
|
+
Symlinks that point outside base paths are also blocked.
|
|
104
|
+
|
|
105
|
+
### File Ownership
|
|
106
|
+
|
|
107
|
+
Filegate can set Unix file ownership on uploaded files:
|
|
108
|
+
|
|
109
|
+
```typescript
|
|
110
|
+
await client.upload.single({
|
|
111
|
+
path: "/data/uploads",
|
|
112
|
+
filename: "file.txt",
|
|
113
|
+
data: buffer,
|
|
114
|
+
uid: 1000, // Owner user ID
|
|
115
|
+
gid: 1000, // Owner group ID
|
|
116
|
+
mode: "644", // Unix permissions (rw-r--r--)
|
|
117
|
+
});
|
|
103
118
|
```
|
|
104
119
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
120
|
+
If ownership is not specified, files are created with the user running Filegate (typically root in Docker).
|
|
121
|
+
|
|
122
|
+
Filegate does not validate whether the specified uid/gid exists on the system, nor does it verify that the requesting user matches the specified ownership. Your backend is responsible for this validation.
|
|
123
|
+
|
|
124
|
+
This feature is intended for scenarios like NFS shares exposed through Filegate, where preserving the original permission structure is required.
|
|
125
|
+
|
|
126
|
+
### Chunked Uploads
|
|
127
|
+
|
|
128
|
+
For large files, use chunked uploads. They support:
|
|
129
|
+
- Resume after connection failure
|
|
130
|
+
- Progress tracking
|
|
131
|
+
- Per-chunk checksum verification
|
|
132
|
+
- Automatic assembly when complete
|
|
133
|
+
|
|
134
|
+
The [Browser Utilities](#browser-utilities) help with checksum calculation, chunking, and progress tracking. They work both in the browser and on the server.
|
|
135
|
+
|
|
136
|
+
```typescript
|
|
137
|
+
// Start or resume upload
|
|
138
|
+
const start = await client.upload.chunked.start({
|
|
139
|
+
path: "/data/uploads",
|
|
140
|
+
filename: "large-file.zip",
|
|
141
|
+
size: file.size,
|
|
142
|
+
checksum: "sha256:abc123...", // Checksum of entire file
|
|
143
|
+
chunkSize: 5 * 1024 * 1024, // 5MB chunks
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
// Upload each chunk
|
|
147
|
+
for (let i = 0; i < start.data.totalChunks; i++) {
|
|
148
|
+
if (start.data.uploadedChunks.includes(i)) continue; // Skip already uploaded
|
|
149
|
+
|
|
150
|
+
await client.upload.chunked.send({
|
|
151
|
+
uploadId: start.data.uploadId,
|
|
152
|
+
index: i,
|
|
153
|
+
data: chunkData,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
109
156
|
```
|
|
110
157
|
|
|
111
|
-
|
|
158
|
+
Uploads automatically expire after 24 hours of inactivity.
|
|
159
|
+
|
|
160
|
+
## Configuration
|
|
161
|
+
|
|
162
|
+
All configuration is done through environment variables.
|
|
163
|
+
|
|
164
|
+
| Variable | Required | Default | Description |
|
|
165
|
+
|----------|----------|---------|-------------|
|
|
166
|
+
| `FILE_PROXY_TOKEN` | Yes | - | Bearer token for API authentication |
|
|
167
|
+
| `ALLOWED_BASE_PATHS` | Yes | - | Comma-separated list of allowed directories |
|
|
168
|
+
| `PORT` | No | 4000 | Server port |
|
|
169
|
+
| `MAX_UPLOAD_MB` | No | 500 | Maximum upload size in MB |
|
|
170
|
+
| `MAX_DOWNLOAD_MB` | No | 5000 | Maximum download size in MB |
|
|
171
|
+
| `MAX_CHUNK_SIZE_MB` | No | 50 | Maximum chunk size in MB |
|
|
172
|
+
| `SEARCH_MAX_RESULTS` | No | 100 | Maximum search results returned |
|
|
173
|
+
| `SEARCH_MAX_RECURSIVE_WILDCARDS` | No | 10 | Maximum `**` wildcards in glob patterns |
|
|
174
|
+
| `UPLOAD_EXPIRY_HOURS` | No | 24 | Hours until incomplete uploads expire |
|
|
175
|
+
| `UPLOAD_TEMP_DIR` | No | /tmp/filegate-uploads | Directory for temporary chunk storage |
|
|
176
|
+
| `DISK_CLEANUP_INTERVAL_HOURS` | No | 6 | Interval for cleaning orphaned chunks |
|
|
177
|
+
|
|
178
|
+
### Development Mode
|
|
179
|
+
|
|
180
|
+
For local development without root permissions, you can override file ownership:
|
|
112
181
|
|
|
113
182
|
```bash
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
183
|
+
DEV_UID_OVERRIDE=1000
|
|
184
|
+
DEV_GID_OVERRIDE=1000
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
This applies the specified uid/gid instead of the requested values. Do not use in production.
|
|
188
|
+
|
|
189
|
+
## Docker Compose Example
|
|
190
|
+
|
|
191
|
+
```yaml
|
|
192
|
+
services:
|
|
193
|
+
filegate:
|
|
194
|
+
image: ghcr.io/valentinkolb/filegate:latest
|
|
195
|
+
ports:
|
|
196
|
+
- "4000:4000"
|
|
197
|
+
environment:
|
|
198
|
+
FILE_PROXY_TOKEN: ${FILE_PROXY_TOKEN}
|
|
199
|
+
ALLOWED_BASE_PATHS: /data
|
|
200
|
+
volumes:
|
|
201
|
+
- ./data:/data
|
|
202
|
+
- filegate-chunks:/tmp/filegate-uploads
|
|
203
|
+
|
|
204
|
+
volumes:
|
|
205
|
+
filegate-chunks:
|
|
137
206
|
```
|
|
138
207
|
|
|
139
208
|
## Client API
|
|
140
209
|
|
|
141
|
-
|
|
210
|
+
The client provides a type-safe interface for all Filegate operations.
|
|
142
211
|
|
|
143
212
|
### Installation
|
|
144
213
|
|
|
145
|
-
```
|
|
146
|
-
|
|
147
|
-
import { chunks, formatBytes } from "@valentinkolb/filegate/utils";
|
|
214
|
+
```bash
|
|
215
|
+
npm install @valentinkolb/filegate
|
|
148
216
|
```
|
|
149
217
|
|
|
150
|
-
### Default Instance
|
|
218
|
+
### Default Instance
|
|
151
219
|
|
|
152
|
-
Set `FILEGATE_URL` and `FILEGATE_TOKEN
|
|
220
|
+
Set `FILEGATE_URL` and `FILEGATE_TOKEN` environment variables, then import the pre-configured client:
|
|
153
221
|
|
|
154
222
|
```typescript
|
|
155
223
|
import { filegate } from "@valentinkolb/filegate/client";
|
|
156
224
|
|
|
157
|
-
|
|
225
|
+
await filegate.info({ path: "/data/uploads" });
|
|
158
226
|
```
|
|
159
227
|
|
|
160
228
|
### Custom Instance
|
|
161
229
|
|
|
230
|
+
For more control or multiple Filegate servers, create instances manually:
|
|
231
|
+
|
|
162
232
|
```typescript
|
|
163
233
|
import { Filegate } from "@valentinkolb/filegate/client";
|
|
164
234
|
|
|
165
235
|
const client = new Filegate({
|
|
166
236
|
url: "http://localhost:4000",
|
|
167
|
-
token: "your-token",
|
|
237
|
+
token: "your-secret-token",
|
|
168
238
|
});
|
|
169
239
|
```
|
|
170
240
|
|
|
171
|
-
###
|
|
241
|
+
### Methods
|
|
172
242
|
|
|
173
243
|
```typescript
|
|
174
|
-
// Get file
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
// Download file (returns
|
|
178
|
-
|
|
179
|
-
if (file.ok) {
|
|
180
|
-
const text = await file.data.text();
|
|
181
|
-
const buffer = await file.data.arrayBuffer();
|
|
182
|
-
// Or stream directly: file.data.body
|
|
183
|
-
}
|
|
244
|
+
// Get file or directory info
|
|
245
|
+
await client.info({ path: "/data/file.txt", showHidden: false });
|
|
246
|
+
|
|
247
|
+
// Download file (returns streaming Response)
|
|
248
|
+
await client.download({ path: "/data/file.txt" });
|
|
184
249
|
|
|
185
|
-
// Download directory as
|
|
186
|
-
|
|
250
|
+
// Download directory as TAR archive
|
|
251
|
+
await client.download({ path: "/data/folder" });
|
|
187
252
|
|
|
188
|
-
//
|
|
253
|
+
// Simple upload
|
|
189
254
|
await client.upload.single({
|
|
190
|
-
path: "/
|
|
191
|
-
filename: "
|
|
192
|
-
data:
|
|
255
|
+
path: "/data/uploads",
|
|
256
|
+
filename: "file.txt",
|
|
257
|
+
data: buffer,
|
|
193
258
|
uid: 1000,
|
|
194
259
|
gid: 1000,
|
|
195
260
|
mode: "644",
|
|
196
261
|
});
|
|
197
262
|
|
|
198
|
-
//
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
filename: "large.zip",
|
|
202
|
-
size: file.size,
|
|
203
|
-
checksum: "sha256:...",
|
|
204
|
-
chunkSize: 5 * 1024 * 1024,
|
|
205
|
-
uid: 1000,
|
|
206
|
-
gid: 1000,
|
|
207
|
-
});
|
|
208
|
-
|
|
209
|
-
// Send chunks
|
|
210
|
-
await client.upload.chunked.send({
|
|
211
|
-
uploadId: startResult.data.uploadId,
|
|
212
|
-
index: 0,
|
|
213
|
-
data: chunkData,
|
|
214
|
-
checksum: "sha256:...",
|
|
215
|
-
});
|
|
263
|
+
// Chunked upload
|
|
264
|
+
await client.upload.chunked.start({ ... });
|
|
265
|
+
await client.upload.chunked.send({ ... });
|
|
216
266
|
|
|
217
267
|
// Create directory
|
|
218
|
-
await client.mkdir({ path: "/
|
|
268
|
+
await client.mkdir({ path: "/data/new-folder", mode: "755" });
|
|
219
269
|
|
|
220
|
-
//
|
|
221
|
-
await client.
|
|
222
|
-
await client.copy({ from: "/export/homes/alice/file.txt", to: "/export/homes/alice/backup.txt" });
|
|
270
|
+
// Delete file or directory
|
|
271
|
+
await client.delete({ path: "/data/old-file.txt" });
|
|
223
272
|
|
|
224
|
-
//
|
|
225
|
-
await client.
|
|
273
|
+
// Move (within same base path)
|
|
274
|
+
await client.move({ from: "/data/old.txt", to: "/data/new.txt" });
|
|
275
|
+
|
|
276
|
+
// Copy (within same base path)
|
|
277
|
+
await client.copy({ from: "/data/file.txt", to: "/data/backup.txt" });
|
|
226
278
|
|
|
227
279
|
// Search with glob patterns
|
|
228
|
-
|
|
229
|
-
paths: ["/
|
|
280
|
+
await client.glob({
|
|
281
|
+
paths: ["/data/uploads"],
|
|
230
282
|
pattern: "**/*.pdf",
|
|
231
|
-
|
|
232
|
-
limit: 100,
|
|
283
|
+
limit: 50,
|
|
233
284
|
});
|
|
234
285
|
```
|
|
235
286
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
Browser-compatible utilities for chunked uploads. Framework-agnostic with reactive state management.
|
|
287
|
+
### Response Format
|
|
239
288
|
|
|
240
|
-
|
|
289
|
+
All methods return a discriminated union:
|
|
241
290
|
|
|
242
291
|
```typescript
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
const upload = await chunks.prepare({ file, chunkSize: 5 * 1024 * 1024 });
|
|
292
|
+
type Response<T> =
|
|
293
|
+
| { ok: true; data: T }
|
|
294
|
+
| { ok: false; error: string; status: number };
|
|
247
295
|
|
|
248
|
-
|
|
249
|
-
upload.file // Original File/Blob
|
|
250
|
-
upload.fileSize // Total size in bytes
|
|
251
|
-
upload.chunkSize // Chunk size in bytes
|
|
252
|
-
upload.totalChunks // Number of chunks
|
|
253
|
-
upload.checksum // "sha256:..." of entire file
|
|
296
|
+
const result = await client.info({ path: "/data/file.txt" });
|
|
254
297
|
|
|
255
|
-
|
|
256
|
-
|
|
298
|
+
if (result.ok) {
|
|
299
|
+
console.log(result.data.size);
|
|
300
|
+
} else {
|
|
301
|
+
console.error(result.error); // "not found", "path not allowed", etc.
|
|
302
|
+
}
|
|
257
303
|
```
|
|
258
304
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
```typescript
|
|
262
|
-
// Subscribe to state changes (framework-agnostic)
|
|
263
|
-
const unsubscribe = upload.subscribe((state) => {
|
|
264
|
-
console.log(`${state.percent}% - ${state.status}`);
|
|
265
|
-
// state: { uploaded: number, total: number, percent: number, status: "pending" | "uploading" | "completed" | "error" }
|
|
266
|
-
});
|
|
267
|
-
|
|
268
|
-
// Mark chunk as completed
|
|
269
|
-
upload.complete({ index: 0 });
|
|
270
|
-
|
|
271
|
-
// Reset state
|
|
272
|
-
upload.reset();
|
|
273
|
-
|
|
274
|
-
// Unsubscribe when done
|
|
275
|
-
unsubscribe();
|
|
276
|
-
```
|
|
305
|
+
## Browser Utilities
|
|
277
306
|
|
|
278
|
-
|
|
307
|
+
Utilities for chunked uploads that work both in the browser and on the server. They handle file chunking, SHA-256 checksum calculation, progress tracking, and retry logic.
|
|
279
308
|
|
|
280
309
|
```typescript
|
|
281
|
-
|
|
282
|
-
const chunk = upload.get({ index: 0 });
|
|
310
|
+
import { chunks, formatBytes } from "@valentinkolb/filegate/utils";
|
|
283
311
|
|
|
284
|
-
//
|
|
285
|
-
const
|
|
312
|
+
// Prepare a file for chunked upload
|
|
313
|
+
const upload = await chunks.prepare({
|
|
314
|
+
file: fileInput.files[0],
|
|
315
|
+
chunkSize: 5 * 1024 * 1024,
|
|
316
|
+
});
|
|
286
317
|
|
|
287
|
-
//
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
}
|
|
291
|
-
```
|
|
318
|
+
console.log(upload.checksum); // "sha256:..."
|
|
319
|
+
console.log(upload.totalChunks); // Number of chunks
|
|
320
|
+
console.log(formatBytes({ bytes: upload.fileSize })); // "52.4 MB"
|
|
292
321
|
|
|
293
|
-
|
|
322
|
+
// Subscribe to progress updates
|
|
323
|
+
upload.subscribe((state) => {
|
|
324
|
+
console.log(`${state.percent}% - ${state.status}`);
|
|
325
|
+
});
|
|
294
326
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
index: 0,
|
|
327
|
+
// Upload all chunks with retry and concurrency
|
|
328
|
+
await upload.sendAll({
|
|
329
|
+
skip: alreadyUploadedChunks,
|
|
299
330
|
retries: 3,
|
|
331
|
+
concurrency: 3,
|
|
300
332
|
fn: async ({ index, data }) => {
|
|
301
333
|
await fetch("/api/upload/chunk", {
|
|
302
334
|
method: "POST",
|
|
303
|
-
headers: {
|
|
335
|
+
headers: {
|
|
336
|
+
"X-Upload-Id": uploadId,
|
|
337
|
+
"X-Chunk-Index": String(index),
|
|
338
|
+
},
|
|
304
339
|
body: data,
|
|
305
340
|
});
|
|
306
341
|
},
|
|
307
342
|
});
|
|
308
|
-
|
|
309
|
-
// Send all chunks (with skip for resume, concurrency, retries)
|
|
310
|
-
await upload.sendAll({
|
|
311
|
-
skip: [0, 1], // Already uploaded chunks (from resume)
|
|
312
|
-
retries: 3,
|
|
313
|
-
concurrency: 3, // Parallel uploads
|
|
314
|
-
fn: async ({ index, data }) => {
|
|
315
|
-
await fetch("/api/upload/chunk", { ... });
|
|
316
|
-
},
|
|
317
|
-
});
|
|
318
343
|
```
|
|
319
344
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
```typescript
|
|
323
|
-
import { chunks } from "@valentinkolb/filegate/utils";
|
|
345
|
+
## API Endpoints
|
|
324
346
|
|
|
325
|
-
|
|
326
|
-
const upload = await chunks.prepare({ file, chunkSize: 5 * 1024 * 1024 });
|
|
327
|
-
|
|
328
|
-
if (onProgress) upload.subscribe(onProgress);
|
|
329
|
-
|
|
330
|
-
// 1. Start/Resume upload
|
|
331
|
-
const { uploadId, uploadedChunks, completed } = await fetch("/api/upload/start", {
|
|
332
|
-
method: "POST",
|
|
333
|
-
headers: { "Content-Type": "application/json" },
|
|
334
|
-
body: JSON.stringify({
|
|
335
|
-
path: targetPath,
|
|
336
|
-
filename: file.name,
|
|
337
|
-
size: upload.fileSize,
|
|
338
|
-
checksum: upload.checksum,
|
|
339
|
-
chunkSize: upload.chunkSize,
|
|
340
|
-
}),
|
|
341
|
-
}).then(r => r.json());
|
|
342
|
-
|
|
343
|
-
if (completed) return; // Already done
|
|
344
|
-
|
|
345
|
-
// 2. Upload all chunks (skipping already uploaded)
|
|
346
|
-
await upload.sendAll({
|
|
347
|
-
skip: uploadedChunks,
|
|
348
|
-
retries: 3,
|
|
349
|
-
fn: async ({ index, data }) => {
|
|
350
|
-
await fetch("/api/upload/chunk", {
|
|
351
|
-
method: "POST",
|
|
352
|
-
headers: {
|
|
353
|
-
"X-Upload-Id": uploadId,
|
|
354
|
-
"X-Chunk-Index": String(index),
|
|
355
|
-
"X-Chunk-Checksum": await upload.hash({ data }),
|
|
356
|
-
},
|
|
357
|
-
body: data,
|
|
358
|
-
});
|
|
359
|
-
},
|
|
360
|
-
});
|
|
361
|
-
}
|
|
347
|
+
All `/files/*` endpoints require `Authorization: Bearer <token>`.
|
|
362
348
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
349
|
+
| Method | Path | Description |
|
|
350
|
+
|--------|------|-------------|
|
|
351
|
+
| GET | `/health` | Health check |
|
|
352
|
+
| GET | `/docs` | OpenAPI documentation (Scalar UI) |
|
|
353
|
+
| GET | `/openapi.json` | OpenAPI specification |
|
|
354
|
+
| GET | `/llms.txt` | LLM-friendly markdown documentation |
|
|
355
|
+
| GET | `/files/info` | Get file or directory info |
|
|
356
|
+
| GET | `/files/content` | Download file or directory (TAR) |
|
|
357
|
+
| PUT | `/files/content` | Upload file |
|
|
358
|
+
| POST | `/files/mkdir` | Create directory |
|
|
359
|
+
| DELETE | `/files/delete` | Delete file or directory |
|
|
360
|
+
| POST | `/files/move` | Move file or directory |
|
|
361
|
+
| POST | `/files/copy` | Copy file or directory |
|
|
362
|
+
| GET | `/files/search` | Search with glob pattern |
|
|
363
|
+
| POST | `/files/upload/start` | Start or resume chunked upload |
|
|
364
|
+
| POST | `/files/upload/chunk` | Upload a chunk |
|
|
374
365
|
|
|
375
|
-
|
|
366
|
+
## Security
|
|
376
367
|
|
|
377
|
-
|
|
368
|
+
Filegate implements multiple security layers:
|
|
378
369
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
const response = await client.download({ path });
|
|
386
|
-
|
|
387
|
-
if (!response.ok) {
|
|
388
|
-
return c.json({ error: response.error }, response.status);
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
// Stream directly to client - no buffering!
|
|
392
|
-
return new Response(response.data.body, {
|
|
393
|
-
headers: {
|
|
394
|
-
"Content-Type": response.data.headers.get("Content-Type") || "application/octet-stream",
|
|
395
|
-
"Content-Disposition": response.data.headers.get("Content-Disposition") || "",
|
|
396
|
-
},
|
|
397
|
-
});
|
|
398
|
-
});
|
|
399
|
-
```
|
|
370
|
+
- **Path validation**: All paths are validated against allowed base paths
|
|
371
|
+
- **Symlink protection**: Symlinks pointing outside base paths are blocked
|
|
372
|
+
- **Path traversal prevention**: Sequences like `../` are normalized and checked
|
|
373
|
+
- **Size limits**: Configurable limits for uploads, downloads, and chunks
|
|
374
|
+
- **Search limits**: Glob pattern complexity is limited to prevent DoS
|
|
375
|
+
- **Secure headers**: X-Frame-Options, X-Content-Type-Options, etc.
|
|
400
376
|
|
|
401
|
-
##
|
|
377
|
+
## Development
|
|
402
378
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
- SHA-256 checksum verification
|
|
407
|
-
- Configurable upload/download size limits
|
|
408
|
-
- Glob pattern limits (max length 500 chars, configurable recursive wildcard limit)
|
|
409
|
-
- Security headers (X-Frame-Options, X-Content-Type-Options, etc.)
|
|
379
|
+
```bash
|
|
380
|
+
# Install dependencies
|
|
381
|
+
bun install
|
|
410
382
|
|
|
411
|
-
|
|
383
|
+
# Run server
|
|
384
|
+
FILE_PROXY_TOKEN=dev ALLOWED_BASE_PATHS=/tmp bun run src/index.ts
|
|
412
385
|
|
|
413
|
-
|
|
414
|
-
# Run unit tests
|
|
386
|
+
# Run tests
|
|
415
387
|
bun run test:unit
|
|
416
|
-
|
|
417
|
-
# Run integration tests (requires Docker)
|
|
418
388
|
bun run test:integration:run
|
|
419
|
-
|
|
420
|
-
# Run all tests
|
|
421
|
-
bun run test:all
|
|
422
389
|
```
|
|
423
390
|
|
|
424
|
-
##
|
|
391
|
+
## License
|
|
425
392
|
|
|
426
|
-
|
|
427
|
-
- **Framework:** Hono
|
|
428
|
-
- **Validation:** Zod
|
|
429
|
-
- **Docs:** hono-openapi + Scalar
|
|
393
|
+
MIT
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@valentinkolb/filegate",
|
|
3
|
-
"version": "0.0.
|
|
4
|
-
"description": "Secure
|
|
3
|
+
"version": "0.0.6",
|
|
4
|
+
"description": "Secure file proxy for building custom file management systems. Streaming uploads, chunked transfers, Unix permissions.",
|
|
5
5
|
"module": "index.ts",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"license": "MIT",
|
package/src/handlers/upload.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Hono } from "hono";
|
|
2
2
|
import { describeRoute } from "hono-openapi";
|
|
3
|
-
import {
|
|
4
|
-
import { mkdir, readdir, rm, stat, rename } from "node:fs/promises";
|
|
3
|
+
import { mkdir, readdir, rm, stat, rename, readFile, writeFile } from "node:fs/promises";
|
|
5
4
|
import { join, dirname } from "node:path";
|
|
6
5
|
import { validatePath } from "../lib/path";
|
|
7
6
|
import { applyOwnership, type Ownership } from "../lib/ownership";
|
|
@@ -25,12 +24,10 @@ const computeUploadId = (path: string, filename: string, checksum: string): stri
|
|
|
25
24
|
return hasher.digest("hex").slice(0, 16);
|
|
26
25
|
};
|
|
27
26
|
|
|
28
|
-
// Single Redis key per upload
|
|
29
|
-
const metaKey = (id: string) => `filegate:upload:${id}`;
|
|
30
|
-
|
|
31
27
|
// Chunk storage paths
|
|
32
28
|
const chunksDir = (id: string) => join(config.uploadTempDir, id);
|
|
33
29
|
const chunkPath = (id: string, idx: number) => join(chunksDir(id), String(idx));
|
|
30
|
+
const metaPath = (id: string) => join(chunksDir(id), "meta.json");
|
|
34
31
|
|
|
35
32
|
type UploadMeta = {
|
|
36
33
|
uploadId: string;
|
|
@@ -41,26 +38,35 @@ type UploadMeta = {
|
|
|
41
38
|
chunkSize: number;
|
|
42
39
|
totalChunks: number;
|
|
43
40
|
ownership: Ownership | null;
|
|
41
|
+
createdAt: number; // Unix timestamp for expiry check
|
|
44
42
|
};
|
|
45
43
|
|
|
46
44
|
const saveMeta = async (meta: UploadMeta): Promise<void> => {
|
|
47
|
-
await
|
|
45
|
+
await mkdir(chunksDir(meta.uploadId), { recursive: true });
|
|
46
|
+
await writeFile(metaPath(meta.uploadId), JSON.stringify(meta));
|
|
48
47
|
};
|
|
49
48
|
|
|
50
49
|
const loadMeta = async (id: string): Promise<UploadMeta | null> => {
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
try {
|
|
51
|
+
const data = await readFile(metaPath(id), "utf-8");
|
|
52
|
+
return JSON.parse(data) as UploadMeta;
|
|
53
|
+
} catch {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
53
56
|
};
|
|
54
57
|
|
|
55
|
-
const refreshExpiry = async (id: string): Promise<void> => {
|
|
56
|
-
|
|
58
|
+
const refreshExpiry = async (id: string, meta: UploadMeta): Promise<void> => {
|
|
59
|
+
// Update createdAt to extend expiry
|
|
60
|
+
meta.createdAt = Date.now();
|
|
61
|
+
await saveMeta(meta);
|
|
57
62
|
};
|
|
58
63
|
|
|
59
|
-
// Get uploaded chunks from filesystem
|
|
64
|
+
// Get uploaded chunks from filesystem
|
|
60
65
|
const getUploadedChunks = async (id: string): Promise<number[]> => {
|
|
61
66
|
try {
|
|
62
67
|
const files = await readdir(chunksDir(id));
|
|
63
68
|
return files
|
|
69
|
+
.filter((f) => f !== "meta.json")
|
|
64
70
|
.map((f) => parseInt(f, 10))
|
|
65
71
|
.filter((n) => !isNaN(n))
|
|
66
72
|
.sort((a, b) => a - b);
|
|
@@ -70,7 +76,7 @@ const getUploadedChunks = async (id: string): Promise<number[]> => {
|
|
|
70
76
|
};
|
|
71
77
|
|
|
72
78
|
const cleanupUpload = async (id: string): Promise<void> => {
|
|
73
|
-
await
|
|
79
|
+
await rm(chunksDir(id), { recursive: true }).catch(() => {});
|
|
74
80
|
};
|
|
75
81
|
|
|
76
82
|
const assembleFile = async (meta: UploadMeta): Promise<string | null> => {
|
|
@@ -155,8 +161,8 @@ app.post(
|
|
|
155
161
|
// Check for existing upload (resume)
|
|
156
162
|
const existingMeta = await loadMeta(uploadId);
|
|
157
163
|
if (existingMeta) {
|
|
158
|
-
// Refresh
|
|
159
|
-
await refreshExpiry(uploadId);
|
|
164
|
+
// Refresh expiry on resume
|
|
165
|
+
await refreshExpiry(uploadId, existingMeta);
|
|
160
166
|
// Get chunks from filesystem
|
|
161
167
|
const uploadedChunks = await getUploadedChunks(uploadId);
|
|
162
168
|
return c.json({
|
|
@@ -186,9 +192,9 @@ app.post(
|
|
|
186
192
|
chunkSize,
|
|
187
193
|
totalChunks,
|
|
188
194
|
ownership,
|
|
195
|
+
createdAt: Date.now(),
|
|
189
196
|
};
|
|
190
197
|
|
|
191
|
-
await mkdir(chunksDir(uploadId), { recursive: true });
|
|
192
198
|
await saveMeta(meta);
|
|
193
199
|
|
|
194
200
|
return c.json({
|
|
@@ -306,23 +312,26 @@ app.post(
|
|
|
306
312
|
},
|
|
307
313
|
);
|
|
308
314
|
|
|
309
|
-
// Cleanup
|
|
315
|
+
// Cleanup expired upload directories
|
|
310
316
|
export const cleanupOrphanedChunks = async () => {
|
|
311
317
|
try {
|
|
312
318
|
const dirs = await readdir(config.uploadTempDir);
|
|
313
319
|
let cleaned = 0;
|
|
320
|
+
const now = Date.now();
|
|
321
|
+
const expiryMs = config.uploadExpirySecs * 1000;
|
|
314
322
|
|
|
315
323
|
for (const dir of dirs) {
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
if
|
|
324
|
+
const meta = await loadMeta(dir);
|
|
325
|
+
|
|
326
|
+
// Remove if no meta or expired
|
|
327
|
+
if (!meta || now - meta.createdAt > expiryMs) {
|
|
319
328
|
await rm(chunksDir(dir), { recursive: true }).catch(() => {});
|
|
320
329
|
cleaned++;
|
|
321
330
|
}
|
|
322
331
|
}
|
|
323
332
|
|
|
324
333
|
if (cleaned > 0) {
|
|
325
|
-
console.log(`[Filegate] Cleaned up ${cleaned}
|
|
334
|
+
console.log(`[Filegate] Cleaned up ${cleaned} expired upload${cleaned === 1 ? "" : "s"}`);
|
|
326
335
|
}
|
|
327
336
|
} catch {
|
|
328
337
|
// Directory doesn't exist yet
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Hono } from "hono";
|
|
2
|
+
import { HTTPException } from "hono/http-exception";
|
|
2
3
|
import { Scalar } from "@scalar/hono-api-reference";
|
|
3
4
|
import { generateSpecs } from "hono-openapi";
|
|
4
5
|
import { createMarkdownFromOpenApi } from "@scalar/openapi-to-markdown";
|
|
@@ -22,7 +23,6 @@ if (config.isDev) {
|
|
|
22
23
|
|
|
23
24
|
console.log(`[Filegate] ALLOWED_BASE_PATHS: ${config.allowedPaths.join(", ")}`);
|
|
24
25
|
console.log(`[Filegate] MAX_UPLOAD_MB: ${config.maxUploadBytes / 1024 / 1024}`);
|
|
25
|
-
console.log(`[Filegate] REDIS_URL: ${process.env.REDIS_URL ?? "redis://localhost:6379 (default)"}`);
|
|
26
26
|
console.log(`[Filegate] PORT: ${config.port}`);
|
|
27
27
|
|
|
28
28
|
// Periodic disk cleanup for orphaned chunks (every 6h by default)
|
|
@@ -70,6 +70,11 @@ app.notFound((c) => c.json({ error: "not found" }, 404));
|
|
|
70
70
|
|
|
71
71
|
// Error handler
|
|
72
72
|
app.onError((err, c) => {
|
|
73
|
+
// HTTPException (from bearerAuth, etc.) - return the proper response
|
|
74
|
+
if (err instanceof HTTPException) {
|
|
75
|
+
return err.getResponse();
|
|
76
|
+
}
|
|
77
|
+
|
|
73
78
|
console.error("[Filegate] Error:", err);
|
|
74
79
|
return c.json({ error: "internal error" }, 500);
|
|
75
80
|
});
|