@bhooai/nexus-core 2.0.13 → 2.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/src/app/Storage.ts +616 -9
- package/src/app/Upload.ts +114 -0
- package/src/app/adminModule.ts +256 -52
- package/src/app/authModule.ts +56 -30
- package/src/app/createNexusApp.ts +171 -28
- package/src/app/databaseAdminModule.ts +171 -2
- package/src/app/discover.ts +4 -2
- package/src/app/errorPages.ts +21 -5
- package/src/app/fusionEngine.ts +53 -0
- package/src/app/index.ts +3 -0
- package/src/app/preflightModule.ts +41 -4
- package/src/app/userStore.ts +321 -0
- package/src/config/dbAccess.ts +26 -0
- package/src/config/defaults.ts +34 -4
- package/src/config/index.ts +1 -0
- package/src/config/schema.ts +72 -6
- package/src/config/types.ts +94 -1
- package/src/http/Server.ts +27 -3
- package/src/http/static.ts +1 -1
- package/tests/config.test.ts +24 -1
- package/tests/fusion-userstore.test.ts +68 -0
package/src/app/Storage.ts
CHANGED
|
@@ -5,14 +5,30 @@
|
|
|
5
5
|
*
|
|
6
6
|
* await storage.disk('uploads').put('avatars', file);
|
|
7
7
|
* const url = await storage.disk('private').signedUrl('invoices/1.pdf', { ttl: 300 });
|
|
8
|
+
*
|
|
9
|
+
* Image variants: when `StorageConfig.imageTransform.enabled` and the file is
|
|
10
|
+
* an image (jpeg/png/webp/avif), `put()` auto-calls the Python image service
|
|
11
|
+
* (Pillow + optional Real-ESRGAN) at `NEXUS_IMAGES_URL` to generate variants
|
|
12
|
+
* per `imageTransform.sizes` (e.g. thumb:150, small:480) in `imageTransform.formats`.
|
|
13
|
+
* No native Node dependency (sharp removed).
|
|
8
14
|
*/
|
|
9
15
|
import { createHash, createHmac, randomUUID } from 'node:crypto';
|
|
10
|
-
import { mkdir, readFile, stat as fsStat, unlink, writeFile } from 'node:fs/promises';
|
|
16
|
+
import { mkdir, readFile, stat as fsStat, unlink, writeFile, symlink, link, lstat } from 'node:fs/promises';
|
|
11
17
|
import { createReadStream } from 'node:fs';
|
|
12
|
-
import { dirname, extname, join, resolve } from 'node:path';
|
|
18
|
+
import { dirname, extname, join, resolve, relative } from 'node:path';
|
|
13
19
|
import { existsSync } from 'node:fs';
|
|
14
20
|
import type { Readable } from 'node:stream';
|
|
15
21
|
|
|
22
|
+
export interface ImageVariant {
|
|
23
|
+
path: string;
|
|
24
|
+
url: string;
|
|
25
|
+
width: number;
|
|
26
|
+
height: number;
|
|
27
|
+
format: string;
|
|
28
|
+
size: number;
|
|
29
|
+
sha256: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
16
32
|
export interface StoredFile {
|
|
17
33
|
id: string;
|
|
18
34
|
path: string;
|
|
@@ -22,6 +38,7 @@ export interface StoredFile {
|
|
|
22
38
|
mime: string;
|
|
23
39
|
originalName: string;
|
|
24
40
|
sha256: string;
|
|
41
|
+
variants?: Record<string, ImageVariant>;
|
|
25
42
|
}
|
|
26
43
|
|
|
27
44
|
export interface SignedUrlOptions {
|
|
@@ -66,9 +83,283 @@ export interface StorageConfig {
|
|
|
66
83
|
formats: string[];
|
|
67
84
|
sizes: Record<string, number>;
|
|
68
85
|
stripMetadata?: boolean;
|
|
86
|
+
quality?: number;
|
|
87
|
+
fit?: 'cover' | 'contain' | 'fill' | 'inside' | 'outside';
|
|
88
|
+
upscale?: {
|
|
89
|
+
enabled?: boolean;
|
|
90
|
+
algorithm?: 'realesrgan' | 'lanczos';
|
|
91
|
+
};
|
|
92
|
+
/** Override Python image service URL; defaults to NEXUS_IMAGES_URL env or http://localhost:8000 */
|
|
93
|
+
imagesUrl?: string;
|
|
69
94
|
};
|
|
70
95
|
}
|
|
71
96
|
|
|
97
|
+
// ---------------------------------------------------------------------------
|
|
98
|
+
// Image helpers — call Python image service to generate variants
|
|
99
|
+
// ---------------------------------------------------------------------------
|
|
100
|
+
|
|
101
|
+
const IMAGE_MIMES = new Set([
|
|
102
|
+
'image/jpeg',
|
|
103
|
+
'image/jpg',
|
|
104
|
+
'image/png',
|
|
105
|
+
'image/webp',
|
|
106
|
+
'image/avif',
|
|
107
|
+
'image/gif',
|
|
108
|
+
'image/bmp',
|
|
109
|
+
]);
|
|
110
|
+
|
|
111
|
+
function isImageMime(mime: string | undefined): boolean {
|
|
112
|
+
if (!mime) return false;
|
|
113
|
+
if (IMAGE_MIMES.has(mime.toLowerCase())) return true;
|
|
114
|
+
return mime.toLowerCase().startsWith('image/');
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function imagesBaseUrl(cfg: StorageConfig | null): string | null {
|
|
118
|
+
const fromCfg = cfg?.imageTransform?.imagesUrl;
|
|
119
|
+
if (fromCfg) return fromCfg.replace(/\/+$/, '');
|
|
120
|
+
const pyPortUrl = process.env.PY_PORT ? `http://127.0.0.1:${process.env.PY_PORT}` : '';
|
|
121
|
+
const fromEnv = process.env.NEXUS_IMAGES_URL || pyPortUrl || process.env.AI_SERVER_URL || '';
|
|
122
|
+
if (fromEnv) return fromEnv.replace(/\/+$/, '');
|
|
123
|
+
// Try pyserver (8001) first, then ai-server (8000) — image service is at pyserver (bhooai_nexus)
|
|
124
|
+
try {
|
|
125
|
+
const ports = (globalThis as any).__nexus_ports ?? null;
|
|
126
|
+
if (ports?.pyserver) return `http://127.0.0.1:${ports.pyserver}`;
|
|
127
|
+
} catch {}
|
|
128
|
+
return 'http://127.0.0.1:8001';
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async function generateImageVariants(
|
|
132
|
+
buffer: Buffer,
|
|
133
|
+
cfg: StorageConfig | null,
|
|
134
|
+
): Promise<Record<string, ImageVariant> | undefined> {
|
|
135
|
+
const tf = cfg?.imageTransform;
|
|
136
|
+
if (!tf?.enabled || !tf.sizes || Object.keys(tf.sizes).length === 0) return undefined;
|
|
137
|
+
const baseUrl = imagesBaseUrl(cfg);
|
|
138
|
+
if (!baseUrl) return undefined;
|
|
139
|
+
const formats = tf.formats?.length ? tf.formats : ['webp'];
|
|
140
|
+
const quality = tf.quality ?? 80;
|
|
141
|
+
const fit = tf.fit ?? 'inside';
|
|
142
|
+
const algorithm = tf.upscale?.algorithm ?? 'realesrgan';
|
|
143
|
+
const stripMetadata = tf.stripMetadata ?? true;
|
|
144
|
+
|
|
145
|
+
const b64 = buffer.toString('base64');
|
|
146
|
+
const controller = new AbortController();
|
|
147
|
+
const timeout = setTimeout(() => controller.abort(), 15_000);
|
|
148
|
+
try {
|
|
149
|
+
const doFetch = (url: string) =>
|
|
150
|
+
fetch(`${url}/images/variants/json`, {
|
|
151
|
+
method: 'POST',
|
|
152
|
+
headers: { 'content-type': 'application/json' },
|
|
153
|
+
body: JSON.stringify({
|
|
154
|
+
b64,
|
|
155
|
+
sizes: tf.sizes,
|
|
156
|
+
formats,
|
|
157
|
+
quality,
|
|
158
|
+
fit,
|
|
159
|
+
algorithm,
|
|
160
|
+
strip_metadata: stripMetadata,
|
|
161
|
+
}),
|
|
162
|
+
signal: controller.signal,
|
|
163
|
+
} as any);
|
|
164
|
+
let res = await doFetch(baseUrl);
|
|
165
|
+
// Fallback to alternate port (pyserver 8001 vs ai-server 8000)
|
|
166
|
+
if (res.status === 404) {
|
|
167
|
+
const alt = baseUrl.includes(':8001') ? baseUrl.replace(':8001', ':8000') : baseUrl.replace(':8000', ':8001');
|
|
168
|
+
if (alt !== baseUrl) {
|
|
169
|
+
const altRes = await doFetch(alt);
|
|
170
|
+
if (altRes.ok || altRes.status !== 404) res = altRes;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (!res.ok) {
|
|
174
|
+
const txt = await res.text().catch(() => '');
|
|
175
|
+
console.warn(`[storage] image variants failed: ${res.status} ${txt.slice(0, 300)}`);
|
|
176
|
+
return undefined;
|
|
177
|
+
}
|
|
178
|
+
const json = (await res.json()) as { variants: Record<string, { b64: string; width: number; height: number; format: string; size: number; sha256: string }> };
|
|
179
|
+
// Return b64 variants for caller to persist; caller maps to paths/urls
|
|
180
|
+
// We keep b64 here — LocalDisk/S3Disk will write files and strip b64.
|
|
181
|
+
// To avoid double-encoding, return as-is with b64 included.
|
|
182
|
+
const out: Record<string, ImageVariant & { b64: string }> = {} as any;
|
|
183
|
+
for (const [k, v] of Object.entries(json.variants ?? {})) {
|
|
184
|
+
(out as any)[k] = v;
|
|
185
|
+
}
|
|
186
|
+
return out as any;
|
|
187
|
+
} catch (e: any) {
|
|
188
|
+
if (e?.name === 'AbortError') console.warn('[storage] image variants timed out (15s) — skipping');
|
|
189
|
+
else console.warn(`[storage] image variants error — skipping: ${e?.message ?? e}`);
|
|
190
|
+
return undefined;
|
|
191
|
+
} finally {
|
|
192
|
+
clearTimeout(timeout);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async function persistVariants(
|
|
197
|
+
variants: Record<string, { b64: string; width: number; height: number; format: string; size: number; sha256: string }>,
|
|
198
|
+
dir: string,
|
|
199
|
+
id: string,
|
|
200
|
+
shard: string,
|
|
201
|
+
cfg: DiskConfig,
|
|
202
|
+
resolveRoot: () => string,
|
|
203
|
+
baseUrlForPath: (rel: string) => string,
|
|
204
|
+
): Promise<Record<string, ImageVariant>> {
|
|
205
|
+
const out: Record<string, ImageVariant> = {};
|
|
206
|
+
for (const [name, v] of Object.entries(variants)) {
|
|
207
|
+
const ext = v.format === 'jpeg' ? 'jpg' : v.format;
|
|
208
|
+
const rel = join(dir, shard, 'variants', name, `${id}.${ext}`).replace(/\\/g, '/');
|
|
209
|
+
const abs = join(resolveRoot(), rel);
|
|
210
|
+
await mkdir(dirname(abs), { recursive: true });
|
|
211
|
+
const buf = Buffer.from(v.b64, 'base64');
|
|
212
|
+
await writeFile(abs, buf);
|
|
213
|
+
out[name] = {
|
|
214
|
+
path: rel,
|
|
215
|
+
url: baseUrlForPath(rel),
|
|
216
|
+
width: v.width,
|
|
217
|
+
height: v.height,
|
|
218
|
+
format: v.format,
|
|
219
|
+
size: buf.length,
|
|
220
|
+
sha256: v.sha256,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
return out;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// ---------------------------------------------------------------------------
|
|
227
|
+
// DB dedup helpers — single entry per sha256, multiple placements, symlink reuse
|
|
228
|
+
// ---------------------------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
function getUploadModelSafe(): any {
|
|
231
|
+
try {
|
|
232
|
+
// Lazy to avoid circular at import time; Upload.ts imports from nexus-data which is already a dep
|
|
233
|
+
// Use dynamic require via eval to keep ESM happy
|
|
234
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
235
|
+
const mod = (globalThis as any).__nexus_upload_model;
|
|
236
|
+
if (mod) return mod;
|
|
237
|
+
return null;
|
|
238
|
+
} catch {
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async function findUploadBySha256(sha256: string): Promise<any | null> {
|
|
244
|
+
try {
|
|
245
|
+
const { getUploadModel } = await import('./Upload.js');
|
|
246
|
+
const M = getUploadModel();
|
|
247
|
+
if (!M) return null;
|
|
248
|
+
const doc = await M.findOne({ sha256 }).exec();
|
|
249
|
+
return doc ?? null;
|
|
250
|
+
} catch {
|
|
251
|
+
return null;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
async function upsertUploadPlacement(doc: any | null, sha256: string, placement: any, canonical: any, variants: Record<string, ImageVariant> | undefined, meta: { originalName: string; mime: string; size: number; width?: number; height?: number }): Promise<any> {
|
|
256
|
+
try {
|
|
257
|
+
const { getUploadModel } = await import('./Upload.js');
|
|
258
|
+
const M = getUploadModel();
|
|
259
|
+
if (!M) return null;
|
|
260
|
+
if (doc) {
|
|
261
|
+
// Existing sha256 — push new placement if not already present (by path)
|
|
262
|
+
const exists = (doc.placements ?? []).some((p: any) => p.path === placement.path);
|
|
263
|
+
if (!exists) {
|
|
264
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
265
|
+
const updated = await M.findOneAndUpdate(
|
|
266
|
+
{ sha256 },
|
|
267
|
+
{ $push: { placements: placement }, $set: { updatedAt: new Date(), ...(variants ? { variants } : {}) } } as any,
|
|
268
|
+
{ returnDocument: 'after' } as any,
|
|
269
|
+
).exec();
|
|
270
|
+
return updated ?? doc;
|
|
271
|
+
}
|
|
272
|
+
return doc;
|
|
273
|
+
}
|
|
274
|
+
const toCreate: any = {
|
|
275
|
+
sha256,
|
|
276
|
+
originalName: meta.originalName,
|
|
277
|
+
mime: meta.mime,
|
|
278
|
+
size: meta.size,
|
|
279
|
+
...(meta.width ? { width: meta.width } : {}),
|
|
280
|
+
...(meta.height ? { height: meta.height } : {}),
|
|
281
|
+
canonical,
|
|
282
|
+
placements: [placement],
|
|
283
|
+
...(variants ? { variants } : {}),
|
|
284
|
+
createdAt: new Date(),
|
|
285
|
+
updatedAt: new Date(),
|
|
286
|
+
};
|
|
287
|
+
const created = await M.create(toCreate);
|
|
288
|
+
return created[0] ?? toCreate;
|
|
289
|
+
} catch (e) {
|
|
290
|
+
// Duplicate key race — fetch existing
|
|
291
|
+
try {
|
|
292
|
+
const { getUploadModel } = await import('./Upload.js');
|
|
293
|
+
const M = getUploadModel();
|
|
294
|
+
if (!M) return null;
|
|
295
|
+
return await M.findOne({ sha256 }).exec();
|
|
296
|
+
} catch {
|
|
297
|
+
return doc;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
async function ensureSymlink(targetAbs: string, linkAbs: string): Promise<boolean> {
|
|
303
|
+
try {
|
|
304
|
+
await mkdir(dirname(linkAbs), { recursive: true });
|
|
305
|
+
// Remove existing file/link if any
|
|
306
|
+
try {
|
|
307
|
+
await unlink(linkAbs);
|
|
308
|
+
} catch {}
|
|
309
|
+
const relTarget = relative(dirname(linkAbs), targetAbs);
|
|
310
|
+
// Windows needs type 'file' and often admin; try symlink then hardlink then copy
|
|
311
|
+
try {
|
|
312
|
+
await symlink(relTarget, linkAbs, 'file');
|
|
313
|
+
return true;
|
|
314
|
+
} catch (e: any) {
|
|
315
|
+
// Fallback to hardlink
|
|
316
|
+
try {
|
|
317
|
+
await link(targetAbs, linkAbs);
|
|
318
|
+
return true;
|
|
319
|
+
} catch {
|
|
320
|
+
// Last fallback: copy
|
|
321
|
+
const buf = await readFile(targetAbs);
|
|
322
|
+
await writeFile(linkAbs, buf);
|
|
323
|
+
return false;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
} catch {
|
|
327
|
+
return false;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
async function removePlacementByPath(path: string): Promise<boolean> {
|
|
332
|
+
try {
|
|
333
|
+
const { getUploadModel } = await import('./Upload.js');
|
|
334
|
+
const M = getUploadModel();
|
|
335
|
+
if (!M) return false;
|
|
336
|
+
const doc = await M.findOne({ $or: [{ 'canonical.path': path }, { 'placements.path': path }] }).exec();
|
|
337
|
+
if (!doc) return false;
|
|
338
|
+
const placements: any[] = (doc as any).placements ?? [];
|
|
339
|
+
const remaining = placements.filter((p: any) => p.path !== path);
|
|
340
|
+
if (remaining.length === 0) {
|
|
341
|
+
// Last placement — delete canonical file + variants physically, then doc
|
|
342
|
+
await M.deleteOne({ sha256: (doc as any).sha256 }).exec();
|
|
343
|
+
return true; // caller should delete physical canonical
|
|
344
|
+
}
|
|
345
|
+
await M.updateOne({ sha256: (doc as any).sha256 }, { $set: { placements: remaining, updatedAt: new Date() } } as any).exec();
|
|
346
|
+
return false; // not last — keep canonical
|
|
347
|
+
} catch {
|
|
348
|
+
return false;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
async function findPlacementDocByPath(path: string): Promise<any | null> {
|
|
353
|
+
try {
|
|
354
|
+
const { getUploadModel } = await import('./Upload.js');
|
|
355
|
+
const M = getUploadModel();
|
|
356
|
+
if (!M) return null;
|
|
357
|
+
return await M.findOne({ $or: [{ 'canonical.path': path }, { 'placements.path': path }] }).exec();
|
|
358
|
+
} catch {
|
|
359
|
+
return null;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
72
363
|
// ---------------------------------------------------------------------------
|
|
73
364
|
// Local driver
|
|
74
365
|
// ---------------------------------------------------------------------------
|
|
@@ -93,6 +384,89 @@ class LocalDisk implements Disk {
|
|
|
93
384
|
const meta = sourceAsMeta(source);
|
|
94
385
|
const originalName = meta.originalName ?? 'file';
|
|
95
386
|
const ext = extname(originalName) || '';
|
|
387
|
+
const sha256 = createHash('sha256').update(buffer).digest('hex');
|
|
388
|
+
const mime = meta.mime ?? 'application/octet-stream';
|
|
389
|
+
|
|
390
|
+
// Try DB dedup — single entry per sha256
|
|
391
|
+
let existing: any = null;
|
|
392
|
+
try {
|
|
393
|
+
existing = await findUploadBySha256(sha256);
|
|
394
|
+
} catch {}
|
|
395
|
+
const isDedupHit = !!existing && existing.canonical?.disk === this.name;
|
|
396
|
+
|
|
397
|
+
if (isDedupHit) {
|
|
398
|
+
// Existing content — create symlink for new placement (separate directory, same content)
|
|
399
|
+
const date = new Date();
|
|
400
|
+
const shard = `${date.getUTCFullYear()}/${String(date.getUTCMonth() + 1).padStart(2, '0')}`;
|
|
401
|
+
const newId = randomUUID();
|
|
402
|
+
const newRel = join(dir, shard, `${newId}${ext}`).replace(/\\/g, '/');
|
|
403
|
+
const newAbs = this.abs(newRel);
|
|
404
|
+
const canonicalAbs = join(this.resolveRoot(), existing.canonical.path);
|
|
405
|
+
// If canonical file missing (e.g. storage cleared), fallback to normal write
|
|
406
|
+
let canLink = false;
|
|
407
|
+
try {
|
|
408
|
+
await fsStat(canonicalAbs);
|
|
409
|
+
canLink = true;
|
|
410
|
+
} catch {
|
|
411
|
+
canLink = false;
|
|
412
|
+
}
|
|
413
|
+
if (canLink) {
|
|
414
|
+
await ensureSymlink(canonicalAbs, newAbs);
|
|
415
|
+
// Symlink variants if any
|
|
416
|
+
let newVariants: Record<string, ImageVariant> | undefined;
|
|
417
|
+
if (existing.variants && Object.keys(existing.variants).length > 0) {
|
|
418
|
+
newVariants = {};
|
|
419
|
+
for (const [name, v] of Object.entries(existing.variants as Record<string, any>)) {
|
|
420
|
+
const vExt = (v as any).format === 'jpeg' ? 'jpg' : (v as any).format;
|
|
421
|
+
const vRel = join(dir, shard, 'variants', name, `${newId}.${vExt}`).replace(/\\/g, '/');
|
|
422
|
+
const vAbs = this.abs(vRel);
|
|
423
|
+
const canonicalVRel = (v as any).path as string;
|
|
424
|
+
const canonicalVAbs = join(this.resolveRoot(), canonicalVRel);
|
|
425
|
+
try {
|
|
426
|
+
await fsStat(canonicalVAbs);
|
|
427
|
+
await ensureSymlink(canonicalVAbs, vAbs);
|
|
428
|
+
newVariants[name] = {
|
|
429
|
+
path: vRel,
|
|
430
|
+
url: this.url(vRel),
|
|
431
|
+
width: (v as any).width,
|
|
432
|
+
height: (v as any).height,
|
|
433
|
+
format: (v as any).format,
|
|
434
|
+
size: (v as any).size,
|
|
435
|
+
sha256: (v as any).sha256,
|
|
436
|
+
};
|
|
437
|
+
} catch {
|
|
438
|
+
// ignore variant link failure
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
// Record new placement in DB
|
|
443
|
+
const placement = {
|
|
444
|
+
id: newId,
|
|
445
|
+
disk: this.name,
|
|
446
|
+
path: newRel,
|
|
447
|
+
url: this.url(newRel),
|
|
448
|
+
dir,
|
|
449
|
+
createdAt: new Date(),
|
|
450
|
+
};
|
|
451
|
+
try {
|
|
452
|
+
await upsertUploadPlacement(existing, sha256, placement, existing.canonical, undefined, { originalName, mime, size: buffer.length });
|
|
453
|
+
} catch {}
|
|
454
|
+
return {
|
|
455
|
+
id: sha256, // single entry id is sha256
|
|
456
|
+
path: newRel,
|
|
457
|
+
url: this.url(newRel),
|
|
458
|
+
disk: this.name,
|
|
459
|
+
size: buffer.length,
|
|
460
|
+
mime,
|
|
461
|
+
originalName,
|
|
462
|
+
sha256,
|
|
463
|
+
...(newVariants && Object.keys(newVariants).length ? { variants: newVariants } : existing.variants ? { variants: existing.variants } : {}),
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
// Fall through to normal write if canonical missing
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// Miss or canLink false — normal write
|
|
96
470
|
const id = randomUUID();
|
|
97
471
|
const date = new Date();
|
|
98
472
|
const shard = `${date.getUTCFullYear()}/${String(date.getUTCMonth() + 1).padStart(2, '0')}`;
|
|
@@ -102,16 +476,60 @@ class LocalDisk implements Disk {
|
|
|
102
476
|
await mkdir(dirname(abs), { recursive: true });
|
|
103
477
|
await writeFile(abs, buffer);
|
|
104
478
|
|
|
105
|
-
|
|
106
|
-
|
|
479
|
+
let variants: Record<string, ImageVariant> | undefined;
|
|
480
|
+
// Auto-generate image variants via Python (Pillow+Real-ESRGAN) — best-effort, never fails upload
|
|
481
|
+
if (isImageMime(mime) && (globalThis as any).__nexus_storage_cfg?.imageTransform?.enabled) {
|
|
482
|
+
const raw = await generateImageVariants(buffer, (globalThis as any).__nexus_storage_cfg as StorageConfig);
|
|
483
|
+
if (raw) {
|
|
484
|
+
try {
|
|
485
|
+
variants = await persistVariants(
|
|
486
|
+
raw as any,
|
|
487
|
+
dir,
|
|
488
|
+
id,
|
|
489
|
+
shard,
|
|
490
|
+
this.cfg,
|
|
491
|
+
() => this.resolveRoot(),
|
|
492
|
+
(p) => this.url(p),
|
|
493
|
+
);
|
|
494
|
+
} catch (e) {
|
|
495
|
+
console.warn(`[storage] persist variants failed: ${(e as Error).message}`);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// Create/Update DB entry (single doc per sha256)
|
|
501
|
+
let width: number | undefined;
|
|
502
|
+
let height: number | undefined;
|
|
503
|
+
if (isImageMime(mime) && variants) {
|
|
504
|
+
const first = Object.values(variants)[0] as any;
|
|
505
|
+
if (first) {
|
|
506
|
+
width = first.width;
|
|
507
|
+
height = first.height;
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
const canonical = { disk: this.name, path: rel, url: this.url(rel) };
|
|
511
|
+
const placement = {
|
|
107
512
|
id,
|
|
513
|
+
disk: this.name,
|
|
514
|
+
path: rel,
|
|
515
|
+
url: this.url(rel),
|
|
516
|
+
dir,
|
|
517
|
+
createdAt: new Date(),
|
|
518
|
+
};
|
|
519
|
+
try {
|
|
520
|
+
await upsertUploadPlacement(existing, sha256, placement, canonical, variants, { originalName, mime, size: buffer.length, width, height });
|
|
521
|
+
} catch {}
|
|
522
|
+
|
|
523
|
+
return {
|
|
524
|
+
id: sha256,
|
|
108
525
|
path: rel,
|
|
109
526
|
url: this.url(rel),
|
|
110
527
|
disk: this.name,
|
|
111
528
|
size: buffer.length,
|
|
112
|
-
mime
|
|
529
|
+
mime,
|
|
113
530
|
originalName,
|
|
114
531
|
sha256,
|
|
532
|
+
...(variants ? { variants } : {}),
|
|
115
533
|
};
|
|
116
534
|
}
|
|
117
535
|
|
|
@@ -124,11 +542,58 @@ class LocalDisk implements Disk {
|
|
|
124
542
|
}
|
|
125
543
|
|
|
126
544
|
async delete(path: string): Promise<void> {
|
|
545
|
+
// DB-aware delete: remove placement, keep canonical if other placements remain
|
|
546
|
+
try {
|
|
547
|
+
const doc: any = await findPlacementDocByPath(path);
|
|
548
|
+
if (doc) {
|
|
549
|
+
const isCanonical = doc.canonical?.path === path;
|
|
550
|
+
const placements: any[] = doc.placements ?? [];
|
|
551
|
+
const remaining = placements.filter((p: any) => p.path !== path);
|
|
552
|
+
if (isCanonical && remaining.length > 0) {
|
|
553
|
+
// Don't delete canonical physical file while other placements reference it; just drop placement entry
|
|
554
|
+
try {
|
|
555
|
+
const { getUploadModel } = await import('./Upload.js');
|
|
556
|
+
const M = getUploadModel();
|
|
557
|
+
if (M) await M.updateOne({ sha256: doc.sha256 }, { $pull: { placements: { path } } as any, $set: { updatedAt: new Date() } }).exec();
|
|
558
|
+
} catch {}
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
561
|
+
// For non-canonical or last placement, unlink the requested path
|
|
562
|
+
try {
|
|
563
|
+
await unlink(this.abs(path));
|
|
564
|
+
} catch {}
|
|
565
|
+
// Also unlink its variant symlinks if any (per-placement variants)
|
|
566
|
+
if (doc.variants) {
|
|
567
|
+
for (const v of Object.values(doc.variants as Record<string, any>)) {
|
|
568
|
+
// Variants are stored per placement? For dedup, variants are canonical-level, not per-placement separate
|
|
569
|
+
// We keep canonical variants until last placement deleted, so don't delete per-placement variant here
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
const shouldDeleteCanonical = await removePlacementByPath(path);
|
|
573
|
+
if (shouldDeleteCanonical) {
|
|
574
|
+
// Last placement — also delete canonical file + variants physically
|
|
575
|
+
try {
|
|
576
|
+
await unlink(this.abs(doc.canonical.path));
|
|
577
|
+
} catch {}
|
|
578
|
+
if (doc.variants) {
|
|
579
|
+
for (const v of Object.values(doc.variants as Record<string, any>)) {
|
|
580
|
+
try {
|
|
581
|
+
await unlink(this.abs((v as any).path));
|
|
582
|
+
} catch {}
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
} catch {}
|
|
127
589
|
try {
|
|
128
590
|
await unlink(this.abs(path));
|
|
129
591
|
} catch {
|
|
130
592
|
/* swallow — deletes are idempotent */
|
|
131
593
|
}
|
|
594
|
+
try {
|
|
595
|
+
await removePlacementByPath(path);
|
|
596
|
+
} catch {}
|
|
132
597
|
}
|
|
133
598
|
|
|
134
599
|
async exists(path: string): Promise<boolean> {
|
|
@@ -190,6 +655,38 @@ class S3Disk implements Disk {
|
|
|
190
655
|
const meta = sourceAsMeta(source);
|
|
191
656
|
const originalName = meta.originalName ?? 'file';
|
|
192
657
|
const ext = extname(originalName) || '';
|
|
658
|
+
const sha256 = createHash('sha256').update(buffer).digest('hex');
|
|
659
|
+
const mime = meta.mime ?? 'application/octet-stream';
|
|
660
|
+
|
|
661
|
+
// DB dedup for S3 — single entry per sha256, no duplicate upload
|
|
662
|
+
let existing: any = null;
|
|
663
|
+
try {
|
|
664
|
+
existing = await findUploadBySha256(sha256);
|
|
665
|
+
} catch {}
|
|
666
|
+
if (existing && existing.canonical?.disk === this.name) {
|
|
667
|
+
const date = new Date();
|
|
668
|
+
const shard = `${date.getUTCFullYear()}/${String(date.getUTCMonth() + 1).padStart(2, '0')}`;
|
|
669
|
+
const newId = randomUUID();
|
|
670
|
+
const newRel = `${dir}/${shard}/${newId}${ext}`;
|
|
671
|
+
// For S3 dedup, don't duplicate object — just add placement pointing to canonical
|
|
672
|
+
const placement = { id: newId, disk: this.name, path: newRel, url: this.url(newRel), dir, createdAt: new Date() };
|
|
673
|
+
// Note: we keep logical newRel but reuse canonical variants
|
|
674
|
+
try {
|
|
675
|
+
await upsertUploadPlacement(existing, sha256, placement, existing.canonical, existing.variants as any, { originalName, mime, size: buffer.length });
|
|
676
|
+
} catch {}
|
|
677
|
+
return {
|
|
678
|
+
id: sha256,
|
|
679
|
+
path: newRel,
|
|
680
|
+
url: this.url(newRel),
|
|
681
|
+
disk: this.name,
|
|
682
|
+
size: buffer.length,
|
|
683
|
+
mime,
|
|
684
|
+
originalName,
|
|
685
|
+
sha256,
|
|
686
|
+
...(existing.variants ? { variants: existing.variants } : {}),
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
|
|
193
690
|
const id = randomUUID();
|
|
194
691
|
const date = new Date();
|
|
195
692
|
const shard = `${date.getUTCFullYear()}/${String(date.getUTCMonth() + 1).padStart(2, '0')}`;
|
|
@@ -200,16 +697,58 @@ class S3Disk implements Disk {
|
|
|
200
697
|
Body: buffer,
|
|
201
698
|
ContentType: meta.mime ?? 'application/octet-stream',
|
|
202
699
|
}));
|
|
203
|
-
|
|
700
|
+
let variants: Record<string, ImageVariant> | undefined;
|
|
701
|
+
if (isImageMime(mime) && (globalThis as any).__nexus_storage_cfg?.imageTransform?.enabled) {
|
|
702
|
+
const raw = await generateImageVariants(buffer, (globalThis as any).__nexus_storage_cfg as StorageConfig);
|
|
703
|
+
if (raw) {
|
|
704
|
+
try {
|
|
705
|
+
// Persist variants to S3 as well
|
|
706
|
+
const out: Record<string, ImageVariant> = {};
|
|
707
|
+
for (const [name, v] of Object.entries(raw as any)) {
|
|
708
|
+
const vv = v as { b64: string; width: number; height: number; format: string; size: number; sha256: string };
|
|
709
|
+
const vext = vv.format === 'jpeg' ? 'jpg' : vv.format;
|
|
710
|
+
const vrel = `${dir}/${shard}/variants/${name}/${id}.${vext}`;
|
|
711
|
+
const vbuf = Buffer.from(vv.b64, 'base64');
|
|
712
|
+
await this.client.send(new PutObjectCommand({
|
|
713
|
+
Bucket: this.cfg.bucket,
|
|
714
|
+
Key: vrel,
|
|
715
|
+
Body: vbuf,
|
|
716
|
+
ContentType: `image/${vext === 'jpg' ? 'jpeg' : vext}`,
|
|
717
|
+
}));
|
|
718
|
+
out[name] = {
|
|
719
|
+
path: vrel,
|
|
720
|
+
url: this.url(vrel),
|
|
721
|
+
width: vv.width,
|
|
722
|
+
height: vv.height,
|
|
723
|
+
format: vv.format,
|
|
724
|
+
size: vbuf.length,
|
|
725
|
+
sha256: vv.sha256,
|
|
726
|
+
};
|
|
727
|
+
}
|
|
728
|
+
variants = out;
|
|
729
|
+
} catch (e) {
|
|
730
|
+
console.warn(`[storage] S3 persist variants failed: ${(e as Error).message}`);
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
// Upsert DB
|
|
735
|
+
{
|
|
736
|
+
const canonical = { disk: this.name, path: rel, url: this.url(rel) };
|
|
737
|
+
const placement = { id, disk: this.name, path: rel, url: this.url(rel), dir, createdAt: new Date() };
|
|
738
|
+
try {
|
|
739
|
+
await upsertUploadPlacement(existing, sha256, placement, canonical, variants, { originalName, mime, size: buffer.length });
|
|
740
|
+
} catch {}
|
|
741
|
+
}
|
|
204
742
|
return {
|
|
205
|
-
id,
|
|
743
|
+
id: sha256,
|
|
206
744
|
path: rel,
|
|
207
745
|
url: this.url(rel),
|
|
208
746
|
disk: this.name,
|
|
209
747
|
size: buffer.length,
|
|
210
|
-
mime
|
|
748
|
+
mime,
|
|
211
749
|
originalName,
|
|
212
750
|
sha256,
|
|
751
|
+
...(variants ? { variants } : {}),
|
|
213
752
|
};
|
|
214
753
|
}
|
|
215
754
|
|
|
@@ -229,7 +768,45 @@ class S3Disk implements Disk {
|
|
|
229
768
|
async delete(path: string): Promise<void> {
|
|
230
769
|
await this.ensure();
|
|
231
770
|
const { DeleteObjectCommand } = (await import('@aws-sdk/client-s3' as string)) as any;
|
|
232
|
-
|
|
771
|
+
// DB-aware: only delete physical object if last placement
|
|
772
|
+
try {
|
|
773
|
+
const doc: any = await findPlacementDocByPath(path);
|
|
774
|
+
if (doc) {
|
|
775
|
+
const isCanonical = doc.canonical?.path === path;
|
|
776
|
+
const placements: any[] = doc.placements ?? [];
|
|
777
|
+
const remaining = placements.filter((p: any) => p.path !== path);
|
|
778
|
+
if (isCanonical && remaining.length > 0) {
|
|
779
|
+
try {
|
|
780
|
+
const { getUploadModel } = await import('./Upload.js');
|
|
781
|
+
const M = getUploadModel();
|
|
782
|
+
if (M) await M.updateOne({ sha256: doc.sha256 }, { $pull: { placements: { path } } as any, $set: { updatedAt: new Date() } }).exec();
|
|
783
|
+
} catch {}
|
|
784
|
+
return;
|
|
785
|
+
}
|
|
786
|
+
// Not canonical or last — proceed to delete S3 object, then update DB
|
|
787
|
+
await this.client.send(new DeleteObjectCommand({ Bucket: this.cfg.bucket, Key: path }));
|
|
788
|
+
const shouldDeleteCanonical = await removePlacementByPath(path);
|
|
789
|
+
if (shouldDeleteCanonical) {
|
|
790
|
+
try {
|
|
791
|
+
await this.client.send(new DeleteObjectCommand({ Bucket: this.cfg.bucket, Key: doc.canonical.path }));
|
|
792
|
+
} catch {}
|
|
793
|
+
if (doc.variants) {
|
|
794
|
+
for (const v of Object.values(doc.variants as Record<string, any>)) {
|
|
795
|
+
try {
|
|
796
|
+
await this.client.send(new DeleteObjectCommand({ Bucket: this.cfg.bucket, Key: (v as any).path }));
|
|
797
|
+
} catch {}
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
return;
|
|
802
|
+
}
|
|
803
|
+
} catch {}
|
|
804
|
+
try {
|
|
805
|
+
await this.client.send(new DeleteObjectCommand({ Bucket: this.cfg.bucket, Key: path }));
|
|
806
|
+
} catch {}
|
|
807
|
+
try {
|
|
808
|
+
await removePlacementByPath(path);
|
|
809
|
+
} catch {}
|
|
233
810
|
}
|
|
234
811
|
|
|
235
812
|
async exists(path: string): Promise<boolean> {
|
|
@@ -299,6 +876,8 @@ class StorageFacade {
|
|
|
299
876
|
configure(cfg: StorageConfig, signingSecret: string): void {
|
|
300
877
|
this.cfg = cfg;
|
|
301
878
|
this.signingSecret = signingSecret || cfg.signingSecret || 'nexus-insecure';
|
|
879
|
+
// Expose for variant generation in drivers (LocalDisk/S3Disk)
|
|
880
|
+
(globalThis as any).__nexus_storage_cfg = cfg;
|
|
302
881
|
for (const [name, diskCfg] of Object.entries(cfg.disks ?? {})) {
|
|
303
882
|
if (diskCfg.driver === 'local') {
|
|
304
883
|
this.disks.set(name, new LocalDisk(name, diskCfg, this.signingSecret));
|
|
@@ -308,6 +887,8 @@ class StorageFacade {
|
|
|
308
887
|
}
|
|
309
888
|
}
|
|
310
889
|
|
|
890
|
+
getConfig(): StorageConfig | null { return this.cfg; }
|
|
891
|
+
|
|
311
892
|
disk(name?: string): Disk {
|
|
312
893
|
const key = name ?? this.cfg?.default ?? Object.keys(this.cfg?.disks ?? {})[0];
|
|
313
894
|
if (!key) throw new Error('Storage has no disks configured');
|
|
@@ -326,6 +907,19 @@ export const storage = new StorageFacade();
|
|
|
326
907
|
|
|
327
908
|
/** Auto-configure from env — used by createNexusApp() boot. */
|
|
328
909
|
export function configureStorageFromEnv(root: string, signingSecret: string): void {
|
|
910
|
+
// Allow disabling or tuning image variants via env
|
|
911
|
+
const imageTransformEnabled = (process.env.NEXUS_IMAGE_TRANSFORM_ENABLED ?? 'true') !== 'false';
|
|
912
|
+
const imageSizesEnv = process.env.NEXUS_IMAGE_SIZES; // e.g. "thumb:150,small:480,medium:1280,large:2048"
|
|
913
|
+
const imagesUrlEnv = process.env.NEXUS_IMAGES_URL || process.env.AI_SERVER_URL || undefined;
|
|
914
|
+
const imageQualEnv = process.env.NEXUS_IMAGE_QUALITY ? parseInt(process.env.NEXUS_IMAGE_QUALITY, 10) : 80;
|
|
915
|
+
let parsedSizes: Record<string, number> | undefined;
|
|
916
|
+
if (imageSizesEnv) {
|
|
917
|
+
parsedSizes = {};
|
|
918
|
+
for (const part of imageSizesEnv.split(',')) {
|
|
919
|
+
const [k, v] = part.split(':').map(s => s.trim());
|
|
920
|
+
if (k && v) parsedSizes[k] = parseInt(v, 10);
|
|
921
|
+
}
|
|
922
|
+
}
|
|
329
923
|
const cfg: StorageConfig = {
|
|
330
924
|
default: 'uploads',
|
|
331
925
|
disks: {
|
|
@@ -351,6 +945,19 @@ export function configureStorageFromEnv(root: string, signingSecret: string): vo
|
|
|
351
945
|
: {}),
|
|
352
946
|
},
|
|
353
947
|
signingSecret,
|
|
948
|
+
imageTransform: {
|
|
949
|
+
enabled: imageTransformEnabled,
|
|
950
|
+
formats: (process.env.NEXUS_IMAGE_FORMATS ?? 'webp').split(',').map(s => s.trim().toLowerCase()).filter(Boolean),
|
|
951
|
+
sizes: parsedSizes ?? { thumb: 150, small: 480, medium: 1280, large: 2048 },
|
|
952
|
+
stripMetadata: (process.env.NEXUS_IMAGE_STRIP_METADATA ?? 'true') !== 'false',
|
|
953
|
+
quality: imageQualEnv,
|
|
954
|
+
fit: (process.env.NEXUS_IMAGE_FIT as any) || 'inside',
|
|
955
|
+
upscale: {
|
|
956
|
+
enabled: (process.env.NEXUS_IMAGE_UPSCALE_ENABLED ?? 'true') !== 'false',
|
|
957
|
+
algorithm: (process.env.NEXUS_IMAGE_UPSCALE_ALGO as any) || 'realesrgan',
|
|
958
|
+
},
|
|
959
|
+
...(imagesUrlEnv ? { imagesUrl: imagesUrlEnv } : {}),
|
|
960
|
+
},
|
|
354
961
|
};
|
|
355
962
|
storage.configure(cfg, signingSecret);
|
|
356
963
|
|