@proteinjs/db-file 1.4.14 → 1.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/db-file",
-  "version": "1.4.14",
+  "version": "1.5.0",
   "description": "File storage in the db",
   "publishConfig": {
     "access": "public"
@@ -18,10 +18,11 @@
   "scripts": {
     "clean": "rm -rf dist/ node_modules/ generated/",
     "build": "reflection-build && tsc",
+    "test": "jest --runInBand --passWithNoTests",
     "watch": "reflection-watch"
   },
   "dependencies": {
-    "@proteinjs/db": "^1.32.3",
+    "@proteinjs/db": "^1.34.0",
     "@proteinjs/logger": "^1.0.21",
     "@proteinjs/reflection": "^1.1.12",
     "@proteinjs/server-api": "^3.0.11",
@@ -30,6 +31,7 @@
     "moment": "2.29.4"
   },
   "devDependencies": {
+    "@proteinjs/db-driver-spanner": "^1.22.0",
     "@proteinjs/reflection-build": "^1.4.10",
     "@types/jest": "29.5.5",
     "@types/node": "14.0.27",
@@ -44,5 +46,5 @@
   },
   "main": "./dist/generated/index.js",
   "types": "./dist/generated/index.d.ts",
-  "gitHead": "3fa64176929e1c9183cf3d998d9d629dcbb669b0"
+  "gitHead": "7a301275151aee8040fd52e15e3fe950a6863783"
 }

package/src/DbFileStorageDriver.ts

@@ -29,14 +29,6 @@ export class DbFileStorageDriver implements FileStorageDriver {
     }
   }
 
-  private splitIntoChunks(data: string): string[] {
-    const chunks = [];
-    for (let i = 0; i < data.length; i += this.chunkSize) {
-      chunks.push(data.substring(i, i + this.chunkSize));
-    }
-    return chunks;
-  }
-
   async getFileData(fileId: string): Promise<string> {
     // Byte reads are keyed by id and UNSCOPED — matching the GCS driver's semantics (a bucket
     // download by id). The driver is a byte store; access control lives with callers and the
@@ -70,4 +62,22 @@ export class DbFileStorageDriver implements FileStorageDriver {
       });
     }
   }
+
+  async deleteFile(fileId: string): Promise<void> {
+    // Byte deletes are keyed by id and UNSCOPED — the same semantics as getFileData above (the
+    // driver is a byte store; access control lives with callers and the scoped File metadata
+    // layer), and system sweeps delete rows outside the current session's scope. Deleting zero
+    // chunks succeeds, which gives this the idempotency the driver contract requires.
+    const db = getScopedDbAsSystem();
+    const deleteQuery = new QueryBuilderFactory().getQueryBuilder(tables.FileData, { file: fileId });
+    await db.delete(tables.FileData, deleteQuery);
+  }
+
+  private splitIntoChunks(data: string): string[] {
+    const chunks = [];
+    for (let i = 0; i < data.length; i += this.chunkSize) {
+      chunks.push(data.substring(i, i + this.chunkSize));
+    }
+    return chunks;
+  }
 }

package/src/FileStorage.ts

@@ -27,9 +27,7 @@ export interface DefaultFileStorageDriverFactory extends Loadable {
  * File data is stored by the `FileStorageDriver`.
  */
 export class FileStorage implements FileStorageService {
-  private static defaultDriver: FileStorageDriver;
-  private driver: FileStorageDriver;
-  private logger: Logger = new Logger({ name: this.constructor.name });
+  private static driver: FileStorageDriver;
 
   public serviceMetadata = {
     auth: {
@@ -37,24 +35,28 @@ export class FileStorage implements FileStorageService {
     },
   };
 
-  constructor(driver?: FileStorageDriver) {
-    this.driver = driver ? driver : this.getDefaultDriver();
-  }
-
-  private getDefaultDriver(): FileStorageDriver {
-    if (!FileStorage.defaultDriver) {
+  /**
+   * The `FileStorageDriver` for this process — provided by the `DefaultFileStorageDriverFactory`
+   * implementation, defaulting to the `DbFileStorageDriver`. One resolution for every byte
+   * operation (`FileStorage` and `FileStorageTableWatcher`), so bytes always live and die in the
+   * same store.
+   */
+  static getDriver(): FileStorageDriver {
+    if (!FileStorage.driver) {
       const defaultDriverFactory = SourceRepository.get().object<DefaultFileStorageDriverFactory>(
         '@proteinjs/db-file/DefaultFileStorageDriverFactory'
       );
       if (defaultDriverFactory) {
-        FileStorage.defaultDriver = defaultDriverFactory.getDriver();
+        FileStorage.driver = defaultDriverFactory.getDriver();
       } else {
-        this.logger.info({ message: `Defaulting to DbFileStorageDriver since no FileStorageDriver was provided` });
-        FileStorage.defaultDriver = new DbFileStorageDriver();
+        new Logger({ name: 'FileStorage' }).info({
+          message: `Defaulting to DbFileStorageDriver since no FileStorageDriver was provided`,
+        });
+        FileStorage.driver = new DbFileStorageDriver();
       }
     }
 
-    return FileStorage.defaultDriver;
+    return FileStorage.driver;
   }
 
   /**
@@ -66,7 +68,7 @@ export class FileStorage implements FileStorageService {
   async createFile(fileMetaData: Omit<File, keyof ScopedRecord>, fileData: string): Promise<File> {
     const db = getScopedDb();
     const file = await db.insert(tables.File, fileMetaData);
-    await this.driver.createFile(file, fileData);
+    await FileStorage.getDriver().createFile(file, fileData);
     return file;
   }
 
@@ -87,7 +89,31 @@ export class FileStorage implements FileStorageService {
    * @returns The file data as a single string.
    */
   async getFileData(fileId: string): Promise<string> {
-    return await this.driver.getFileData(fileId);
+    return await FileStorage.getDriver().getFileData(fileId);
+  }
+
+  /**
+   * Mint a short-lived, read-only URL for the file's bytes (server-only; not part of the
+   * browser-facing `FileStorageService`). A signed URL is a bearer capability, so minting
+   * verifies the caller can read the file row (scoped read) before signing.
+   * @param fileId - The `id` of the file.
+   * @param options.ttlMs - How long the URL stays valid; the driver applies its default when omitted.
+   * @returns The signed URL, or `undefined` when the driver has no external URL space
+   *          (`DbFileStorageDriver`) — the caller then serves bytes through the proxy route.
+   * @throws When the file row does not exist or is not readable by the caller.
+   */
+  async getSignedUrl(fileId: string, options?: { ttlMs?: number }): Promise<string | undefined> {
+    const file = await this.getFile(fileId);
+    if (!file) {
+      throw new Error(`File not found: ${fileId}`);
+    }
+
+    const driver = FileStorage.getDriver();
+    if (!driver.getSignedUrl) {
+      return undefined;
+    }
+
+    return await driver.getSignedUrl(fileId, options);
   }
 
   /**
@@ -96,7 +122,7 @@ export class FileStorage implements FileStorageService {
    * @param data - The new data string to replace the existing data.
    */
   async updateFileData(fileId: string, data: string): Promise<void> {
-    await this.driver.updateFileData(fileId, data);
+    await FileStorage.getDriver().updateFileData(fileId, data);
   }
 
   /**
@@ -107,22 +133,21 @@ export class FileStorage implements FileStorageService {
     const db = getScopedDb();
     await db.update(tables.File, file);
 
-    if (this.driver.updateFile) {
-      await this.driver.updateFile(file as File);
+    const driver = FileStorage.getDriver();
+    if (driver.updateFile) {
+      await driver.updateFile(file as File);
     }
   }
 
   /**
    * Deletes a file and its data.
-   * The file data is deleted by a cascade delete rule defined on the `FileTable`
+   * The file data is deleted by the `FileStorageTableWatcher` when the row delete runs — the
+   * watcher fires for every file-row delete path (this one, reference cascades, system sweeps),
+   * so the bytes die with the row no matter where the delete originates.
    * @param fileId - The `id` of the file to delete.
    */
   async deleteFile(fileId: string): Promise<void> {
     const db = getScopedDb();
-    if (this.driver.deleteFile) {
-      await this.driver.deleteFile(fileId);
-    }
-
     await db.delete(tables.File, { id: fileId });
   }
 }

package/src/FileStorageDriver.ts

@@ -1,9 +1,32 @@
 import { File } from './tables/FileTable';
 
+/**
+ * Byte store behind `FileStorage`.
+ *
+ * Encoding contract: every `fileData`/`data` string crossing this interface is BASE64 of the
+ * file's true bytes — for text and binary files alike. Drivers own their at-rest representation,
+ * with one hard requirement: a driver that serves blobs directly to clients (`getSignedUrl`)
+ * MUST store the true bytes at rest, because clients read its objects without any decode step.
+ */
 export interface FileStorageDriver {
   createFile(file: File, fileData: string): Promise<void>;
   getFileData(fileId: string): Promise<string>;
   updateFileData(fileId: string, data: string): Promise<void>;
   updateFile?(file: File): Promise<void>;
-  deleteFile?(fileId: string): Promise<void>;
+  /**
+   * Mint a short-lived, read-only URL for the file's bytes, when the driver's store has an
+   * external URL space (e.g. GCS signed URLs). Serving rides this: `GET /file/:id` 302-redirects
+   * to the minted URL, so clients get native Range/seek and caching and bytes stop transiting
+   * the app server. Absent on drivers with no external URL space (`DbFileStorageDriver`) — their
+   * bytes serve through the proxy route.
+   */
+  getSignedUrl?(fileId: string, options?: { ttlMs?: number }): Promise<string>;
+  /**
+   * Delete the stored bytes for `fileId`.
+   *
+   * Contract: idempotent on the goal state — bytes already gone is success, since file-row
+   * deletes are retryable (a re-run after a half-completed prior attempt must not wedge on the
+   * missing bytes). Every other failure throws loudly; the file row survives to drive a retry.
+   */
+  deleteFile(fileId: string): Promise<void>;
 }

package/src/FileStorageTableWatcher.ts

@@ -0,0 +1,43 @@
+import { QueryBuilder, Table, TableWatcher } from '@proteinjs/db';
+import { Logger } from '@proteinjs/logger';
+import { File, FileTable } from './tables/FileTable';
+import { FileStorage } from './FileStorage';
+
+/**
+ * Deletes stored file bytes whenever file rows are deleted. Lives at the Db layer so every
+ * delete path reaches the byte store — `FileStorage.deleteFile`, reference cascades (e.g.
+ * `file.preview`), and system sweeps that delete file rows directly.
+ *
+ * Bytes go first (`beforeDelete`): if the byte delete fails, the row survives to drive a retry;
+ * the reverse order would strand bytes with no row left to retry from. A retried delete is safe
+ * because `FileStorageDriver.deleteFile` is idempotent on the goal state.
+ */
+export class FileStorageTableWatcher implements TableWatcher<File> {
+  private logger = new Logger({ name: this.constructor.name });
+
+  name(): string {
+    return this.constructor.name;
+  }
+
+  table(): Table<File> {
+    return new FileTable();
+  }
+
+  async beforeDelete<T extends File>(
+    recordsToDelete: T[],
+    initialQb: QueryBuilder<T>,
+    deleteQb: QueryBuilder<T>
+  ): Promise<void> {
+    const driver = FileStorage.getDriver();
+    const deletedFileIds: string[] = [];
+    for (const record of recordsToDelete) {
+      await driver.deleteFile(record.id);
+      deletedFileIds.push(record.id);
+    }
+
+    this.logger.info({
+      message: `Deleted stored bytes for file records being deleted`,
+      obj: { deletedFileIds },
+    });
+  }
+}

package/src/routes/getFile.ts

@@ -2,6 +2,17 @@ import { Route } from '@proteinjs/server-api';
 import { getFileStorage } from '../FileStorage';
 import { UserAuth } from '@proteinjs/user';
 
+/**
+ * Serves a file's bytes. Auth first (logged-in + scoped row read), then one of two paths:
+ *
+ * - **302 redirect** to a short-lived signed URL when the driver's store has an external URL
+ *   space (GCS). The app URL stays the one stable reference every `<img>`/`<video>`/chip uses;
+ *   the client follows the redirect and reads the store directly — native Range/206 for video
+ *   seeking, real caching, bytes never transit the app server. The redirect itself is cached
+ *   briefly (client-private, well under the signed TTL) so repeated loads reuse one URL.
+ * - **Proxy** for drivers with no external URL space (`DbFileStorageDriver`): the interface's
+ *   base64 is decoded so every mime — binary or text — serves its true bytes.
+ */
 export const getFile: Route = {
   path: '/file/:id',
   method: 'get',
@@ -14,18 +25,25 @@ export const getFile: Route = {
     const fileId = request.params.id;
     const fileStorage = getFileStorage();
     try {
+      // The file row decides existence (a scoped read, so it is also the access check).
       const file = await fileStorage.getFile(fileId);
-      const fileDataBase64 = await fileStorage.getFileData(fileId);
-      // The file row decides existence; empty data is a legitimate zero-byte file, not a 404.
       if (!file) {
         response.status(404).send('File not found');
-      } else {
-        const safeFilename = encodeURIComponent(file.name);
-        const fileData = file.type.startsWith('image/') ? Buffer.from(fileDataBase64, 'base64') : fileDataBase64;
-        response.setHeader('Content-Disposition', `inline; filename="${safeFilename}"`);
-        response.setHeader('Content-Type', file.type);
-        response.send(fileData);
+        return;
+      }
+
+      const signedUrl = await fileStorage.getSignedUrl(fileId);
+      if (signedUrl) {
+        response.setHeader('Cache-Control', 'private, max-age=300');
+        response.redirect(302, signedUrl);
+        return;
       }
+
+      const fileDataBase64 = await fileStorage.getFileData(fileId);
+      const safeFilename = encodeURIComponent(file.name);
+      response.setHeader('Content-Disposition', `inline; filename="${safeFilename}"`);
+      response.setHeader('Content-Type', file.type);
+      response.send(Buffer.from(fileDataBase64, 'base64'));
     } catch (error) {
       console.error(`Error fetching file (${fileId}):`, error);
       response.status(500).send('Internal Server Error');

package/src/tables/FileTable.ts

@@ -1,6 +1,5 @@
 import { Table, StringColumn, IntegerColumn, ReferenceColumn, Reference } from '@proteinjs/db';
 import { ScopedRecord, withScopedRecordColumns } from '@proteinjs/user';
-import { FileDataTable } from './FileDataTable';
 
 const FILE_TABLE_NAME = 'file';
 
@@ -14,6 +13,15 @@ export interface File extends ScopedRecord {
    * the bytes stay out of the DB row (consistent with the storage model); deleted with this file.
    */
   preview?: Reference<File>;
+  /**
+   * Media metadata — generic file facts (an image/video's pixel dimensions, a video/audio
+   * duration) every consumer needs to render without loading bytes, e.g. reserving a media box's
+   * aspect ratio before any bytes arrive. Set at ingest for media files; absent for everything
+   * else.
+   */
+  width?: number;
+  height?: number;
+  durationMs?: number;
 }
 
 export class FileTable extends Table<File> {
@@ -32,11 +40,10 @@ export class FileTable extends Table<File> {
     size: new IntegerColumn('size'),
     // Self-reference (the preview is itself a File). cascadeDelete: removing a file removes its preview.
     preview: new ReferenceColumn<File>('preview', FILE_TABLE_NAME, true),
+    width: new IntegerColumn('width'),
+    height: new IntegerColumn('height'),
+    durationMs: new IntegerColumn('duration_ms'),
   });
-  public cascadeDeleteReferences = () => [
-    {
-      table: new FileDataTable().name,
-      referenceColumn: new FileDataTable().columns.file.name,
-    },
-  ];
+  // No cascadeDeleteReferences for FileData: byte cleanup (FileData rows included) is owned by
+  // FileStorageDriver.deleteFile, invoked for every file-row delete by FileStorageTableWatcher.
 }

package/test/FileMediaMetadata.test.ts

@@ -0,0 +1,74 @@
+import { getDbAsSystem } from '@proteinjs/db';
+import { File } from '../src/tables/FileTable';
+import { tables } from '../src/tables/tables';
+import { FileStorage } from '../src/FileStorage';
+import { FileStorageDriver } from '../src/FileStorageDriver';
+import { FileTestEnvironment } from './FileTestEnvironment';
+
+class TestMemoryFileStorageDriver implements FileStorageDriver {
+  readonly store = new Map<string, string>();
+
+  async createFile(file: File, fileData: string): Promise<void> {
+    this.store.set(file.id, fileData);
+  }
+
+  async getFileData(fileId: string): Promise<string> {
+    const data = this.store.get(fileId);
+    if (data === undefined) {
+      throw new Error(`No such object: ${fileId}`);
+    }
+    return data;
+  }
+
+  async updateFileData(fileId: string, data: string): Promise<void> {
+    this.store.set(fileId, data);
+  }
+
+  async deleteFile(fileId: string): Promise<void> {
+    this.store.delete(fileId);
+  }
+}
+
+const testEnv = new FileTestEnvironment();
+
+beforeAll(async () => {
+  await testEnv.beforeAll();
+  const user = await testEnv.createUser({ name: 'File owner', email: 'media-metadata@test.local' });
+  testEnv.actAs(user);
+  testEnv.setDriver(new TestMemoryFileStorageDriver());
+});
+
+afterAll(async () => {
+  await testEnv.afterAll();
+});
+
+/**
+ * Media metadata lives on the File row (`width`/`height`/`durationMs`) — generic file facts every
+ * consumer needs to render without loading bytes (aspect-ratio-reserved layout). This proves the
+ * columns persist and round-trip through the real schema on the emulator.
+ */
+describe('File media metadata columns', () => {
+  it('round-trips width/height/durationMs on a media file', async () => {
+    const file = await new FileStorage().createFile(
+      { name: 'clip.mp4', type: 'video/mp4', size: 4, width: 1280, height: 720, durationMs: 24_500 } as File,
+      Buffer.from('clip').toString('base64')
+    );
+
+    const row = await getDbAsSystem().get(tables.File, { id: file.id });
+    expect(row.width).toEqual(1280);
+    expect(row.height).toEqual(720);
+    expect(row.durationMs).toEqual(24_500);
+  });
+
+  it('leaves the fields absent for non-media files', async () => {
+    const file = await new FileStorage().createFile(
+      { name: 'a.txt', type: 'text/plain', size: 5 } as File,
+      Buffer.from('hello').toString('base64')
+    );
+
+    const row = await getDbAsSystem().get(tables.File, { id: file.id });
+    expect(row.width ?? undefined).toBeUndefined();
+    expect(row.height ?? undefined).toBeUndefined();
+    expect(row.durationMs ?? undefined).toBeUndefined();
+  });
+});

package/test/FileStorageDeleteFile.test.ts

@@ -0,0 +1,128 @@
+import { Reference, getDbAsSystem } from '@proteinjs/db';
+import { getScopedDbAsSystem, ScopedRecord } from '@proteinjs/user';
+import { File } from '../src/tables/FileTable';
+import { tables } from '../src/tables/tables';
+import { FileStorage } from '../src/FileStorage';
+import { FileStorageDriver } from '../src/FileStorageDriver';
+import { DbFileStorageDriver } from '../src/DbFileStorageDriver';
+import { FileTestEnvironment } from './FileTestEnvironment';
+
+/**
+ * In-memory byte store shaped like the GoogleCloudStorageDriver: bytes keyed by file id,
+ * reads of a missing object fail loudly (GCS 404 semantics).
+ */
+class TestMemoryFileStorageDriver implements FileStorageDriver {
+  readonly store = new Map<string, string>();
+
+  async createFile(file: File, fileData: string): Promise<void> {
+    this.store.set(file.id, fileData);
+  }
+
+  async getFileData(fileId: string): Promise<string> {
+    const data = this.store.get(fileId);
+    if (data === undefined) {
+      throw new Error(`No such object: ${fileId}`);
+    }
+    return data;
+  }
+
+  async updateFileData(fileId: string, data: string): Promise<void> {
+    this.store.set(fileId, data);
+  }
+
+  async updateFile(_file: File): Promise<void> {}
+
+  async deleteFile(fileId: string): Promise<void> {
+    this.store.delete(fileId);
+  }
+}
+
+const testEnv = new FileTestEnvironment();
+
+beforeAll(async () => {
+  await testEnv.beforeAll();
+  const user = await testEnv.createUser({ name: 'File owner', email: 'file-owner@test.local' });
+  testEnv.actAs(user);
+});
+
+afterAll(async () => {
+  await testEnv.afterAll();
+});
+
+describe('FileStorage.deleteFile', () => {
+  const driver = new TestMemoryFileStorageDriver();
+
+  beforeAll(() => {
+    testEnv.setDriver(driver);
+  });
+
+  it('deletes the stored bytes along with the file row', async () => {
+    const fileStorage = new FileStorage();
+    const file = await fileStorage.createFile({ name: 'a.txt', type: 'text/plain', size: 5 } as File, 'hello');
+    expect(driver.store.has(file.id)).toBe(true);
+
+    await fileStorage.deleteFile(file.id);
+
+    expect(await getDbAsSystem().get(tables.File, { id: file.id })).toBeUndefined();
+    expect(driver.store.has(file.id)).toBe(false);
+  });
+
+  it('deletes the stored bytes when file rows are deleted directly through Db.delete (system sweep path)', async () => {
+    const fileStorage = new FileStorage();
+    const file = await fileStorage.createFile({ name: 'b.txt', type: 'text/plain', size: 5 } as File, 'swept');
+    expect(driver.store.has(file.id)).toBe(true);
+
+    await getScopedDbAsSystem<File>().delete(tables.File, { id: file.id });
+
+    expect(await getDbAsSystem().get(tables.File, { id: file.id })).toBeUndefined();
+    expect(driver.store.has(file.id)).toBe(false);
+  });
+
+  it('deletes the preview file bytes through the reference cascade', async () => {
+    const fileStorage = new FileStorage();
+    const preview = await fileStorage.createFile({ name: 'c.gif.preview', type: 'image/png', size: 4 } as File, 'PREV');
+    const file = await fileStorage.createFile(
+      {
+        name: 'c.gif',
+        type: 'image/gif',
+        size: 4,
+        preview: new Reference<File>(tables.File.name, preview.id),
+      } as Omit<File, keyof ScopedRecord>,
+      'MAIN'
+    );
+
+    await fileStorage.deleteFile(file.id);
+
+    expect(await getDbAsSystem().get(tables.File, { id: file.id })).toBeUndefined();
+    expect(await getDbAsSystem().get(tables.File, { id: preview.id })).toBeUndefined();
+    expect(driver.store.has(file.id)).toBe(false);
+    expect(driver.store.has(preview.id)).toBe(false);
+  });
+});
+
+describe('FileStorage.deleteFile with DbFileStorageDriver', () => {
+  beforeAll(() => {
+    // 4-byte chunks so a small payload exercises multi-chunk cleanup.
+    testEnv.setDriver(new DbFileStorageDriver(4));
+  });
+
+  it('deletes all file_data chunk rows along with the file row', async () => {
+    const fileStorage = new FileStorage();
+    const file = await fileStorage.createFile({ name: 'd.txt', type: 'text/plain', size: 11 } as File, 'hello world');
+    const chunksBefore = await getDbAsSystem().query(tables.FileData, { file: file.id });
+    expect(chunksBefore.length).toBeGreaterThan(1);
+
+    await fileStorage.deleteFile(file.id);
+
+    expect(await getDbAsSystem().get(tables.File, { id: file.id })).toBeUndefined();
+    expect(await getDbAsSystem().query(tables.FileData, { file: file.id })).toHaveLength(0);
+  });
+
+  it('deleteFile is idempotent: deleting bytes that are already gone succeeds', async () => {
+    const fileStorage = new FileStorage();
+    const file = await fileStorage.createFile({ name: 'e.txt', type: 'text/plain', size: 5 } as File, 'again');
+
+    await fileStorage.deleteFile(file.id);
+    await expect(FileStorage.getDriver().deleteFile(file.id)).resolves.toBeUndefined();
+  });
+});