@proteinjs/db-file-storage-driver-gcs 1.4.32 → 1.5.0

package/src/GoogleCloudStorageDriver.ts

@@ -1,5 +1,5 @@
 import { Storage } from '@google-cloud/storage';
-import { File, FileStorageDriver } from '@proteinjs/db-file';
+import { File, FileStorageDriver, FileStorageError } from '@proteinjs/db-file';
 import {
   GoogleCloudStorageConfig,
   getDefaultGoogleCloudStorageConfigFactory,
@@ -9,73 +9,170 @@ import {
  *  a video, short enough that a leaked URL goes stale quickly. */
 const DEFAULT_SIGNED_URL_TTL_MS = 15 * 60 * 1000;
 
+/** Failures of the connection itself, as Node names them — the store was never reached. */
+const NETWORK_ERROR_CODES = ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EAI_AGAIN', 'EPIPE'];
+
+/**
+ * ERRORS: every operation runs inside {@link GoogleCloudStorageDriver.guarded}, so what leaves this
+ * driver on a failure is a `FileStorageError` — a code, a plain message, the HTTP status — and
+ * never the storage client's own error. The client's error holds the request it made (`config`,
+ * `response.request`, `response.config`), and that request holds this driver's access token in its
+ * `Authorization` header: an application that logged the error it caught would write a live
+ * credential. A missing object is the `not-found` code on every read and overwrite.
+ */
 export class GoogleCloudStorageDriver implements FileStorageDriver {
   private storage: Storage;
   private bucketName: string;
+  private objectPrefix: string;
 
   constructor(config?: GoogleCloudStorageConfig) {
-    const { projectId, bucketName, storageOptions } = config ? config : this.getDefaultConfig();
+    const { projectId, bucketName, objectPrefix, storageOptions } = config ? config : this.getDefaultConfig();
     this.storage = new Storage({ ...storageOptions, projectId });
     this.bucketName = bucketName;
+    this.objectPrefix = objectPrefix ?? '';
   }
 
   async createFile(file: File, fileData: string): Promise<void> {
-    const bucket = this.storage.bucket(this.bucketName);
-    const gcsFile = bucket.file(file.id);
+    const gcsFile = this.object(file.id);
 
     // The GCS object stores the file's TRUE bytes (the interface string is base64 transport).
     // Raw bytes at rest are what make signed-URL serving correct: the browser reads the object
     // directly, so base64 text at rest would corrupt every binary served that way.
-    await gcsFile.save(Buffer.from(fileData, 'base64'), {
-      metadata: {
-        contentType: file.type,
+    await this.guarded('createFile', file.id, () =>
+      gcsFile.save(Buffer.from(fileData, 'base64'), {
         metadata: {
-          fileId: file.id,
-          fileName: file.name,
-          fileSize: file.size.toString(),
+          contentType: file.type,
+          metadata: {
+            fileId: file.id,
+            fileName: file.name,
+            fileSize: file.size.toString(),
+          },
         },
-      },
-    });
+      })
+    );
   }
 
   async getFileData(fileId: string): Promise<string> {
-    const file = this.storage.bucket(this.bucketName).file(fileId);
-    const [fileContent] = await file.download();
+    const file = this.object(fileId);
+    const [fileContent] = await this.guarded('getFileData', fileId, () => file.download());
     return fileContent.toString('base64');
   }
 
   async updateFileData(fileId: string, data: string): Promise<void> {
-    const file = this.storage.bucket(this.bucketName).file(fileId);
-    await file.save(Buffer.from(data, 'base64'));
+    const gcsFile = this.object(fileId);
+    const bytes = Buffer.from(data, 'base64');
+
+    // Writing an object's bytes creates a new generation, and a new generation holds only the
+    // metadata sent with that write. The object is named by file id, so there is no extension to
+    // infer a content type from: an overwrite that sends none is served as
+    // `application/octet-stream`, and a browser following `GET /file/:id` downloads a picture
+    // instead of showing it. So the overwrite restates what the object already says about
+    // itself. `contentEncoding` is left out on purpose — it describes the bytes being replaced.
+    const [existing] = await this.guarded('updateFileData', fileId, () => gcsFile.getMetadata());
+    await this.guarded('updateFileData', fileId, () =>
+      gcsFile.save(bytes, {
+        metadata: {
+          contentType: existing.contentType,
+          cacheControl: existing.cacheControl,
+          contentDisposition: existing.contentDisposition,
+          contentLanguage: existing.contentLanguage,
+          metadata: { ...existing.metadata, fileSize: bytes.length.toString() },
+        },
+        // The metadata read and the byte write are two requests. Pinning the write to the
+        // generation that was read makes the pair one decision: an overwrite that raced another
+        // writer fails loudly instead of restating metadata that is no longer the object's.
+        preconditionOpts: { ifGenerationMatch: existing.generation },
+      })
+    );
   }
 
   async updateFile(file: File): Promise<void> {
-    const gcsFile = this.storage.bucket(this.bucketName).file(file.id);
-    await gcsFile.setMetadata({
-      contentType: file.type,
-      metadata: {
-        fileId: file.id,
-        fileName: file.name,
-        fileSize: file.size.toString(),
-      },
-    });
+    const gcsFile = this.object(file.id);
+    await this.guarded('updateFile', file.id, () =>
+      gcsFile.setMetadata({
+        contentType: file.type,
+        metadata: {
+          fileId: file.id,
+          fileName: file.name,
+          fileSize: file.size.toString(),
+        },
+      })
+    );
   }
 
   async getSignedUrl(fileId: string, options?: { ttlMs?: number }): Promise<string> {
-    const gcsFile = this.storage.bucket(this.bucketName).file(fileId);
-    const [url] = await gcsFile.getSignedUrl({
-      version: 'v4',
-      action: 'read',
-      expires: Date.now() + (options?.ttlMs ?? DEFAULT_SIGNED_URL_TTL_MS),
-    });
+    const gcsFile = this.object(fileId);
+    const [url] = await this.guarded('getSignedUrl', fileId, () =>
+      gcsFile.getSignedUrl({
+        version: 'v4',
+        action: 'read',
+        expires: Date.now() + (options?.ttlMs ?? DEFAULT_SIGNED_URL_TTL_MS),
+      })
+    );
     return url;
   }
 
   async deleteFile(fileId: string): Promise<void> {
-    const gcsFile = this.storage.bucket(this.bucketName).file(fileId);
+    const gcsFile = this.object(fileId);
     // ignoreNotFound implements the driver contract's idempotency: a retried row delete must not
     // wedge because a prior attempt already removed the blob. Every other failure throws loudly.
-    await gcsFile.delete({ ignoreNotFound: true });
+    await this.guarded('deleteFile', fileId, () => gcsFile.delete({ ignoreNotFound: true }));
+  }
+
+  /**
+   * THE one place an object's name is decided (`<objectPrefix><file id>`): every operation
+   * addresses its object through here, so a configured prefix holds for all of them or none.
+   */
+  private object(fileId: string) {
+    return this.storage.bucket(this.bucketName).file(`${this.objectPrefix}${fileId}`);
+  }
+
+  /**
+   * THE one door a storage-client call leaves through. Whatever the client throws is replaced by
+   * this driver's own error; nothing of the client's error object rides on it.
+   */
+  private async guarded<T>(operation: string, fileId: string, call: () => Promise<T>): Promise<T> {
+    try {
+      return await call();
+    } catch (clientError: unknown) {
+      throw this.driverError(operation, fileId, clientError);
+    }
+  }
+
+  /**
+   * Reads exactly three plain facts off the client's error — an HTTP status, a connection error
+   * code, its message TEXT — and builds the driver's error from them. The client's error itself,
+   * its `config`, `request`, `response` and `errors` are never copied, wrapped or kept as a cause.
+   */
+  private driverError(operation: string, fileId: string, clientError: unknown): FileStorageError {
+    const status = this.httpStatus(clientError);
+    const connectionFailed = NETWORK_ERROR_CODES.includes((clientError as { code?: unknown } | null)?.code as string);
+    const message = (clientError as { message?: unknown } | null)?.message;
+    return new FileStorageError(
+      connectionFailed ? 'unavailable' : FileStorageError.codeForStatus(status),
+      `${operation} failed for file ${fileId}`,
+      { status, detail: typeof message === 'string' ? message : undefined }
+    );
+  }
+
+  /** The client reports the HTTP status as its error's `code`; an upload's error carries it as `status` or on its `response`. */
+  private httpStatus(clientError: unknown): number | undefined {
+    const candidate = clientError as
+      | { code?: unknown; status?: unknown; response?: { status?: unknown; statusCode?: unknown } }
+      | null
+      | undefined;
+    for (const value of [
+      candidate?.code,
+      candidate?.status,
+      candidate?.response?.status,
+      candidate?.response?.statusCode,
+    ]) {
+      const status = typeof value === 'string' && /^\d{3}$/.test(value) ? Number(value) : value;
+      if (typeof status === 'number' && status >= 100 && status <= 599) {
+        return status;
+      }
+    }
+    return undefined;
   }
 
   private getDefaultConfig(): GoogleCloudStorageConfig {

package/test/GoogleCloudStorageDriverByteEncoding.test.ts

@@ -2,7 +2,8 @@ import { GoogleCloudStorageDriver } from '../src/GoogleCloudStorageDriver';
 
 const saveMock = jest.fn();
 const downloadMock = jest.fn();
-const fileMock = jest.fn(() => ({ save: saveMock, download: downloadMock }));
+const getMetadataMock = jest.fn();
+const fileMock = jest.fn(() => ({ save: saveMock, download: downloadMock, getMetadata: getMetadataMock }));
 const bucketMock = jest.fn(() => ({ file: fileMock }));
 
 jest.mock('@google-cloud/storage', () => ({
@@ -25,6 +26,7 @@ describe('GoogleCloudStorageDriver byte encoding', () => {
   beforeEach(() => {
     saveMock.mockReset();
     downloadMock.mockReset();
+    getMetadataMock.mockReset();
     fileMock.mockClear();
     bucketMock.mockClear();
   });
@@ -55,6 +57,7 @@ describe('GoogleCloudStorageDriver byte encoding', () => {
 
   it('updateFileData stores the decoded true bytes', async () => {
     saveMock.mockResolvedValue(undefined);
+    getMetadataMock.mockResolvedValue([{ contentType: 'video/mp4', generation: '1' }, {}]);
 
     await driver.updateFileData('file-1', base64);
 

package/test/GoogleCloudStorageDriverDefaultConfig.test.ts

@@ -0,0 +1,97 @@
+import { SourceRepository } from '@proteinjs/reflection';
+import { GoogleCloudStorageDriver } from '../src/GoogleCloudStorageDriver';
+import type {
+  DefaultGoogleCloudStorageConfigFactory,
+  GoogleCloudStorageConfig,
+} from '../src/DefaultGoogleCloudStorageConfigFactory';
+
+/** An in-memory bucket keyed by OBJECT NAME: whatever name the driver decides is the key here. */
+const objects = new Map<string, Buffer>();
+const buckets: string[] = [];
+
+const fileMock = jest.fn((name: string) => ({
+  save: async (bytes: Buffer) => {
+    objects.set(name, bytes);
+  },
+  download: async () => {
+    const existing = objects.get(name);
+    if (!existing) {
+      throw Object.assign(new Error(`No such object: ${name}`), { code: 404 });
+    }
+    return [existing];
+  },
+  getSignedUrl: async () => [`https://storage.example/${name}?signed`],
+  delete: async () => {
+    objects.delete(name);
+  },
+}));
+
+jest.mock('@google-cloud/storage', () => ({
+  Storage: jest.fn(() => ({
+    bucket: (name: string) => {
+      buckets.push(name);
+      return { file: fileMock };
+    },
+  })),
+}));
+
+const FACTORY_TYPE = '@proteinjs/db-file-storage-driver-gcs/DefaultGoogleCloudStorageConfigFactory';
+type SourceRepositoryInternals = { objectCache: Record<string, unknown[]> };
+
+/**
+ * An application rarely hands the driver a config: it implements
+ * `DefaultGoogleCloudStorageConfigFactory` and the framework builds `new GoogleCloudStorageDriver()`
+ * with NO argument. That is the path most deployments run, so it is the path the prefix has to hold
+ * on: a prefix honoured only for an explicit config would leave those deployments writing bare ids
+ * while believing their files are grouped.
+ */
+describe('GoogleCloudStorageDriver built from the default config factory (no constructor argument)', () => {
+  const bytes = Buffer.from('the bytes of a picture');
+  const picture = (id: string) => ({ id, name: 'holiday.jpg', type: 'image/jpeg', size: bytes.length }) as never;
+  const objectCache = () => (SourceRepository.get() as unknown as SourceRepositoryInternals).objectCache;
+  /** The implementation an application would ship, seeded where the framework looks it up. */
+  const implementFactory = (config: GoogleCloudStorageConfig) => {
+    const factory: DefaultGoogleCloudStorageConfigFactory = { getConfig: () => config } as never;
+    objectCache()[FACTORY_TYPE] = [factory];
+  };
+
+  beforeEach(() => {
+    objects.clear();
+    buckets.length = 0;
+  });
+
+  afterEach(() => {
+    delete objectCache()[FACTORY_TYPE];
+  });
+
+  it('honours the factory’s objectPrefix on every operation: write, read, sign, delete', async () => {
+    implementFactory({ projectId: 'test-project', bucketName: 'test-bucket', objectPrefix: 'deployment-a/' });
+    const driver = new GoogleCloudStorageDriver();
+
+    await driver.createFile(picture('file-1'), bytes.toString('base64'));
+    expect(Array.from(objects.keys())).toEqual(['deployment-a/file-1']);
+    expect(buckets).toEqual(['test-bucket']);
+
+    expect(await driver.getFileData('file-1')).toBe(bytes.toString('base64'));
+    expect(await driver.getSignedUrl('file-1')).toBe('https://storage.example/deployment-a/file-1?signed');
+
+    await driver.deleteFile('file-1');
+    expect(Array.from(objects.keys())).toEqual([]);
+  });
+
+  it('a default-config driver and an explicit-config driver with the same prefix reach the same object', async () => {
+    const config = { projectId: 'test-project', bucketName: 'test-bucket', objectPrefix: 'deployment-a/' };
+    implementFactory(config);
+    await new GoogleCloudStorageDriver().createFile(picture('file-1'), bytes.toString('base64'));
+
+    expect(await new GoogleCloudStorageDriver(config).getFileData('file-1')).toBe(bytes.toString('base64'));
+  });
+
+  it('a factory that names no prefix keeps objects at the file id alone', async () => {
+    implementFactory({ projectId: 'test-project', bucketName: 'test-bucket' });
+
+    await new GoogleCloudStorageDriver().createFile(picture('file-1'), bytes.toString('base64'));
+
+    expect(Array.from(objects.keys())).toEqual(['file-1']);
+  });
+});

package/test/GoogleCloudStorageDriverErrors.test.ts

@@ -0,0 +1,244 @@
+import http from 'http';
+import util from 'util';
+import { AddressInfo } from 'net';
+import { OAuth2Client } from 'google-auth-library';
+import { FileStorageError } from '@proteinjs/db-file';
+import { GoogleCloudStorageDriver } from '../src/GoogleCloudStorageDriver';
+
+/** An obviously fake access token — what the storage client sends as `Authorization: Bearer …`. */
+const MARKER = 'FAKE-TOKEN-MARKER';
+
+type DriverInternals = { storage: unknown };
+
+/** Every way an application reads an error it caught: printed, serialised, inspected to any depth. */
+function everyReadingOf(error: unknown): string {
+  return [
+    String(error),
+    (error as Error)?.stack ?? '',
+    JSON.stringify(error),
+    JSON.stringify({ error }),
+    util.inspect(error),
+    util.inspect(error, { depth: null, showHidden: true, getters: true }),
+  ].join('\n');
+}
+
+/** What the operation threw — anything at all; the assertions decide what it is allowed to be. */
+async function failureOf(run: () => Promise<unknown>): Promise<FileStorageError> {
+  return await run().then(
+    () => {
+      throw new Error('expected the operation to fail');
+    },
+    (error) => error
+  );
+}
+
+function expectDriverOwned(error: unknown) {
+  expect(everyReadingOf(error)).not.toContain(MARKER);
+  expect(FileStorageError.is(error)).toBe(true);
+  expect(
+    Object.getOwnPropertyNames(error)
+      .filter((name) => name !== 'status')
+      .sort()
+  ).toEqual(['code', 'message', 'name', 'stack']);
+}
+
+/**
+ * THE REAL STORAGE CLIENT against a stub server on the loopback interface (nothing leaves the
+ * machine), authorised by a client whose token is the fake marker. The client's own error holds
+ * the request it made — headers included — so the marker is inside it; an application that logs
+ * the error it caught with a deep inspect (a development log writer does) writes the token out.
+ * What leaves the driver is the driver's own error.
+ */
+describe('GoogleCloudStorageDriver errors, through the real storage client', () => {
+  let server: http.Server;
+  let driver: GoogleCloudStorageDriver;
+  let answerWith = 404;
+  let authorizedRequests = 0;
+
+  beforeAll(async () => {
+    server = http.createServer((request, response) => {
+      if (request.headers.authorization === `Bearer ${MARKER}`) {
+        authorizedRequests++;
+      }
+      request.resume();
+      request.on('end', () => {
+        response.writeHead(answerWith, { 'content-type': 'application/json' });
+        response.end(JSON.stringify({ error: { code: answerWith, message: `the store answered ${answerWith}` } }));
+      });
+    });
+    await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+
+    const tokenHolder = new OAuth2Client();
+    tokenHolder.setCredentials({ access_token: MARKER, expiry_date: Date.now() + 60 * 60 * 1000 });
+    driver = new GoogleCloudStorageDriver({
+      projectId: 'test-project',
+      bucketName: 'test-bucket',
+      storageOptions: {
+        apiEndpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
+        useAuthWithCustomEndpoint: true,
+        authClient: tokenHolder,
+        retryOptions: { autoRetry: false },
+      },
+    });
+  });
+
+  afterAll(async () => {
+    await new Promise((resolve) => server.close(resolve));
+  });
+
+  beforeEach(() => {
+    answerWith = 404;
+    authorizedRequests = 0;
+  });
+
+  const picture = { id: 'file-1', name: 'holiday.jpg', type: 'image/jpeg', size: 5 } as never;
+  const operations: [string, () => Promise<unknown>][] = [
+    ['getFileData', () => driver.getFileData('file-1')],
+    ['updateFileData', () => driver.updateFileData('file-1', Buffer.from('bytes').toString('base64'))],
+    ['updateFile', () => driver.updateFile(picture)],
+  ];
+
+  it.each(operations)('%s on a missing object: not-found, and the token is nowhere in the error', async (name, run) => {
+    const failure = await failureOf(run);
+
+    expect(authorizedRequests).toBeGreaterThan(0); // the token WAS on the request that failed
+    expectDriverOwned(failure);
+    expect(failure.code).toEqual('not-found');
+    expect(failure.status).toEqual(404);
+    expect(failure.message).toContain(`${name} failed for file file-1: not-found (HTTP 404)`);
+  });
+
+  it('createFile refused by the store (the upload path, its own client error): the driver’s error', async () => {
+    answerWith = 403;
+
+    const failure = await failureOf(() => driver.createFile(picture, Buffer.from('bytes').toString('base64')));
+
+    expectDriverOwned(failure);
+    expect(failure.code).toEqual('forbidden');
+    expect(failure.status).toEqual(403);
+    expect(failure.message).toContain('createFile failed for file file-1: forbidden (HTTP 403)');
+  });
+
+  it('deleteFile refused by the store: forbidden, and the token is nowhere in the error', async () => {
+    answerWith = 403;
+
+    const failure = await failureOf(() => driver.deleteFile('file-1'));
+
+    expect(authorizedRequests).toBeGreaterThan(0);
+    expectDriverOwned(failure);
+    expect(failure.code).toEqual('forbidden');
+  });
+
+  it('deleteFile on a missing object is still success — deletes are idempotent', async () => {
+    await expect(driver.deleteFile('file-1')).resolves.toBeUndefined();
+  });
+
+  it.each([
+    [401, 'forbidden'],
+    [403, 'forbidden'],
+    [412, 'precondition-failed'],
+    [429, 'unavailable'],
+    [503, 'unavailable'],
+    [400, 'unknown'],
+  ])('the store answering %i reads as %s', async (status, code) => {
+    answerWith = status;
+
+    const failure = await failureOf(() => driver.getFileData('file-1'));
+
+    expectDriverOwned(failure);
+    expect(failure.code).toEqual(code);
+    expect(failure.status).toEqual(status);
+  });
+
+  it('a store that cannot be reached reads as unavailable', async () => {
+    const closed = http.createServer();
+    await new Promise<void>((resolve) => closed.listen(0, '127.0.0.1', resolve));
+    const port = (closed.address() as AddressInfo).port;
+    await new Promise((resolve) => closed.close(resolve));
+    const unreachable = new GoogleCloudStorageDriver({
+      projectId: 'test-project',
+      bucketName: 'test-bucket',
+      storageOptions: { apiEndpoint: `http://127.0.0.1:${port}`, retryOptions: { autoRetry: false } },
+    });
+
+    const failure = await failureOf(() => unreachable.getFileData('file-1'));
+
+    expectDriverOwned(failure);
+    expect(failure.code).toEqual('unavailable');
+    expect(failure.status).toBeUndefined();
+  });
+
+  it('a signing failure is the driver’s error too', async () => {
+    const failure = await failureOf(() => driver.getSignedUrl('file-1'));
+
+    expectDriverOwned(failure);
+    expect(failure.message).toContain('getSignedUrl failed for file file-1');
+  });
+});
+
+/**
+ * THE WORST CASE, whatever client version is installed: an error that carries the credential
+ * everywhere one can — its message (a signed URL), an enumerable request config, the request, the
+ * response's config and request, its error list, its cause. None of it leaves the driver.
+ */
+describe('GoogleCloudStorageDriver errors, whatever the client’s error carries', () => {
+  const authorization = { Authorization: `Bearer ${MARKER}` };
+  const signedUrl = `https://storage.example/test-bucket/file-1?X-Goog-Signature=${MARKER}&X-Goog-Expires=900`;
+  const clientError = () =>
+    Object.assign(new Error(`Request to ${signedUrl} failed with Authorization: Bearer ${MARKER}`), {
+      code: 404,
+      config: { url: signedUrl, headers: authorization },
+      request: { headers: authorization },
+      response: { status: 404, config: { headers: authorization }, request: { headers: authorization } },
+      errors: [{ message: `denied for ${signedUrl}` }],
+      cause: { config: { headers: authorization } },
+    });
+
+  function driverWhoseClientThrows(): GoogleCloudStorageDriver {
+    const failing = async () => {
+      throw clientError();
+    };
+    const driver = new GoogleCloudStorageDriver({ projectId: 'test-project', bucketName: 'test-bucket' });
+    (driver as unknown as DriverInternals).storage = {
+      bucket: () => ({
+        file: () => ({
+          save: failing,
+          download: failing,
+          getMetadata: failing,
+          setMetadata: failing,
+          getSignedUrl: failing,
+          delete: failing,
+        }),
+      }),
+    };
+    return driver;
+  }
+
+  const picture = { id: 'file-1', name: 'holiday.jpg', type: 'image/jpeg', size: 5 } as never;
+  const operations: [string, (driver: GoogleCloudStorageDriver) => Promise<unknown>][] = [
+    ['createFile', (driver) => driver.createFile(picture, '')],
+    ['getFileData', (driver) => driver.getFileData('file-1')],
+    ['updateFileData', (driver) => driver.updateFileData('file-1', '')],
+    ['updateFile', (driver) => driver.updateFile(picture)],
+    ['getSignedUrl', (driver) => driver.getSignedUrl('file-1')],
+    ['deleteFile', (driver) => driver.deleteFile('file-1')],
+  ];
+
+  it('the stand-in error really does carry the marker everywhere', () => {
+    expect(String(clientError())).toContain(MARKER);
+    expect(JSON.stringify(clientError())).toContain(MARKER);
+    expect(util.inspect(clientError())).toContain(MARKER);
+  });
+
+  it.each(operations)(
+    '%s: a caller printing, serialising or inspecting the error never reads the marker',
+    async (name, run) => {
+      const failure = await failureOf(() => run(driverWhoseClientThrows()));
+
+      expectDriverOwned(failure);
+      expect(failure.code).toEqual('not-found');
+      expect(failure.message).toContain(`${name} failed for file file-1: not-found (HTTP 404)`);
+      expect(failure.message).toContain('https://storage.example/test-bucket/file-1 failed');
+    }
+  );
+});