@storecraft/storage-google 1.0.0

package/README.md

@@ -0,0 +1,46 @@
+# Storecraft Google Cloud Storage
+
+`fetch` ready support for an `GCP` **Storage**
+
+Features:
+- Works in any `js` runtime and platform that supports `fetch`
+- Supports streaming `Get` / `Put` / `Delete`
+- Supports `presigned` `Get` / `Put` requests to offload to client
+
+## How-to
+1. Create a bucket at `GCP console` or even at `firebase`
+2. Download the `service json file`
+
+Use the values of the service file.
+
+Note:
+- You can use an empty constructor and upon `StoreCraft` init, the platform
+environment variables will be used by this storage if needed.
+
+```js
+import { GoogleStorage } from '@storecraft/storage-google';
+
+const storage = new GoogleStorage({
+  bucket: process.env.GS_BUCKET, 
+  client_email: process.env.GS_CLIENT_EMAIL, 
+  private_key: process.env.GS_PRIVATE_KEY, 
+  private_key_id: process.env.GS_PRIVATE_KEY_ID
+});
+
+// write
+await storage.putBlob(
+  'folder1/tomer.txt', 
+  new Blob(['this is some text from tomer :)'])
+);
+
+// read
+const { value } = await storage.getBlob('folder1/tomer.txt');
+const url = await storage.getSigned('folder1/tomer.txt');
+console.log('presign GET url ', url);
+
+```
+
+
+```text
+Author: Tomer Shalev (tomer.shalev@gmail.com)
+```

package/adapter.js

@@ -0,0 +1,280 @@
+import { getJWTFromServiceAccount, presign } from './adapter.utils.js';
+
+const types = {
+  'png': 'image/png',
+  'gif': 'image/gif',
+  'jpeg': 'image/jpeg',
+  'jpg': 'image/jpeg',
+  'tiff': 'image/tiff',
+  'webp': 'image/webp',
+  'txt': 'text/plain',
+  'json': 'application/json',
+}
+
+/**
+ * 
+ * @param {string} name 
+ */
+const infer_content_type = (name) => {
+  const idx = name.lastIndexOf('.');
+  if(!idx) return 'application/octet-stream';
+  const type = types[name.substring(idx + 1).trim()]
+  return type ?? 'application/octet-stream';
+}
+
+
+/**
+ * @typedef {import('./types.public.js').ServiceFile} ServiceFile
+ * @typedef {import('./types.public.js').Config} Config
+ */
+
+/**
+ * Google Storage
+ * @typedef {import('@storecraft/core/v-storage').storage_driver} storage
+ * 
+ * @implements {storage}
+ */
+export class GoogleStorage {
+  
+  /** @type {Config} */ #_config;
+
+  /**
+   * @param {Config} [config]
+   */
+  constructor(config) {
+    this.#_config = config;
+  }
+
+  get bucket() { return this.config.bucket; }
+  get config() { return this.#_config; }
+
+  /**
+   * @type {storage["init"]}
+   */
+  async init(app) {
+    if(!app)
+      return this;
+
+    this.#_config = this.#_config ?? {
+      bucket: app.platform.env.GS_BUCKET,
+      client_email: app.platform.env.GS_CLIENT_EMAIL,
+      private_key: app.platform.env.GS_PRIVATE_KEY,
+      private_key_id: app.platform.env.GS_PRIVATE_KEY_ID,
+    }
+
+    return this; 
+  }
+
+  features() {
+    /** @type {import('@storecraft/core/v-storage').StorageFeatures} */
+    const f = {
+      supports_signed_urls: true
+    }
+
+    return f;
+  }
+
+  // puts
+
+  /**
+   * @param {string} key 
+   */
+  put_file_url(key) {
+    const base = 'https://storage.googleapis.com/upload/storage/v1';
+    return `${base}/b/${this.bucket}/o?uploadType=media&name=${encodeURIComponent(key)}`
+  }
+
+  /**
+   * 
+   * @param {string} key 
+   * @param {BodyInit} body 
+   */
+  async #put_internal(key, body) {
+    const auth = 'Bearer ' + await getJWTFromServiceAccount(this.config);
+
+    const r = await fetch(
+      this.put_file_url(key),
+      {
+        method: 'POST',
+        body,
+        headers: {
+          Authorization: auth,
+          'Content-Type': 'image/png'
+        },
+        duplex: 'half'
+      }
+    );
+
+    return r.ok;
+    // console.log(r)
+    // console.log(JSON.stringify(await r.json(), null, 2))
+  }
+
+  /**
+   * 
+   * @param {string} key 
+   * @param {Blob} blob 
+   */
+  async putBlob(key, blob) {
+    return this.#put_internal(key, blob);
+  }
+
+  /**
+   * 
+   * @param {string} key 
+   * @param {ArrayBuffer} buffer 
+   */
+  async putArraybuffer(key, buffer) {
+    return this.#put_internal(key, buffer);
+  }  
+
+  /**
+   * 
+   * @param {string} key 
+   * @param {ReadableStream} stream 
+   */
+  async putStream(key, stream) {
+    return this.#put_internal(key, stream);
+  }
+
+  /**
+   * 
+   * @param {string} key 
+   */
+  async putSigned(key) {
+    const ct = infer_content_type(key);
+    const sf = this.config;
+
+    const url_signed = await presign({
+      pem_private_key: sf.private_key,
+      client_id_email: sf.client_email,
+      gcs_api_endpoint: 'https://storage.googleapis.com',
+      path: `/${this.bucket}/${key}`,
+      verb: 'PUT',
+      content_md5: '',
+      content_type: ct
+    });
+
+    return {
+      url: url_signed,
+      method: 'PUT',
+      headers: {
+        'Content-Type': ct
+      }
+    }
+
+  }
+
+  // gets
+
+  /** @param {string} key  */
+  get_file_url(key) {
+    const base = 'https://storage.googleapis.com/storage/v1';
+    return `${base}/b/${this.bucket}/o/${encodeURIComponent(key)}`
+  }
+
+  /** @param {string} key  */
+  async #get_request(key) {
+    const auth = 'Bearer ' + await getJWTFromServiceAccount(this.config);
+    return fetch(
+      this.get_file_url(key) + '?alt=media',
+      {
+        method: 'GET',
+        headers: {
+          Authorization: auth,
+        }
+      }
+    );
+  }
+
+  /**
+   * 
+   * @param {string} key 
+   */
+  async getArraybuffer(key) {
+    const r = await this.#get_request(key);
+    const b = await r.arrayBuffer();
+    return {
+      value: b, 
+      metadata: {
+        contentType: infer_content_type(key)
+      }
+    };
+  }
+
+  /**
+   * 
+   * @param {string} key 
+   */
+  async getBlob(key) {
+    const r = await this.#get_request(key);
+    // console.log(await r.json())
+    // console.log(r)
+    const b = await r.blob();
+    return {
+      value: b, 
+      metadata: {
+        contentType: infer_content_type(key)
+      }
+    };
+  }
+
+  /**
+   * 
+   * @param {string} key 
+   * @param {Response} key 
+   */
+  async getStream(key) {
+
+    const s = (await this.#get_request(key)).body
+    return {
+      value: s, 
+      metadata: {
+        contentType: infer_content_type(key)
+      }
+    };
+  }
+
+  /**
+   * 
+   * @param {string} key 
+   */
+  async getSigned(key) {
+    const sf = this.config;
+    const url_signed = await presign({
+      pem_private_key: sf.private_key,
+      client_id_email: sf.client_email,
+      gcs_api_endpoint: 'https://storage.googleapis.com',
+      path: `/${this.bucket}/${key}`,
+      verb: 'GET',
+    });
+
+    return {
+      url: url_signed,
+      method: 'GET',
+    }
+  }
+
+  // // remove
+
+  /**
+   * 
+   * @param {string} key 
+   */
+  async remove(key) {
+    const Authorization = 'Bearer ' + await getJWTFromServiceAccount(this.config);
+    const r = await fetch(
+      this.get_file_url(key), 
+      { 
+        method: 'DELETE',
+        headers: {
+          Authorization
+        }
+      }
+    );
+
+    return r.ok;
+    // console.log(r)
+  }
+}
+

package/adapter.utils.js

@@ -0,0 +1,83 @@
+import { base64, jwt } from '@storecraft/core/v-crypto';
+
+/**
+ * 
+ * @param {import('./types.public.js').ServiceFile} sf Google service account json
+ * @param {string} [aud]
+ * @returns 
+ */
+export async function getJWTFromServiceAccount(sf, aud=undefined) {
+  /** @type {Partial<import('@storecraft/core/v-crypto').jwt.JWTClaims> & Record<string, string>} */
+  const claims = {
+    scope: [
+      // 'https://www.googleapis.com/auth/cloud-platform',
+      // 'https://www.googleapis.com/auth/firebase.database',
+      // 'https://www.googleapis.com/auth/firebase.messaging',
+      // 'https://www.googleapis.com/auth/identitytoolkit',
+      // 'https://www.googleapis.com/auth/userinfo.email',
+      'https://www.googleapis.com/auth/iam',
+      'https://www.googleapis.com/auth/cloud-platform',
+      'https://www.googleapis.com/auth/devstorage.full_control'
+    ].join(' '),
+    iss: sf.client_email,
+    sub: sf.client_email,
+  };
+
+  if(aud) claims.aud=aud;
+
+  const r = await jwt.create_with_pem(
+    sf.private_key, 
+    claims, 3600, 
+    {
+      kid: sf.private_key_id
+    }
+  );
+  
+  return r.token;
+}
+
+/**
+ * Presign Google Storage resource
+ * @param {{
+ *  pem_private_key: string, 
+ *  client_id_email: string, 
+ *  gcs_api_endpoint: string, 
+ *  path: string, 
+ *  verb: string,
+ *  content_md5?: string,
+ *  content_type?: string,
+ *  expiration_delta?: number,
+ * }} signature
+ */
+export const presign = async ({pem_private_key, client_id_email, gcs_api_endpoint, 
+  path, verb, content_md5 = '', content_type='', expiration_delta=3600 }) => {
+
+  const expiration = Math.floor(Date.now()/1000) + expiration_delta;
+  const digest = [
+    verb, content_md5 ?? '', content_type ?? '', 
+    expiration, path
+  ].join('\n');
+
+  const algorithm = {
+    name: 'RSASSA-PKCS1-v1_5',
+    hash: { name: 'SHA-256' },
+  }
+  // console.log(digest)
+  const privateKey = await jwt.import_pem_pkcs8(pem_private_key, algorithm);
+  const digest_buffer = new TextEncoder().encode(digest);
+  const signature_buffer = await crypto.subtle.sign(
+    algorithm,
+    privateKey,
+    digest_buffer
+  );
+
+  const signature_b64 = base64.fromUint8Array(new Uint8Array(signature_buffer));
+  const qp = {
+    'GoogleAccessId': client_id_email,
+    'Expires': expiration.toString(),
+    'Signature': signature_b64
+  };
+  const qp_string = new URLSearchParams(qp).toString();
+
+  return`${gcs_api_endpoint}${path}?${qp_string}`;
+}

package/index.js

@@ -0,0 +1 @@
+export * from './adapter.js'

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@storecraft/storage-google",
+  "version": "1.0.0",
+  "description": "Google Storage support",
+  "license": "MIT",
+  "author": "Tomer Shalev (https://github.com/store-craft)",
+  "homepage": "https://github.com/store-craft/storecraft",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/store-craft/storecraft.git",
+    "directory": "packages/storage-google"
+  },
+  "keywords": [
+    "commerce",
+    "dashboard",
+    "code",
+    "storecraft"
+  ],
+  "scripts": {
+    "storage-google:test": "uvu -c",
+    "storage-google:publish": "npm publish --access public"
+  },
+  "type": "module",
+  "main": "index.js",
+  "types": "./types.public.d.ts",
+  "dependencies": {
+    "@storecraft/core": "^1.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.0",
+    "uvu": "^0.5.6",
+    "dotenv": "^16.3.1"
+  }
+}

package/tests/storage.test.js

@@ -0,0 +1,85 @@
+import 'dotenv/config';
+import { test } from 'uvu';
+import * as assert from 'uvu/assert';
+import { GoogleStorage } from '../adapter.js'
+import { readFile } from 'node:fs/promises';
+
+const areBlobsEqual = async (blob1, blob2) => {
+  return !Buffer.from(await blob1.arrayBuffer()).compare(
+    Buffer.from(await blob2.arrayBuffer())
+  );
+};
+
+const storage = new GoogleStorage({
+  bucket: process.env.GS_BUCKET, client_email: process.env.GS_CLIENT_EMAIL, 
+  private_key: process.env.GS_PRIVATE_KEY, private_key_id: process.env.GS_PRIVATE_KEY_ID
+});
+
+test.before(async () => { await storage.init(undefined) });
+
+test('blob put/get/delete', async () => {
+  const data = [
+    // {
+    //   key: 'folder1/tomer.txt',
+    //   blob: new Blob(['this is some text from tomer :)']),
+    // },
+    {
+      key: 'node2222.png',
+      blob: new Blob([await readFile('./node.png')])
+    }
+  ];
+
+  data.forEach(
+    async d => {
+      // write
+      await storage.putBlob(d.key, d.blob);
+      // read
+      const { value: blob_read } = await storage.getBlob(d.key);
+      const url = await storage.getSigned(d.key);
+      console.log('presign GET url ', url);
+
+      // compare
+      const equal = await areBlobsEqual(blob_read, d.blob);
+      assert.ok(equal, 'Blobs are not equal !!!');
+
+      // delete
+      // await storage.remove(d.key);
+    }
+  );
+  
+});
+
+test('blob put (presign)', async () => {
+  const data = [
+    // {
+    //   key: 'folder1/tomer.txt',
+    //   blob: new Blob(['this is some text from tomer :)']),
+    // },
+    {
+      key: 'node_test2.png',
+      blob: new Blob([await readFile('./node.png')])
+    }
+  ];
+
+  data.forEach(
+    async d => {
+      // get put presigned url
+      const { url, method, headers } = await storage.putSigned(d.key);
+      // now let's use it to upload
+      const r = await fetch(
+        url, {
+          method,
+          headers,
+          body: d.blob
+        }
+      );
+
+      console.log(url)
+
+      assert.ok(r.ok, 'upload failed')
+    }
+  );
+  
+});
+
+test.run();

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compileOnSave": false,
+  "compilerOptions": {
+    "noEmit": true,
+    "allowJs": true,
+    "checkJs": true,
+    "target": "ESNext",
+    "resolveJsonModule": true,
+    "moduleResolution": "NodeNext",
+    "module": "NodeNext",
+    "composite": true,
+  },
+  "include": ["*", "*/*", "src/*"]
+}

package/types.public.d.ts

@@ -0,0 +1,26 @@
+export * from './index.js';
+
+export type ServiceFile = {
+  type?: string;
+  project_id?: string;
+  private_key_id?: string;
+  private_key?: string;
+  client_email?: string;
+  client_id?: string;
+  auth_uri?: string;
+  token_uri?: string;
+  auth_provider_x509_cert_url?: string;
+  client_x509_cert_url?: string;
+  universe_domain?: string;
+}
+
+export type Config = {
+  /** bucket name */
+  bucket: string;
+  /** client email from the service file */
+  client_email: string;
+  /** private key */
+  private_key: string;
+  /** private key id */
+  private_key_id: string;
+}