@naturalcycles/cloud-storage-lib 1.0.0

package/src/commonStorageKeyValueDB.ts

@@ -0,0 +1,109 @@
+import * as Buffer from 'buffer'
+import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib'
+import { pMap, StringMap } from '@naturalcycles/js-lib'
+import { ReadableTyped, transformMapSimple } from '@naturalcycles/nodejs-lib'
+import { CommonStorage, FileEntry } from './commonStorage'
+
+export interface CommonStorageKeyValueDBCfg {
+  storage: CommonStorage
+  bucketName: string
+}
+
+/**
+ * CommonKeyValueDB, backed up by a CommonStorage implementation.
+ *
+ * Each Table is represented as a Folder.
+ * Each Item is represented as a File:
+ * fileName is ${id} (without extension)
+ * file contents is ${v} (Buffer)
+ */
+export class CommonStorageKeyValueDB implements CommonKeyValueDB {
+  constructor(public cfg: CommonStorageKeyValueDBCfg) {}
+
+  async ping(): Promise<void> {
+    await this.cfg.storage.ping(this.cfg.bucketName)
+  }
+
+  async createTable(_table: string, _opt?: CommonDBCreateOptions): Promise<void> {
+    // no-op
+  }
+
+  /**
+   * Allows to pass `SomeBucket.SomeTable` in `table`, to override a Bucket.
+   */
+  private getBucketAndPrefix(table: string): { bucketName: string; prefix: string } {
+    const [part1, part2] = table.split('.')
+
+    if (part2) {
+      return {
+        bucketName: part1!,
+        prefix: part2,
+      }
+    }
+
+    // As is
+    return {
+      bucketName: this.cfg.bucketName,
+      prefix: table,
+    }
+  }
+
+  async deleteByIds(table: string, ids: string[]): Promise<void> {
+    const { bucketName, prefix } = this.getBucketAndPrefix(table)
+    await pMap(ids, async id => {
+      await this.cfg.storage.deletePath(bucketName, [prefix, id].join('/'))
+    })
+  }
+
+  async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
+    const { bucketName, prefix } = this.getBucketAndPrefix(table)
+
+    const map: StringMap<Buffer> = {}
+
+    await pMap(ids, async id => {
+      const buf = await this.cfg.storage.getFile(bucketName, [prefix, id].join('/'))
+      if (buf) map[id] = buf
+    })
+
+    return ids.map(id => [id, map[id]] as KeyValueDBTuple).filter(t => t[1])
+  }
+
+  async saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void> {
+    const { bucketName, prefix } = this.getBucketAndPrefix(table)
+
+    await pMap(entries, async ([id, content]) => {
+      await this.cfg.storage.saveFile(bucketName, [prefix, id].join('/'), content)
+    })
+  }
+
+  streamIds(table: string, limit?: number): ReadableTyped<string> {
+    const { bucketName, prefix } = this.getBucketAndPrefix(table)
+    const index = prefix.length + 1
+
+    return this.cfg.storage
+      .getFileNamesStream(bucketName, prefix, { limit })
+      .pipe(transformMapSimple<string, string>(f => f.slice(index)))
+  }
+
+  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
+    const { bucketName, prefix } = this.getBucketAndPrefix(table)
+
+    return this.cfg.storage
+      .getFilesStream(bucketName, prefix, { limit })
+      .pipe(transformMapSimple<FileEntry, Buffer>(f => f.content))
+  }
+
+  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
+    const { bucketName, prefix } = this.getBucketAndPrefix(table)
+    const index = prefix.length + 1
+
+    return this.cfg.storage
+      .getFilesStream(bucketName, prefix, { limit })
+      .pipe(
+        transformMapSimple<FileEntry, KeyValueDBTuple>(({ filePath, content }) => [
+          filePath.slice(index),
+          content,
+        ]),
+      )
+  }
+}

package/src/inMemoryCommonStorage.ts

@@ -0,0 +1,115 @@
+import { Readable, Writable } from 'stream'
+import { StringMap } from '@naturalcycles/js-lib'
+import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
+
+export class InMemoryCommonStorage implements CommonStorage {
+  /**
+   * data[bucketName][filePath] = Buffer
+   */
+  data: StringMap<StringMap<Buffer>> = {}
+
+  publicMap: StringMap<StringMap<boolean>> = {}
+
+  async ping(): Promise<void> {}
+
+  async getBucketNames(): Promise<string[]> {
+    return Object.keys(this.data)
+  }
+
+  getBucketNamesStream(): ReadableTyped<string> {
+    return Readable.from(Object.keys(this.data))
+  }
+
+  async fileExists(bucketName: string, filePath: string): Promise<boolean> {
+    return !!this.data[bucketName]?.[filePath]
+  }
+
+  async getFile(bucketName: string, filePath: string): Promise<Buffer | null> {
+    return this.data[bucketName]?.[filePath] || null
+  }
+
+  async saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void> {
+    this.data[bucketName] ||= {}
+    this.data[bucketName]![filePath] = content
+  }
+
+  async deletePath(bucketName: string, prefix: string): Promise<void> {
+    Object.keys(this.data[bucketName] || {}).forEach(filePath => {
+      if (filePath.startsWith(prefix)) {
+        delete this.data[bucketName]![filePath]
+      }
+    })
+  }
+
+  async getFileNames(bucketName: string, prefix: string): Promise<string[]> {
+    return Object.keys(this.data[bucketName] || {}).filter(filePath => filePath.startsWith(prefix))
+  }
+
+  getFileNamesStream(
+    bucketName: string,
+    prefix: string,
+    opt: CommonStorageGetOptions = {},
+  ): ReadableTyped<string> {
+    return Readable.from(
+      Object.keys(this.data[bucketName] || {})
+        .filter(filePath => filePath.startsWith(prefix))
+        .slice(0, opt.limit),
+    )
+  }
+
+  getFilesStream(
+    bucketName: string,
+    prefix: string,
+    opt: CommonStorageGetOptions = {},
+  ): ReadableTyped<FileEntry> {
+    return Readable.from(
+      Object.entries(this.data[bucketName] || {})
+        .map(([filePath, content]) => ({ filePath, content }))
+        .filter(f => f.filePath.startsWith(prefix))
+        .slice(0, opt.limit),
+    )
+  }
+
+  getFileReadStream(bucketName: string, filePath: string): Readable {
+    return Readable.from(this.data[bucketName]![filePath]!)
+  }
+
+  getFileWriteStream(_bucketName: string, _filePath: string): Writable {
+    throw new Error('Method not implemented.')
+  }
+
+  async setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void> {
+    this.publicMap[bucketName] ||= {}
+    this.publicMap[bucketName]![filePath] = isPublic
+  }
+
+  async getFileVisibility(bucketName: string, filePath: string): Promise<boolean> {
+    return !!this.publicMap[bucketName]?.[filePath]
+  }
+
+  async copyFile(
+    fromBucket: string,
+    fromPath: string,
+    toPath: string,
+    toBucket?: string,
+  ): Promise<void> {
+    const tob = toBucket || fromBucket
+    this.data[fromBucket] ||= {}
+    this.data[tob] ||= {}
+    this.data[tob]![toPath] = this.data[fromBucket]![fromPath]
+  }
+
+  async moveFile(
+    fromBucket: string,
+    fromPath: string,
+    toPath: string,
+    toBucket?: string,
+  ): Promise<void> {
+    const tob = toBucket || fromBucket
+    this.data[fromBucket] ||= {}
+    this.data[tob] ||= {}
+    this.data[tob]![toPath] = this.data[fromBucket]![fromPath]
+    delete this.data[fromBucket]![fromPath]
+  }
+}

package/src/index.ts

@@ -0,0 +1,24 @@
+import { CloudStorage, CloudStorageCfg } from './cloudStorage'
+import { CommonStorage, CommonStorageGetOptions } from './commonStorage'
+import { CommonStorageBucket, CommonStorageBucketCfg } from './commonStorageBucket'
+import { CommonStorageKeyValueDB, CommonStorageKeyValueDBCfg } from './commonStorageKeyValueDB'
+import { InMemoryCommonStorage } from './inMemoryCommonStorage'
+import { GCPServiceAccount } from './model'
+import { runCommonStorageTest } from './testing/commonStorageTest'
+
+export type {
+  CommonStorage,
+  CloudStorageCfg,
+  CommonStorageGetOptions,
+  GCPServiceAccount,
+  CommonStorageBucketCfg,
+  CommonStorageKeyValueDBCfg,
+}
+
+export {
+  CloudStorage,
+  CommonStorageKeyValueDB,
+  CommonStorageBucket,
+  InMemoryCommonStorage,
+  runCommonStorageTest,
+}

package/src/model.ts

@@ -0,0 +1,5 @@
+export interface GCPServiceAccount {
+  client_email: string
+  private_key: string
+  project_id: string
+}

package/src/testing/commonStorageTest.ts

@@ -0,0 +1,134 @@
+import { _range, pMap, StringMap } from '@naturalcycles/js-lib'
+import { readableToArray } from '@naturalcycles/nodejs-lib'
+import { CommonStorage, FileEntry } from '../commonStorage'
+
+const TEST_FOLDER = 'test/subdir'
+
+const TEST_ITEMS = _range(10).map(n => ({
+  id: `id_${n + 1}`,
+  n,
+  even: n % 2 === 0,
+}))
+
+const TEST_ITEMS2 = _range(10).map(n => ({
+  fileType: 2,
+  id: `id_${n + 1}`,
+  n,
+  even: n % 2 === 0,
+}))
+
+const TEST_ITEMS3 = _range(10).map(n => ({
+  fileType: 3,
+  id: `id_${n + 1}`,
+  n,
+  even: n % 2 === 0,
+}))
+
+const TEST_FILES: FileEntry[] = [TEST_ITEMS, TEST_ITEMS2, TEST_ITEMS3].map((obj, i) => ({
+  filePath: `${TEST_FOLDER}/file_${i + 1}.json`,
+  content: Buffer.from(JSON.stringify(obj)),
+}))
+
+/**
+ * This test suite must be idempotent.
+ */
+export function runCommonStorageTest(storage: CommonStorage, bucketName: string): void {
+  // test('createBucket', async () => {
+  //   await storage.createBucket(bucketName)
+  // })
+
+  test('ping', async () => {
+    await storage.ping()
+  })
+
+  test('listBuckets', async () => {
+    const buckets = await storage.getBucketNames()
+    console.log(buckets)
+  })
+
+  test('streamBuckets', async () => {
+    const buckets = await readableToArray(storage.getBucketNamesStream())
+    console.log(buckets)
+  })
+
+  test('prepare: clear bucket', async () => {
+    await pMap(
+      TEST_FILES.map(f => f.filePath),
+      async filePath => await storage.deletePath(bucketName, filePath),
+    )
+  })
+
+  test('listFileNames on root should return empty', async () => {
+    const fileNames = await storage.getFileNames(bucketName, '')
+    expect(fileNames).toEqual([])
+  })
+
+  test(`listFileNames on ${TEST_FOLDER} should return empty`, async () => {
+    const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER)
+    expect(fileNames).toEqual([])
+  })
+
+  test('streamFileNames on root should return empty', async () => {
+    const fileNames = await readableToArray(storage.getFileNamesStream(bucketName, ''))
+    expect(fileNames).toEqual([])
+  })
+
+  test(`exists should return empty array`, async () => {
+    await pMap(TEST_FILES, async f => {
+      const exists = await storage.fileExists(bucketName, f.filePath)
+      expect(exists).toBe(false)
+    })
+  })
+
+  test(`saveFiles, then listFileNames, streamFileNames and getFiles should return just saved files`, async () => {
+    const testFilesMap = Object.fromEntries(TEST_FILES.map(f => [f.filePath, f.content]))
+
+    // It's done in the same test to ensure "strong consistency"
+    await pMap(TEST_FILES, async f => await storage.saveFile(bucketName, f.filePath, f.content))
+
+    const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER)
+    expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort())
+
+    const streamedFileNames = await readableToArray(
+      storage.getFileNamesStream(bucketName, TEST_FOLDER),
+    )
+    expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort())
+
+    const filesMap: StringMap<Buffer> = {}
+
+    await pMap(fileNames, async filePath => {
+      filesMap[filePath] = (await storage.getFile(bucketName, filePath))!
+    })
+
+    expect(filesMap).toEqual(testFilesMap)
+
+    await pMap(fileNames, async filePath => {
+      const exists = await storage.fileExists(bucketName, filePath)
+      expect(exists).toBe(true)
+    })
+  })
+
+  test('cleanup', async () => {
+    await storage.deletePath(bucketName, '')
+  })
+
+  // Cannot update access control for an object when uniform bucket-level access is enabled. Read more at https://cloud.google.com/storage/docs/uniform-bucket-level-access
+  /*
+  test(`get/set FilesVisibility`, async () => {
+    const fileNames = TEST_FILES.map(f => f.filePath)
+
+    let map = await storage.getFilesVisibility(bucketName, fileNames)
+    expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, false])))
+
+    await storage.setFilesVisibility(bucketName, fileNames, true)
+
+    map = await storage.getFilesVisibility(bucketName, fileNames)
+    expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, true])))
+
+    await storage.setFilesVisibility(bucketName, fileNames, false)
+
+    map = await storage.getFilesVisibility(bucketName, fileNames)
+    expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, false])))
+  })
+   */
+}