@rsdk/nats.object-storage 5.4.0-next.10

package/src/nats-object-storage.module.ts

@@ -0,0 +1,58 @@
+import type {
+  DynamicModule,
+  OnModuleDestroy,
+  OnModuleInit,
+  Provider,
+} from '@nestjs/common';
+import { Module } from '@nestjs/common';
+import { DiscoveryModule } from '@nestjs/core';
+import {
+  createNatsClientToken,
+  NATS_CLIENT_INJECTION_TOKEN,
+  NATS_CLIENT_INJECTION_TOKEN_FOR_PACKAGE,
+} from '@rsdk/nats.common';
+
+import { NatsObjectStorageService } from './nats-object-storage.service';
+import { createObjectStorageServiceToken } from './tokens.fn';
+import { WatchObjectStorageService } from './watch-object-storage.service';
+
+@Module({
+  imports: [DiscoveryModule],
+  providers: [NatsObjectStorageService, WatchObjectStorageService],
+})
+export class NatsObjectStorageModule implements OnModuleInit, OnModuleDestroy {
+  constructor(private readonly watchService: WatchObjectStorageService) {}
+
+  public static forFeature(connectionName?: string): DynamicModule {
+    const clientProvider: Provider = {
+      provide: createObjectStorageServiceToken(connectionName),
+      useClass: NatsObjectStorageService,
+    };
+
+    return {
+      module: NatsObjectStorageModule,
+      providers: [
+        {
+          provide: NATS_CLIENT_INJECTION_TOKEN_FOR_PACKAGE,
+          useExisting: connectionName
+            ? createNatsClientToken(connectionName)
+            : NATS_CLIENT_INJECTION_TOKEN,
+        },
+        {
+          provide: 'CONNECTION_NAME',
+          useFactory: () => connectionName,
+        },
+        clientProvider,
+      ],
+      exports: [clientProvider],
+    };
+  }
+
+  async onModuleInit(): Promise<void> {
+    await this.watchService?.runWatches();
+  }
+
+  async onModuleDestroy(): Promise<void> {
+    await this.watchService?.stopWatches();
+  }
+}

package/src/nats-object-storage.service.ts

@@ -0,0 +1,141 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { delay, Timespan } from '@rsdk/common';
+import { InjectLogger, InternalException, NotFoundException } from '@rsdk/core';
+import { ILogger } from '@rsdk/logging';
+import { NATS_CLIENT_INJECTION_TOKEN_FOR_PACKAGE } from '@rsdk/nats.common';
+import { NatsConnection } from 'nats';
+import type { ObjectStoreOptions } from 'nats/lib/jetstream/types';
+
+import { ObjectStorage } from './object-storage';
+
+@Injectable()
+export class NatsObjectStorageService {
+  constructor(
+    @InjectLogger(NatsObjectStorageService)
+    private readonly logger: ILogger,
+    @Inject(NATS_CLIENT_INJECTION_TOKEN_FOR_PACKAGE)
+    private readonly nats: NatsConnection,
+  ) {}
+
+  /**
+   * Get or create object storage
+   */
+  public async create({
+    bucketName,
+    domain,
+    ...opts
+  }: Partial<ObjectStoreOptions> & {
+    bucketName: string;
+    domain?: string;
+  }): Promise<ObjectStorage> {
+    const jsOptions = domain ? { domain } : undefined;
+    const js = this.nats.jetstream(jsOptions);
+
+    try {
+      const storage = await js.views.os(bucketName, opts);
+
+      this.logger.trace('Object storage initialized', {
+        connectionName: this.nats.info,
+        bucket: bucketName,
+      });
+
+      return new ObjectStorage(storage);
+    } catch (error) {
+      this.logger.error('Failed to get or create object storage', {
+        err: error,
+        bucket: bucketName,
+      });
+      throw error;
+    }
+  }
+
+  /**
+   * Get object storage
+   */
+  public async bind({
+    bucketName,
+    domain,
+    ...opts
+  }: Partial<ObjectStoreOptions> & {
+    bucketName: string;
+    domain?: string;
+  }): Promise<ObjectStorage> {
+    const jsOptions = domain ? { domain } : undefined;
+    const js = this.nats.jetstream(jsOptions);
+
+    try {
+      if (!(await this.isBucketExists(bucketName, jsOptions))) {
+        throw new NotFoundException(
+          `Object storage bucket "${bucketName}" not found`,
+        );
+      }
+
+      const storage = await js.views.os(bucketName, opts);
+
+      this.logger.trace('Object storage bound', {
+        connectionName: this.nats.info,
+        bucket: bucketName,
+      });
+
+      return new ObjectStorage(storage);
+    } catch (error) {
+      this.logger.error('Failed to get object storage', {
+        err: error,
+        bucket: bucketName,
+      });
+      throw error;
+    }
+  }
+
+  /**
+   * Waits for the specified bucket to exist.
+   *
+   * @param bucketName - The name of the bucket to wait for.
+   * @param timeout - The maximum time to wait for the bucket to exist, in milliseconds. Defaults to infinity.
+   * @throws {InternalException} If the bucket does not exist within the specified timeout.
+   */
+  async waitForBucketExists(
+    bucketName: string,
+    timeout: number = Number.POSITIVE_INFINITY,
+  ): Promise<void> {
+    const start = Date.now();
+
+    while (true) {
+      if (await this.isBucketExists(bucketName)) {
+        this.logger.trace('Object storage bucket exists');
+        return;
+      }
+
+      if (timeout && Date.now() - start > timeout) {
+        throw new InternalException(`Bucket ${bucketName} does not exist`);
+      }
+
+      this.logger.trace(
+        'Object storage bucket does not exist, retry after 3 seconds...',
+        {
+          bucketName,
+        },
+      );
+      await delay(Timespan.second(3));
+    }
+  }
+
+  /**
+   * Checks if a bucket exists.
+   *
+   * @param bucket - The name of the bucket to check.
+   * @returns A promise that resolves to a boolean indicating whether the bucket exists or not.
+   */
+  async isBucketExists(
+    bucket: string,
+    opts?: { domain?: string },
+  ): Promise<boolean> {
+    try {
+      await this.nats.jetstream(opts).streams.get(`OBJ_${bucket}`);
+    } catch {
+      return false;
+    }
+
+    return true;
+  }
+}

package/src/object-storage.ts

@@ -0,0 +1,53 @@
+import type {
+  ObjectInfo,
+  ObjectResult,
+  ObjectStore,
+  ObjectStoreMeta,
+  ObjectStorePutOpts,
+  PurgeResponse,
+} from 'nats';
+
+export class ObjectStorage {
+  constructor(private readonly storage: ObjectStore) {}
+
+  /**
+   * Возвращает метаданные и стрим с объектом
+   */
+  public get(key: string): Promise<ObjectResult | null> {
+    return this.storage.get(key);
+  }
+
+  /**
+   * Удаляет объект
+   */
+  public delete(key: string): Promise<PurgeResponse> {
+    return this.storage.delete(key);
+  }
+
+  /**
+   * Загружает объект в хранилище
+   */
+  public put(
+    meta: ObjectStoreMeta,
+    rs: ReadableStream<Uint8Array>,
+    opts?: ObjectStorePutOpts,
+  ): Promise<ObjectInfo> {
+    return this.storage.put(meta, rs, opts);
+  }
+
+  /**
+   * Возвращает массив всех объектов в хранилище
+   */
+  public list(): Promise<ObjectInfo[]> {
+    return this.storage.list();
+  }
+
+  /**
+   * Возвращает сырой object storage Nats. Используйте этот метод
+   * только в крайнем случае! Рекомендуется работать с оберткой
+   * для безопасного взаимодействия.
+   */
+  public getRawStorage(): ObjectStore {
+    return this.storage;
+  }
+}

package/src/patch.ts

@@ -0,0 +1,292 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck issue - https://github.com/nats-io/nats.js/issues/155
+import type {
+  MsgHdrs,
+  ObjectInfo,
+  ObjectResult,
+  ObjectStoreMetaOptions,
+} from 'nats';
+import { AckPolicy, consumerOpts, deferred, MsgHdrsImpl } from 'nats';
+import type { ServerObjectInfo } from 'nats/lib/jetstream/objectstore';
+import { digestType, ObjectStoreImpl } from 'nats/lib/jetstream/objectstore';
+import { JSONCodec } from 'nats/lib/nats-base-client/codec';
+import { QueuedIteratorImpl } from 'nats/lib/nats-base-client/queued_iterator';
+import { SHA256 } from 'nats/lib/nats-base-client/sha256';
+
+function emptyReadableStream(): ReadableStream {
+  return new ReadableStream({
+    // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+    pull(c) {
+      c.enqueue(new Uint8Array(0));
+      c.close();
+    },
+  });
+}
+
+class ObjectInfoImpl implements ObjectInfo {
+  info: ServerObjectInfo;
+  hdrs!: MsgHdrs;
+  constructor(oi: ServerObjectInfo) {
+    this.info = oi;
+  }
+
+  get name(): string {
+    return this.info.name;
+  }
+
+  get description(): string {
+    return this.info.description ?? '';
+  }
+
+  get headers(): MsgHdrs {
+    if (!this.hdrs) {
+      this.hdrs = MsgHdrsImpl.fromRecord(this.info.headers || {});
+    }
+    return this.hdrs;
+  }
+
+  get options(): ObjectStoreMetaOptions | undefined {
+    return this.info.options;
+  }
+
+  get bucket(): string {
+    return this.info.bucket;
+  }
+
+  get chunks(): number {
+    return this.info.chunks;
+  }
+
+  get deleted(): boolean {
+    return this.info.deleted ?? false;
+  }
+
+  get digest(): string {
+    return this.info.digest;
+  }
+
+  get mtime(): string {
+    return this.info.mtime;
+  }
+
+  get nuid(): string {
+    return this.info.nuid;
+  }
+
+  get size(): number {
+    return this.info.size;
+  }
+
+  get revision(): number {
+    return this.info.revision;
+  }
+
+  get metadata(): Record<string, string> {
+    return this.info.metadata || {};
+  }
+
+  // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+  isLink() {
+    return (
+      this.info.options?.link !== undefined && this.info.options?.link !== null
+    );
+  }
+}
+
+ObjectStoreImpl.prototype.get = async function get(
+  name: string,
+): Promise<ObjectResult | null> {
+  const info = await this.rawInfo(name);
+  if (info === null) {
+    return null;
+  }
+
+  if (info.deleted) {
+    return null;
+  }
+
+  if (info.options && info.options.link) {
+    const ln = info.options.link.name || ' ';
+    if (ln === '') {
+      throw new Error('link is a bucket');
+    }
+    const os =
+      info.options.link.bucket === this.name
+        ? this
+        : await ObjectStoreImpl.create(this.js, info.options.link.bucket);
+
+    return os.get(ln);
+  }
+
+  const d = deferred<Error | null>();
+
+  const r: Partial<ObjectResult> = {
+    info: new ObjectInfoImpl(info),
+    error: d,
+  };
+  if (info.size === 0) {
+    r.data = emptyReadableStream();
+    d.resolve(null);
+    return r as ObjectResult;
+  }
+
+  let controller: ReadableStreamDefaultController;
+
+  const oc = consumerOpts();
+
+  oc.orderedConsumer();
+  const sha = new SHA256();
+  const subj = `$O.${this.name}.C.${info.nuid}`;
+  let sub;
+
+  try {
+    // eslint-disable-next-line deprecation/deprecation
+    sub = await this.js.subscribe(subj, oc);
+  } catch (error) {
+    if (error.message !== 'no stream matches subject') {
+      throw error;
+    }
+    const oc = consumerOpts({
+      ack_policy: AckPolicy.None,
+    });
+
+    oc.orderedConsumer();
+    oc.bindStream('OBJ_' + this.name);
+
+    // HACK: for mirrored ObjectStore
+    // eslint-disable-next-line deprecation/deprecation
+    sub = await this.js.subscribe(subj, oc);
+  }
+  // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+  (async () => {
+    for await (const jm of sub) {
+      if (jm.data.length > 0) {
+        sha.update(jm.data);
+        controller!.enqueue(jm.data);
+      }
+      if (jm.info.pending === 0) {
+        const hash = sha.digest('base64');
+
+        // go pads the hash - which should be multiple of 3 - otherwise pads with '='
+        const pad = hash.length % 3;
+        const padding = pad > 0 ? '='.repeat(pad) : '';
+        const digest = `${digestType}${hash}${padding}`;
+        if (digest === info.digest) {
+          controller!.close();
+        } else {
+          controller!.error(
+            new Error(
+              `received a corrupt object, digests do not match received: ${info.digest} calculated ${digest}`,
+            ),
+          );
+        }
+        sub.unsubscribe();
+      }
+    }
+  })()
+    .then(() => {
+      d.resolve();
+    })
+    .catch((error) => {
+      controller!.error(error);
+      d.reject(error);
+    });
+
+  r.data = new ReadableStream({
+    // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+    start(c) {
+      controller = c;
+    },
+    // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+    cancel() {
+      sub.unsubscribe();
+    },
+  });
+
+  return r as ObjectResult;
+};
+
+ObjectStoreImpl.prototype.watch = async function watch(
+  opts: Partial<{
+    ignoreDeletes?: boolean;
+    includeHistory?: boolean;
+  }> = {},
+): Promise<QueuedIterator<ObjectInfo | null>> {
+  opts.includeHistory = opts.includeHistory ?? false;
+  opts.ignoreDeletes = opts.ignoreDeletes ?? false;
+  let initialized = false;
+  const qi = new QueuedIteratorImpl<ObjectInfo | null>();
+  const subj = this._metaSubjectAll();
+
+  try {
+    await this.jsm.streams.getMessage(this.stream, { last_by_subj: subj });
+  } catch (error) {
+    if (error.code === '404') {
+      qi.push(null);
+      initialized = true;
+    } else {
+      qi.stop(error);
+    }
+  }
+  const jc = JSONCodec<ObjectInfo>();
+  const copts = consumerOpts();
+
+  copts.orderedConsumer();
+  if (opts.includeHistory) {
+    copts.deliverLastPerSubject();
+  } else {
+    // FIXME: Go's implementation doesn't seem correct - if history is not desired
+    //  the watch should only be giving notifications on new entries
+    initialized = true;
+    copts.deliverNew();
+  }
+  copts.callback((err: NatsError | null, jm: JsMsg | null) => {
+    if (err) {
+      qi.stop(err);
+      return;
+    }
+    if (jm !== null) {
+      const oi = jc.decode(jm.data);
+      if (oi.deleted && opts.ignoreDeletes === true) {
+        // do nothing
+      } else {
+        qi.push(oi);
+      }
+      if (jm.info?.pending === 0 && !initialized) {
+        initialized = true;
+        qi.push(null);
+      }
+    }
+  });
+
+  let sub;
+
+  try {
+    // eslint-disable-next-line deprecation/deprecation
+    sub = await this.js.subscribe(subj, copts);
+  } catch (error) {
+    if (error.message !== 'no stream matches subject') {
+      throw error;
+    }
+
+    copts.bindStream('OBJ_' + this.name);
+
+    // HACK: for mirrored ObjectStore
+    // eslint-disable-next-line deprecation/deprecation
+    sub = await this.js.subscribe(subj, copts);
+  }
+
+  qi._data = sub;
+  qi.iterClosed.then(() => {
+    sub.unsubscribe();
+  });
+  sub.closed
+    .then(() => {
+      qi.stop();
+    })
+    .catch((error) => {
+      qi.stop(error);
+    });
+
+  return qi;
+};

package/src/tokens.fn.ts

@@ -0,0 +1,10 @@
+export const createObjectStorageServiceToken = (
+  connectionName?: string,
+): string =>
+  `NATS_OBJECT_STORAGE_SERVICE${connectionName ? `_${connectionName}` : ''}`;
+
+export const createObjectStorageToken = (
+  bucketName: string,
+  connectionName?: string,
+): string =>
+  `NATS_OBJECT_STORAGE_${bucketName}${connectionName ? `_${connectionName}` : ''}`;

package/src/watch-object-storage.decorator.ts

@@ -0,0 +1,30 @@
+import { Reflector } from '@nestjs/core';
+import type { ObjectInfo } from 'nats';
+
+export type WatchConfig = {
+  bucket: string;
+  ignoreDeletes?: boolean;
+  includeHistory?: boolean;
+  connectionName?: string;
+};
+
+export interface ObjectStorageWatcher {
+  onDelete(entry: ObjectInfo | null): Promise<void>;
+  onPut(entry: ObjectInfo | null): Promise<void>;
+}
+
+/**
+ * Key used to mark class as a watcher for object storage.
+ */
+export const WATCH_NATS_OBJECT_STORAGE_KEY = 'WATCH_NATS_OBJECT_STORAGE';
+
+/**
+ * Decorator used to mark class as a watcher for object storage.
+ * Needs to be used in conjunction with the `ObjectStorageWatcher` interface.
+ *
+ * @param {WatchConfig} config - The configuration for watching object storage.
+ * @returns {void}
+ */
+export const WatchObjectStorage = Reflector.createDecorator<WatchConfig>({
+  key: WATCH_NATS_OBJECT_STORAGE_KEY,
+});

package/src/watch-object-storage.service.ts

@@ -0,0 +1,129 @@
+import { Inject } from '@nestjs/common';
+import { DiscoveryService } from '@nestjs/core';
+import { InjectLogger } from '@rsdk/core';
+import { ILogger } from '@rsdk/logging';
+import type { ObjectInfo, ObjectStore } from 'nats';
+
+import { NatsObjectStorageService } from './nats-object-storage.service';
+import type {
+  ObjectStorageWatcher,
+  WatchConfig,
+} from './watch-object-storage.decorator';
+import { WatchObjectStorage } from './watch-object-storage.decorator';
+
+export class WatchObjectStorageService {
+  constructor(
+    @InjectLogger(WatchObjectStorageService)
+    private readonly logger: ILogger,
+    private readonly discovery: DiscoveryService,
+    private readonly client: NatsObjectStorageService,
+    @Inject('CONNECTION_NAME')
+    private readonly connectionName?: string,
+  ) {}
+
+  /**
+   * Runs the watches for the object storage service.
+   * Waits for buckets to exist and starts the watch operation for each bucket.
+   */
+  public async runWatches(): Promise<void> {
+    const watchers = await this.discoverWatchers();
+
+    await Promise.all(
+      watchers.map(async ({ instance, metadata }) => {
+        await this.client.waitForBucketExists(metadata.bucket, 5000);
+        await this.runWatch(metadata, instance);
+        this.logger.info('Nats object storage watcher started', { metadata });
+      }),
+    );
+  }
+
+  /**
+   * Stops all watches.
+   *
+   * @note This method is not implemented yet.
+   */
+  public async stopWatches(): Promise<void> {
+    // TODO: Implement
+  }
+
+  /**
+   * Register watcher callback for object storage
+   */
+  public async watch(
+    storage: ObjectStore,
+    callback: (entry: ObjectInfo | null) => Promise<void>,
+    opts?: {
+      ignoreDeletes?: boolean;
+      includeHistory?: boolean;
+    },
+  ): Promise<void> {
+    for await (const entry of await storage.watch(opts)) {
+      callback(entry).catch((error) => {
+        this.logger.error('Error processing object storage entry', {
+          entry: entry,
+          err: error,
+        });
+      });
+    }
+  }
+
+  /**
+   * Runs the watch operation for the given metadata and watcher.
+   *
+   * @param config - The watch configuration.
+   * @param watcher - The ObjectStorageWatcher instance.
+   * @returns A Promise that resolves when the watch operation is complete.
+   */
+  private async runWatch(
+    { bucket, ...options }: WatchConfig,
+    watcher: ObjectStorageWatcher,
+  ): Promise<void> {
+    const storage = await this.client.bind({ bucketName: bucket });
+
+    this.watch(
+      storage.getRawStorage(),
+      async (event) => {
+        if (!event) return;
+
+        await (event.deleted ? watcher.onDelete(event) : watcher.onPut(event));
+      },
+      options,
+    );
+  }
+
+  /**
+   * Discovers and returns an array of ObjectStorageWatcher instances
+   * along with their corresponding metadata.
+   *
+   * @returns An array of objects containing the ObjectStorageWatcher
+   * instance and its metadata.
+   */
+  private discoverWatchers(): {
+    instance: ObjectStorageWatcher;
+    metadata: WatchConfig;
+  }[] {
+    return this.discovery.getProviders().reduce(
+      (acc, watcher) => {
+        /**
+         * NOTE: This is a fix for TypeError in call .getMetadataByDecorator() above
+         * because some tokens are not instances of classes and have no constructor.
+         * */
+        const clsRef = watcher.instance?.constructor ?? watcher.metatype;
+        if (!clsRef) {
+          return acc;
+        }
+
+        const metadata = this.discovery.getMetadataByDecorator(
+          WatchObjectStorage,
+          watcher,
+        );
+
+        if (metadata && metadata.connectionName === this.connectionName) {
+          acc.push({ instance: watcher.instance, metadata });
+        }
+        return acc;
+      },
+      [] as { instance: ObjectStorageWatcher; metadata: WatchConfig }[],
+    );
+  }
+}