@bitblit/ratchet-aws-node-only 6.0.145-alpha → 6.0.147-alpha

package/src/cli/site-uploader/site-uploader.ts

@@ -0,0 +1,143 @@
+import fs from 'fs';
+import walk from 'walk';
+import { S3Client } from '@aws-sdk/client-s3';
+import path from 'path';
+import mime from 'mime-types';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { Upload } from '@aws-sdk/lib-storage';
+
+export class SiteUploader {
+  private srcDir: string;
+  private bucketName: string;
+  private config: any;
+  private readonly s3: S3Client = new S3Client({ region: 'us-east-1' });
+
+  constructor(srcDir: string, bucketName: string, configFile: string) {
+    this.srcDir = srcDir;
+    this.bucketName = bucketName;
+    this.config = JSON.parse(fs.readFileSync(configFile).toString('ascii'));
+  }
+
+  public static createFromArgs(args: string[]): SiteUploader {
+    if (args && args.length === 3) {
+      const src = args[0];
+      const bucket = args[1];
+      const configFile = args[2];
+
+      return new SiteUploader(src, bucket, configFile);
+    } else {
+      console.log('Usage : node ratchet-site-uploader {srcDir} {bucket} {configFile} (Found ' + args + ' arguments, need 3)');
+      return null;
+    }
+  }
+
+  public static async runFromCliArgs(args: string[]): Promise<void> {
+    const inst: SiteUploader = SiteUploader.createFromArgs(args);
+    return inst.runPump();
+  }
+
+  findMatch(prefix: string, fileName: string, config: any): any {
+    let found = null;
+
+    if (prefix != null && fileName != null && config != null && config.mapping != null) {
+      config.mapping.forEach((entry) => {
+        if (found == null) {
+          if (entry.prefixMatch == null || prefix.match(entry.prefixMatch)) {
+            if (entry.fileMatch == null || fileName.match(entry.fileMatch)) {
+              found = entry;
+            }
+          }
+        }
+      });
+    }
+
+    return found;
+  }
+
+  findMime(fileName: string, config: any): string {
+    let found = null;
+
+    if (config != null && config.customMimeTypeMapping != null) {
+      Object.keys(config.customMimeTypeMapping).forEach((k) => {
+        if (found == null && fileName.endsWith(k)) {
+          found = config.customMimeTypeMapping[k];
+        }
+      });
+    }
+
+    if (found == null) {
+      found = mime.lookup(fileName);
+    }
+
+    if (found == null) {
+      found = 'binary/octet-stream';
+    }
+
+    return found;
+  }
+
+  runPump(): Promise<any> {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    return new Promise<any>((resolve, reject) => {
+      Logger.info('Uploading contents of %s to %s using %j as config', this.srcDir, this.bucketName, this.config);
+      // bucket = boto3.resource("s3").Bucket(bucket)
+
+      const options = {};
+      const walker = walk.walk(this.srcDir, options);
+
+      walker.on(
+        'file',
+        function (root, fileStats, next) {
+          Logger.info('Processing %j', fileStats.name);
+          const prefix: string = root == this.srcDir ? '' : root.substring(this.srcDir.length + 1) + '/';
+
+          const proc: any = this.findMatch(prefix, fileStats.name, this.config);
+          const key: string = prefix + fileStats.name;
+          Logger.info('Uploading file : %s/%s to key %s with %j', root, fileStats.name, key, proc);
+
+          const params: any = proc && proc.putParams ? JSON.parse(JSON.stringify(proc.putParams)) : {};
+
+          params.Bucket = this.bucketName;
+          params.Key = key;
+          params.Body = fs.readFileSync(path.join(root, fileStats.name));
+
+          if (!params.ContentType) {
+            params.ContentType = this.findMime(fileStats.name, this.config);
+          }
+
+          const upload: Upload = new Upload({
+            client: this.s3,
+            params: params,
+            tags: [],
+            queueSize: 4,
+            partSize: 1024 * 1024 * 5,
+            leavePartsOnError: false,
+          });
+
+          upload.on('httpUploadProgress', (progress) => {
+            Logger.debug('Uploading : %s', progress);
+          });
+          upload
+            .done()
+            .then((result) => {
+              Logger.info('Finished upload of %s: %j', key, result);
+              next();
+            })
+            .catch((err) => {
+              Logger.warn('%s failed to upload : %s : Continuing', key, err);
+              next();
+            });
+        }.bind(this),
+      );
+
+      walker.on('errors', function (root, nodeStatsArray, next) {
+        next();
+      });
+
+      walker.on('end', function () {
+        Logger.info('All done');
+        resolve(true);
+      });
+    });
+  }
+}

package/src/cli/start-instance-and-ssh.ts

@@ -0,0 +1,70 @@
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { Ec2Ratchet } from '@bitblit/ratchet-aws/ec2/ec2-ratchet';
+import { spawnSync, SpawnSyncReturns } from 'child_process';
+import os from 'os';
+import path from 'path';
+import { Instance } from '@aws-sdk/client-ec2';
+import { Ec2InstanceUtil } from '../ec2/ec2-instance-util.js';
+
+export class StartInstanceAndSsh {
+  private instanceId: string;
+  private publicKeyFile: string;
+  private instanceOsUser: string;
+  private region: string;
+  private availabilityZone: string;
+  private ec2Ratchet: Ec2Ratchet;
+  private instanceUtil: Ec2InstanceUtil;
+
+  constructor(
+    instanceId: string,
+    publicKeyFile: string = path.join(os.homedir(), '.ssh', 'id_rsa.pub'),
+    instanceOsUser: string = 'ec2-user',
+    region: string = 'us-east-1',
+    availabilityZone: string = 'us-east-1a',
+  ) {
+    this.instanceId = instanceId;
+    this.publicKeyFile = publicKeyFile;
+    this.instanceOsUser = instanceOsUser;
+    this.region = region;
+    this.availabilityZone = availabilityZone;
+
+    this.ec2Ratchet = new Ec2Ratchet(this.region, this.availabilityZone);
+    this.instanceUtil = new Ec2InstanceUtil(this.ec2Ratchet);
+  }
+
+  public static createFromArgs(args: string[]): StartInstanceAndSsh {
+    if (args?.length === 1 || args?.length === 2) {
+      const instanceId = args[0];
+      //const publicKeyFile = args[1];
+
+      return new StartInstanceAndSsh(instanceId); // , publicKeyFile);
+    } else {
+      Logger.info('Usage : ratchet-start-instance-and-ssh {instanceId} {publicKeyFile} (Found %s arguments, need 1 or 2)', args);
+      return null;
+    }
+  }
+
+  public static async runFromCliArgs(args: string[]): Promise<void> {
+    const inst: StartInstanceAndSsh = StartInstanceAndSsh.createFromArgs(args);
+    return inst.run();
+  }
+
+  public async run(): Promise<any> {
+    //return new Promise<any>(async (res, rej) => {
+    const instance: Instance = await this.instanceUtil.startInstanceAndUploadPublicKeyFile(
+      this.instanceId,
+      this.publicKeyFile,
+      this.instanceOsUser,
+    );
+    if (instance) {
+      Logger.info('Instance IP address is %s', instance.PublicIpAddress);
+      const ret: SpawnSyncReturns<Buffer> = spawnSync('ssh', [this.instanceOsUser + '@' + instance.PublicIpAddress], {
+        stdio: 'inherit',
+      });
+      Logger.info('%j', ret);
+    } else {
+      Logger.info('No such instance found - check your AWS keys? : %s', this.instanceId);
+    }
+    //});
+  }
+}

package/src/daemon/daemon-like.ts

@@ -0,0 +1,34 @@
+import { DaemonProcessState } from './daemon-process-state.js';
+import { DaemonProcessCreateOptions } from './daemon-process-create-options.js';
+
+/*
+    Classes implementing this interface provide the ability to monitor and update long-running
+    processes in S3.
+
+    For all functions, if group is not provided the default group will be used.
+*/
+export interface DaemonLike {
+  get defaultGroup(): string;
+
+  keyToPublicToken(key: string, expirationSeconds: number): Promise<string>;
+
+  start(options: DaemonProcessCreateOptions): Promise<DaemonProcessState>;
+
+  clean(group?: string, olderThanSeconds?: number): Promise<DaemonProcessState[]>;
+
+  listKeys(group?: string): Promise<string[]>;
+
+  list(group?: string): Promise<DaemonProcessState[]>;
+
+  updateMessage(id: string, newMessage: string): Promise<DaemonProcessState>;
+
+  statFromPublicToken(publicToken: string): Promise<DaemonProcessState>;
+
+  stat(key: string): Promise<DaemonProcessState>;
+
+  abort(id: string): Promise<DaemonProcessState>;
+
+  error(id: string, error: string): Promise<DaemonProcessState>;
+
+  finalize(id: string, contents: Buffer): Promise<DaemonProcessState>;
+}

package/src/daemon/daemon-process-create-options.ts

@@ -0,0 +1,7 @@
+export interface DaemonProcessCreateOptions {
+  title: string;
+  contentType: string;
+  group: string;
+  meta: any;
+  targetFileName: string;
+}

package/src/daemon/daemon-process-state-public-token.ts

@@ -0,0 +1,5 @@
+import { JwtTokenBase } from '@bitblit/ratchet-common/jwt/jwt-token-base';
+
+export interface DaemonProcessStatePublicToken extends JwtTokenBase {
+  daemonKey: string;
+}

package/src/daemon/daemon-process-state.ts

@@ -0,0 +1,16 @@
+export interface DaemonProcessState {
+  id: string;
+
+  title: string;
+  targetFileName: string;
+
+  lastUpdatedEpochMS: number;
+  lastUpdatedMessage: string;
+
+  startedEpochMS: number;
+  completedEpochMS: number;
+  meta: any;
+  error: string;
+  link: string;
+  contentType: string;
+}

package/src/daemon/daemon-stream-data-options.ts

@@ -0,0 +1,6 @@
+import { Progress } from '@aws-sdk/lib-storage';
+
+export interface DaemonStreamDataOptions {
+  overrideTargetFileName?: string;
+  progressFn?: (progress: Progress) => void;
+}

package/src/daemon/daemon-util.spec.ts

@@ -0,0 +1,133 @@
+import { DaemonProcessState } from './daemon-process-state.js';
+import { DaemonUtil } from './daemon-util.js';
+import fs, { ReadStream } from 'fs';
+import { DaemonProcessCreateOptions } from './daemon-process-create-options.js';
+
+import { Subject } from 'rxjs';
+import { PassThrough } from 'stream';
+import { S3Client } from '@aws-sdk/client-s3';
+import { beforeEach, describe, expect, test } from 'vitest';
+import { mock, MockProxy } from 'vitest-mock-extended';
+
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { CsvRatchet } from '@bitblit/ratchet-node-only/csv/csv-ratchet';
+import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
+import { LoggerLevelName } from '@bitblit/ratchet-common/logger/logger-level-name';
+import { S3CacheRatchetLike } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet-like';
+import { S3CacheRatchet } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet';
+
+let mockS3CR: MockProxy<S3CacheRatchetLike>;
+
+describe('#DaemonUtil', function () {
+  beforeEach(() => {
+    mockS3CR = mock<S3CacheRatchetLike>();
+  });
+
+  test('should test the daemon util', async () => {
+    mockS3CR.getDefaultBucket.mockReturnValueOnce('TEST-BUCKET');
+    mockS3CR.fetchMetaForCacheFile.mockResolvedValue({
+      Metadata: { daemon_meta: '{"id":"testid", "completedEpochMS":123456}' },
+      $metadata: null,
+    });
+    mockS3CR.preSignedDownloadUrlForCacheFile.mockResolvedValue('https://test-link');
+
+    const t1: DaemonProcessState = await DaemonUtil.stat(mockS3CR, 'test1.csv');
+    Logger.info('Got : %j', t1);
+    expect(t1).not.toBeNull();
+    expect(t1.link).not.toBeNull();
+
+    /*
+        let id = 'test';
+        const newDaemonOptions: DaemonProcessCreateOptions = {
+            title: 'test',
+            contentType: 'text/csv',
+            group: 'NA',
+            meta: {},
+            targetFileName: 'test.csv'
+        };
+
+        const t2: DaemonProcessState = await DaemonUtil.start(cache, id,'test1.csv', newDaemonOptions);
+        Logger.info('Got : %j', t2);
+
+
+    const t2: DaemonProcessState = await DaemonUtil.updateMessage(mockS3CR, 'test1.csv', 'msg : ' + new Date());
+    Logger.info('Got : %j', t2);
+
+    const result: DaemonProcessState = await DaemonUtil.stat(mockS3CR, 'test1.csv');
+
+    Logger.info('Got : %j', result);
+
+    expect(result).toBeTruthy();
+    Logger.info('Got objects : %j', result);
+         */
+  });
+
+  test.skip('should test the daemon util streaming', async () => {
+    const s3: S3Client = new S3Client({ region: 'us-east-1' });
+    const cache: S3CacheRatchetLike = new S3CacheRatchet(s3, 'test-bucket');
+    const key: string = 's3-cache-ratchet.spec.ts';
+
+    const newDaemonOptions: DaemonProcessCreateOptions = {
+      title: 'test',
+      contentType: 'text/plain',
+      group: 'NA',
+      meta: {},
+      targetFileName: 's3-cache-ratchet.spec.ts',
+    };
+
+    const _t2: DaemonProcessState = await DaemonUtil.start(cache, key, 's3-cache-ratchet.spec.ts', newDaemonOptions);
+
+    const t1: DaemonProcessState = await DaemonUtil.stat(cache, key);
+    Logger.info('Got : %j', t1);
+
+    const stream: ReadStream = fs.createReadStream('test/aws/s3-cache-ratchet.spec.ts');
+    const result: DaemonProcessState = await DaemonUtil.streamDataAndFinish(cache, key, stream);
+
+    expect(result).toBeTruthy();
+    Logger.info('Got objects : %j', result);
+  });
+
+  test.skip('should stream objects to a csv', async () => {
+    Logger.setLevel(LoggerLevelName.debug);
+    const sub: Subject<TestItem> = new Subject<TestItem>();
+    const out: PassThrough = new PassThrough();
+    const s3: S3Client = new S3Client({ region: 'us-east-1' });
+    const cache: S3CacheRatchet = new S3CacheRatchet(s3, 'test-bucket');
+    const key: string = 'test.csv';
+
+    const newDaemonOptions: DaemonProcessCreateOptions = {
+      title: 'test',
+      contentType: 'text/csv',
+      group: 'NA',
+      meta: {},
+      targetFileName: 'test.csv',
+    };
+    const _t2: DaemonProcessState = await DaemonUtil.start(cache, key, key, newDaemonOptions);
+
+    const dProm: Promise<DaemonProcessState> = DaemonUtil.streamDataAndFinish(cache, key, out);
+
+    const prom: Promise<number> = CsvRatchet.streamObjectsToCsv<TestItem>(sub, out); //, opts);
+
+    for (let i = 1; i < 6; i++) {
+      Logger.debug('Proc : %d', i);
+      sub.next({ a: i, b: 'test ' + i + ' ,,' });
+      await PromiseRatchet.wait(10);
+    }
+    sub.complete();
+
+    Logger.debug('Waiting on write');
+
+    const result: number = await prom;
+    Logger.debug('Write complete');
+
+    const val: DaemonProcessState = await dProm;
+
+    expect(result).toEqual(5);
+    Logger.debug('Have res : %d and val : \n%j', result, val);
+  });
+});
+
+export interface TestItem {
+  a: number;
+  b: string;
+}

package/src/daemon/daemon-util.ts

@@ -0,0 +1,204 @@
+import { DaemonProcessState } from './daemon-process-state.js';
+import { DaemonProcessCreateOptions } from './daemon-process-create-options.js';
+import {
+  CompleteMultipartUploadCommandOutput,
+  HeadObjectOutput,
+  PutObjectCommand,
+  PutObjectCommandInput,
+  PutObjectOutput,
+  PutObjectRequest,
+} from '@aws-sdk/client-s3';
+import { Readable } from 'stream';
+import { Upload } from '@aws-sdk/lib-storage';
+import { DaemonStreamDataOptions } from './daemon-stream-data-options.js';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { S3CacheRatchetLike } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet-like';
+
+/**
+ * Internal utilities which are here for the USE OF THE DAEMON OBJECT ONLY - if you are trying to use this
+ * class outside of Ratchet, you are doing it wrong.  Instantiate a Daemon object and use that instead.
+ *
+ * The details of storage and retrieval of a DaemonProcessState is meant to be transparent to the user
+ */
+export class DaemonUtil {
+  public static DEFAULT_CONTENT: Buffer = Buffer.from('DAEMON_PLACEHOLDER');
+  public static DAEMON_METADATA_KEY: string = 'daemon_meta'; // Must be lowercase for s3
+
+  public static async start(
+    cache: S3CacheRatchetLike,
+    id: string,
+    s3Key: string,
+    options: DaemonProcessCreateOptions,
+  ): Promise<DaemonProcessState> {
+    try {
+      options.meta ??= {};
+
+      Logger.info('Starting daemon, key: %s, options: %j', s3Key, options);
+      const now: number = new Date().getTime();
+
+      const newState: DaemonProcessState = {
+        id: id,
+        title: options.title,
+        lastUpdatedEpochMS: now,
+        lastUpdatedMessage: 'Created',
+        targetFileName: options.targetFileName,
+
+        startedEpochMS: now,
+        completedEpochMS: null,
+        meta: options.meta,
+        error: null,
+        link: null,
+        contentType: options.contentType,
+      };
+
+      const rval: DaemonProcessState = await DaemonUtil.writeState(cache, s3Key, newState, DaemonUtil.DEFAULT_CONTENT);
+      return rval;
+    } catch (err) {
+      Logger.error('Error while trying to start a daemon: %j %s', options, err);
+      throw err;
+    }
+  }
+
+  public static async writeState(
+    cache: S3CacheRatchetLike,
+    s3Key: string,
+    newState: DaemonProcessState,
+    contents: Uint8Array, // This was Buffer before, moving to Uint8array to be node/browser ok.  Need a streaming version
+  ): Promise<DaemonProcessState> {
+    try {
+      const s3meta: any = {};
+      newState.lastUpdatedEpochMS = new Date().getTime();
+      s3meta[DaemonUtil.DAEMON_METADATA_KEY] = JSON.stringify(newState);
+
+      const params: PutObjectCommandInput = {
+        Bucket: cache.getDefaultBucket(),
+        Key: s3Key,
+        ContentType: newState.contentType,
+        Metadata: s3meta,
+        Body: contents,
+      };
+      if (newState.targetFileName) {
+        params.ContentDisposition = 'attachment;filename="' + newState.targetFileName + '"';
+      }
+
+      const written: PutObjectOutput = await cache.getS3Client().send(new PutObjectCommand(params));
+      Logger.silly('Daemon wrote : %s', written);
+
+      return DaemonUtil.stat(cache, s3Key);
+    } catch (err) {
+      Logger.error('Error while trying to write a daemon stat: %j %s', newState, err);
+      throw err;
+    }
+  }
+
+  public static async streamDataAndFinish(
+    cache: S3CacheRatchetLike,
+    s3Key: string,
+    data: Readable,
+    options?: DaemonStreamDataOptions,
+  ): Promise<DaemonProcessState> {
+    Logger.debug('Streaming data to %s', s3Key);
+    const inStat: DaemonProcessState = await DaemonUtil.updateMessage(cache, s3Key, 'Streaming data');
+    inStat.completedEpochMS = new Date().getTime();
+    inStat.lastUpdatedMessage = 'Complete';
+
+    const s3meta: any = {};
+    s3meta[DaemonUtil.DAEMON_METADATA_KEY] = JSON.stringify(inStat);
+
+    const params: PutObjectRequest = {
+      Bucket: cache.getDefaultBucket(),
+      Key: s3Key,
+      ContentType: inStat.contentType,
+      Metadata: s3meta,
+      Body: data,
+    };
+    const targetFileName: string =
+      StringRatchet.trimToNull(options?.overrideTargetFileName) || StringRatchet.trimToNull(inStat?.targetFileName);
+    if (targetFileName) {
+      params.ContentDisposition = 'attachment;filename="' + targetFileName + '"';
+    }
+
+    const upload: Upload = new Upload({
+      client: cache.getS3Client(),
+      params: params,
+      tags: [],
+      queueSize: 4,
+      partSize: 1024 * 1024 * 5,
+      leavePartsOnError: false,
+    });
+
+    if (options?.progressFn) {
+      upload.on('httpUploadProgress', options.progressFn);
+    }
+    const written: CompleteMultipartUploadCommandOutput = await upload.done();
+
+    Logger.silly('Daemon wrote : %s', written);
+
+    return DaemonUtil.stat(cache, s3Key);
+  }
+
+  public static async updateMessage(cache: S3CacheRatchetLike, s3Key: string, newMessage: string): Promise<DaemonProcessState> {
+    try {
+      const inStat: DaemonProcessState = await DaemonUtil.stat(cache, s3Key);
+      inStat.lastUpdatedMessage = newMessage;
+      return DaemonUtil.writeState(cache, s3Key, inStat, DaemonUtil.DEFAULT_CONTENT);
+    } catch (err) {
+      Logger.error('Error while trying to update a daemon message: %j %s', s3Key, err);
+      throw err;
+    }
+  }
+
+  public static async stat(s3Cache: S3CacheRatchetLike, path: string): Promise<DaemonProcessState> {
+    try {
+      Logger.debug('Daemon stat for path %s / %s', s3Cache.getDefaultBucket(), path);
+      let stat: DaemonProcessState = null;
+
+      const meta: HeadObjectOutput = await s3Cache.fetchMetaForCacheFile(path);
+      Logger.debug('Daemon: Meta is %j', meta);
+      const metaString: string = meta && meta.Metadata ? meta.Metadata[DaemonUtil.DAEMON_METADATA_KEY] : null;
+      if (metaString) {
+        stat = JSON.parse(metaString) as DaemonProcessState;
+
+        if (stat.completedEpochMS && !stat.error) {
+          stat.link = await s3Cache.preSignedDownloadUrlForCacheFile(path);
+        }
+      } else {
+        Logger.warn('No metadata found! (Head was %j)', meta);
+      }
+      return stat;
+    } catch (err) {
+      Logger.error('Error while trying to fetch a daemon state: %j %s', path, err);
+      throw err;
+    }
+  }
+
+  public static async abort(s3Cache: S3CacheRatchetLike, path: string): Promise<DaemonProcessState> {
+    return DaemonUtil.error(s3Cache, path, 'Aborted');
+  }
+  public static async error(s3Cache: S3CacheRatchetLike, path: string, error: string): Promise<DaemonProcessState> {
+    try {
+      const inStat: DaemonProcessState = await DaemonUtil.stat(s3Cache, path);
+      inStat.error = error;
+      inStat.completedEpochMS = new Date().getTime();
+      return DaemonUtil.writeState(s3Cache, path, inStat, DaemonUtil.DEFAULT_CONTENT);
+    } catch (err) {
+      Logger.error('Error while trying to write a daemon error: %j %s', path, err);
+      throw err;
+    }
+  }
+
+  public static async finalize(s3Cache: S3CacheRatchetLike, path: string, contents: Buffer): Promise<DaemonProcessState> {
+    try {
+      Logger.info('Finalizing daemon %s with %d bytes', path, contents.length);
+      const inStat: DaemonProcessState = await DaemonUtil.stat(s3Cache, path);
+      inStat.completedEpochMS = new Date().getTime();
+      inStat.lastUpdatedMessage = 'Complete';
+
+      return DaemonUtil.writeState(s3Cache, path, inStat, contents);
+    } catch (err) {
+      Logger.error('Error while trying to finalize a daemon: %j %s', path, err);
+      throw err;
+    }
+  }
+}