@bitblit/ratchet-aws-node-only 6.0.146-alpha → 6.0.147-alpha

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bitblit/ratchet-aws-node-only",
-  "version": "6.0.146-alpha",
+  "version": "6.0.147-alpha",
   "description": "Common tools for use with AWS (Node only)",
   "note-on-side-effects": "Technically the entries in 'bin' below might be side effects, but they are called explicitly",
   "sideEffects": false,
@@ -9,6 +9,7 @@
   },
   "type": "module",
   "files": [
+    "src/**",
     "lib/**",
     "bin/**"
   ],
@@ -56,8 +57,8 @@
   },
   "license": "Apache-2.0",
   "dependencies": {
-    "@bitblit/ratchet-aws": "6.0.146-alpha",
-    "@bitblit/ratchet-common": "6.0.146-alpha"
+    "@bitblit/ratchet-aws": "6.0.147-alpha",
+    "@bitblit/ratchet-common": "6.0.147-alpha"
   },
   "optionalDependencies": {
     "@aws-sdk/client-athena": "3.922.0",
@@ -76,8 +77,8 @@
     "@aws-sdk/client-athena": "^3.922.0",
     "@aws-sdk/client-sts": "^3.922.0",
     "@aws-sdk/types": "^3.922.0",
-    "@bitblit/ratchet-aws": "6.0.146-alpha",
-    "@bitblit/ratchet-common": "6.0.146-alpha",
+    "@bitblit/ratchet-aws": "6.0.147-alpha",
+    "@bitblit/ratchet-common": "6.0.147-alpha",
     "@smithy/abort-controller": "^4.2.3",
     "@smithy/smithy-client": "^4.9.1",
     "@smithy/util-waiter": "^4.2.3",

package/src/athena/alb-athena-log-ratchet.spec.ts

@@ -0,0 +1,36 @@
+import { AthenaRatchet } from './athena-ratchet.js';
+
+import { AlbAthenaLogRatchet, AlbLogRecord, AlbLogRecordQuery } from './alb-athena-log-ratchet.js';
+import { S3Client } from '@aws-sdk/client-s3';
+import { AthenaClient } from '@aws-sdk/client-athena';
+import { describe, expect, test } from 'vitest';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { LoggerLevelName } from '@bitblit/ratchet-common/logger/logger-level-name';
+import { TimeZoneRatchet } from '@bitblit/ratchet-common/lang/time-zone-ratchet';
+
+describe('#AlbAthenaLogRatchet', function () {
+  test('Placeholder', async () => {
+    expect(2).toEqual(2);
+  });
+
+  test.skip('should test a query', async () => {
+    Logger.setLevel(LoggerLevelName.debug);
+    const athena: AthenaClient = new AthenaClient({ region: 'us-east-1' });
+    const s3: S3Client = new S3Client({ region: 'us-east-1' });
+
+    const outputDir: string = 's3://alb-log-bucket/temp';
+    const athRatchet: AthenaRatchet = new AthenaRatchet(athena, s3, outputDir);
+    const srv: AlbAthenaLogRatchet = new AlbAthenaLogRatchet(athRatchet, 'alb_logs.log_table');
+
+    const qry: AlbLogRecordQuery = {
+      startTimeEpochMS: TimeZoneRatchet.PACIFIC.startOfTodayEpochMS(),
+      endTimeEpochMS: TimeZoneRatchet.PACIFIC.startOfTodayEpochMS() + 1000 * 60 * 10,
+      limit: 10,
+    };
+
+    const result: AlbLogRecord[] = await srv.fetchAlbLogRecords(qry);
+
+    expect(result).toBeTruthy();
+    Logger.info('Got objects : %j', result);
+  });
+});

package/src/athena/alb-athena-log-ratchet.ts

@@ -0,0 +1,223 @@
+import { readFileSync, ReadStream } from 'fs';
+import path from 'path';
+import { AthenaRatchet } from './athena-ratchet.js';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { EsmRatchet } from '@bitblit/ratchet-common/lang/esm-ratchet';
+import { CsvRatchet } from '@bitblit/ratchet-node-only/csv/csv-ratchet';
+import { S3Client } from '@aws-sdk/client-s3';
+import { S3Ratchet } from '@bitblit/ratchet-aws/s3/s3-ratchet';
+
+// A class to simplify reading an Athena table based on ALB Logs
+// NOTE: This class only runs on Node since it depends on fs and path
+export class AlbAthenaLogRatchet {
+  constructor(
+    private athena: AthenaRatchet,
+    private athenaTableName: string,
+  ) {
+    RequireRatchet.notNullOrUndefined(athena, 'athena');
+    RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(athenaTableName), 'athenaTableName');
+  }
+
+  public async updatePartitions(
+    rootPath: string,
+    s3: S3Client,
+    startTimeEpochMS: number = new Date().getTime() - 1000 * 60 * 60 * 24,
+    endTimeEpochMS: number = new Date().getTime(),
+  ): Promise<string[]> {
+    RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
+    RequireRatchet.notNullOrUndefined(s3, 's3');
+    Logger.info('Updating partitions for %s from %s', this.athenaTableName, rootPath);
+    const _bucketName: string = S3Ratchet.extractBucketFromURL(rootPath);
+    const _rootKey: string = S3Ratchet.extractKeyFromURL(rootPath);
+
+    let current: number = startTimeEpochMS;
+    const clauses: string[] = [];
+    while (current < endTimeEpochMS) {
+      const dateUtcVal: string = new Date(current).toISOString().substring(0, 10);
+      Logger.info('d:%s', dateUtcVal);
+      const dateParts: string[] = dateUtcVal.split('-');
+      clauses.push(
+        "PARTITION (date_utc_partition='" +
+          dateUtcVal +
+          "') LOCATION '" +
+          rootPath +
+          '/' +
+          dateParts[0] +
+          '/' +
+          dateParts[1] +
+          '/' +
+          dateParts[2] +
+          "'",
+      );
+      current += 1000 * 60 * 60 * 24;
+    }
+    if (clauses.length > 0) {
+      const stmt: string = 'ALTER TABLE ' + this.athenaTableName + ' ADD IF NOT EXISTS \n' + clauses.join('\n');
+      await this.athena.runQueryToObjects<any>(stmt);
+    } else {
+      Logger.warn('Not updating partitions - no time between time clauses');
+    }
+    return clauses;
+  }
+
+  public async createTable(rootPath: string, replaceIfPresent: boolean = false): Promise<boolean> {
+    RequireRatchet.true(S3Ratchet.checkS3UrlForValidity(rootPath), 'root path not valid');
+    let rval: boolean = false;
+    Logger.info('Creating ALB table %s', this.athenaTableName);
+    if (replaceIfPresent) {
+      Logger.info('Replace if present specified, removed old table');
+      try {
+        await this.athena.runQueryToObjects<any>('drop table ' + this.athenaTableName);
+      } catch (err) {
+        Logger.info('Drop error : %j', err);
+      }
+    }
+
+    let tableCreateQry: string = readFileSync(
+      path.join(EsmRatchet.fetchDirName(import.meta.url), '../static/albAthenaTableCreate.txt'),
+    ).toString();
+    tableCreateQry = StringRatchet.simpleTemplateFill(tableCreateQry, { TABLE_NAME: this.athenaTableName, ALB_LOG_ROOT: rootPath }, true);
+    Logger.info('Creating table with %s', tableCreateQry);
+
+    try {
+      await this.athena.runQueryToObjects<any>(tableCreateQry);
+      rval = true;
+    } catch (err) {
+      Logger.error('Error creating table : %s', err);
+    }
+    return rval;
+  }
+
+  public static async readLogObjectsFromCsvStream(readStream: ReadStream): Promise<AlbLogRecord[]> {
+    return CsvRatchet.streamParse(readStream, (p) => p);
+  }
+
+  public static async readLogObjectsFromFile(fileName: string): Promise<AlbLogRecord[]> {
+    return CsvRatchet.fileParse(fileName, (p) => p);
+  }
+
+  public async fetchAlbLogRecords(qry: AlbLogRecordQuery): Promise<AlbLogRecord[]> {
+    const tempFile: string = await this.fetchAlbLogRecordsToFile(qry);
+    return AlbAthenaLogRatchet.readLogObjectsFromFile(tempFile);
+  }
+
+  public async fetchAlbLogRecordsToFile(qry: AlbLogRecordQuery, outputFileName: string = null): Promise<string> {
+    Logger.info('Querying %s : %j', this.athenaTableName, qry);
+
+    let qrySt: string = 'select * from ' + this.athenaTableName + ' where 1=1 ';
+    if (qry.startTimeEpochMS) {
+      // Dates use the partition for speed and time column for accuracy
+      if (qry.startTimeEpochMS) {
+        qrySt += " AND time >= '" + new Date(qry.startTimeEpochMS).toISOString() + "'";
+        qrySt += " AND date_utc_partition >='" + new Date(qry.startTimeEpochMS).toISOString().substring(0, 10) + "'";
+      }
+      if (qry.endTimeEpochMS) {
+        qrySt += " AND time < '" + new Date(qry.endTimeEpochMS).toISOString() + "'";
+        qrySt += " AND date_utc_partition <='" + new Date(qry.endTimeEpochMS).toISOString().substring(0, 10) + "'";
+      }
+      if (qry.requestUrlFilter) {
+        qrySt += " AND request_url LIKE '" + qry.requestUrlFilter + "'";
+      }
+      if (qry.limit) {
+        qrySt += ' LIMIT ' + qry.limit;
+      }
+    }
+
+    const result: string = await this.athena.runQueryToFile(qrySt, null, outputFileName);
+    return result;
+  }
+
+  public static readonly CREATE_TABLE_STATEMENT: string =
+    'CREATE EXTERNAL TABLE IF NOT EXISTS `{{TABLE NAME}}`(\n' +
+    "  `type` string COMMENT '',\n" +
+    "  `time` string COMMENT '',\n" +
+    "  `elb` string COMMENT '',\n" +
+    "  `client_ip` string COMMENT '',\n" +
+    "  `client_port` int COMMENT '',\n" +
+    "  `target_ip` string COMMENT '',\n" +
+    "  `target_port` int COMMENT '',\n" +
+    "  `request_processing_time` double COMMENT '',\n" +
+    "  `target_processing_time` double COMMENT '',\n" +
+    "  `response_processing_time` double COMMENT '',\n" +
+    "  `elb_status_code` string COMMENT '',\n" +
+    "  `target_status_code` string COMMENT '',\n" +
+    "  `received_bytes` bigint COMMENT '',\n" +
+    "  `sent_bytes` bigint COMMENT '',\n" +
+    "  `request_verb` string COMMENT '',\n" +
+    "  `request_url` string COMMENT '',\n" +
+    "  `request_proto` string COMMENT '',\n" +
+    "  `user_agent` string COMMENT '',\n" +
+    "  `ssl_cipher` string COMMENT '',\n" +
+    "  `ssl_protocol` string COMMENT '',\n" +
+    "  `target_group_arn` string COMMENT '',\n" +
+    "  `trace_id` string COMMENT '',\n" +
+    "  `domain_name` string COMMENT '',\n" +
+    "  `chosen_cert_arn` string COMMENT '',\n" +
+    "  `matched_rule_priority` string COMMENT '',\n" +
+    "  `request_creation_time` string COMMENT '',\n" +
+    "  `actions_executed` string COMMENT '',\n" +
+    "  `redirect_url` string COMMENT '',\n" +
+    "  `lambda_error_reason` string COMMENT '',\n" +
+    "  `target_port_list` string COMMENT '',\n" +
+    "  `target_status_code_list` string COMMENT '',\n" +
+    "  `new_field` string COMMENT '')\n" +
+    'PARTITIONED BY (\n' +
+    '  `date_utc_partition` string\n' +
+    ')\n' +
+    'ROW FORMAT SERDE\n' +
+    "  'org.apache.hadoop.hive.serde2.RegexSerDe'\n" +
+    'WITH SERDEPROPERTIES (\n' +
+    '  \'input.regex\'=\'([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \\"([^ ]*) ([^ ]*) (- |[^ ]*)\\" \\"([^\\"]*)\\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^\\"]*)\\" ([-.0-9]*) ([^ ]*) \\"([^\\"]*)\\" \\"([^\\"]*)\\" \\"([^ ]*)\\" \\"([^s]+)\\" \\"([^s]+)\\"(.*)\')\n' +
+    'STORED AS INPUTFORMAT\n' +
+    "  'org.apache.hadoop.mapred.TextInputFormat'\n" +
+    'OUTPUTFORMAT\n' +
+    "  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" +
+    'LOCATION\n' +
+    "  '{{ALB_LOG_ROOT}}'\n";
+}
+
+export interface AlbLogRecordQuery {
+  startTimeEpochMS?: number;
+  endTimeEpochMS?: number;
+  requestUrlFilter?: string;
+  limit?: number;
+}
+
+// Matches the fields that Alb stores natively
+export interface AlbLogRecord {
+  type: string;
+  time: string;
+  elb: string;
+  client_ip: string;
+  client_port: string;
+  target_ip: string;
+  target_port: string;
+  request_processing_time: string;
+  target_processing_time: string;
+  response_processing_time: string;
+  elb_status_code: string;
+  target_status_code: string;
+  received_bytes: string;
+  sent_bytes: string;
+  request_verb: string;
+  request_url: string;
+  request_proto: string;
+  user_agent: string;
+  ssl_cipher: string;
+  ssl_protocol: string;
+  target_group_arn: string;
+  trace_id: string;
+  domain_name: string;
+  chosen_cert_arn: string;
+  matched_rule_priority: string;
+  request_creation_time: string;
+  actions_executed: string;
+  redirect_url: string;
+  lambda_error_reason: string;
+  target_port_list: string;
+  target_status_code_list: string;
+  new_field: string;
+  date_utc_partition: string;
+}

package/src/athena/athena-ratchet.spec.ts

@@ -0,0 +1,43 @@
+import { AthenaRatchet } from './athena-ratchet.js';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import {
+  AthenaClient,
+  GetQueryExecutionCommand,
+  GetQueryExecutionOutput,
+  StartQueryExecutionCommand,
+  StartQueryExecutionOutput,
+} from '@aws-sdk/client-athena';
+import { S3Client } from '@aws-sdk/client-s3';
+import { mockClient } from 'aws-sdk-client-mock';
+import { beforeEach, describe, expect, test } from 'vitest';
+
+let mockAthena;
+let mockS3;
+
+describe('#AthenaRatchet', function () {
+  mockAthena = mockClient(AthenaClient);
+  mockS3 = mockClient(S3Client);
+
+  beforeEach(() => {
+    mockAthena.reset();
+    mockS3.reset();
+  });
+
+  test.skip('should test a query', async () => {
+    const outputDir: string = 's3://your-bucket/your-prefix';
+    const qry: string = 'select * from test limit 20000';
+
+    mockAthena.on(StartQueryExecutionCommand).resolves({ jobName: 'b' } as StartQueryExecutionOutput);
+    mockAthena.on(GetQueryExecutionCommand).resolves({
+      QueryExecution: { ResultConfiguration: { OutputLocation: 'test' }, Status: { State: 'SUCCEEDED' } },
+    } as GetQueryExecutionOutput);
+
+    const ratchet: AthenaRatchet = new AthenaRatchet(mockAthena, mockS3, outputDir);
+
+    const result: any[] = await ratchet.runQueryToObjects(qry);
+    //const result: string = await ratchet.runQueryToFile(qry, null, 'testfile.csv');
+
+    expect(result).toBeTruthy();
+    Logger.info('Got objects : %j', result);
+  });
+});

package/src/athena/athena-ratchet.ts

@@ -0,0 +1,194 @@
+import {
+  AthenaClient,
+  GetNamedQueryCommand,
+  GetNamedQueryCommandOutput,
+  GetQueryExecutionCommand,
+  GetQueryExecutionCommandOutput,
+  ListNamedQueriesCommand,
+  ListNamedQueriesCommandOutput,
+  NamedQuery,
+  Row,
+  StartQueryExecutionCommand,
+  StartQueryExecutionCommandOutput,
+  StartQueryExecutionInput,
+} from '@aws-sdk/client-athena';
+import { GetObjectCommand, GetObjectCommandOutput, GetObjectRequest, S3Client } from '@aws-sdk/client-s3';
+import tmp from 'tmp';
+import fs, { WriteStream } from 'fs';
+import { Readable } from 'stream';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { CsvRatchet } from '@bitblit/ratchet-node-only/csv/csv-ratchet';
+import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
+import { StopWatch } from '@bitblit/ratchet-common/lang/stop-watch';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+
+export class AthenaRatchet {
+  constructor(
+    private athena: AthenaClient,
+    private s3: S3Client,
+    private outputLocation: string,
+  ) {
+    RequireRatchet.notNullOrUndefined(athena);
+    RequireRatchet.notNullOrUndefined(s3);
+    RequireRatchet.notNullOrUndefined(outputLocation);
+    RequireRatchet.true(outputLocation.startsWith('s3://'));
+  }
+
+  public static athenaRowsToObject<T>(input: Row[]): T[] {
+    const colNames: string[] = input[0].Data.map((d) => d.VarCharValue);
+    const temp: Row[] = input.slice(1);
+    const rval: T[] = temp.map((t) => {
+      const newItem: T = {} as T;
+      for (let i = 0; i < t.Data.length; i++) {
+        newItem[colNames[i]] = t.Data[i].VarCharValue;
+      }
+      return newItem;
+    });
+    return rval;
+  }
+
+  public async fetchQueryIds(): Promise<string[]> {
+    const params = {
+      // MaxResults: 0,
+      NextToken: null,
+    };
+
+    let rval: string[] = [];
+    let next: ListNamedQueriesCommandOutput = null;
+
+    do {
+      next = await this.athena.send(new ListNamedQueriesCommand(params));
+      rval = rval.concat(next.NamedQueryIds);
+      params.NextToken = next.NextToken;
+    } while (params.NextToken);
+
+    return rval;
+  }
+
+  public async listQueries(): Promise<NamedQuery[]> {
+    const rval: NamedQuery[] = [];
+    const ids: string[] = await this.fetchQueryIds();
+    Logger.debug('Finding %d items', ids.length);
+    for (const id of ids) {
+      //for (let i = 0; i < ids.length; i++) {
+      const params = {
+        NamedQueryId: id,
+      };
+      const val: GetNamedQueryCommandOutput = await this.athena.send(new GetNamedQueryCommand(params));
+      rval.push(val.NamedQuery);
+    }
+
+    return rval;
+  }
+
+  public async findQueryByName(name: string): Promise<NamedQuery> {
+    const all: NamedQuery[] = await this.listQueries();
+    const rval: NamedQuery = all.find((a) => a.Name.toLowerCase() == name.toLowerCase());
+    return rval;
+  }
+
+  public async runQueryToObjects<T>(queryIn: string, queryParams: any = {}, pingTimeMS = 2000): Promise<T[]> {
+    Logger.info('Running query to objects');
+    const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
+    Logger.info('Query succeeded, processing file from %s', outputLoc);
+
+    const bucketName: string = outputLoc.substring(5, outputLoc.indexOf('/', 5));
+    const obKey: string = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
+
+    const req: GetObjectRequest = {
+      Bucket: bucketName,
+      Key: obKey,
+    };
+    const getFileOut: GetObjectCommandOutput = await this.s3.send(new GetObjectCommand(req));
+
+    const bodyAsString: string = await getFileOut.Body.transformToString();
+
+    const rval: T[] = await CsvRatchet.stringParse<T>(
+      bodyAsString,
+      (p) => {
+        return p;
+      },
+      { columns: true, skip_empty_lines: true },
+    );
+
+    return rval;
+  }
+
+  public async runQueryToFile(queryIn: string, queryParams: any = {}, targetDataFileIn: string = null, pingTimeMS = 2000): Promise<string> {
+    Logger.info('Running query to file');
+    const outputLoc = await this.runQueryToOutputLocation(queryIn, queryParams, pingTimeMS);
+    Logger.info('Query succeeded, pulling file from %s', outputLoc);
+
+    const bucketName: string = outputLoc.substring(5, outputLoc.indexOf('/', 5));
+    const obKey: string = outputLoc.substring(outputLoc.indexOf('/', 5) + 1);
+
+    const req: GetObjectRequest = {
+      Bucket: bucketName,
+      Key: obKey,
+    };
+
+    const targetDataFile: string = targetDataFileIn || tmp.fileSync({ postfix: '.csv', keep: false }).name;
+    const fileStream: WriteStream = fs.createWriteStream(targetDataFile);
+    const output: GetObjectCommandOutput = await this.s3.send(new GetObjectCommand(req));
+
+    const readStream: Readable = output.Body as Readable;
+    readStream.pipe(fileStream);
+
+    const rval: string = await PromiseRatchet.resolveOnEvent<string>(readStream, ['finish', 'close'], ['error'], targetDataFile);
+    Logger.silly('Response: %s', rval);
+    return targetDataFile;
+  }
+
+  private async runQueryToOutputLocation(queryIn: string, queryParams: any = {}, pingTimeMS = 2000): Promise<string> {
+    let rval: string = null;
+    const timer: StopWatch = new StopWatch();
+    const query: string = StringRatchet.simpleTemplateFill(queryIn, queryParams, true, '{', '}');
+
+    try {
+      Logger.info('Starting query : %s', query);
+
+      const token: string = StringRatchet.createType4Guid();
+      const params: StartQueryExecutionInput = {
+        QueryString: query,
+        ResultConfiguration: {
+          /* required */
+          OutputLocation: this.outputLocation,
+          EncryptionConfiguration: {
+            EncryptionOption: 'SSE_S3' /* required */,
+            // KmsKey: 'STRING_VALUE'
+          },
+        },
+        ClientRequestToken: token,
+        QueryExecutionContext: {
+          Database: 'default',
+        },
+      };
+
+      const startToken: StartQueryExecutionCommandOutput = await this.athena.send(new StartQueryExecutionCommand(params));
+
+      const getExecParams: any = {
+        QueryExecutionId: startToken.QueryExecutionId,
+      };
+
+      const finalStates: string[] = ['FAILED', 'CANCELLED', 'SUCCEEDED'];
+      let curState: GetQueryExecutionCommandOutput = await this.athena.send(new GetQueryExecutionCommand(getExecParams));
+      while (finalStates.indexOf(curState.QueryExecution.Status.State) === -1) {
+        await PromiseRatchet.createTimeoutPromise('wait', pingTimeMS);
+        Logger.debug('%s : %s : %s', curState.QueryExecution.Status.State, timer.dump(), query);
+        curState = await this.athena.send(new GetQueryExecutionCommand(getExecParams));
+      }
+
+      if (curState.QueryExecution.Status.State === 'FAILED') {
+        Logger.warn('Query failed : %s', curState.QueryExecution.Status.StateChangeReason);
+      } else if (curState.QueryExecution.Status.State === 'SUCCEEDED') {
+        rval = curState.QueryExecution.ResultConfiguration.OutputLocation;
+      }
+    } catch (err) {
+      Logger.warn('Failure : %s', err, err);
+    }
+    Logger.info('Query took %s : %s', timer.dump(), query);
+
+    return rval;
+  }
+}

package/src/build/ratchet-aws-node-only-info.ts

@@ -0,0 +1,19 @@
+import { BuildInformation } from '@bitblit/ratchet-common/build/build-information';
+
+export class RatchetAwsNodeOnlyInfo {
+  // Empty constructor prevents instantiation
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  private constructor() {}
+
+  public static buildInformation(): BuildInformation {
+    const val: BuildInformation = {
+      version: 'LOCAL-SNAPSHOT',
+      hash: 'LOCAL-HASH',
+      branch: 'LOCAL-BRANCH',
+      tag: 'LOCAL-TAG',
+      timeBuiltISO: 'LOCAL-TIME-ISO',
+      notes: 'LOCAL-NOTES',
+    };
+    return val;
+  }
+}

package/src/cli/dynamo-exporter.ts

@@ -0,0 +1,104 @@
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { PromiseRatchet } from '@bitblit/ratchet-common/lang/promise-ratchet';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import fs, { WriteStream } from 'fs';
+import { DynamoRatchet } from '@bitblit/ratchet-aws/dynamodb/dynamo-ratchet';
+
+import readline from 'readline';
+import { QueryCommandInput, ScanCommandInput } from '@aws-sdk/lib-dynamodb';
+
+export class DynamoExporter {
+  // Prevent instantiation
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  private constructor() {}
+
+  public static async importJsonLFileToTable(dynamo: DynamoRatchet, tableName: string, filename: string): Promise<number> {
+    RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+    RequireRatchet.notNullOrUndefined(tableName, 'tableName');
+    RequireRatchet.notNullOrUndefined(filename, 'filename');
+
+    const fileStream = fs.createReadStream(filename);
+
+    const rl = readline.createInterface({
+      input: fileStream,
+      crlfDelay: Infinity,
+    });
+    // Note: we use the crlfDelay option to recognize all instances of CR LF
+    // ('\r\n') in input.txt as a single line break.
+
+    let rval: number = 0;
+    for await (const line of rl) {
+      if (rval % 100 === 0) {
+        Logger.info('Importing line %d', rval);
+      }
+      if (StringRatchet.trimToNull(line)) {
+        const parsed: any = JSON.parse(line);
+        await dynamo.simplePut(tableName, parsed);
+        rval++;
+      }
+    }
+
+    return rval;
+  }
+
+  public static async exportScanToJsonLFile(dynamo: DynamoRatchet, scan: ScanCommandInput, filename: string): Promise<number> {
+    RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+    RequireRatchet.notNullOrUndefined(scan, 'scan');
+    RequireRatchet.notNullOrUndefined(filename, 'filename');
+    const ws: WriteStream = fs.createWriteStream(filename);
+    ws.on('end', () => {
+      Logger.debug('Write complete');
+    });
+
+    const rval: number = await DynamoExporter.exportScanToJsonLWriteStream(dynamo, scan, ws);
+
+    await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
+    ws.close();
+    return rval;
+  }
+
+  public static async exportQueryToJsonLFile(dynamo: DynamoRatchet, qry: QueryCommandInput, filename: string): Promise<number> {
+    RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+    RequireRatchet.notNullOrUndefined(qry, 'qry');
+    RequireRatchet.notNullOrUndefined(filename, 'filename');
+
+    const ws: WriteStream = fs.createWriteStream(filename);
+    ws.on('end', () => {
+      Logger.debug('Write complete');
+    });
+
+    const rval: number = await DynamoExporter.exportQueryToJsonLWriteStream(dynamo, qry, ws);
+
+    await PromiseRatchet.resolveOnEvent(ws, ['finish', 'close'], ['error']);
+    ws.close();
+    return rval;
+  }
+
+  public static async exportScanToJsonLWriteStream(dynamo: DynamoRatchet, scan: ScanCommandInput, target: WriteStream): Promise<number> {
+    RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+    RequireRatchet.notNullOrUndefined(scan, 'scan');
+    RequireRatchet.notNullOrUndefined(target, 'target');
+
+    const rval: number = await dynamo.fullyExecuteProcessOverScan(scan, async (row) =>
+      DynamoExporter.writeItemToJsonLStream(row, target, false),
+    );
+    return rval;
+  }
+
+  public static async exportQueryToJsonLWriteStream(dynamo: DynamoRatchet, qry: QueryCommandInput, target: WriteStream): Promise<number> {
+    RequireRatchet.notNullOrUndefined(dynamo, 'dynamo');
+    RequireRatchet.notNullOrUndefined(qry, 'qry');
+    RequireRatchet.notNullOrUndefined(target, 'target');
+
+    const rval: number = await dynamo.fullyExecuteProcessOverQuery(qry, async (row) =>
+      DynamoExporter.writeItemToJsonLStream(row, target, false),
+    );
+    return rval;
+  }
+  public static writeItemToJsonLStream(item: any, target: WriteStream, includeNulls: boolean = false): void {
+    if (!!item || includeNulls) {
+      target.write(JSON.stringify(item) + '\n');
+    }
+  }
+}

package/src/cli/ratchet-cli-handler.ts

@@ -0,0 +1,18 @@
+import { AbstractRatchetCliHandler } from '@bitblit/ratchet-node-only/cli/abstract-ratchet-cli-handler';
+import { SiteUploader } from './site-uploader/site-uploader.js';
+import { StartInstanceAndSsh } from './start-instance-and-ssh.js';
+import { BuildInformation } from '@bitblit/ratchet-common/build/build-information';
+import { RatchetAwsNodeOnlyInfo } from '../build/ratchet-aws-node-only-info.js';
+
+export class RatchetCliHandler extends AbstractRatchetCliHandler {
+  fetchHandlerMap(): Record<string, any> {
+    return {
+      'site-uploader': SiteUploader.runFromCliArgs,
+      'start-instance-and-ssh': StartInstanceAndSsh.runFromCliArgs,
+    };
+  }
+
+  fetchVersionInfo(): BuildInformation {
+    return RatchetAwsNodeOnlyInfo.buildInformation();
+  }
+}