@bitblit/ratchet-aws-node-only 6.0.146-alpha → 6.0.148-alpha

package/src/daemon/daemon.ts

@@ -0,0 +1,150 @@
+import { DaemonProcessState } from './daemon-process-state.js';
+import { DaemonProcessCreateOptions } from './daemon-process-create-options.js';
+import { DaemonUtil } from './daemon-util.js';
+import { DaemonLike } from './daemon-like.js';
+import { DaemonProcessStatePublicToken } from './daemon-process-state-public-token.js';
+import { S3Client } from '@aws-sdk/client-s3';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { JwtRatchetLike } from '@bitblit/ratchet-node-only/jwt/jwt-ratchet-like';
+import { S3CacheRatchetLike } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet-like';
+import { S3CacheRatchet } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet';
+
+export class Daemon implements DaemonLike {
+  public static DEFAULT_DEFAULT_GROUP: string = 'DEFAULT';
+
+  private cache: S3CacheRatchetLike;
+
+  constructor(
+    private s3: S3Client,
+    private bucket: string,
+    private prefix: string = '',
+    private _defaultGroup: string = Daemon.DEFAULT_DEFAULT_GROUP,
+    private jwtRatchet?: JwtRatchetLike,
+  ) {
+    this.cache = new S3CacheRatchet(this.s3, this.bucket);
+  }
+
+  public get defaultGroup(): string {
+    return this._defaultGroup;
+  }
+
+  public async keyToPublicToken(key: string, expirationSeconds: number): Promise<string> {
+    RequireRatchet.notNullOrUndefined(this.jwtRatchet, 'You must set jwtRatchet if you wish to use public tokens');
+    RequireRatchet.notNullOrUndefined(key, 'key');
+    RequireRatchet.true(expirationSeconds > 0, 'Expiration seconds must be larger than 0');
+
+    const token: DaemonProcessStatePublicToken = { daemonKey: key };
+    const publicToken: string = await this.jwtRatchet.createTokenString(token, expirationSeconds);
+    return publicToken;
+  }
+
+  private keyToPath(key: string): string {
+    return Buffer.from(key, 'base64').toString();
+  }
+
+  private pathToKey(path: string): string {
+    return Buffer.from(path).toString('base64');
+  }
+
+  private generatePath(group: string = this._defaultGroup): string {
+    return this.generatePrefix(group) + StringRatchet.createType4Guid();
+  }
+
+  private generatePrefix(group: string = this._defaultGroup): string {
+    return this.prefix + group + '/';
+  }
+
+  public async start(options: DaemonProcessCreateOptions): Promise<DaemonProcessState> {
+    options.group ??= this._defaultGroup;
+    const path: string = this.generatePath(options.group);
+    const key: string = this.pathToKey(path);
+    return DaemonUtil.start(this.cache, key, path, options);
+  }
+
+  private async writeState(newState: DaemonProcessState, contents: Buffer): Promise<DaemonProcessState> {
+    const key: string = this.keyToPath(newState.id);
+    return DaemonUtil.writeState(this.cache, key, newState, contents);
+  }
+
+  public async clean(group: string = this._defaultGroup, olderThanSeconds: number = 60 * 60 * 24 * 7): Promise<DaemonProcessState[]> {
+    try {
+      Logger.info('Daemon removing items older than %d seconds from group %s', olderThanSeconds, group);
+      const original: DaemonProcessState[] = await this.list(group);
+      const now: number = new Date().getTime();
+      const removed: DaemonProcessState[] = [];
+      for (const test of original) {
+        //for (let i = 0; i < original.length; i++) {
+        //const test: DaemonProcessState = original[i];
+        const ageSeconds: number = (now - test.startedEpochMS) / 1000;
+        if (ageSeconds > olderThanSeconds) {
+          const _remove: any = await this.cache.removeCacheFile(this.keyToPath(test.id));
+          removed.push(test);
+        }
+      }
+      Logger.debug('Removed %d of %d items', removed.length, original.length);
+      return removed;
+    } catch (err) {
+      Logger.error('Error while trying to clean a daemon: %j %s', group, err);
+      throw err;
+    }
+  }
+
+  public async listKeys(group: string = this._defaultGroup): Promise<string[]> {
+    try {
+      const prefix: string = this.generatePrefix(group);
+      Logger.info('Fetching children of %s', prefix);
+      const rval: string[] = await this.cache.directChildrenOfPrefix(prefix);
+      Logger.debug('Found : %j', rval);
+      return rval;
+    } catch (err) {
+      Logger.error('Error while trying to list daemon keys: %j %s', group, err);
+      throw err;
+    }
+  }
+
+  public async list(group: string = this._defaultGroup): Promise<DaemonProcessState[]> {
+    try {
+      const prefix: string = this.generatePrefix(group);
+      Logger.info('Fetching children of %s', prefix);
+      const keys: string[] = await this.listKeys(group);
+      const proms: Promise<DaemonProcessState>[] = keys.map((k) => this.stat(this.pathToKey(this.generatePrefix(group) + k)));
+      const rval: DaemonProcessState[] = await Promise.all(proms);
+
+      return rval;
+    } catch (err) {
+      Logger.error('Error while trying to list daemon states: %j %s', group, err);
+      throw err;
+    }
+  }
+
+  public async updateMessage(id: string, newMessage: string): Promise<DaemonProcessState> {
+    const itemPath: string = this.keyToPath(id);
+    return DaemonUtil.updateMessage(this.cache, itemPath, newMessage);
+  }
+
+  public async stat(key: string): Promise<DaemonProcessState> {
+    const itemPath: string = this.keyToPath(key);
+    return DaemonUtil.stat(this.cache, itemPath);
+  }
+
+  public async statFromPublicToken(publicToken: string): Promise<DaemonProcessState> {
+    RequireRatchet.notNullOrUndefined(this.jwtRatchet, 'You must set jwtRatchet if you wish to use public tokens');
+    RequireRatchet.notNullOrUndefined(publicToken, 'publicToken');
+    const parsed: DaemonProcessStatePublicToken = await this.jwtRatchet.decodeToken<DaemonProcessStatePublicToken>(publicToken);
+    const key: string = parsed?.daemonKey;
+    return key ? this.stat(key) : null;
+  }
+
+  public async abort(id: string): Promise<DaemonProcessState> {
+    return DaemonUtil.abort(this.cache, this.keyToPath(id));
+  }
+  public async error(id: string, error: string): Promise<DaemonProcessState> {
+    return DaemonUtil.error(this.cache, this.keyToPath(id), error);
+  }
+
+  public async finalize(id: string, contents: Buffer): Promise<DaemonProcessState> {
+    return DaemonUtil.finalize(this.cache, this.keyToPath(id), contents);
+  }
+}

package/src/ec2/ec2-instance-util.ts

@@ -0,0 +1,60 @@
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { Ec2Ratchet } from '@bitblit/ratchet-aws/ec2/ec2-ratchet';
+import fs from 'fs';
+import { Instance } from '@aws-sdk/client-ec2';
+import { SendSSHPublicKeyResponse } from '@aws-sdk/client-ec2-instance-connect';
+
+export class Ec2InstanceUtil {
+  constructor(private ec2Ratchet: Ec2Ratchet) {}
+
+  public async startInstanceAndUploadPublicKeyFile(
+    instanceId: string,
+    filePath: string,
+    instanceOsUser: string = 'ec2-user',
+  ): Promise<Instance> {
+    RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(instanceId, 'instanceId');
+    RequireRatchet.notNullUndefinedOrOnlyWhitespaceString(filePath, 'filePath');
+    RequireRatchet.true(fs.existsSync(filePath), 'File does not exist');
+    Logger.info('Starting instance %s and uploading contents of public key file %s', instanceId, filePath);
+    const publicKeyText: string = fs.readFileSync(filePath).toString();
+    return this.startInstanceAndUploadPublicKey(instanceId, publicKeyText, instanceOsUser);
+  }
+
+  public async startInstanceAndUploadPublicKey(
+    instanceId: string,
+    publicKeyText: string,
+    instanceOsUser: string = 'ec2-user',
+  ): Promise<Instance> {
+    Logger.info('Starting instance %s, public key length %d, user %s', instanceId, publicKeyText.length, instanceOsUser);
+    let instance: Instance = await this.ec2Ratchet.describeInstance(instanceId);
+    if (instance) {
+      let launched: boolean = false;
+      if (instance.State.Code == 16) {
+        Logger.info('Instance is already running...');
+        launched = true;
+      } else {
+        Logger.info('Instance is not running... starting up : %s', instanceId);
+        launched = await this.ec2Ratchet.launchInstance(instanceId, 1000 * 30); // 30 seconds
+      }
+
+      if (launched) {
+        Logger.info('Uploading public key...');
+        const publicKeyResponse: SendSSHPublicKeyResponse = await this.ec2Ratchet.sendPublicKeyToEc2Instance(
+          instanceId,
+          publicKeyText,
+          instanceOsUser,
+        );
+        Logger.info('Key response : %j', publicKeyResponse);
+
+        instance = instance && instance.PublicIpAddress ? instance : await this.ec2Ratchet.describeInstance(instanceId);
+        Logger.info('Instance IP address is %s', instance.PublicIpAddress);
+      } else {
+        Logger.info('Instance could not start - check logs');
+      }
+    } else {
+      Logger.info('No such instance found - check your AWS keys? : %s', instanceId);
+    }
+    return instance;
+  }
+}

package/src/mail/inbound/email-to-db-insert-processor.ts

@@ -0,0 +1,105 @@
+import { ParsedEmailProcessor } from './parsed-email-processor.js';
+import { ParsedMail } from 'mailparser';
+import unzipper from 'unzipper';
+import { DateTime } from 'luxon';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { CsvRatchet } from '@bitblit/ratchet-node-only/csv/csv-ratchet';
+import { MultiStream } from '@bitblit/ratchet-node-only/stream/multi-stream';
+
+export class EmailToDbInsertProcessor implements ParsedEmailProcessor<EmailToDbStatement[]> {
+  public canProcess(_mail: ParsedMail): boolean {
+    return true;
+  }
+
+  public async processEmail(msg: ParsedMail): Promise<EmailToDbStatement[]> {
+    const rval: EmailToDbStatement[] = [];
+    try {
+      RequireRatchet.notNullOrUndefined(msg, 'msg');
+      Logger.info('Processing Broadsign reach inbound inventory email');
+      const data: Buffer = msg.attachments[0].content;
+
+      Logger.info('Unzipping attachment');
+      const rs: MultiStream = new MultiStream(data);
+      let wBuf: Buffer = null;
+      const prom: Promise<any> = rs
+        .pipe(unzipper.Parse())
+        .on('entry', async (entry) => {
+          if (entry.path.toLowerCase().endsWith('csv')) {
+            wBuf = await entry.buffer();
+          } else {
+            Logger.info('Pass: %s', entry.path);
+            entry.autodrain();
+          }
+        })
+        .promise();
+      await prom;
+      const csvParsed: any[] = await CsvRatchet.stringParse(wBuf.toString(), (o) => o, {
+        columns: false,
+        skip_empty_lines: true,
+      });
+
+      if (csvParsed.length > 1) {
+        const dropTable: string = 'drop table if exists sample';
+        let createTable: string = 'create table sample (pump_date varchar(255),';
+        const colNames: string[] = csvParsed[0];
+
+        let insertPrefix: string = 'insert into sample (pump_date,';
+        let insertQ: string = '?,';
+
+        for (let i = 0; i < colNames.length; i++) {
+          if (i > 0) {
+            createTable += ', ';
+            insertPrefix += ', ';
+            insertQ += ', ';
+          }
+          const kOut: string = colNames[i].toLowerCase().split(' ').join('_');
+          insertPrefix += kOut;
+          insertQ += '?';
+          createTable += kOut + ' varchar(255)';
+          if (kOut === 'id') {
+            createTable += ' primary key';
+          } else if (kOut === 'device_id') {
+            createTable += ' unique';
+          }
+        }
+        createTable += ')';
+        insertPrefix += ') values ';
+        // ('+insertQ+')';
+
+        Logger.info('Recreating table');
+        const _dropRes: any = rval.push({ statement: dropTable });
+        const _createRes: any = rval.push({ statement: createTable });
+        const pumpDate: string = DateTime.utc().toISO();
+        let insertStmt: string = insertPrefix;
+        let insertParams: any[] = [];
+
+        for (let i = 1; i < csvParsed.length; i++) {
+          if (insertStmt > insertPrefix) {
+            insertStmt += ',';
+          }
+          insertStmt += '(' + insertQ + ')';
+          insertParams = insertParams.concat(pumpDate, csvParsed[i]);
+
+          if (i % 25 === 0 || i === csvParsed.length - 1) {
+            rval.push({ statement: insertStmt, params: insertParams });
+            insertStmt = insertPrefix;
+            insertParams = [];
+            Logger.info('Inserted %d of %d rows', i, csvParsed.length);
+          }
+        }
+
+        Logger.info('Finished insertion of %d rows', csvParsed.length);
+      }
+    } catch (err) {
+      Logger.error('Failure: %s : %j', err, rval, err);
+    }
+
+    return rval;
+  }
+}
+
+export interface EmailToDbStatement {
+  statement: string;
+  params?: any[];
+}

package/src/mail/inbound/inbound-email-ratchet.spec.ts

@@ -0,0 +1,28 @@
+import { S3CacheRatchet } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet';
+import { InboundEmailRatchet } from './inbound-email-ratchet.js';
+
+import { SampleEmailProcessor } from './sample-email-processor.js';
+import { beforeEach, describe, expect, test } from 'vitest';
+import { mock, MockProxy } from 'vitest-mock-extended';
+
+let mockS3CR: MockProxy<S3CacheRatchet>;
+
+describe('#inboundEmailService', () => {
+  beforeEach(() => {
+    mockS3CR = mock<S3CacheRatchet>();
+  });
+
+  test('should process an email from S3', async () => {
+    mockS3CR.getDefaultBucket.mockReturnValueOnce('TEST-BUCKET');
+    mockS3CR.fileExists.mockResolvedValueOnce(true);
+    mockS3CR.fetchCacheFileAsString.mockResolvedValue('TEST');
+
+    const svc: InboundEmailRatchet = new InboundEmailRatchet(mockS3CR, [new SampleEmailProcessor()]);
+
+    //const buf: Buffer = fs.readFileSync('testemail.txt');
+    //const res: boolean = await svc.processEmailFromBuffer(buf);
+    const res: boolean = await svc.processEmailFromS3('some-key');
+
+    expect(res).not.toBeUndefined();
+  });
+});

package/src/mail/inbound/inbound-email-ratchet.ts

@@ -0,0 +1,55 @@
+import { ParsedMail, simpleParser } from 'mailparser';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { ParsedEmailProcessor } from './parsed-email-processor.js';
+import { S3CacheRatchet } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet';
+
+/**
+ * Service for handling inbound emails
+ */
+export class InboundEmailRatchet {
+  constructor(
+    private cache: S3CacheRatchet,
+    private processors: ParsedEmailProcessor<any>[],
+  ) {
+    RequireRatchet.notNullOrUndefined(this.cache, 'cache');
+    RequireRatchet.notNullOrUndefined(this.cache.getDefaultBucket(), 'cache.defaultBucket');
+  }
+
+  public async processEmailFromS3(key: string): Promise<boolean> {
+    const rval: boolean = false;
+    if (await this.cache.fileExists(key)) {
+      const data: string = await this.cache.fetchCacheFileAsString(key);
+      return this.processEmailFromBuffer(new Buffer(data));
+    } else {
+      Logger.warn('Cannot process inbound email - no such key : %s', key);
+    }
+
+    return rval;
+  }
+
+  public async processEmailFromBuffer(buf: Buffer): Promise<boolean> {
+    let rval: boolean = false;
+    RequireRatchet.notNullOrUndefined(buf, 'buf');
+    Logger.info('Processing inbound email - size %d bytes', buf.length);
+
+    const message: ParsedMail = await simpleParser(buf);
+    Logger.info(
+      'Found mail from "%s" subject "%s" with %d attachments',
+      message?.from?.text,
+      message?.subject,
+      message?.attachments?.length,
+    );
+
+    for (let i = 0; i < this.processors.length && !rval; i++) {
+      if (this.processors[i].canProcess(message)) {
+        Logger.info('Processing message with processor %d', i);
+        const result: any = await this.processors[i].processEmail(message);
+        Logger.info('Result was : %j', result);
+        rval = true;
+      }
+    }
+
+    return rval;
+  }
+}

package/src/mail/inbound/parsed-email-processor.ts

@@ -0,0 +1,6 @@
+import { ParsedMail } from 'mailparser';
+
+export interface ParsedEmailProcessor<T> {
+  canProcess(mail: ParsedMail): boolean;
+  processEmail(msg: ParsedMail): Promise<T>;
+}

package/src/mail/inbound/sample-email-processor.ts

@@ -0,0 +1,12 @@
+import { ParsedEmailProcessor } from './parsed-email-processor.js';
+import { ParsedMail } from 'mailparser';
+
+export class SampleEmailProcessor implements ParsedEmailProcessor<string> {
+  public canProcess(_mail: ParsedMail): boolean {
+    return true;
+  }
+
+  public async processEmail(msg: ParsedMail): Promise<string> {
+    return msg.body;
+  }
+}

package/src/s3/s3-cache-to-local-disk-ratchet.ts

@@ -0,0 +1,118 @@
+import crypto from 'crypto';
+import fs from 'fs';
+import path from 'path';
+import { S3CacheRatchetLike } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet-like';
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+
+/**
+ * Use this when you want a lambda to cache a remote S3 bucket locally on disk for faster access
+ */
+export class S3CacheToLocalDiskRatchet {
+  private static readonly DEFAULT_CACHE_TIMEOUT_SEC = 7 * 24 * 3600;
+
+  private currentlyLoading: Map<string, Promise<Buffer>> = new Map<string, Promise<Buffer>>();
+
+  constructor(
+    private s3: S3CacheRatchetLike,
+    private tmpFolder: string,
+    private cacheTimeoutSeconds: number = S3CacheToLocalDiskRatchet.DEFAULT_CACHE_TIMEOUT_SEC,
+  ) {
+    RequireRatchet.notNullOrUndefined(s3, 's3');
+    RequireRatchet.notNullOrUndefined(StringRatchet.trimToNull(tmpFolder));
+    RequireRatchet.true(fs.existsSync(tmpFolder), 'folder must exist : ' + tmpFolder);
+  }
+
+  public async getFileString(key: string): Promise<string> {
+    const buf: Buffer = await this.getFileBuffer(key);
+    return buf ? buf.toString() : null;
+  }
+
+  private keyToLocalCachePath(key: string): string {
+    const cachedHash: string = this.generateCacheHash(this.s3.getDefaultBucket() + '/' + key);
+    const rval: string = path.join(this.tmpFolder, cachedHash);
+    return rval;
+  }
+
+  public removeCacheFileForKey(key: string): void {
+    const localCachePath = this.keyToLocalCachePath(key);
+    Logger.info('Removing cache file for %s : %s', key, localCachePath);
+    if (fs.existsSync(localCachePath)) {
+      fs.unlinkSync(localCachePath);
+    } else {
+      Logger.debug('Skipping delete for %s - does not exist', localCachePath);
+    }
+  }
+
+  public async getFileBuffer(key: string): Promise<Buffer> {
+    const localCachePath: string = this.keyToLocalCachePath(key);
+
+    let rval: Buffer = null;
+    rval = this.getCacheFileAsBuffer(localCachePath);
+
+    if (!rval) {
+      Logger.info('No cache. Downloading File s3://%s/%s to %s', this.s3.getDefaultBucket(), key, localCachePath);
+      try {
+        let prom: Promise<Buffer> = this.currentlyLoading.get(key);
+        if (prom) {
+          Logger.info('Already running - wait for that');
+        } else {
+          Logger.info('Not running - start');
+          prom = this.updateLocalCacheFile(key, localCachePath);
+          this.currentlyLoading.set(key, prom);
+        }
+        rval = await prom;
+        this.currentlyLoading.delete(key); // Clear the cache
+      } catch (err) {
+        Logger.warn('File %s/%s does not exist. Err code: %s', this.s3.getDefaultBucket(), key, err);
+      }
+    } else {
+      Logger.info('Found cache file for s3://%s/%s. Local path %s', this.s3.getDefaultBucket(), key, localCachePath);
+    }
+    return rval;
+  }
+
+  private async updateLocalCacheFile(key: string, localCachePath: string): Promise<Buffer> {
+    const rval: Buffer = await this.s3.fetchCacheFileAsBuffer(key);
+    if (rval && rval.length > 0) {
+      Logger.info('Saving %d bytes to disk for cache', rval.length);
+      fs.writeFileSync(localCachePath, rval);
+    }
+    return rval;
+  }
+
+  public getCacheFileAsString(filePath: string): string {
+    const buf: Buffer = this.getCacheFileAsBuffer(filePath);
+    return buf ? buf.toString() : null;
+  }
+
+  public getCacheFileAsBuffer(filePath: string): Buffer {
+    if (!fs.existsSync(filePath)) {
+      return null;
+    }
+
+    try {
+      const stats = fs.statSync(filePath);
+      const now = new Date().getTime();
+
+      const duration: number = (now - stats.ctimeMs) / 1000;
+      if (duration >= this.cacheTimeoutSeconds) {
+        return null;
+      } else {
+        const rval: Buffer = fs.readFileSync(filePath);
+        return rval;
+      }
+    } catch (err) {
+      Logger.warn('Error getting s3 cache file %s', err);
+    }
+
+    return null;
+  }
+
+  private generateCacheHash(hashVal: string): string {
+    const rval: string = crypto.createHash('md5').update(hashVal).digest('hex');
+
+    return rval;
+  }
+}

package/src/s3/s3-cache-to-local-dist-ratchet.spec.ts

@@ -0,0 +1,31 @@
+import { S3CacheToLocalDiskRatchet } from './s3-cache-to-local-disk-ratchet.js';
+import { S3CacheRatchetLike } from '@bitblit/ratchet-aws/s3/s3-cache-ratchet-like';
+import { tmpdir } from 'os';
+import { beforeEach, describe, expect, test } from 'vitest';
+import { mock, MockProxy } from 'vitest-mock-extended';
+
+let mockS3CR: MockProxy<S3CacheRatchetLike>;
+
+describe('#S3CacheToLocalDiskRatchet', () => {
+  beforeEach(() => {
+    mockS3CR = mock<S3CacheRatchetLike>();
+  });
+
+  test('should download file and store in tmp', async () => {
+    mockS3CR.fetchCacheFileAsBuffer.mockResolvedValue(Buffer.from(JSON.stringify({ a: 'b' })));
+
+    const pth: string = 'test-path';
+
+    const svc: S3CacheToLocalDiskRatchet = new S3CacheToLocalDiskRatchet(mockS3CR, tmpdir()); //, 'tmp', 1000);
+    svc.removeCacheFileForKey(pth);
+
+    const proms: Promise<Buffer>[] = [];
+    for (let i = 0; i < 5; i++) {
+      proms.push(svc.getFileBuffer(pth));
+    }
+
+    const all: Buffer[] = await Promise.all(proms);
+
+    expect(all.length).toEqual(5);
+  });
+});

package/src/static/albAthenaTableCreate.txt

@@ -0,0 +1,46 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS `${TABLE_NAME}`(
+  `type` string COMMENT '',
+  `time` string COMMENT '',
+  `elb` string COMMENT '',
+  `client_ip` string COMMENT '',
+  `client_port` int COMMENT '',
+  `target_ip` string COMMENT '',
+  `target_port` int COMMENT '',
+  `request_processing_time` double COMMENT '',
+  `target_processing_time` double COMMENT '',
+  `response_processing_time` double COMMENT '',
+  `elb_status_code` string COMMENT '',
+  `target_status_code` string COMMENT '',
+  `received_bytes` bigint COMMENT '',
+  `sent_bytes` bigint COMMENT '',
+  `request_verb` string COMMENT '',
+  `request_url` string COMMENT '',
+  `request_proto` string COMMENT '',
+  `user_agent` string COMMENT '',
+  `ssl_cipher` string COMMENT '',
+  `ssl_protocol` string COMMENT '',
+  `target_group_arn` string COMMENT '',
+  `trace_id` string COMMENT '',
+  `domain_name` string COMMENT '',
+  `chosen_cert_arn` string COMMENT '',
+  `matched_rule_priority` string COMMENT '',
+  `request_creation_time` string COMMENT '',
+  `actions_executed` string COMMENT '',
+  `redirect_url` string COMMENT '',
+  `lambda_error_reason` string COMMENT '',
+  `target_port_list` string COMMENT '',
+  `target_status_code_list` string COMMENT '',
+  `new_field` string COMMENT '')
+PARTITIONED BY (
+  `date_utc_partition` string
+)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.serde2.RegexSerDe'
+WITH SERDEPROPERTIES (
+  'input.regex'='([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) ([^ ]*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^s]+)\" \"([^s]+)\"(.*)')
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+  '${ALB_LOG_ROOT}'