@wireio/shared 1.0.1 → 1.0.3

package/src/logging/appenders/FileAppender.ts

@@ -1,281 +0,0 @@
-import { assign, defaults, padStart } from "lodash"
-import * as Path from "path"
-import * as Fs from "fs"
-import type { LogRecord } from "../LogRecord"
-import type { Appender } from "../Appender"
-import { Option, Future } from "prelude-ts"
-import { Buffer } from "buffer"
-import Debug from "debug"
-import Bluebird from "bluebird"
-
-const FsAsync = Fs.promises
-
-const debug = Debug("3fv:logger:FileAppender")
-const osTempDir = process.env.TMP ?? process.env.TEMP ?? "/tmp"
-const tempDir = Fs.mkdtempSync(Path.join(osTempDir, "logger-proxy-example")) //
-const getDefaultConfig = (): FileAppenderConfig => ({
-  filename: Path.join(tempDir, "app.log"),
-  prettyPrint: process.env.NODE_ENV !== "production",
-  maxSize: -1,
-  maxFiles: 5,
-  enableRolling: false,
-  sync: false
-})
-
-function applyConfigDefaults(options: FileAppenderOptions): FileAppenderConfig {
-  return defaults(options, getDefaultConfig())
-}
-
-export interface FileAppenderConfig<Record extends LogRecord = any> {
-  filename: string
-
-  /**
-   * For rolling
-   */
-  maxFiles: number
-
-  /**
-   * For rolling
-   */
-  maxSize: number
-
-  enableRolling: boolean
-  prettyPrint: boolean
-  sync: boolean
-}
-
-export type FileAppenderOptions<Record extends LogRecord = any> = Partial<
-  FileAppenderConfig<Record>
->
-
-export class FileAppender<Record extends LogRecord>
-  implements Appender<Record>
-{
-  readonly config: FileAppenderConfig<Record>
-
-  private readonly state: {
-    filename: string
-    file: number
-    flushing: boolean
-    queue: Array<Buffer>
-    ready: boolean
-    error?: Error
-    currentSize: number
-    archivedFilenames: string[]
-  } = {
-    archivedFilenames: [],
-    filename: undefined,
-    file: undefined,
-    ready: false,
-    currentSize: 0,
-    flushing: false,
-    queue: []
-  }
-
-  get enableRolling() {
-    return this.config.enableRolling
-  }
-
-  isReady() {
-    return this.state.ready
-  }
-
-  /**
-   * Initialize and setup the appender
-   *
-   * @returns {FileAppender<Record>}
-   */
-  setup(): FileAppender<Record> {
-    const { state } = this
-    if (!!state.ready) {
-      return this
-    }
-
-    try {
-      const { filename } = this
-      const file = Fs.openSync(filename, "a")
-
-      assign(state, {
-        filename,
-        file,
-        ready: true
-      })
-
-      return this
-    } catch (err) {
-      assign(state, {
-        error: err,
-        file: undefined,
-        ready: false
-      })
-      throw err
-    }
-  }
-
-  /**
-   * Only exposed for the sake of tests & examples
-   * DO NOT USE
-   *
-   * This should all be async, but i was in a rush :(
-   *
-   * @returns
-   */
-  rollFile() {
-    const { state, config } = this
-    let { file, filename, archivedFilenames } = state
-    if (!this.isReady() || !file) {
-      debug(`Can not roll file before ready`)
-      return
-    }
-
-    if (!this.enableRolling) {
-      return
-    }
-
-    const fileCount = archivedFilenames.length + 1
-    if (config.maxFiles < 1 || fileCount < config.maxFiles) {
-      archivedFilenames.push(
-        filename + "." + padStart(fileCount.toString(10), 4, "0")
-      )
-    }
-    Fs.fdatasyncSync(file)
-    Fs.closeSync(file)
-    file = state.file = undefined
-    const filenames = [filename, ...archivedFilenames]
-
-    let dest: string = filenames.pop()
-    while (filenames.length) {
-      let src = filenames.pop()
-      if (!Fs.existsSync(src)) {
-        debug(`src does not exist, skipping ${src}`)
-      } else {
-        debug(`Moving src (${src}) to dest (${dest})`)
-        if (Fs.existsSync(dest)) {
-          Fs.unlinkSync(dest)
-        }
-        Fs.renameSync(src, dest)
-        // if (src === filename) {
-        //   Fs.copyFileSync(src, dest)
-        //   Fs.truncateSync(src, 0)
-        // } else {
-        //   Fs.renameSync(src, dest)
-        // }
-      }
-      dest = src
-    }
-
-    assign(state, {
-      currentSize: 0,
-      file: Fs.openSync(filename, "a")
-    })
-  }
-
-  /**
-   * Close the handler
-   *
-   * @returns {Promise<void>}
-   */
-  close() {
-    return this.flush().onComplete(() =>
-      Option.of(this.state.file).map(Fs.closeSync)
-    ).toPromise()
-  }
-
-  get queue() {
-    return this.state.queue
-  }
-
-  get file() {
-    return this.state.file
-  }
-
-  get flushing() {
-    return this.state.flushing
-  }
-
-  get filename() {
-    return (this.state.filename = this.state.filename ?? this.config.filename)
-  }
-
-  /**
-   * Appends the log queue records to the file
-   */
-  private flush() {
-    const { state } = this
-    return Future.do(async () => {
-      state.flushing = true
-      try {
-        const { file } = this
-        if (!file) {
-          throw Error(`${this.filename} was not opened properly`)
-        }
-        while (this.queue.length) {
-          const buf: Buffer = this.queue.shift()
-
-          await Bluebird.fromCallback((cb) =>
-            Fs.appendFile(file, buf, "utf-8", cb)
-          )
-
-          state.currentSize += buf.byteLength
-        }
-
-        const { maxSize } = this.config
-        if (maxSize > 0 && state.currentSize >= maxSize) {
-          this.rollFile()
-        }
-
-        //await Bluebird.fromCallback((cb) => Fs.fdatasync(this.file, cb))
-      } catch (err) {
-        console.error(`Failed to append file ${this.filename}`, err)
-      } finally {
-        this.state.flushing = false
-        if (this.queue.length) {
-          queueMicrotask(() => this.flush())
-        }
-      }
-    })
-  }
-
-  /**
-   * Handle log records, transform, push to ES
-   *
-   * @param record
-   */
-  append(record: Record): void {
-    try {
-      const { queue, filename } = this.state
-      const count = queue.length
-      if (count > 999) {
-        debug(
-          `Too many log records (${count}) are in the queue without the file (${filename}) opening, skipping %O`,
-          record
-        )
-        return
-      }
-
-      const data = this.config.prettyPrint
-        ? JSON.stringify(record, null, 2)
-        : JSON.stringify(record)
-      queue.push(Buffer.from(data + "\n", "utf-8"))
-      this.flush()
-    } catch (err) {
-      console.warn(`Failed to synchronize `, err)
-    }
-  }
-
-  /**
-   *
-   * @param {Partial<FileAppenderOptions<Record>>} options
-   */
-  constructor(options: Partial<FileAppenderOptions<Record>> = {}) {
-    this.config = applyConfigDefaults(options)
-
-    assign(this.state, {
-      filename: this.config.filename
-    })
-
-    this.setup()
-    debug(`File appender is ready, logging to ${this.state.filename}`)
-    //      return this
-  }
-}

package/src/logging/appenders/aws-firehose/AWSFirehoseAppender.ts

@@ -1,176 +0,0 @@
-/* eslint-disable typescript/no-explicit-any */
-import { isString } from '../../../guards/primitive.js'
-import { FirehoseClient, PutRecordBatchCommand, PutRecordBatchInput } from "@aws-sdk/client-firehose"
-import {
-  Credentials as IAWSTemporaryCredentials,
-  ExpiredTokenException,
-} from "@aws-sdk/client-sts"
-import { Level } from "../../Level.js"
-
-import AWSFirehoseCredentialManager from "./AWSFirehoseCredentialManager.js"
-import { getInternalLogger } from "../../InternalLogger.js"
-import { getValue, guard } from "../../../guards/index.js"
-import { pick } from "lodash"
-import { Deferred } from "../../../helpers/index.js"
-import { LogRecord } from "../../LogRecord.js"
-import { Appender } from "../../Appender.js"
-
-const LogQueueMaxRecords = 10000
-
-const log = getInternalLogger(import.meta.filename)
-
-
-export class AWSFirehoseAppender<Record extends LogRecord = LogRecord>
-  implements Appender<Record> {
-  private pendingRecords = Array<LogRecord>()
-  private flushDeferred: Deferred<void>
-  private flushIntervalMs = 1000;
-
-  constructor(public readonly credentialManager: AWSFirehoseCredentialManager = new AWSFirehoseCredentialManager()) {
-    credentialManager.on("received", () => {
-      log.info("Credentials received, initializing firehose client")
-      if (!this.pendingRecords.length || !!this.flushDeferred) {
-        log.debug("Nothing to flush upon credential receipt OR flush already in progress")
-        return
-      }
-      this.flush()
-        .catch((e: Error) => {
-          log.error(`Error flushing logs after credential receipt: ${e.message}`, e)
-        })
-    })
-  }
-  append(record: Record): void {
-    const
-      pendingCount = this.pendingRecords.length + 1,
-      removeCount = LogQueueMaxRecords - pendingCount
-
-    if (removeCount < 0) {
-      this.pendingRecords.splice(0, Math.abs(removeCount))
-    }
-
-    this.pendingRecords.push({
-      ...record,
-      app: "web",
-      timestamp: Date.now(),
-      env: typeof process === "undefined" ? "WEB" : process.env.WIRE_ENV ?? "SHARED",
-      url: typeof window === "undefined" ? "local://node" : window.location.href,
-    })
-
-    this.flush()
-  }
-
-  /**
-   * Create the firehose client and return
-   *
-   * @param {IAWSTemporaryCredentials} credentials
-   * @param {boolean} flush
-   * @returns {Firehose}
-   */
-  private getFirehose(): FirehoseClient {
-    const creds = this.credentialManager.getCredentials()
-
-    if (!creds) {
-      log.warn("No credentials available for flush")
-      return null
-    }
-
-    const { region, streamName } = creds
-
-    if (!region || !streamName) {
-      throw new Error("Region or stream name not available")
-    }
-
-    const client = new FirehoseClient({
-      region: creds.region,
-      credentials: pick(creds, "accessKeyId", "secretAccessKey", "sessionToken"),
-    })
-
-    return client
-  }
-
-  /**
-   * Flush pending records
-   *
-   * @returns {Promise<void>}
-   */
-  private async flush(): Promise<void> {
-    if (!!this.flushDeferred) {
-      log.debug("Flush already in progress, skipping")
-      return
-    }
-
-    if (!this.pendingRecords.length) {
-      log.debug("No pending records to flush")
-      return
-    }
-    const creds = this.credentialManager.getCredentials()
-    if (!creds) {
-      log.warn("No credentials available for flush")
-      return
-    }
-    const { streamName } = creds,
-      client = this.getFirehose()
-    if (!client) {
-      log.warn("No firehose client available, skipping flush")
-      return
-    }
-
-    const deferred = this.flushDeferred = new Deferred<void>()
-    try {
-      while (this.pendingRecords.length) {
-        const
-          chunkSize = Math.min(this.pendingRecords.length, 10),
-          records = this.pendingRecords.splice(0, chunkSize),
-          recordJsons = records.map(record => JSON.stringify(record))
-
-        // log.info("Record JSONs", recordJsons)
-
-        const
-          textEncoder = new TextEncoder(),
-          batch: PutRecordBatchInput["Records"] = recordJsons.map(jsonStr => ({
-            Data: textEncoder.encode(jsonStr)
-          }))
-
-        log.info(`Pushing ${records.length} records to firehose`)
-
-        const cmd = new PutRecordBatchCommand({
-          DeliveryStreamName: streamName,
-          Records: batch,
-        })
-
-        const resp = await client.send(cmd)
-        const failed = resp.FailedPutCount || 0
-        if (failed) {
-          const rr = resp.RequestResponses || []
-          for (let i = 0; i < rr.length; i++) {
-            if (rr[i].ErrorCode) {
-              log.warn(`Record failed: ${rr[i].ErrorCode} ${rr[i].ErrorMessage}`)
-              //this.pendingRecords.unshift(records[i])
-            }
-          }
-          log.info(`PutRecordBatch failed=${failed}; requeued failed records`)
-        } else {
-          log.info(`PutRecordBatch ok count=${records.length}`)
-        }
-
-      }
-    } catch (err) {
-      log.error("Failed to push logs", err)
-      if (err.code === ExpiredTokenException) {
-        this.credentialManager.forceUpdateCredentials()
-      }
-      deferred.reject(err)
-    } finally {
-      this.flushDeferred = null
-
-      // if (this.pendingRecords.length)
-      //   this.flush()
-    }
-
-    return deferred.promise
-  }
-
-
-}
-
-export default AWSFirehoseAppender

package/src/logging/appenders/aws-firehose/AWSFirehoseCredentialManager.ts

@@ -1,132 +0,0 @@
-import { FirehoseClient, PutRecordBatchCommand } from "@aws-sdk/client-firehose"
-import { Deferred } from "../../../helpers/index.js"
-import { getInternalLogger } from "../../InternalLogger.js"
-import EventEmitter3 from "eventemitter3"
-
-const API_BASE = "https://log-ingest.shared.wire-dev.com"
-const CRED_BUFFER_PADDING_MS = 3 * 60 * 1000 // 3 minutes
-
-const log = getInternalLogger(import.meta.filename)
-
-export interface AWSCredentials {
-  accessKeyId: string
-  secretAccessKey: string
-  sessionToken: string
-  streamName: string
-  region: string
-  expiration: string
-}
-
-export interface AWSFirehoseCredentialManagerEventMap {
-  received: (creds: AWSCredentials) => void
-  expired: () => void
-}
-
-export class AWSFirehoseCredentialManager extends EventEmitter3<AWSFirehoseCredentialManagerEventMap> {
-
-  private refreshTimer: ReturnType<typeof setTimeout> = null;
-  private loadDeferred: Deferred<AWSCredentials> = null;
-
-  forceUpdateCredentials(): void {
-    if (this.loadDeferred && !this.loadDeferred.isSettled()) {
-      this.loadDeferred.reject(new Error("force update"))
-    }
-
-    this.loadDeferred = null
-    this.scheduleRefresh(1)
-  }
-
-  getCredentials(): AWSCredentials {
-    if (this.loadDeferred && this.loadDeferred.isFulfilled()) {
-      const creds = this.loadDeferred.getResult(),
-        expMs = Date.parse(creds.expiration)
-
-      if (expMs <= Date.now()) {
-        log.info("Credentials expired")
-        this.loadDeferred = null
-        this.scheduleRefresh(1)
-        this.emit("expired")
-        return null
-      }
-      return creds
-    }
-
-    this.scheduleRefresh(1)
-    return null
-  }
-
-  hasCredentials(): boolean {
-    return this.getCredentials() != null
-  }
-
-  private async load(): Promise<AWSCredentials> {
-    if (this.loadDeferred && !this.loadDeferred.isRejected())
-      return this.loadDeferred.promise
-
-    const deferred = this.loadDeferred = new Deferred<AWSCredentials>()
-    try {
-      const r = await fetch(API_BASE + "/creds", {
-        method: "GET",
-        credentials: "include",
-      })
-      if (!r.ok) throw new Error(`creds http ${r.status}`)
-      const creds: AWSCredentials = await r.json()
-
-
-      this.scheduleRefresh(creds)
-      log.info(`Got creds, expire=${creds.expiration}, stream=${creds.streamName}, region=${creds.region}`)
-      deferred.resolve(creds)
-      this.emit("received", creds)
-      return deferred.promise
-    } catch (e) {
-      deferred.reject(e)
-      this.loadDeferred = null
-      throw e
-    }
-  }
-  scheduleRefresh(credsOrMillis: AWSCredentials | number): void {
-    if (this.refreshTimer) {
-      clearTimeout(this.refreshTimer)
-      this.refreshTimer = null
-    }
-    if (!credsOrMillis)
-      return
-
-    let refreshInMs: number
-    if (typeof credsOrMillis === "number") {
-      refreshInMs = credsOrMillis
-    } else {
-      const
-        creds = credsOrMillis,
-        expMs = Date.parse(creds.expiration)
-
-      refreshInMs = expMs - Date.now() - CRED_BUFFER_PADDING_MS
-
-    }
-
-    refreshInMs = Math.max(100, refreshInMs)
-
-    this.refreshTimer = setTimeout(() => {
-      this.loadDeferred = null
-      this.load().catch((e: Error) => log.error(`refresh failed: ${e.message}`, e))
-    }, refreshInMs)
-  }
-
-  getStreamName(): string {
-    return this.getCredentials()?.streamName
-  }
-
-  getRegion(): string {
-    return this.getCredentials()?.region
-  }
-
-  destroy(): void {
-    this.scheduleRefresh(null)
-    if (!this.loadDeferred.isSettled())
-      this.loadDeferred.reject(new Error("destroyed"))
-    this.loadDeferred = null
-  }
-}
-
-
-export default AWSFirehoseCredentialManager

package/src/logging/appenders/index.ts

@@ -1,3 +0,0 @@
-export * from "./aws-firehose/AWSFirehoseAppender.js"
-export * from "./ConsoleAppender.js"
-export * from "./DebugAppender.js"

package/src/logging/context/LogContext.ts

@@ -1,74 +0,0 @@
-import { Appender } from "../Appender.js"
-import { Level } from "../Level.js"
-import { LogContextProvider } from "./LogContextProvider.js"
-import { LogContextContainer } from "./LogContextContainer.js"
-
-
-
-
-
-
-export class LogContext {
-  /**
-   * Threshold override level
-   *
-   * @type {Level}
-   */
-  readonly thresholdLevel: Level
-
-  /**
-   * If specified, only categories that match will
-   * use this context
-   *
-   * @type {RegExp}
-   */
-  readonly pattern: RegExp
-
-  /**
-   * Exclusive, only use this config in context.  Defaults to `false`
-   *
-   * @type {boolean}
-   */
-  readonly exclusive: RegExp
-  
-  /**
-   * Use a logger context within
-   *
-   * @param {() => Promise<T>} fn
-   * @returns {Promise<T>}
-   */
-  use<T = unknown>(fn: () => Promise<T>) {
-    return LogContextContainer.runInContext(this, fn)
-  }
-  
-  /**
-   * Creates a new logger context
-   *
-   * @param {Appender[]} appenders - additional appenders for the context
-   * @param {LoggerContextOptions} options -
-   */
-  protected constructor(
-    readonly appenders: Appender[],
-    options: LoggerContextOptions
-  ) {
-    Object.assign(this, {
-      exclusive: false,
-      thresholdLevel: null,
-      pattern: null,
-      ...options
-    })
-  }
-
-  /**
-   * Create a new logger context
-   *
-   * @param {Appender[]} appenders
-   * @param {LoggerContextOptions} options
-   * @returns {LogContext}
-   */
-  static with(appenders: Appender[], options: LoggerContextOptions = {}) {
-    return new LogContext(appenders, options)
-  }
-}
-
-export type LoggerContextOptions = Partial<Pick<LogContext, "pattern" | "exclusive">>

package/src/logging/context/LogContextContainer.ts

@@ -1,28 +0,0 @@
-import { LogContext } from "./LogContext.js"
-import { DefaultLogContextProvider, LogContextProvider } from "./LogContextProvider.js"
-
-const state = {
-  provider: DefaultLogContextProvider as LogContextProvider
-}
-
-
-export namespace LogContextContainer  {
-  
-  export function setProvider(provider: LogContextProvider) {
-    state.provider = provider ?? DefaultLogContextProvider
-  }
-  
-  export function getProvider():LogContextProvider {
-    return state.provider ?? DefaultLogContextProvider
-  }
-  
-  export function currentContext(): Array<LogContext> {
-    return getProvider().currentContext?.() ?? []
-  }
-  
-  export function runInContext<T = unknown>(context: LogContext, fn: () => Promise<T>): Promise<T> {
-    const provider = getProvider()
-    return provider.runInContext(context,fn)
-  }
-  
-}