@multiplayer-app/ai-agent-node 0.1.0-beta.1 → 0.1.0-beta.10

package/src/libs/kafka/consumer.ts

@@ -1,161 +0,0 @@
-import { Consumer, ConsumerRunConfig, IHeaders, Message } from 'kafkajs'
-import logger from '../logger'
-import { kafka } from './kafka'
-import { KAFKA_SESSION_TIMEOUT } from './config'
-
-const startTimer = () => {
-  const startTime = process.hrtime()
-  return startTime
-}
-const getDuration = (startTime) => {
-  const diff = process.hrtime(startTime)
-  return diff[0] * 1e3 + diff[1] * 1e-6
-}
-
-export type KafkaConsumerListener = (key, value, headers?: IHeaders) => void | Promise<void>
-export class KafkaConsumer {
-  private consumer: Consumer
-  private listeners: Record<string, KafkaConsumerListener[]> = {}
-  private lastHeartbeat: number = 0
-
-  constructor(groupId: string) {
-    this.consumer = kafka.consumer({
-      groupId,
-      sessionTimeout: KAFKA_SESSION_TIMEOUT,
-    })
-    this.consumer.on(this.consumer.events.HEARTBEAT, ({ timestamp }) => {
-      this.lastHeartbeat = timestamp
-    })
-    this.consumer.on(this.consumer.events.REQUEST_TIMEOUT, ({ timestamp }) => {
-      this.lastHeartbeat = 0
-    })
-    this.consumer.on(this.consumer.events.CRASH, ({ timestamp }) => {
-      this.lastHeartbeat = 0
-    })
-  }
-
-  public async isConnected() {
-    if (Date.now() - this.lastHeartbeat < KAFKA_SESSION_TIMEOUT) {
-      return true
-    }
-    try {
-      const { state } = await this.consumer.describeGroup()
-      return ['CompletingRebalance', 'PreparingRebalance', 'Stable'].includes(state)
-    } catch (err) {
-      return false
-    }
-  }
-
-  public async connect() {
-    await this.consumer.connect()
-  }
-
-  public async disconnect() {
-    await this.consumer.disconnect()
-  }
-
-  public async subscribe(topic: string, listener: KafkaConsumerListener) {
-    logger.info({ topic }, '[KAFKAJS] Subscribing for topic')
-
-    if (!this.listeners[topic]) {
-      this.listeners[topic] = []
-    }
-    this.listeners[topic].push(listener)
-  }
-
-  private async listenFnWrapper(
-    topic: string,
-    message: Message,
-  ) {
-
-    const parseStartTime = startTimer()
-    try {
-      if (!this.listeners[topic]?.length) {
-        logger.error(`[KAFKAJS] Missing handler for topic ${topic}`)
-      }
-
-      if (!message?.value) {
-        return
-      }
-
-      const key = message?.key?.toString('utf8')
-      const value = JSON.parse(message.value.toString('utf8'))
-      const headers = message.headers
-
-      logger.debug(
-        {
-          topic,
-          key,
-          value,
-        },
-        `[KAFKAJS] Received message from topic ${topic}`,
-      )
-
-      await Promise.all(this.listeners[topic].map(func => func(key, value, headers)))
-
-      logger.debug({
-        key,
-        topic,
-        duration: getDuration(parseStartTime),
-      }, '[KAFKAJS] Done with processing data from kafka')
-    } catch (err) {
-      logger.error(err, '[KAFKAJS] Error on handling incoming message')
-    }
-  }
-
-  public async listen(options: Omit<ConsumerRunConfig, 'eachMessage' | 'eachBatch'> = {
-    partitionsConsumedConcurrently: 3, autoCommit: true,
-  }) {
-    const topics = Object.keys(this.listeners)
-
-    await this.consumer.subscribe({
-      topics,
-      fromBeginning: true,
-    })
-
-    await this.consumer.run({
-      autoCommit: true,
-      ...options,
-      eachMessage: async ({ topic, message }) => this.listenFnWrapper(topic, message),
-    })
-  }
-
-  public async listenBatch(options: Omit<ConsumerRunConfig, 'eachMessage' | 'eachBatch'>) {
-    const topics = Object.keys(this.listeners)
-
-    await this.consumer.subscribe({
-      topics,
-      fromBeginning: true,
-    })
-
-    await this.consumer.run({
-      eachBatchAutoResolve: true,
-      ...options,
-      eachBatch: async ({
-        batch,
-        resolveOffset,
-        heartbeat,
-        commitOffsetsIfNecessary,
-        uncommittedOffsets,
-        isRunning,
-        isStale,
-        pause,
-      }) => {
-        for (const message of batch.messages) {
-          if (!this.listeners[batch.topic]?.length) {
-            logger.error(`[KAFKAJS] Missing handler for topic ${batch.topic}`)
-          }
-
-          try {
-            await Promise.all(batch.messages.map(message => this.listenFnWrapper(batch.topic, message)))
-          } catch (batchError) {
-            logger.error(batchError, '[KAFKAJS] Batch fn error')
-          }
-
-          resolveOffset(message.offset)
-          await heartbeat()
-        }
-      },
-    })
-  }
-}

package/src/libs/kafka/index.ts

@@ -1,2 +0,0 @@
-export * from './consumer'
-export * from './producer'

package/src/libs/kafka/kafka.ts

@@ -1,27 +0,0 @@
-import { Kafka, logLevel } from 'kafkajs'
-import { KAFKA_CLIENT_ID, KAFKA_URI } from './config'
-import { KafkaJsLogCreator } from '../logger'
-
-
-const toBunyanLogLevel = level => {
-  switch (level) {
-    case logLevel.ERROR:
-    case logLevel.NOTHING:
-      return 'error'
-    case logLevel.WARN:
-      return 'warn'
-    case logLevel.INFO:
-      return 'info'
-    case logLevel.DEBUG:
-      return 'debug'
-    default:
-      return 'info'
-  }
-}
-
-
-export const kafka = new Kafka({
-  clientId: KAFKA_CLIENT_ID,
-  brokers: KAFKA_URI,
-  logCreator: KafkaJsLogCreator(toBunyanLogLevel),
-})

package/src/libs/kafka/producer.ts

@@ -1,48 +0,0 @@
-import { IHeaders, Producer, ProducerConfig } from 'kafkajs'
-import { kafka } from './kafka'
-
-export class KafkaProducer {
-  private producer: Producer
-  private _isConnected = false
-  constructor(config: ProducerConfig = {}) {
-    this.producer = kafka.producer(config)
-  }
-
-  public async connect() {
-    try {
-      await this.producer.connect()
-      this._isConnected = true
-    } catch (err) {
-      this._isConnected = false
-      throw err
-    }
-  }
-
-  public isConnected() {
-    return this._isConnected
-  }
-
-  public async disconnect() {
-    await this.producer.disconnect()
-    this._isConnected = false
-  }
-
-  public async send(topic: string, value: Record<string, any>, key?: string, headers?: IHeaders) {
-    try {
-      await this.producer.send({
-        topic,
-        messages: [
-          {
-            key,
-            value: JSON.stringify(value),
-            headers: headers
-          },
-        ],
-      })
-      this._isConnected = true
-    } catch (err) {
-      this._isConnected = false
-      throw err
-    }
-  }
-}

package/src/libs/logger/config.ts

@@ -1,4 +0,0 @@
-export const NODE_ENV = process.env.NODE_ENV || 'development'
-export const isProduction = NODE_ENV === 'production'
-export const APP_NAME = process.env.npm_package_name?.split('/').pop() as string || 'tests'
-export const LOG_LEVEL = process.env.LOG_LEVEL || (isProduction ? 'info' : 'debug')

package/src/libs/logger/index.ts

@@ -1,21 +0,0 @@
-import { getLogger, setLogger, type AgentLogger } from './logger'
-export { getLogger, setLogger, type AgentLogger }
-export * from './kafkajs-logger-creator'
-
-/**
- * Logger object for backward compatibility.
- * All method calls proxy to the current logger instance.
- */
-export const logger = new Proxy({} as AgentLogger, {
-  get(_target, prop) {
-    const currentLogger = getLogger()
-    const method = (currentLogger as any)[prop]
-    if (typeof method === 'function') {
-      return method.bind(currentLogger)
-    }
-    // Return no-op for missing methods
-    return () => {}
-  }
-})
-
-export default logger

package/src/libs/logger/kafkajs-logger-creator.ts

@@ -1,28 +0,0 @@
-import { getLogger } from './logger'
-
-/**
- * Creates a KafkaJS log creator that uses the injected logger.
- * If no logger is set, logs are silently discarded.
- * 
- * @param logLevelConverter - Function to convert KafkaJS log levels to logger levels
- * @returns KafkaJS log creator function
- */
-export const KafkaJsLogCreator = (logLevelConverter: any) => (level: any) => {
-  const logger = getLogger()
-
-  return ({ level, log }) => {
-    const { message, ...extra } = log
-    const logLevel = logLevelConverter(level)
-    
-    // Use the logger's error, warn, info, or debug method based on level
-    if (logLevel === 'error' && logger.error) {
-      logger.error({ extra }, message)
-    } else if (logLevel === 'warn' && logger.warn) {
-      logger.warn({ extra }, message)
-    } else if (logLevel === 'info' && logger.info) {
-      logger.info({ extra }, message)
-    } else if (logLevel === 'debug' && logger.debug) {
-      logger.debug({ extra }, message)
-    }
-  }
-}

package/src/libs/logger/logger.ts

@@ -1,60 +0,0 @@
-/**
- * Logger interface for AI Agent Node operations.
- * Compatible with bunyan-style loggers and most common Node.js loggers (pino, winston, bunyan, etc.)
- */
-export interface AgentLogger {
-  info(messageOrFields: string | object, message?: string): void;
-  error(messageOrFields: string | object | Error | unknown, message?: string | unknown): void;
-  error(message: string, error: unknown): void;
-  warn(messageOrFields: string | object, message?: string): void;
-  warn(message: string, fields: object): void;
-  debug(messageOrFields: string | object, message?: string): void;
-  debug(message: string, fields: object): void;
-}
-
-/**
- * No-op logger that discards all log messages.
- * Used as default when no logger is provided.
- */
-const noOpLogger: AgentLogger = {
-  info: () => {},
-  error: () => {},
-  warn: () => {},
-  debug: () => {},
-};
-
-let logger: AgentLogger = noOpLogger;
-
-/**
- * Sets a custom logger for AI Agent Node operations.
- * If not called, all logging is silently discarded.
- * 
- * @param customLogger - Logger instance implementing AgentLogger interface
- * 
- * @example
- * ```ts
- * import { setLogger } from '@multiplayer-app/ai-agent-node';
- * import bunyan from 'bunyan';
- * 
- * setLogger(bunyan.createLogger({ name: 'my-app' }));
- * ```
- * 
- * @example
- * ```ts
- * import { setLogger } from '@multiplayer-app/ai-agent-node';
- * import pino from 'pino';
- * 
- * setLogger(pino());
- * ```
- */
-export function setLogger(customLogger: AgentLogger): void {
-  logger = customLogger;
-}
-
-/**
- * Gets the current logger instance.
- * @internal
- */
-export function getLogger(): AgentLogger {
-  return logger;
-}

package/src/libs/s3/config.ts

@@ -1,7 +0,0 @@
-export const S3_HOST = process.env.S3_HOST || 'https://s3.amazonaws.com'
-export const S3_EXPORT_HOST = process.env.S3_EXPORT_HOST || S3_HOST
-export const S3_PRESIGNED_URL_EXPIRES = Number(process.env.S3_PRESIGNED_URL_EXPIRES) || 120
-
-export const AWS_REGION = process.env.AWS_REGION as string
-export const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID as string
-export const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY as string

package/src/libs/s3/index.ts

@@ -1,3 +0,0 @@
-export { ObjectCannedACL } from '@aws-sdk/client-s3'
-export * as s3 from './s3.lib'
-export * from './config'

package/src/libs/s3/s3.lib.ts

@@ -1,284 +0,0 @@
-import {
-  S3,
-  GetObjectCommand,
-  PutObjectCommand,
-  CompleteMultipartUploadOutput,
-  ObjectCannedACL, GetObjectOutput,
-  GetObjectCommandOutput, ListObjectsV2CommandOutput,
-  ListObjectsV2Command,
-  HeadBucketCommand,
-  CreateBucketCommand,
-} from '@aws-sdk/client-s3'
-import { S3RequestPresigner } from '@aws-sdk/s3-request-presigner'
-import { createRequest } from '@aws-sdk/util-create-request'
-import { formatUrl } from '@aws-sdk/util-format-url'
-import { Upload } from '@aws-sdk/lib-storage'
-import stream from 'stream'
-import {
-  S3_HOST,
-  S3_PRESIGNED_URL_EXPIRES,
-  AWS_REGION,
-} from './config'
-import logger from '../logger'
-
-const S3Client = new S3({
-  ...S3_HOST ? {
-    endpoint: S3_HOST,
-    forcePathStyle: true,
-  }
-    : {},
-})
-
-const signer = new S3RequestPresigner({
-  ...S3Client.config,
-})
-
-export const uploadFile = (Key: string, Bucket: string, Body: any, Expires?: Date) => {
-  return S3Client.putObject({
-    Key,
-    Bucket,
-    Body,
-    Expires,
-  })
-}
-
-export const downloadFile = (Key: string, Bucket: string): Promise<GetObjectOutput> => {
-  const command = new GetObjectCommand({
-    Bucket,
-    Key,
-  })
-
-  return S3Client.send(command)
-}
-
-export const downloadFileAsString = async (
-  Key: string,
-  Bucket: string,
-): Promise<string | undefined> => {
-  try {
-    const command = new GetObjectCommand({
-      Bucket,
-      Key,
-    })
-
-    const response = await S3Client.send(command)
-    return response.Body?.transformToString()
-  } catch (err) {
-    if ((err as any)?.name === 'NoSuchKey') {
-      return undefined
-    }
-    throw err
-  }
-}
-
-export const downloadFileAsByteArray = async (Key: string, Bucket: string): Promise<Uint8Array | undefined> => {
-  try {
-    const command = new GetObjectCommand({
-      Bucket,
-      Key,
-    })
-
-    const response = await S3Client.send(command)
-    return response.Body?.transformToByteArray()
-  } catch (err) {
-    if ((err as any)?.name === 'NoSuchKey') {
-      return undefined
-    }
-    throw err
-  }
-}
-
-export const streamUpload = (Key: string, Bucket: string, ACL?: ObjectCannedACL) => {
-  const passThroughStream = new stream.PassThrough()
-
-  const options: any = {
-    client: S3Client,
-    params: {
-      Bucket,
-      Key,
-      Body: passThroughStream,
-    },
-    leavePartsOnError: false,
-  }
-
-  if (ACL) {
-    options.params.ACL = ACL
-  }
-
-  const upload = new Upload(options)
-
-  return {
-    writeStream: passThroughStream,
-    promise: upload.done() as Promise<CompleteMultipartUploadOutput>,
-  }
-}
-
-export const getPresignedUploadUrl = async (
-  Key: string,
-  Bucket: string,
-  expiresIn?: number,
-) => {
-  const request = await createRequest(
-    S3Client,
-    new PutObjectCommand({ Key, Bucket }),
-  )
-
-  const signedUrl = formatUrl(await signer.presign(
-    request,
-    {
-      expiresIn: expiresIn || S3_PRESIGNED_URL_EXPIRES,
-    },
-  ))
-
-  return signedUrl
-}
-
-export const getPresignedDownloadUrl = async (Key: string, Bucket: string) => {
-  const request = await createRequest(
-    S3Client,
-    new GetObjectCommand({ Key, Bucket }),
-  )
-
-  const signedUrl = formatUrl(
-    await signer.presign(
-      request,
-      {
-        expiresIn: S3_PRESIGNED_URL_EXPIRES,
-      },
-    ),
-  )
-
-  return signedUrl
-}
-
-export const getDownloadUrl = (Key: string, Bucket: string) => {
-  if (S3_HOST) {
-    return `${S3_HOST}/${Bucket}/${Key}`
-  }
-
-  const regionString = AWS_REGION.includes('us-east-1') ? '' : '-' + AWS_REGION
-  return `https://${Bucket}.s3${regionString}.amazonaws.com/${Key}`
-}
-
-export const copy = (
-  Bucket: string,
-  KeyFrom: string,
-  KeyTo: string,
-) => {
-  return S3Client.copyObject({
-    CopySource: `/${Bucket}/${KeyFrom}`,
-    Bucket,
-    Key: KeyTo,
-  })
-}
-
-export const copyBetweenBuckets = (
-  BucketFrom: string,
-  KeyFrom: string,
-  BucketTo: string,
-  KeyTo: string,
-) => {
-  return S3Client.copyObject({
-    CopySource: `/${BucketFrom}/${KeyFrom}`,
-    Bucket: BucketTo,
-    Key: KeyTo,
-  })
-}
-
-export const getObject = async (
-  Bucket: string,
-  Key: string,
-): Promise<GetObjectCommandOutput> => {
-  return S3Client.getObject({
-    Bucket,
-    Key,
-  })
-}
-
-export const listObjectsByPrefix = async (
-  Bucket: string,
-  Prefix: string,
-): Promise<ListObjectsV2CommandOutput> => {
-  return S3Client.send(
-    new ListObjectsV2Command({
-      Bucket,
-      Prefix,
-    }),
-  )
-}
-
-export const headObject = async (
-  Bucket: string,
-  Key: string,
-): Promise<GetObjectCommandOutput> => {
-  return S3Client.headObject({
-    Bucket,
-    Key,
-  })
-}
-
-export const deleteObject = async (
-  Bucket: string,
-  Key: string,
-): Promise<GetObjectCommandOutput> => {
-  return S3Client.deleteObject({
-    Bucket,
-    Key,
-  })
-}
-
-export const deleteObjectsByPrefix = async (
-  Bucket: string,
-  Prefix: string,
-  ContinuationToken?: string,
-): Promise<void> => {
-  const objects = await S3Client.listObjectsV2({ Bucket, Prefix, ContinuationToken })
-  if (!objects.Contents?.length) return
-
-  const deleteParams = {
-    Bucket,
-    Delete: { Objects: [] as { Key: string }[] },
-  }
-
-  objects.Contents.forEach(({ Key }) => {
-    if (!Key) return
-    deleteParams.Delete.Objects.push({ Key })
-  })
-  logger.info(deleteParams, '[S3] Deleting objects')
-  const deletedResp = await S3Client.deleteObjects(deleteParams)
-  if (deletedResp.Errors?.length) {
-    logger.error(deletedResp.Errors, '[S3] Error during deleteObjectsByPrefix')
-  }
-  if (objects.IsTruncated) {
-    await deleteObjectsByPrefix(Bucket, Prefix, objects.NextContinuationToken)
-  }
-}
-
-/**
- * Ensures a bucket exists, creating it if it doesn't
- */
-export const ensureBucketExists = async (Bucket: string): Promise<void> => {
-  try {
-    await S3Client.send(new HeadBucketCommand({ Bucket }))
-    // Bucket exists
-    return
-  } catch (error: any) {
-    // If bucket doesn't exist, create it
-    if (error.$metadata?.httpStatusCode === 404) {
-      try {
-        await S3Client.send(new CreateBucketCommand({ Bucket }))
-        logger.info({ Bucket }, '[S3] Created bucket')
-      } catch (createError: any) {
-        // Ignore if bucket was created by another process
-        if (createError.$metadata?.httpStatusCode !== 409) {
-          logger.error({ Bucket, error: createError }, '[S3] Failed to create bucket')
-          throw createError
-        }
-      }
-    } else {
-      // Some other error (permissions, etc.)
-      logger.error({ Bucket, error }, '[S3] Failed to check bucket existence')
-      throw error
-    }
-  }
-}