@elizaos/core 1.5.1 → 1.5.2

package/src/types/model.ts

@@ -0,0 +1,359 @@
+import type { IAgentRuntime } from './runtime';
+
+export type ModelTypeName = (typeof ModelType)[keyof typeof ModelType] | string;
+
+/**
+ * Defines the recognized types of models that the agent runtime can use.
+ * These include models for text generation (small, large, reasoning, completion),
+ * text embedding, tokenization (encode/decode), image generation and description,
+ * audio transcription, text-to-speech, and generic object generation.
+ * This constant is used throughout the system, particularly in `AgentRuntime.useModel`,
+ * `AgentRuntime.registerModel`, and in `ModelParamsMap` / `ModelResultMap` to ensure
+ * type safety and clarity when working with different AI models.
+ * String values are used for extensibility with custom model types.
+ */
+export const ModelType = {
+  SMALL: 'TEXT_SMALL', // kept for backwards compatibility
+  MEDIUM: 'TEXT_LARGE', // kept for backwards compatibility
+  LARGE: 'TEXT_LARGE', // kept for backwards compatibility
+  TEXT_SMALL: 'TEXT_SMALL',
+  TEXT_LARGE: 'TEXT_LARGE',
+  TEXT_EMBEDDING: 'TEXT_EMBEDDING',
+  TEXT_TOKENIZER_ENCODE: 'TEXT_TOKENIZER_ENCODE',
+  TEXT_TOKENIZER_DECODE: 'TEXT_TOKENIZER_DECODE',
+  TEXT_REASONING_SMALL: 'REASONING_SMALL',
+  TEXT_REASONING_LARGE: 'REASONING_LARGE',
+  TEXT_COMPLETION: 'TEXT_COMPLETION',
+  IMAGE: 'IMAGE',
+  IMAGE_DESCRIPTION: 'IMAGE_DESCRIPTION',
+  TRANSCRIPTION: 'TRANSCRIPTION',
+  TEXT_TO_SPEECH: 'TEXT_TO_SPEECH',
+  AUDIO: 'AUDIO',
+  VIDEO: 'VIDEO',
+  OBJECT_SMALL: 'OBJECT_SMALL',
+  OBJECT_LARGE: 'OBJECT_LARGE',
+} as const;
+
+/**
+ * Model configuration setting keys used in character settings.
+ * These constants define the keys for accessing model parameters
+ * from character configuration with support for per-model-type settings.
+ *
+ * Setting Precedence (highest to lowest):
+ * 1. Parameters passed directly to useModel()
+ * 2. Model-specific settings (e.g., TEXT_SMALL_TEMPERATURE)
+ * 3. Default settings (e.g., DEFAULT_TEMPERATURE)
+ *
+ * Example character settings:
+ * ```
+ * settings: {
+ *   DEFAULT_TEMPERATURE: 0.7,              // Applies to all models
+ *   TEXT_SMALL_TEMPERATURE: 0.5,           // Overrides default for TEXT_SMALL
+ *   TEXT_LARGE_MAX_TOKENS: 4096,           // Specific to TEXT_LARGE
+ *   OBJECT_SMALL_TEMPERATURE: 0.3,         // Specific to OBJECT_SMALL
+ * }
+ * ```
+ */
+export const MODEL_SETTINGS = {
+  // Default settings - apply to all model types unless overridden
+  DEFAULT_MAX_TOKENS: 'DEFAULT_MAX_TOKENS',
+  DEFAULT_TEMPERATURE: 'DEFAULT_TEMPERATURE',
+  DEFAULT_FREQUENCY_PENALTY: 'DEFAULT_FREQUENCY_PENALTY',
+  DEFAULT_PRESENCE_PENALTY: 'DEFAULT_PRESENCE_PENALTY',
+
+  // TEXT_SMALL specific settings
+  TEXT_SMALL_MAX_TOKENS: 'TEXT_SMALL_MAX_TOKENS',
+  TEXT_SMALL_TEMPERATURE: 'TEXT_SMALL_TEMPERATURE',
+  TEXT_SMALL_FREQUENCY_PENALTY: 'TEXT_SMALL_FREQUENCY_PENALTY',
+  TEXT_SMALL_PRESENCE_PENALTY: 'TEXT_SMALL_PRESENCE_PENALTY',
+
+  // TEXT_LARGE specific settings
+  TEXT_LARGE_MAX_TOKENS: 'TEXT_LARGE_MAX_TOKENS',
+  TEXT_LARGE_TEMPERATURE: 'TEXT_LARGE_TEMPERATURE',
+  TEXT_LARGE_FREQUENCY_PENALTY: 'TEXT_LARGE_FREQUENCY_PENALTY',
+  TEXT_LARGE_PRESENCE_PENALTY: 'TEXT_LARGE_PRESENCE_PENALTY',
+
+  // OBJECT_SMALL specific settings
+  OBJECT_SMALL_MAX_TOKENS: 'OBJECT_SMALL_MAX_TOKENS',
+  OBJECT_SMALL_TEMPERATURE: 'OBJECT_SMALL_TEMPERATURE',
+  OBJECT_SMALL_FREQUENCY_PENALTY: 'OBJECT_SMALL_FREQUENCY_PENALTY',
+  OBJECT_SMALL_PRESENCE_PENALTY: 'OBJECT_SMALL_PRESENCE_PENALTY',
+
+  // OBJECT_LARGE specific settings
+  OBJECT_LARGE_MAX_TOKENS: 'OBJECT_LARGE_MAX_TOKENS',
+  OBJECT_LARGE_TEMPERATURE: 'OBJECT_LARGE_TEMPERATURE',
+  OBJECT_LARGE_FREQUENCY_PENALTY: 'OBJECT_LARGE_FREQUENCY_PENALTY',
+  OBJECT_LARGE_PRESENCE_PENALTY: 'OBJECT_LARGE_PRESENCE_PENALTY',
+
+  // Legacy keys for backwards compatibility (will be treated as defaults)
+  MODEL_MAX_TOKEN: 'MODEL_MAX_TOKEN',
+  MODEL_TEMPERATURE: 'MODEL_TEMPERATURE',
+  MODEL_FREQ_PENALTY: 'MODEL_FREQ_PENALTY',
+  MODEL_PRESENCE_PENALTY: 'MODEL_PRESENCE_PENALTY',
+} as const;
+
+/**
+ * Helper to get the model-specific setting key for a given model type and parameter.
+ * @param modelType The model type (e.g., TEXT_SMALL, TEXT_LARGE)
+ * @param param The parameter name (e.g., MAX_TOKENS, TEMPERATURE)
+ * @returns The appropriate setting key or null if not a supported model type
+ */
+export function getModelSpecificSettingKey(
+  modelType: ModelTypeName,
+  param: 'MAX_TOKENS' | 'TEMPERATURE' | 'FREQUENCY_PENALTY' | 'PRESENCE_PENALTY'
+): string | null {
+  const supportedModelTypes = ['TEXT_SMALL', 'TEXT_LARGE', 'OBJECT_SMALL', 'OBJECT_LARGE'];
+
+  if (!supportedModelTypes.includes(modelType)) {
+    return null;
+  }
+
+  return `${modelType}_${param}`;
+}
+
+/**
+ * Parameters for generating text using a language model.
+ * This structure is typically passed to `AgentRuntime.useModel` when the `modelType` is one of
+ * `ModelType.TEXT_SMALL`, `ModelType.TEXT_LARGE`, `ModelType.TEXT_REASONING_SMALL`,
+ * `ModelType.TEXT_REASONING_LARGE`, or `ModelType.TEXT_COMPLETION`.
+ * It includes essential information like the prompt, model type, and various generation controls.
+ */
+export type GenerateTextParams = {
+  /** The `AgentRuntime` instance, providing access to models and other services. */
+  runtime: IAgentRuntime;
+  /** The input string or prompt that the language model will use to generate text. */
+  prompt: string;
+  /** Specifies the type of text generation model to use (e.g., TEXT_LARGE, REASONING_SMALL). */
+  modelType: ModelTypeName;
+  /** Optional. The maximum number of tokens to generate in the response. */
+  maxTokens?: number;
+  /** Optional. Controls randomness (0.0-1.0). Lower values are more deterministic, higher are more creative. */
+  temperature?: number;
+  /** Optional. Penalizes new tokens based on their existing frequency in the text so far. */
+  frequencyPenalty?: number;
+  /** Optional. Penalizes new tokens based on whether they appear in the text so far. */
+  presencePenalty?: number;
+  /** Optional. A list of sequences at which the model will stop generating further tokens. */
+  stopSequences?: string[];
+};
+
+/**
+ * Parameters for detokenizing text, i.e., converting a sequence of numerical tokens back into a string.
+ * This is the reverse operation of tokenization.
+ * This structure is used with `AgentRuntime.useModel` when the `modelType` is `ModelType.TEXT_TOKENIZER_DECODE`.
+ */
+export interface DetokenizeTextParams {
+  /** An array of numerical tokens to be converted back into text. */
+  tokens: number[];
+  /** The model type used for detokenization, ensuring consistency with the original tokenization. */
+  modelType: ModelTypeName;
+}
+
+/**
+ * Base parameters common to all model types
+ */
+export interface BaseModelParams {
+  /** The agent runtime for accessing services and utilities */
+  runtime: IAgentRuntime;
+}
+
+/**
+ * Parameters for text generation models
+ */
+export interface TextGenerationParams extends BaseModelParams {
+  /** The prompt to generate text from */
+  prompt: string;
+  /** Model temperature (0.0 to 1.0, lower is more deterministic) */
+  temperature?: number;
+  /** Maximum number of tokens to generate */
+  maxTokens?: number;
+  /** Sequences that should stop generation when encountered */
+  stopSequences?: string[];
+  /** Frequency penalty to apply */
+  frequencyPenalty?: number;
+  /** Presence penalty to apply */
+  presencePenalty?: number;
+}
+
+/**
+ * Parameters for text embedding models
+ */
+export interface TextEmbeddingParams extends BaseModelParams {
+  /** The text to create embeddings for */
+  text: string;
+}
+
+/**
+ * Parameters for text tokenization models
+ */
+export interface TokenizeTextParams extends BaseModelParams {
+  /** The text to tokenize */
+  prompt: string;
+  /** The model type to use for tokenization */
+  modelType: ModelTypeName;
+}
+
+/**
+ * Parameters for image generation models
+ */
+export interface ImageGenerationParams extends BaseModelParams {
+  /** The prompt describing the image to generate */
+  prompt: string;
+  /** The dimensions of the image to generate */
+  size?: string;
+  /** Number of images to generate */
+  count?: number;
+}
+
+/**
+ * Parameters for image description models
+ */
+export interface ImageDescriptionParams extends BaseModelParams {
+  /** The URL or path of the image to describe */
+  imageUrl: string;
+  /** Optional prompt to guide the description */
+  prompt?: string;
+}
+
+/**
+ * Parameters for transcription models
+ */
+export interface TranscriptionParams extends BaseModelParams {
+  /** The URL or path of the audio file to transcribe */
+  audioUrl: string;
+  /** Optional prompt to guide transcription */
+  prompt?: string;
+}
+
+/**
+ * Parameters for text-to-speech models
+ */
+export interface TextToSpeechParams extends BaseModelParams {
+  /** The text to convert to speech */
+  text: string;
+  /** The voice to use */
+  voice?: string;
+  /** The speaking speed */
+  speed?: number;
+}
+
+/**
+ * Parameters for audio processing models
+ */
+export interface AudioProcessingParams extends BaseModelParams {
+  /** The URL or path of the audio file to process */
+  audioUrl: string;
+  /** The type of audio processing to perform */
+  processingType: string;
+}
+
+/**
+ * Parameters for video processing models
+ */
+export interface VideoProcessingParams extends BaseModelParams {
+  /** The URL or path of the video file to process */
+  videoUrl: string;
+  /** The type of video processing to perform */
+  processingType: string;
+}
+
+/**
+ * Optional JSON schema for validating generated objects
+ */
+export type JSONSchema = {
+  type: string;
+  properties?: Record<string, any>;
+  required?: string[];
+  items?: JSONSchema;
+  [key: string]: any;
+};
+
+/**
+ * Parameters for object generation models
+ * @template T - The expected return type, inferred from schema if provided
+ */
+export interface ObjectGenerationParams extends BaseModelParams {
+  /** The prompt describing the object to generate */
+  prompt: string;
+  /** Optional JSON schema for validation */
+  schema?: JSONSchema;
+  /** Type of object to generate */
+  output?: 'object' | 'array' | 'enum';
+  /** For enum type, the allowed values */
+  enumValues?: string[];
+  /** Model type to use */
+  modelType?: ModelTypeName;
+  /** Model temperature (0.0 to 1.0) */
+  temperature?: number;
+  /** Sequences that should stop generation */
+  stopSequences?: string[];
+}
+
+/**
+ * Map of model types to their parameter types
+ */
+export interface ModelParamsMap {
+  [ModelType.TEXT_SMALL]: TextGenerationParams;
+  [ModelType.TEXT_LARGE]: TextGenerationParams;
+  [ModelType.TEXT_EMBEDDING]: TextEmbeddingParams | string | null;
+  [ModelType.TEXT_TOKENIZER_ENCODE]: TokenizeTextParams;
+  [ModelType.TEXT_TOKENIZER_DECODE]: DetokenizeTextParams;
+  [ModelType.TEXT_REASONING_SMALL]: TextGenerationParams;
+  [ModelType.TEXT_REASONING_LARGE]: TextGenerationParams;
+  [ModelType.IMAGE]: ImageGenerationParams;
+  [ModelType.IMAGE_DESCRIPTION]: ImageDescriptionParams | string;
+  [ModelType.TRANSCRIPTION]: TranscriptionParams | Buffer | string;
+  [ModelType.TEXT_TO_SPEECH]: TextToSpeechParams | string;
+  [ModelType.AUDIO]: AudioProcessingParams;
+  [ModelType.VIDEO]: VideoProcessingParams;
+  [ModelType.OBJECT_SMALL]: ObjectGenerationParams;
+  [ModelType.OBJECT_LARGE]: ObjectGenerationParams;
+  // Allow string index for custom model types
+  [key: string]: BaseModelParams | any;
+}
+
+/**
+ * Map of model types to their return value types
+ */
+export interface ModelResultMap {
+  [ModelType.TEXT_SMALL]: string;
+  [ModelType.TEXT_LARGE]: string;
+  [ModelType.TEXT_EMBEDDING]: number[];
+  [ModelType.TEXT_TOKENIZER_ENCODE]: number[];
+  [ModelType.TEXT_TOKENIZER_DECODE]: string;
+  [ModelType.TEXT_REASONING_SMALL]: string;
+  [ModelType.TEXT_REASONING_LARGE]: string;
+  [ModelType.IMAGE]: { url: string }[];
+  [ModelType.IMAGE_DESCRIPTION]: { title: string; description: string };
+  [ModelType.TRANSCRIPTION]: string;
+  [ModelType.TEXT_TO_SPEECH]: any | Buffer;
+  [ModelType.AUDIO]: any; // Specific return type depends on processing type
+  [ModelType.VIDEO]: any; // Specific return type depends on processing type
+  [ModelType.OBJECT_SMALL]: any;
+  [ModelType.OBJECT_LARGE]: any;
+  // Allow string index for custom model types
+  [key: string]: any;
+}
+
+/**
+ * Defines the structure for a model handler registration within the `AgentRuntime`.
+ * Each model (e.g., for text generation, embedding) is associated with a handler function,
+ * the name of the provider (plugin or system) that registered it, and an optional priority.
+ * The `priority` (higher is more preferred) helps in selecting which handler to use if multiple
+ * handlers are registered for the same model type. The `registrationOrder` (not in type, but used in runtime)
+ * serves as a tie-breaker. See `AgentRuntime.registerModel` and `AgentRuntime.getModel`.
+ */
+export interface ModelHandler {
+  /** The function that executes the model, taking runtime and parameters, and returning a Promise. */
+  handler: (runtime: IAgentRuntime, params: Record<string, unknown>) => Promise<unknown>;
+  /** The name of the provider (e.g., plugin name) that registered this model handler. */
+  provider: string;
+  /**
+   * Optional priority for this model handler. Higher numbers indicate higher priority.
+   * This is used by `AgentRuntime.getModel` to select the most appropriate handler
+   * when multiple are available for a given model type. Defaults to 0 if not specified.
+   */
+  priority?: number; // Optional priority for selection order
+
+  registrationOrder?: number;
+}

package/src/types/pdf.ts

@@ -0,0 +1,77 @@
+import { Service, ServiceType } from './service';
+
+export interface PdfExtractionResult {
+  text: string;
+  pageCount: number;
+  metadata?: {
+    title?: string;
+    author?: string;
+    createdAt?: Date;
+    modifiedAt?: Date;
+  };
+}
+
+export interface PdfGenerationOptions {
+  format?: 'A4' | 'A3' | 'Letter';
+  orientation?: 'portrait' | 'landscape';
+  margins?: {
+    top?: number;
+    bottom?: number;
+    left?: number;
+    right?: number;
+  };
+  header?: string;
+  footer?: string;
+}
+
+export interface PdfConversionOptions {
+  quality?: 'high' | 'medium' | 'low';
+  outputFormat?: 'pdf' | 'pdf/a';
+  compression?: boolean;
+}
+
+/**
+ * Interface for PDF processing services
+ */
+export abstract class IPdfService extends Service {
+  static override readonly serviceType = ServiceType.PDF;
+
+  public readonly capabilityDescription = 'PDF processing, extraction, and generation capabilities';
+
+  /**
+   * Extract text and metadata from a PDF file
+   * @param pdfPath - Path to the PDF file or buffer
+   * @returns Promise resolving to extracted text and metadata
+   */
+  abstract extractText(pdfPath: string | Buffer): Promise<PdfExtractionResult>;
+
+  /**
+   * Generate a PDF from HTML content
+   * @param htmlContent - HTML content to convert to PDF
+   * @param options - PDF generation options
+   * @returns Promise resolving to PDF buffer
+   */
+  abstract generatePdf(htmlContent: string, options?: PdfGenerationOptions): Promise<Buffer>;
+
+  /**
+   * Convert a document to PDF format
+   * @param filePath - Path to the document file
+   * @param options - Conversion options
+   * @returns Promise resolving to PDF buffer
+   */
+  abstract convertToPdf(filePath: string, options?: PdfConversionOptions): Promise<Buffer>;
+
+  /**
+   * Merge multiple PDF files into one
+   * @param pdfPaths - Array of PDF file paths or buffers
+   * @returns Promise resolving to merged PDF buffer
+   */
+  abstract mergePdfs(pdfPaths: (string | Buffer)[]): Promise<Buffer>;
+
+  /**
+   * Split a PDF into individual pages
+   * @param pdfPath - Path to the PDF file or buffer
+   * @returns Promise resolving to array of page buffers
+   */
+  abstract splitPdf(pdfPath: string | Buffer): Promise<Buffer[]>;
+}

package/src/types/plugin.ts

@@ -0,0 +1,78 @@
+import type { Character } from './agent';
+import type { Action, Evaluator, Provider } from './components';
+import type { IDatabaseAdapter } from './database';
+import type { EventHandler, EventPayloadMap } from './events';
+import type { IAgentRuntime } from './runtime';
+import type { Service } from './service';
+import type { TestSuite } from './testing';
+
+export type Route = {
+  type: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'STATIC';
+  path: string;
+  filePath?: string;
+  public?: boolean;
+  name?: string extends { public: true } ? string : string | undefined;
+  handler?: (req: any, res: any, runtime: IAgentRuntime) => Promise<void>;
+  isMultipart?: boolean; // Indicates if the route expects multipart/form-data (file uploads)
+};
+
+/**
+ * Plugin for extending agent functionality
+ */
+
+export type PluginEvents = {
+  [K in keyof EventPayloadMap]?: EventHandler<K>[];
+} & {
+  [key: string]: ((params: any) => Promise<any>)[];
+};
+
+export interface Plugin {
+  name: string;
+  description: string;
+
+  // Initialize plugin with runtime services
+  init?: (config: Record<string, string>, runtime: IAgentRuntime) => Promise<void>;
+
+  // Configuration
+  config?: { [key: string]: any };
+
+  services?: (typeof Service)[];
+
+  // Entity component definitions
+  componentTypes?: {
+    name: string;
+    schema: Record<string, unknown>;
+    validator?: (data: any) => boolean;
+  }[];
+
+  // Optional plugin features
+  actions?: Action[];
+  providers?: Provider[];
+  evaluators?: Evaluator[];
+  adapter?: IDatabaseAdapter;
+  models?: {
+    [key: string]: (...args: any[]) => Promise<any>;
+  };
+  events?: PluginEvents;
+  routes?: Route[];
+  tests?: TestSuite[];
+
+  dependencies?: string[];
+
+  testDependencies?: string[];
+
+  priority?: number;
+
+  schema?: any;
+}
+
+export interface ProjectAgent {
+  character: Character;
+  init?: (runtime: IAgentRuntime) => Promise<void>;
+  plugins?: Plugin[];
+  tests?: TestSuite | TestSuite[];
+}
+
+export interface Project {
+  agents: ProjectAgent[];
+}