@gmickel/gno 0.3.0

package/src/llm/nodeLlamaCpp/adapter.ts

@@ -0,0 +1,133 @@
+/**
+ * Main LLM adapter for node-llama-cpp.
+ * Factory for creating port instances.
+ *
+ * @module src/llm/nodeLlamaCpp/adapter
+ */
+
+import type { Config } from '../../config/types';
+import { ModelCache } from '../cache';
+import { getActivePreset, getModelConfig } from '../registry';
+import type {
+  EmbeddingPort,
+  GenerationPort,
+  LlmResult,
+  RerankPort,
+} from '../types';
+import { NodeLlamaCppEmbedding } from './embedding';
+import { NodeLlamaCppGeneration } from './generation';
+import { getModelManager, type ModelManager } from './lifecycle';
+import { NodeLlamaCppRerank } from './rerank';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Adapter
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class LlmAdapter {
+  private readonly manager: ModelManager;
+  private readonly cache: ModelCache;
+  private readonly config: Config;
+
+  constructor(config: Config, cacheDir?: string) {
+    this.config = config;
+    const modelConfig = getModelConfig(config);
+    this.manager = getModelManager(modelConfig);
+    this.cache = new ModelCache(cacheDir);
+  }
+
+  /**
+   * Create an embedding port.
+   */
+  async createEmbeddingPort(
+    modelUri?: string
+  ): Promise<LlmResult<EmbeddingPort>> {
+    const preset = getActivePreset(this.config);
+    const uri = modelUri ?? preset.embed;
+
+    // Resolve model path from cache
+    const resolved = await this.cache.resolve(uri, 'embed');
+    if (!resolved.ok) {
+      return resolved;
+    }
+
+    return {
+      ok: true,
+      value: new NodeLlamaCppEmbedding(this.manager, uri, resolved.value),
+    };
+  }
+
+  /**
+   * Create a generation port.
+   */
+  async createGenerationPort(
+    modelUri?: string
+  ): Promise<LlmResult<GenerationPort>> {
+    const preset = getActivePreset(this.config);
+    const uri = modelUri ?? preset.gen;
+
+    // Resolve model path from cache
+    const resolved = await this.cache.resolve(uri, 'gen');
+    if (!resolved.ok) {
+      return resolved;
+    }
+
+    return {
+      ok: true,
+      value: new NodeLlamaCppGeneration(this.manager, uri, resolved.value),
+    };
+  }
+
+  /**
+   * Create a rerank port.
+   */
+  async createRerankPort(modelUri?: string): Promise<LlmResult<RerankPort>> {
+    const preset = getActivePreset(this.config);
+    const uri = modelUri ?? preset.rerank;
+
+    // Resolve model path from cache
+    const resolved = await this.cache.resolve(uri, 'rerank');
+    if (!resolved.ok) {
+      return resolved;
+    }
+
+    return {
+      ok: true,
+      value: new NodeLlamaCppRerank(this.manager, uri, resolved.value),
+    };
+  }
+
+  /**
+   * Get the model cache instance.
+   */
+  getCache(): ModelCache {
+    return this.cache;
+  }
+
+  /**
+   * Get the model manager instance.
+   */
+  getManager(): ModelManager {
+    return this.manager;
+  }
+
+  /**
+   * Dispose all resources.
+   */
+  async dispose(): Promise<void> {
+    await this.manager.disposeAll();
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Factory
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Create an LLM adapter instance.
+ */
+export function createLlmAdapter(
+  config: Config,
+  cacheDir?: string
+): LlmAdapter {
+  return new LlmAdapter(config, cacheDir);
+}

package/src/llm/nodeLlamaCpp/embedding.ts

@@ -0,0 +1,165 @@
+/**
+ * Embedding port implementation using node-llama-cpp.
+ *
+ * @module src/llm/nodeLlamaCpp/embedding
+ */
+
+import { inferenceFailedError } from '../errors';
+import type { EmbeddingPort, LlmResult } from '../types';
+import type { ModelManager } from './lifecycle';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+// LlamaModel type from node-llama-cpp
+type LlamaModel = Awaited<
+  ReturnType<
+    Awaited<ReturnType<typeof import('node-llama-cpp').getLlama>>['loadModel']
+  >
+>;
+
+type LlamaEmbeddingContext = Awaited<
+  ReturnType<LlamaModel['createEmbeddingContext']>
+>;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Implementation
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class NodeLlamaCppEmbedding implements EmbeddingPort {
+  private context: LlamaEmbeddingContext | null = null;
+  private contextPromise: Promise<LlmResult<LlamaEmbeddingContext>> | null =
+    null;
+  private dims: number | null = null;
+  private readonly manager: ModelManager;
+  readonly modelUri: string;
+  private readonly modelPath: string;
+
+  constructor(manager: ModelManager, modelUri: string, modelPath: string) {
+    this.manager = manager;
+    this.modelUri = modelUri;
+    this.modelPath = modelPath;
+  }
+
+  async init(): Promise<LlmResult<void>> {
+    const ctx = await this.getContext();
+    if (!ctx.ok) {
+      return ctx;
+    }
+    return { ok: true, value: undefined };
+  }
+
+  async embed(text: string): Promise<LlmResult<number[]>> {
+    const ctx = await this.getContext();
+    if (!ctx.ok) {
+      return ctx;
+    }
+
+    try {
+      const embedding = await ctx.value.getEmbeddingFor(text);
+      const vector = Array.from(embedding.vector) as number[];
+
+      // Cache dimensions on first call
+      if (this.dims === null) {
+        this.dims = vector.length;
+      }
+
+      return { ok: true, value: vector };
+    } catch (e) {
+      return { ok: false, error: inferenceFailedError(this.modelUri, e) };
+    }
+  }
+
+  async embedBatch(texts: string[]): Promise<LlmResult<number[][]>> {
+    const ctx = await this.getContext();
+    if (!ctx.ok) {
+      return ctx;
+    }
+
+    try {
+      const results: number[][] = [];
+      for (const text of texts) {
+        const embedding = await ctx.value.getEmbeddingFor(text);
+        const vector = Array.from(embedding.vector) as number[];
+        results.push(vector);
+
+        // Cache dimensions on first call
+        if (this.dims === null) {
+          this.dims = vector.length;
+        }
+      }
+      return { ok: true, value: results };
+    } catch (e) {
+      return { ok: false, error: inferenceFailedError(this.modelUri, e) };
+    }
+  }
+
+  dimensions(): number {
+    if (this.dims === null) {
+      throw new Error('Call init() or embed() first to initialize dimensions');
+    }
+    return this.dims;
+  }
+
+  async dispose(): Promise<void> {
+    // Clear promise first to prevent reuse of disposed context
+    this.contextPromise = null;
+    if (this.context) {
+      try {
+        await this.context.dispose();
+      } catch {
+        // Ignore disposal errors
+      }
+      this.context = null;
+    }
+  }
+
+  // ───────────────────────────────────────────────────────────────────────────
+  // Private
+  // ───────────────────────────────────────────────────────────────────────────
+
+  private getContext(): Promise<LlmResult<LlamaEmbeddingContext>> {
+    // Return cached context
+    if (this.context) {
+      return Promise.resolve({ ok: true, value: this.context });
+    }
+
+    // Reuse in-flight promise to prevent concurrent context creation
+    if (this.contextPromise) {
+      return this.contextPromise;
+    }
+
+    this.contextPromise = this.createContext();
+    return this.contextPromise;
+  }
+
+  private async createContext(): Promise<LlmResult<LlamaEmbeddingContext>> {
+    const model = await this.manager.loadModel(
+      this.modelPath,
+      this.modelUri,
+      'embed'
+    );
+    if (!model.ok) {
+      this.contextPromise = null; // Allow retry
+      return model;
+    }
+
+    try {
+      // Cast to access createEmbeddingContext
+      const llamaModel = model.value.model as LlamaModel;
+      this.context = await llamaModel.createEmbeddingContext();
+
+      // Cache dimensions from model (available without running embed)
+      const size = llamaModel.embeddingVectorSize;
+      if (this.dims === null && typeof size === 'number' && size > 0) {
+        this.dims = size;
+      }
+
+      return { ok: true, value: this.context };
+    } catch (e) {
+      this.contextPromise = null; // Allow retry
+      return { ok: false, error: inferenceFailedError(this.modelUri, e) };
+    }
+  }
+}

package/src/llm/nodeLlamaCpp/generation.ts

@@ -0,0 +1,88 @@
+/**
+ * Generation port implementation using node-llama-cpp.
+ *
+ * @module src/llm/nodeLlamaCpp/generation
+ */
+
+import { inferenceFailedError } from '../errors';
+import type { GenerationPort, GenParams, LlmResult } from '../types';
+import type { ModelManager } from './lifecycle';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+type LlamaModel = Awaited<
+  ReturnType<
+    Awaited<ReturnType<typeof import('node-llama-cpp').getLlama>>['loadModel']
+  >
+>;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Default Parameters (for determinism)
+// ─────────────────────────────────────────────────────────────────────────────
+
+const DEFAULT_TEMPERATURE = 0;
+const DEFAULT_SEED = 42;
+const DEFAULT_MAX_TOKENS = 256;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Implementation
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class NodeLlamaCppGeneration implements GenerationPort {
+  private readonly manager: ModelManager;
+  readonly modelUri: string;
+  private readonly modelPath: string;
+
+  constructor(manager: ModelManager, modelUri: string, modelPath: string) {
+    this.manager = manager;
+    this.modelUri = modelUri;
+    this.modelPath = modelPath;
+  }
+
+  async generate(
+    prompt: string,
+    params?: GenParams
+  ): Promise<LlmResult<string>> {
+    const model = await this.manager.loadModel(
+      this.modelPath,
+      this.modelUri,
+      'gen'
+    );
+    if (!model.ok) {
+      return model;
+    }
+
+    const llamaModel = model.value.model as LlamaModel;
+    const context = await llamaModel.createContext();
+
+    try {
+      // Import LlamaChatSession dynamically
+      const { LlamaChatSession } = await import('node-llama-cpp');
+      const session = new LlamaChatSession({
+        contextSequence: context.getSequence(),
+      });
+
+      // Note: stop sequences not yet supported - requires stopOnTrigger API
+      const response = await session.prompt(prompt, {
+        temperature: params?.temperature ?? DEFAULT_TEMPERATURE,
+        seed: params?.seed ?? DEFAULT_SEED,
+        maxTokens: params?.maxTokens ?? DEFAULT_MAX_TOKENS,
+      });
+
+      return { ok: true, value: response };
+    } catch (e) {
+      return { ok: false, error: inferenceFailedError(this.modelUri, e) };
+    } finally {
+      await context.dispose().catch(() => {
+        // Ignore disposal errors
+      });
+    }
+  }
+
+  async dispose(): Promise<void> {
+    // Generation doesn't hold persistent context
+    // Model cleanup is handled by ModelManager
+  }
+}

package/src/llm/nodeLlamaCpp/lifecycle.ts

@@ -0,0 +1,317 @@
+/**
+ * Model lifecycle manager.
+ * Handles lazy loading, caching, and disposal of LLM models.
+ *
+ * @module src/llm/nodeLlamaCpp/lifecycle
+ */
+
+import type { ModelConfig } from '../../config/types';
+import { loadFailedError, outOfMemoryError, timeoutError } from '../errors';
+import type { LlmResult, LoadedModel, ModelType } from '../types';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+type Llama = Awaited<ReturnType<typeof import('node-llama-cpp').getLlama>>;
+type LlamaModel = Awaited<ReturnType<Llama['loadModel']>>;
+
+interface CachedModel {
+  uri: string;
+  type: ModelType;
+  model: LlamaModel;
+  loadedAt: number;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// ModelManager
+// ─────────────────────────────────────────────────────────────────────────────
+
+export class ModelManager {
+  private llama: Llama | null = null;
+  private readonly models: Map<string, CachedModel> = new Map();
+  private readonly disposalTimers: Map<string, ReturnType<typeof setTimeout>> =
+    new Map();
+  private readonly inflightLoads: Map<string, Promise<LlmResult<LoadedModel>>> =
+    new Map();
+  private readonly config: ModelConfig;
+
+  constructor(config: ModelConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Get or initialize the Llama instance.
+   * Uses lazy loading - only imports node-llama-cpp on first use.
+   */
+  async getLlama(): Promise<Llama> {
+    if (!this.llama) {
+      const { getLlama, LlamaLogLevel } = await import('node-llama-cpp');
+      // Suppress model loading warnings (vocab tokens, pooling type)
+      this.llama = await getLlama({ logLevel: LlamaLogLevel.error });
+    }
+    return this.llama;
+  }
+
+  /**
+   * Load a model by path.
+   * Uses caching, inflight deduplication, and TTL-based disposal.
+   */
+  loadModel(
+    modelPath: string,
+    uri: string,
+    type: ModelType
+  ): Promise<LlmResult<LoadedModel>> {
+    // Check cache first
+    const cached = this.models.get(uri);
+    if (cached) {
+      this.resetDisposalTimer(uri);
+      return Promise.resolve({
+        ok: true as const,
+        value: {
+          uri: cached.uri,
+          type: cached.type,
+          model: cached.model,
+          loadedAt: cached.loadedAt,
+        },
+      });
+    }
+
+    // Check for inflight load (deduplicate concurrent requests)
+    const inflight = this.inflightLoads.get(uri);
+    if (inflight) {
+      return inflight;
+    }
+
+    // Start new load with cleanup
+    const loadPromise = this.loadModelInternal(modelPath, uri, type).finally(
+      () => {
+        this.inflightLoads.delete(uri);
+      }
+    );
+    this.inflightLoads.set(uri, loadPromise);
+    return loadPromise;
+  }
+
+  /**
+   * Internal model loading with timeout handling.
+   */
+  private async loadModelInternal(
+    modelPath: string,
+    uri: string,
+    type: ModelType
+  ): Promise<LlmResult<LoadedModel>> {
+    const timeoutMs = this.config.loadTimeout;
+    let timeoutId: ReturnType<typeof setTimeout> | null = null;
+    let timedOut = false;
+
+    // Capture loadPromise outside try block so we can dispose it on timeout
+    let loadPromise: Promise<LlamaModel> | null = null;
+
+    try {
+      const llama = await this.getLlama();
+      loadPromise = llama.loadModel({ modelPath });
+
+      // Create timeout with proper cleanup
+      const timeoutPromise = new Promise<never>((_, reject) => {
+        timeoutId = setTimeout(() => {
+          timedOut = true;
+          reject(new Error(`Load timeout after ${timeoutMs}ms`));
+        }, timeoutMs);
+      });
+
+      const model = await Promise.race([loadPromise, timeoutPromise]);
+
+      // Clear timeout on success
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+      }
+
+      const now = Date.now();
+      const cachedModel: CachedModel = {
+        uri,
+        type,
+        model,
+        loadedAt: now,
+      };
+
+      this.models.set(uri, cachedModel);
+      this.setDisposalTimer(uri);
+
+      return {
+        ok: true,
+        value: {
+          uri,
+          type,
+          model,
+          loadedAt: now,
+        },
+      };
+    } catch (e) {
+      // Clear timeout on error
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+      }
+
+      // Dispose late-arriving model after timeout to prevent memory leak
+      if (timedOut && loadPromise) {
+        loadPromise.then(
+          (model) => {
+            // Dispose model that arrived after timeout
+            model.dispose().catch(() => {
+              // Ignore dispose errors
+            });
+          },
+          () => {
+            // Ignore load errors after timeout
+          }
+        );
+      }
+
+      if (e instanceof Error) {
+        if (e.message.includes('timeout')) {
+          return {
+            ok: false,
+            error: timeoutError(uri, 'load', this.config.loadTimeout),
+          };
+        }
+        if (e.message.includes('out of memory') || e.message.includes('OOM')) {
+          return { ok: false, error: outOfMemoryError(uri, e) };
+        }
+      }
+      return { ok: false, error: loadFailedError(uri, e) };
+    }
+  }
+
+  /**
+   * Get a loaded model by URI (no loading).
+   */
+  getLoadedModel(uri: string): CachedModel | undefined {
+    const model = this.models.get(uri);
+    if (model) {
+      this.resetDisposalTimer(uri);
+    }
+    return model;
+  }
+
+  /**
+   * Check if a model is loaded.
+   */
+  isLoaded(uri: string): boolean {
+    return this.models.has(uri);
+  }
+
+  /**
+   * Dispose a specific model.
+   */
+  async dispose(uri: string): Promise<void> {
+    const cached = this.models.get(uri);
+    if (!cached) {
+      return;
+    }
+
+    // Clear disposal timer
+    const timer = this.disposalTimers.get(uri);
+    if (timer) {
+      clearTimeout(timer);
+      this.disposalTimers.delete(uri);
+    }
+
+    // Dispose the model
+    try {
+      await cached.model.dispose();
+    } catch {
+      // Ignore disposal errors
+    }
+
+    this.models.delete(uri);
+  }
+
+  /**
+   * Dispose all loaded models.
+   */
+  async disposeAll(): Promise<void> {
+    // Clear all timers
+    for (const timer of this.disposalTimers.values()) {
+      clearTimeout(timer);
+    }
+    this.disposalTimers.clear();
+
+    // Dispose all models
+    for (const [uri, cached] of this.models) {
+      try {
+        await cached.model.dispose();
+      } catch {
+        // Ignore disposal errors
+      }
+      this.models.delete(uri);
+    }
+
+    // Clear llama instance
+    this.llama = null;
+  }
+
+  /**
+   * Get list of loaded models.
+   */
+  getLoadedModels(): Array<{ uri: string; type: ModelType; loadedAt: number }> {
+    return Array.from(this.models.values()).map((m) => ({
+      uri: m.uri,
+      type: m.type,
+      loadedAt: m.loadedAt,
+    }));
+  }
+
+  // ───────────────────────────────────────────────────────────────────────────
+  // Private
+  // ───────────────────────────────────────────────────────────────────────────
+
+  private setDisposalTimer(uri: string): void {
+    const timer = setTimeout(() => {
+      this.dispose(uri).catch(() => {
+        // Ignore disposal errors in timer callback
+      });
+    }, this.config.warmModelTtl);
+
+    // Allow CLI processes to exit without waiting for TTL timer
+    if (typeof timer.unref === 'function') {
+      timer.unref();
+    }
+
+    this.disposalTimers.set(uri, timer);
+  }
+
+  private resetDisposalTimer(uri: string): void {
+    const existing = this.disposalTimers.get(uri);
+    if (existing) {
+      clearTimeout(existing);
+    }
+    this.setDisposalTimer(uri);
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Singleton
+// ─────────────────────────────────────────────────────────────────────────────
+
+let defaultManager: ModelManager | null = null;
+
+/**
+ * Get the default ModelManager singleton.
+ */
+export function getModelManager(config: ModelConfig): ModelManager {
+  if (!defaultManager) {
+    defaultManager = new ModelManager(config);
+  }
+  return defaultManager;
+}
+
+/**
+ * Reset the default manager (for testing).
+ */
+export async function resetModelManager(): Promise<void> {
+  if (defaultManager) {
+    await defaultManager.disposeAll();
+    defaultManager = null;
+  }
+}