@lov3kaizen/agentsea-cache 0.5.1

package/dist/streaming/index.js

@@ -0,0 +1,794 @@
+import { nanoid } from 'nanoid';
+import EventEmitter from 'eventemitter3';
+import murmurhash from 'murmurhash';
+
+// src/streaming/ChunkBuffer.ts
+var DEFAULT_CONFIG = {
+  maxChunks: 100,
+  maxBytes: 64 * 1024,
+  // 64KB
+  flushIntervalMs: 1e3,
+  preserveOrder: true
+};
+var ChunkBuffer = class {
+  chunks = [];
+  currentBytes = 0;
+  config;
+  flushTimer = null;
+  onFlush;
+  constructor(config, onFlush) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+    this.onFlush = onFlush;
+    if (this.config.flushIntervalMs > 0) {
+      this.startFlushTimer();
+    }
+  }
+  /**
+   * Add a chunk to the buffer
+   */
+  add(chunk) {
+    if (this.config.preserveOrder) {
+      if (chunk.index === void 0) {
+        chunk.index = this.chunks.length;
+      }
+    }
+    this.chunks.push(chunk);
+    this.currentBytes += this.estimateChunkSize(chunk);
+    if (this.shouldFlush()) {
+      this.flush();
+    }
+  }
+  /**
+   * Add multiple chunks
+   */
+  addAll(chunks) {
+    for (const chunk of chunks) {
+      this.add(chunk);
+    }
+  }
+  /**
+   * Flush all buffered chunks
+   */
+  flush() {
+    const flushed = this.chunks;
+    if (this.config.preserveOrder) {
+      flushed.sort((a, b) => a.index - b.index);
+    }
+    this.chunks = [];
+    this.currentBytes = 0;
+    if (this.onFlush && flushed.length > 0) {
+      this.onFlush(flushed);
+    }
+    return flushed;
+  }
+  /**
+   * Get current buffer size
+   */
+  size() {
+    return this.chunks.length;
+  }
+  /**
+   * Get current buffer bytes
+   */
+  bytes() {
+    return this.currentBytes;
+  }
+  /**
+   * Check if buffer is empty
+   */
+  isEmpty() {
+    return this.chunks.length === 0;
+  }
+  /**
+   * Peek at buffered chunks without flushing
+   */
+  peek() {
+    return this.chunks;
+  }
+  /**
+   * Clear the buffer without flushing
+   */
+  clear() {
+    this.chunks = [];
+    this.currentBytes = 0;
+  }
+  /**
+   * Stop the flush timer
+   */
+  stop() {
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer);
+      this.flushTimer = null;
+    }
+  }
+  /**
+   * Destroy the buffer
+   */
+  destroy() {
+    this.stop();
+    this.clear();
+  }
+  shouldFlush() {
+    return this.chunks.length >= this.config.maxChunks || this.currentBytes >= this.config.maxBytes;
+  }
+  startFlushTimer() {
+    this.flushTimer = setInterval(() => {
+      if (!this.isEmpty()) {
+        this.flush();
+      }
+    }, this.config.flushIntervalMs);
+  }
+  estimateChunkSize(chunk) {
+    let size = 0;
+    if (chunk.content) {
+      size += chunk.content.length * 2;
+    }
+    if (chunk.toolCall) {
+      size += JSON.stringify(chunk.toolCall).length;
+    }
+    if (chunk.toolResult) {
+      size += JSON.stringify(chunk.toolResult).length;
+    }
+    if (chunk.metadata) {
+      size += JSON.stringify(chunk.metadata).length;
+    }
+    return size + 50;
+  }
+};
+function createChunkBuffer(config, onFlush) {
+  return new ChunkBuffer(config, onFlush);
+}
+function generateId(prefix) {
+  const id = nanoid(16);
+  return prefix ? `${prefix}_${id}` : id;
+}
+function now() {
+  return Date.now();
+}
+
+// src/streaming/StreamRecorder.ts
+var DEFAULT_CONFIG2 = {
+  buffer: {
+    maxChunks: 100,
+    maxBytes: 64 * 1024,
+    flushIntervalMs: 0,
+    // No auto-flush during recording
+    preserveOrder: true
+  },
+  captureToolCalls: true,
+  captureMetadata: true,
+  maxDurationMs: 3e5,
+  // 5 minutes
+  maxChunks: 1e4
+};
+var StreamRecorder = class {
+  config;
+  buffer;
+  recording = false;
+  startTime = 0;
+  model = "";
+  messages = [];
+  key = "";
+  totalChars = 0;
+  chunkIndex = 0;
+  timeoutId = null;
+  constructor(config) {
+    this.config = {
+      ...DEFAULT_CONFIG2,
+      ...config,
+      buffer: { ...DEFAULT_CONFIG2.buffer, ...config?.buffer }
+    };
+    this.buffer = new ChunkBuffer(this.config.buffer);
+  }
+  /**
+   * Start recording a new stream
+   */
+  start(model, messages, key) {
+    if (this.recording) {
+      throw new Error("Recording already in progress");
+    }
+    this.recording = true;
+    this.startTime = now();
+    this.model = model;
+    this.messages = messages;
+    this.key = key ?? generateId();
+    this.totalChars = 0;
+    this.chunkIndex = 0;
+    this.buffer.clear();
+    if (this.config.maxDurationMs > 0) {
+      this.timeoutId = setTimeout(() => {
+        if (this.recording) {
+          this.abort("Recording exceeded maximum duration");
+        }
+      }, this.config.maxDurationMs);
+    }
+  }
+  /**
+   * Record a text chunk
+   */
+  recordText(content, metadata) {
+    this.recordChunk({
+      type: "text",
+      content,
+      metadata: this.config.captureMetadata ? metadata : void 0,
+      timestamp: now(),
+      index: this.chunkIndex++
+    });
+    this.totalChars += content.length;
+  }
+  /**
+   * Record a tool call chunk
+   */
+  recordToolCall(id, name, args, metadata) {
+    if (!this.config.captureToolCalls) return;
+    this.recordChunk({
+      type: "tool_call",
+      toolCall: { id, name, arguments: args },
+      metadata: this.config.captureMetadata ? metadata : void 0,
+      timestamp: now(),
+      index: this.chunkIndex++
+    });
+  }
+  /**
+   * Record a tool result chunk
+   */
+  recordToolResult(callId, content, metadata) {
+    if (!this.config.captureToolCalls) return;
+    this.recordChunk({
+      type: "tool_result",
+      toolResult: { callId, content },
+      metadata: this.config.captureMetadata ? metadata : void 0,
+      timestamp: now(),
+      index: this.chunkIndex++
+    });
+  }
+  /**
+   * Record metadata
+   */
+  recordMetadata(metadata) {
+    if (!this.config.captureMetadata) return;
+    this.recordChunk({
+      type: "metadata",
+      metadata,
+      timestamp: now(),
+      index: this.chunkIndex++
+    });
+  }
+  /**
+   * Record a generic chunk
+   */
+  recordChunk(chunk) {
+    if (!this.recording) {
+      throw new Error("Not currently recording");
+    }
+    if (chunk.index >= this.config.maxChunks) {
+      throw new Error("Maximum chunks exceeded");
+    }
+    this.buffer.add(chunk);
+  }
+  /**
+   * Complete the recording and return the recorded stream
+   */
+  complete(tokenUsage) {
+    if (!this.recording) {
+      throw new Error("Not currently recording");
+    }
+    this.clearTimeout();
+    const endTime = now();
+    const chunks = this.buffer.flush();
+    const stream = {
+      id: generateId(),
+      key: this.key,
+      chunks,
+      model: this.model,
+      messages: this.messages,
+      startTime: this.startTime,
+      endTime,
+      durationMs: endTime - this.startTime,
+      totalChars: this.totalChars,
+      tokenUsage,
+      complete: true
+    };
+    this.reset();
+    return stream;
+  }
+  /**
+   * Abort the recording
+   */
+  abort(reason) {
+    if (!this.recording) {
+      throw new Error("Not currently recording");
+    }
+    this.clearTimeout();
+    const endTime = now();
+    const chunks = this.buffer.flush();
+    const stream = {
+      id: generateId(),
+      key: this.key,
+      chunks,
+      model: this.model,
+      messages: this.messages,
+      startTime: this.startTime,
+      endTime,
+      durationMs: endTime - this.startTime,
+      totalChars: this.totalChars,
+      complete: false,
+      error: reason ?? "Recording aborted"
+    };
+    this.reset();
+    return stream;
+  }
+  /**
+   * Check if currently recording
+   */
+  isRecording() {
+    return this.recording;
+  }
+  /**
+   * Get current chunk count
+   */
+  getChunkCount() {
+    return this.chunkIndex;
+  }
+  /**
+   * Get current recording duration in ms
+   */
+  getDuration() {
+    if (!this.recording) return 0;
+    return now() - this.startTime;
+  }
+  /**
+   * Destroy the recorder
+   */
+  destroy() {
+    this.clearTimeout();
+    this.buffer.destroy();
+    this.reset();
+  }
+  reset() {
+    this.recording = false;
+    this.startTime = 0;
+    this.model = "";
+    this.messages = [];
+    this.key = "";
+    this.totalChars = 0;
+    this.chunkIndex = 0;
+  }
+  clearTimeout() {
+    if (this.timeoutId) {
+      clearTimeout(this.timeoutId);
+      this.timeoutId = null;
+    }
+  }
+};
+function createStreamRecorder(config) {
+  return new StreamRecorder(config);
+}
+
+// src/streaming/StreamReplayer.ts
+var DEFAULT_CONFIG3 = {
+  speedMultiplier: 1,
+  minDelayMs: 0,
+  maxDelayMs: 100,
+  simulateTiming: false,
+  onChunk: () => {
+  },
+  onComplete: () => {
+  },
+  onError: () => {
+  }
+};
+var StreamReplayer = class {
+  config;
+  abortController = null;
+  constructor(config) {
+    this.config = { ...DEFAULT_CONFIG3, ...config };
+  }
+  /**
+   * Replay a recorded stream as an async iterable
+   */
+  async *replay(stream) {
+    this.abortController = new AbortController();
+    const signal = this.abortController.signal;
+    try {
+      const chunks = [...stream.chunks].sort((a, b) => a.index - b.index);
+      let lastTimestamp = chunks[0]?.timestamp ?? 0;
+      for (let i = 0; i < chunks.length; i++) {
+        if (signal.aborted) {
+          break;
+        }
+        const chunk = chunks[i];
+        if (this.config.simulateTiming && i > 0) {
+          const timeDiff = chunk.timestamp - lastTimestamp;
+          const delay = Math.min(
+            Math.max(
+              timeDiff / this.config.speedMultiplier,
+              this.config.minDelayMs
+            ),
+            this.config.maxDelayMs
+          );
+          if (delay > 0) {
+            await this.delay(delay, signal);
+          }
+        }
+        lastTimestamp = chunk.timestamp;
+        this.config.onChunk(chunk);
+        yield chunk;
+      }
+      this.config.onComplete(stream);
+    } catch (error) {
+      if (error.name !== "AbortError") {
+        this.config.onError(error);
+        throw error;
+      }
+    } finally {
+      this.abortController = null;
+    }
+  }
+  /**
+   * Replay as a full async iterable of text content only
+   */
+  async *replayText(stream) {
+    for await (const chunk of this.replay(stream)) {
+      if (chunk.type === "text" && chunk.content) {
+        yield chunk.content;
+      }
+    }
+  }
+  /**
+   * Replay synchronously (no timing simulation)
+   */
+  *replaySync(stream) {
+    const chunks = [...stream.chunks].sort((a, b) => a.index - b.index);
+    for (const chunk of chunks) {
+      this.config.onChunk(chunk);
+      yield chunk;
+    }
+    this.config.onComplete(stream);
+  }
+  /**
+   * Get all chunks at once
+   */
+  getAllChunks(stream) {
+    return [...stream.chunks].sort((a, b) => a.index - b.index);
+  }
+  /**
+   * Get full text content from stream
+   */
+  getFullText(stream) {
+    return stream.chunks.filter((c) => c.type === "text" && c.content).sort((a, b) => a.index - b.index).map((c) => c.content).join("");
+  }
+  /**
+   * Get tool calls from stream
+   */
+  getToolCalls(stream) {
+    return stream.chunks.filter((c) => c.type === "tool_call" && c.toolCall).sort((a, b) => a.index - b.index).map((c) => c.toolCall);
+  }
+  /**
+   * Stop current replay
+   */
+  stop() {
+    if (this.abortController) {
+      this.abortController.abort();
+    }
+  }
+  /**
+   * Update configuration
+   */
+  configure(config) {
+    this.config = { ...this.config, ...config };
+  }
+  delay(ms, signal) {
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(resolve, ms);
+      signal.addEventListener("abort", () => {
+        clearTimeout(timeoutId);
+        reject(new DOMException("Aborted", "AbortError"));
+      });
+    });
+  }
+};
+function createStreamReplayer(config) {
+  return new StreamReplayer(config);
+}
+var DEFAULT_KEY_OPTIONS = {
+  includeTemperature: false,
+  includeTools: false,
+  normalizeWhitespace: true,
+  extractUserMessage: false
+};
+function generateCacheKey(model, messages, options = {}) {
+  const opts = { ...DEFAULT_KEY_OPTIONS, ...options };
+  const normalized = normalizeRequest(model, messages, opts);
+  const hash = murmurhash.v3(JSON.stringify(normalized)).toString(16);
+  return `cache:${model}:${hash}`;
+}
+function normalizeRequest(model, messages, options = {}) {
+  const normalizedMessages = messages.map((m) => ({
+    role: m.role,
+    content: options.normalizeWhitespace ? normalizeWhitespace(m.content) : m.content
+  }));
+  return {
+    model,
+    messages: options.extractUserMessage ? extractUserMessage(normalizedMessages) : normalizedMessages
+  };
+}
+function normalizeWhitespace(text) {
+  return text.trim().replace(/\r\n/g, "\n").replace(/\s+/g, " ");
+}
+function extractUserMessage(messages) {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (messages[i].role === "user") {
+      return messages[i].content;
+    }
+  }
+  return "";
+}
+
+// src/streaming/StreamCache.ts
+var DEFAULT_CONFIG4 = {
+  recorder: {},
+  replayer: {},
+  cacheIncomplete: false,
+  minLengthToCache: 10,
+  streamTtl: 3600
+  // 1 hour
+};
+var StreamCache = class extends EventEmitter {
+  store;
+  similarity;
+  config;
+  recorder;
+  replayer;
+  stats = {
+    totalLookups: 0,
+    totalHits: 0,
+    totalMisses: 0,
+    hitRate: 0,
+    avgReplayLatencyMs: 0,
+    totalStreamsCached: 0,
+    totalBytesCached: 0,
+    avgStreamDurationMs: 0
+  };
+  replayLatencies = [];
+  streamDurations = [];
+  constructor(store, config, similarity) {
+    super();
+    this.store = store;
+    this.similarity = similarity;
+    this.config = { ...DEFAULT_CONFIG4, ...config };
+    this.recorder = new StreamRecorder(this.config.recorder);
+    this.replayer = new StreamReplayer(this.config.replayer);
+  }
+  /**
+   * Look up a cached stream
+   */
+  async lookup(model, messages) {
+    const startTime = performance.now();
+    this.stats.totalLookups++;
+    const key = this.generateStreamKey(model, messages);
+    try {
+      const entry = await this.store.get(key);
+      if (entry) {
+        const stream = this.deserializeStream(entry.response.content);
+        if (stream) {
+          this.stats.totalHits++;
+          this.updateHitRate();
+          const latencyMs2 = performance.now() - startTime;
+          this.replayLatencies.push(latencyMs2);
+          const result = {
+            hit: true,
+            stream,
+            similarity: 1,
+            source: "exact",
+            latencyMs: latencyMs2
+          };
+          this.emit("hit", result);
+          return result;
+        }
+      }
+      if (this.similarity) {
+        const userMessage = messages.find((m) => m.role === "user")?.content;
+        if (userMessage) {
+          const embedding = await this.similarity.embed(userMessage);
+          const results = await this.store.query(embedding, {
+            topK: 1,
+            minSimilarity: 0.92
+          });
+          if (results.entries.length > 0) {
+            const entry2 = results.entries[0];
+            const stream = this.deserializeStream(entry2.response.content);
+            if (stream && entry2.score >= 0.92) {
+              this.stats.totalHits++;
+              this.updateHitRate();
+              const latencyMs2 = performance.now() - startTime;
+              this.replayLatencies.push(latencyMs2);
+              const result = {
+                hit: true,
+                stream,
+                similarity: entry2.score,
+                source: "semantic",
+                latencyMs: latencyMs2
+              };
+              this.emit("hit", result);
+              return result;
+            }
+          }
+        }
+      }
+      this.stats.totalMisses++;
+      this.updateHitRate();
+      const latencyMs = performance.now() - startTime;
+      this.emit("miss", key);
+      return {
+        hit: false,
+        source: "miss",
+        latencyMs
+      };
+    } catch (error) {
+      this.emit("error", error);
+      return {
+        hit: false,
+        source: "miss",
+        latencyMs: performance.now() - startTime
+      };
+    }
+  }
+  /**
+   * Cache a recorded stream
+   */
+  async cache(stream, embedding) {
+    if (!stream.complete && !this.config.cacheIncomplete) {
+      return;
+    }
+    if (stream.totalChars < this.config.minLengthToCache) {
+      return;
+    }
+    const key = this.generateStreamKey(stream.model, stream.messages);
+    try {
+      const cacheMessages = stream.messages.map((m) => ({
+        role: m.role,
+        content: m.content
+      }));
+      await this.store.set(key, {
+        id: generateId(),
+        key,
+        request: {
+          messages: cacheMessages,
+          model: stream.model
+        },
+        response: {
+          content: this.serializeStream(stream),
+          model: stream.model,
+          finishReason: stream.complete ? "stop" : "error",
+          usage: {
+            promptTokens: stream.tokenUsage?.prompt ?? 0,
+            completionTokens: stream.tokenUsage?.completion ?? 0,
+            totalTokens: stream.tokenUsage?.total ?? 0
+          }
+        },
+        embedding,
+        metadata: {
+          createdAt: now(),
+          accessedAt: now(),
+          accessCount: 1,
+          hitCount: 0,
+          ttl: this.config.streamTtl
+        }
+      });
+      this.stats.totalStreamsCached++;
+      this.stats.totalBytesCached += this.estimateStreamSize(stream);
+      this.streamDurations.push(stream.durationMs);
+      this.updateAvgStreamDuration();
+      this.emit("record", stream);
+    } catch (error) {
+      this.emit("error", error);
+    }
+  }
+  /**
+   * Wrap a streaming function with caching
+   */
+  async *wrapStream(model, messages, streamFn, options) {
+    const lookupResult = await this.lookup(model, messages);
+    if (lookupResult.hit && lookupResult.stream) {
+      for await (const chunk of this.replayer.replay(lookupResult.stream)) {
+        yield { content: chunk.content };
+      }
+      return;
+    }
+    const key = this.generateStreamKey(model, messages);
+    this.recorder.start(model, messages, key);
+    try {
+      for await (const chunk of streamFn()) {
+        if (chunk.content) {
+          this.recorder.recordText(chunk.content);
+        }
+        yield chunk;
+      }
+      const stream = this.recorder.complete();
+      await this.cache(stream, options?.embedding);
+    } catch (error) {
+      if (this.recorder.isRecording()) {
+        this.recorder.abort(error.message);
+      }
+      throw error;
+    }
+  }
+  /**
+   * Replay a cached stream
+   */
+  async *replay(stream) {
+    for await (const chunk of this.replayer.replay(stream)) {
+      yield chunk;
+    }
+  }
+  /**
+   * Get cache statistics
+   */
+  getStats() {
+    return { ...this.stats };
+  }
+  /**
+   * Clear the stream cache
+   */
+  async clear() {
+    await this.store.clear();
+    this.stats = {
+      totalLookups: 0,
+      totalHits: 0,
+      totalMisses: 0,
+      hitRate: 0,
+      avgReplayLatencyMs: 0,
+      totalStreamsCached: 0,
+      totalBytesCached: 0,
+      avgStreamDurationMs: 0
+    };
+    this.replayLatencies = [];
+    this.streamDurations = [];
+  }
+  /**
+   * Destroy the stream cache
+   */
+  destroy() {
+    this.recorder.destroy();
+    this.replayer.stop();
+    this.removeAllListeners();
+  }
+  generateStreamKey(model, messages) {
+    const cacheMessages = messages.map((m) => ({
+      role: m.role,
+      content: m.content
+    }));
+    return generateCacheKey(model, cacheMessages);
+  }
+  serializeStream(stream) {
+    return JSON.stringify(stream);
+  }
+  deserializeStream(content) {
+    try {
+      return JSON.parse(content);
+    } catch {
+      return null;
+    }
+  }
+  estimateStreamSize(stream) {
+    return JSON.stringify(stream).length;
+  }
+  updateHitRate() {
+    if (this.stats.totalLookups > 0) {
+      this.stats.hitRate = this.stats.totalHits / this.stats.totalLookups * 100;
+    }
+  }
+  updateAvgStreamDuration() {
+    if (this.streamDurations.length > 0) {
+      this.stats.avgStreamDurationMs = this.streamDurations.reduce((a, b) => a + b, 0) / this.streamDurations.length;
+    }
+  }
+};
+function createStreamCache(store, config, similarity) {
+  return new StreamCache(store, config, similarity);
+}
+
+export { ChunkBuffer, StreamCache, StreamRecorder, StreamReplayer, createChunkBuffer, createStreamCache, createStreamRecorder, createStreamReplayer };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map