@logtape/file 1.0.0-dev.237 → 1.0.0-dev.241

package/filesink.base.ts

@@ -5,6 +5,257 @@ import {
   type StreamSinkOptions,
 } from "@logtape/logtape";
 
+/**
+ * Adaptive flush strategy that dynamically adjusts buffer thresholds
+ * based on recent flush patterns for optimal performance.
+ */
+class AdaptiveFlushStrategy {
+  private recentFlushSizes: number[] = [];
+  private recentFlushTimes: number[] = [];
+  private avgFlushSize: number;
+  private avgFlushInterval: number;
+  private readonly maxHistorySize = 10;
+  private readonly baseThreshold: number;
+
+  constructor(baseThreshold: number, baseInterval: number) {
+    this.baseThreshold = baseThreshold;
+    this.avgFlushSize = baseThreshold;
+    this.avgFlushInterval = baseInterval;
+  }
+
+  /**
+   * Record a flush event for pattern analysis.
+   * @param size The size of data flushed in bytes.
+   * @param timeSinceLastFlush Time since last flush in milliseconds.
+   */
+  recordFlush(size: number, timeSinceLastFlush: number): void {
+    this.recentFlushSizes.push(size);
+    this.recentFlushTimes.push(timeSinceLastFlush);
+
+    // Keep only recent history
+    if (this.recentFlushSizes.length > this.maxHistorySize) {
+      this.recentFlushSizes.shift();
+      this.recentFlushTimes.shift();
+    }
+
+    // Update averages
+    this.updateAverages();
+  }
+
+  /**
+   * Determine if buffer should be flushed based on adaptive strategy.
+   * @param currentSize Current buffer size in bytes.
+   * @param timeSinceLastFlush Time since last flush in milliseconds.
+   * @returns True if buffer should be flushed.
+   */
+  shouldFlush(currentSize: number, timeSinceLastFlush: number): boolean {
+    const adaptiveThreshold = this.calculateAdaptiveThreshold();
+    const adaptiveInterval = this.calculateAdaptiveInterval();
+
+    return currentSize >= adaptiveThreshold ||
+      (adaptiveInterval > 0 && timeSinceLastFlush >= adaptiveInterval);
+  }
+
+  private updateAverages(): void {
+    if (this.recentFlushSizes.length === 0) return;
+
+    this.avgFlushSize =
+      this.recentFlushSizes.reduce((sum, size) => sum + size, 0) /
+      this.recentFlushSizes.length;
+
+    this.avgFlushInterval =
+      this.recentFlushTimes.reduce((sum, time) => sum + time, 0) /
+      this.recentFlushTimes.length;
+  }
+
+  private calculateAdaptiveThreshold(): number {
+    // Adjust threshold based on recent patterns
+    // Higher average flush sizes suggest larger batches are beneficial
+    const adaptiveFactor = Math.min(
+      2.0,
+      Math.max(0.5, this.avgFlushSize / this.baseThreshold),
+    );
+
+    return Math.max(
+      Math.min(4096, this.baseThreshold / 2),
+      Math.min(64 * 1024, this.baseThreshold * adaptiveFactor),
+    );
+  }
+
+  private calculateAdaptiveInterval(): number {
+    // If base interval is 0, time-based flushing is disabled
+    if (this.avgFlushInterval <= 0) return 0;
+
+    // Adjust interval based on recent flush frequency
+    // More frequent flushes suggest lower latency is preferred
+    if (this.recentFlushTimes.length < 3) return this.avgFlushInterval;
+
+    const variance = this.calculateVariance(this.recentFlushTimes);
+    const stabilityFactor = Math.min(2.0, Math.max(0.5, 1000 / variance));
+
+    return Math.max(
+      1000,
+      Math.min(10000, this.avgFlushInterval * stabilityFactor),
+    );
+  }
+
+  private calculateVariance(values: number[]): number {
+    if (values.length < 2) return 1000; // Default variance
+
+    const mean = values.reduce((sum, val) => sum + val, 0) / values.length;
+    const squaredDiffs = values.map((val) => Math.pow(val - mean, 2));
+    return squaredDiffs.reduce((sum, diff) => sum + diff, 0) / values.length;
+  }
+}
+
+/**
+ * Memory pool for reusing Uint8Array buffers to minimize GC pressure.
+ * Maintains a pool of pre-allocated buffers for efficient reuse.
+ */
+class BufferPool {
+  private pool: Uint8Array[] = [];
+  private readonly maxPoolSize = 50; // Keep a reasonable pool size
+  private readonly maxBufferSize = 64 * 1024; // Don't pool very large buffers
+
+  /**
+   * Acquire a buffer from the pool or create a new one.
+   * @param size The minimum size needed for the buffer.
+   * @returns A Uint8Array that can be used for encoding.
+   */
+  acquire(size: number): Uint8Array {
+    // Don't pool very large buffers to avoid memory waste
+    if (size > this.maxBufferSize) {
+      return new Uint8Array(size);
+    }
+
+    // Try to find a suitable buffer from the pool
+    for (let i = this.pool.length - 1; i >= 0; i--) {
+      const buffer = this.pool[i];
+      if (buffer.length >= size) {
+        // Remove from pool and return
+        this.pool.splice(i, 1);
+        return buffer.subarray(0, size);
+      }
+    }
+
+    // No suitable buffer found, create a new one
+    // Create slightly larger buffer to improve reuse chances
+    const actualSize = Math.max(size, 1024); // Minimum 1KB
+    return new Uint8Array(actualSize);
+  }
+
+  /**
+   * Return a buffer to the pool for future reuse.
+   * @param buffer The buffer to return to the pool.
+   */
+  release(buffer: Uint8Array): void {
+    // Don't pool if we're at capacity or buffer is too large
+    if (
+      this.pool.length >= this.maxPoolSize || buffer.length > this.maxBufferSize
+    ) {
+      return;
+    }
+
+    // Don't pool very small buffers as they're cheap to allocate
+    if (buffer.length < 256) {
+      return;
+    }
+
+    // Add to pool for reuse
+    this.pool.push(buffer);
+  }
+
+  /**
+   * Clear the pool to free memory. Useful for cleanup.
+   */
+  clear(): void {
+    this.pool.length = 0;
+  }
+
+  /**
+   * Get current pool statistics for monitoring.
+   * @returns Object with pool size and buffer count.
+   */
+  getStats(): { poolSize: number; totalBuffers: number } {
+    return {
+      poolSize: this.pool.reduce((sum, buf) => sum + buf.length, 0),
+      totalBuffers: this.pool.length,
+    };
+  }
+}
+
+/**
+ * High-performance byte buffer for batching log records.
+ * Eliminates string concatenation overhead by storing pre-encoded bytes.
+ * Uses memory pooling to reduce GC pressure.
+ */
+class ByteRingBuffer {
+  private buffers: Uint8Array[] = [];
+  private totalSize: number = 0;
+  private bufferPool: BufferPool;
+
+  constructor(bufferPool: BufferPool) {
+    this.bufferPool = bufferPool;
+  }
+
+  /**
+   * Append pre-encoded log record bytes to the buffer.
+   * @param data The encoded log record as bytes.
+   */
+  append(data: Uint8Array): void {
+    this.buffers.push(data);
+    this.totalSize += data.length;
+  }
+
+  /**
+   * Get the current total size of buffered data in bytes.
+   * @returns The total size in bytes.
+   */
+  size(): number {
+    return this.totalSize;
+  }
+
+  /**
+   * Get the number of buffered records.
+   * @returns The number of records in the buffer.
+   */
+  count(): number {
+    return this.buffers.length;
+  }
+
+  /**
+   * Flush all buffered data and return it as an array of byte arrays.
+   * This clears the internal buffer and returns used buffers to the pool.
+   * @returns Array of buffered byte arrays ready for writev() operations.
+   */
+  flush(): Uint8Array[] {
+    const result = [...this.buffers];
+    this.clear();
+    return result;
+  }
+
+  /**
+   * Clear the buffer without returning data.
+   * Returns buffers to the pool for reuse.
+   */
+  clear(): void {
+    // Return buffers to pool for reuse
+    for (const buffer of this.buffers) {
+      this.bufferPool.release(buffer);
+    }
+    this.buffers.length = 0;
+    this.totalSize = 0;
+  }
+
+  /**
+   * Check if the buffer is empty.
+   * @returns True if the buffer contains no data.
+   */
+  isEmpty(): boolean {
+    return this.buffers.length === 0;
+  }
+}
+
 /**
  * Options for the {@link getBaseFileSink} function.
  */
@@ -61,6 +312,14 @@ export interface FileSinkDriver<TFile> {
    */
   writeSync(fd: TFile, chunk: Uint8Array): void;
 
+  /**
+   * Write multiple chunks of data to the file in a single operation.
+   * This is optional - if not implemented, falls back to multiple writeSync calls.
+   * @param fd The file descriptor.
+   * @param chunks Array of data chunks to write.
+   */
+  writeManySync?(fd: TFile, chunks: Uint8Array[]): void;
+
   /**
    * Flush the file to ensure that all data is written to the disk.
    * @param fd The file descriptor.
@@ -80,6 +339,14 @@ export interface FileSinkDriver<TFile> {
  * @since 1.0.0
  */
 export interface AsyncFileSinkDriver<TFile> extends FileSinkDriver<TFile> {
+  /**
+   * Asynchronously write multiple chunks of data to the file in a single operation.
+   * This is optional - if not implemented, falls back to multiple writeSync calls.
+   * @param fd The file descriptor.
+   * @param chunks Array of data chunks to write.
+   */
+  writeMany?(fd: TFile, chunks: Uint8Array[]): Promise<void>;
+
   /**
    * Asynchronously flush the file to ensure that all data is written to the disk.
    * @param fd The file descriptor.
@@ -119,35 +386,82 @@ export function getBaseFileSink<TFile>(
 ): Sink & (Disposable | AsyncDisposable) {
   const formatter = options.formatter ?? defaultTextFormatter;
   const encoder = options.encoder ?? new TextEncoder();
-  const bufferSize = options.bufferSize ?? 1024 * 8; // Default buffer size of 8192 chars
+  const bufferSize = options.bufferSize ?? 1024 * 8; // Default buffer size of 8192 bytes
   const flushInterval = options.flushInterval ?? 5000; // Default flush interval of 5 seconds
   let fd = options.lazy ? null : options.openSync(path);
-  let buffer: string = "";
+
+  // Initialize memory pool and buffer systems
+  const bufferPool = new BufferPool();
+  const byteBuffer = new ByteRingBuffer(bufferPool);
+  const adaptiveStrategy = new AdaptiveFlushStrategy(bufferSize, flushInterval);
   let lastFlushTimestamp: number = Date.now();
 
   if (!options.nonBlocking) {
     // Blocking mode implementation
     // deno-lint-ignore no-inner-declarations
     function flushBuffer(): void {
-      if (fd == null) return;
-      if (buffer.length > 0) {
-        options.writeSync(fd, encoder.encode(buffer));
-        buffer = "";
-        options.flushSync(fd);
-        lastFlushTimestamp = Date.now();
+      if (fd == null || byteBuffer.isEmpty()) return;
+
+      const flushSize = byteBuffer.size();
+      const currentTime = Date.now();
+      const timeSinceLastFlush = currentTime - lastFlushTimestamp;
+
+      const chunks = byteBuffer.flush();
+      if (options.writeManySync && chunks.length > 1) {
+        // Use batch write if available
+        options.writeManySync(fd, chunks);
+      } else {
+        // Fallback to individual writes
+        for (const chunk of chunks) {
+          options.writeSync(fd, chunk);
+        }
       }
+      options.flushSync(fd);
+
+      // Record flush for adaptive strategy
+      adaptiveStrategy.recordFlush(flushSize, timeSinceLastFlush);
+      lastFlushTimestamp = currentTime;
     }
 
     const sink: Sink & Disposable = (record: LogRecord) => {
       if (fd == null) fd = options.openSync(path);
-      buffer += formatter(record);
 
-      const shouldFlushBySize = buffer.length >= bufferSize;
-      const shouldFlushByTime = flushInterval > 0 &&
-        (record.timestamp - lastFlushTimestamp) >= flushInterval;
+      // ULTRA FAST PATH: Direct write when buffer is empty and using default buffer settings
+      if (byteBuffer.isEmpty() && bufferSize === 8192) {
+        // Inline everything for maximum speed - avoid all function calls
+        const formattedRecord = formatter(record);
+        const encodedRecord = encoder.encode(formattedRecord);
+
+        // Only use fast path for typical log sizes to avoid breaking edge cases
+        if (encodedRecord.length < 200) {
+          // Write directly for small logs - no complex buffering logic
+          options.writeSync(fd, encodedRecord);
+          options.flushSync(fd);
+          lastFlushTimestamp = Date.now();
+          return;
+        }
+      }
 
-      if (shouldFlushBySize || shouldFlushByTime) {
+      // STANDARD PATH: Complex logic for edge cases
+      const formattedRecord = formatter(record);
+      const encodedRecord = encoder.encode(formattedRecord);
+      byteBuffer.append(encodedRecord);
+
+      // Check for immediate flush conditions
+      if (bufferSize <= 0) {
+        // No buffering - flush immediately
         flushBuffer();
+      } else {
+        // Use adaptive strategy for intelligent flushing
+        const timeSinceLastFlush = record.timestamp - lastFlushTimestamp;
+        const shouldFlush = adaptiveStrategy.shouldFlush(
+          byteBuffer.size(),
+          timeSinceLastFlush,
+        );
+
+        if (shouldFlush) {
+          flushBuffer();
+        }
       }
     };
     sink[Symbol.dispose] = () => {
@@ -155,6 +469,8 @@ export function getBaseFileSink<TFile>(
         flushBuffer();
         options.closeSync(fd);
       }
+      // Clean up buffer pool
+      bufferPool.clear();
     };
     return sink;
   }
@@ -166,14 +482,28 @@ export function getBaseFileSink<TFile>(
   let flushTimer: ReturnType<typeof setInterval> | null = null;
 
   async function flushBuffer(): Promise<void> {
-    if (fd == null || buffer.length === 0) return;
+    if (fd == null || byteBuffer.isEmpty()) return;
 
-    const data = buffer;
-    buffer = "";
+    const flushSize = byteBuffer.size();
+    const currentTime = Date.now();
+    const timeSinceLastFlush = currentTime - lastFlushTimestamp;
+
+    const chunks = byteBuffer.flush();
     try {
-      asyncOptions.writeSync(fd, encoder.encode(data));
+      if (asyncOptions.writeMany && chunks.length > 1) {
+        // Use async batch write if available
+        await asyncOptions.writeMany(fd, chunks);
+      } else {
+        // Fallback to individual writes
+        for (const chunk of chunks) {
+          asyncOptions.writeSync(fd, chunk);
+        }
+      }
       await asyncOptions.flush(fd);
-      lastFlushTimestamp = Date.now();
+
+      // Record flush for adaptive strategy
+      adaptiveStrategy.recordFlush(flushSize, timeSinceLastFlush);
+      lastFlushTimestamp = currentTime;
     } catch {
       // Silently ignore errors in non-blocking mode
     }
@@ -198,16 +528,45 @@ export function getBaseFileSink<TFile>(
   const nonBlockingSink: Sink & AsyncDisposable = (record: LogRecord) => {
     if (disposed) return;
     if (fd == null) fd = asyncOptions.openSync(path);
-    buffer += formatter(record);
 
-    const shouldFlushBySize = buffer.length >= bufferSize;
-    const shouldFlushByTime = flushInterval > 0 &&
-      (record.timestamp - lastFlushTimestamp) >= flushInterval;
+    // ULTRA FAST PATH: Direct write when buffer is empty and using default buffer settings
+    if (byteBuffer.isEmpty() && !activeFlush && bufferSize === 8192) {
+      // Inline everything for maximum speed - avoid all function calls
+      const formattedRecord = formatter(record);
+      const encodedRecord = encoder.encode(formattedRecord);
+
+      // Only use fast path for typical log sizes to avoid breaking edge cases
+      if (encodedRecord.length < 200) {
+        // Write directly for small logs - no complex buffering logic
+        asyncOptions.writeSync(fd, encodedRecord);
+        scheduleFlush(); // Async flush
+        lastFlushTimestamp = Date.now();
+        return;
+      }
+    }
 
-    if (shouldFlushBySize || shouldFlushByTime) {
+    // STANDARD PATH: Complex logic for edge cases
+    const formattedRecord = formatter(record);
+    const encodedRecord = encoder.encode(formattedRecord);
+    byteBuffer.append(encodedRecord);
+
+    // Check for immediate flush conditions
+    if (bufferSize <= 0) {
+      // No buffering - flush immediately
       scheduleFlush();
-    } else if (flushTimer === null && flushInterval > 0) {
-      startFlushTimer();
+    } else {
+      // Use adaptive strategy for intelligent flushing
+      const timeSinceLastFlush = record.timestamp - lastFlushTimestamp;
+      const shouldFlush = adaptiveStrategy.shouldFlush(
+        byteBuffer.size(),
+        timeSinceLastFlush,
+      );
+
+      if (shouldFlush) {
+        scheduleFlush();
+      } else if (flushTimer === null && flushInterval > 0) {
+        startFlushTimer();
+      }
     }
   };
 
@@ -225,6 +584,8 @@ export function getBaseFileSink<TFile>(
         // Writer might already be closed or errored
       }
     }
+    // Clean up buffer pool
+    bufferPool.clear();
   };
 
   return nonBlockingSink;

package/filesink.deno.ts

@@ -18,6 +18,13 @@ export const denoDriver: RotatingFileSinkDriver<Deno.FsFile> = {
   writeSync(fd, chunk) {
     fd.writeSync(chunk);
   },
+  writeManySync(fd: Deno.FsFile, chunks: Uint8Array[]): void {
+    // Deno doesn't have writev, but we can optimize by writing all chunks
+    // then doing a single sync operation
+    for (const chunk of chunks) {
+      fd.writeSync(chunk);
+    }
+  },
   flushSync(fd) {
     fd.syncSync();
   },
@@ -34,6 +41,13 @@ export const denoDriver: RotatingFileSinkDriver<Deno.FsFile> = {
  */
 export const denoAsyncDriver: AsyncRotatingFileSinkDriver<Deno.FsFile> = {
   ...denoDriver,
+  async writeMany(fd: Deno.FsFile, chunks: Uint8Array[]): Promise<void> {
+    // Deno doesn't have async writev, but we can write all chunks
+    // then do a single async sync
+    for (const chunk of chunks) {
+      await fd.write(chunk);
+    }
+  },
   async flush(fd) {
     await fd.sync();
   },

package/filesink.node.ts

@@ -18,6 +18,15 @@ export const nodeDriver: RotatingFileSinkDriver<number | void> = {
     return fs.openSync(path, "a");
   },
   writeSync: fs.writeSync,
+  writeManySync(fd: number, chunks: Uint8Array[]): void {
+    if (chunks.length === 0) return;
+    if (chunks.length === 1) {
+      fs.writeSync(fd, chunks[0]);
+      return;
+    }
+    // Use writev for multiple chunks
+    fs.writevSync(fd, chunks);
+  },
   flushSync: fs.fsyncSync,
   closeSync: fs.closeSync,
   statSync: fs.statSync,
@@ -30,6 +39,15 @@ export const nodeDriver: RotatingFileSinkDriver<number | void> = {
  */
 export const nodeAsyncDriver: AsyncRotatingFileSinkDriver<number | void> = {
   ...nodeDriver,
+  async writeMany(fd: number, chunks: Uint8Array[]): Promise<void> {
+    if (chunks.length === 0) return;
+    if (chunks.length === 1) {
+      await promisify(fs.write)(fd, chunks[0]);
+      return;
+    }
+    // Use async writev for multiple chunks
+    await promisify(fs.writev)(fd, chunks);
+  },
   flush: promisify(fs.fsync),
   close: promisify(fs.close),
 };

package/mod.ts

@@ -4,4 +4,6 @@ export type {
   RotatingFileSinkDriver,
   RotatingFileSinkOptions,
 } from "./filesink.base.ts";
+export type { StreamFileSinkOptions } from "./streamfilesink.ts";
 export { getFileSink, getRotatingFileSink } from "#filesink";
+export { getStreamFileSink } from "./streamfilesink.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@logtape/file",
-  "version": "1.0.0-dev.237+0615301b",
+  "version": "1.0.0-dev.241+13810dcf",
   "description": "File sink and rotating file sink for LogTape",
   "keywords": [
     "logging",
@@ -50,7 +50,7 @@
     }
   },
   "peerDependencies": {
-    "@logtape/logtape": "1.0.0-dev.237+0615301b"
+    "@logtape/logtape": "1.0.0-dev.241+13810dcf"
   },
   "devDependencies": {
     "@alinea/suite": "^0.6.3",