@logbrew/sdk 0.1.28 → 0.1.29

package/README.md

@@ -502,9 +502,13 @@ const client = LogBrewClient.create({
 });
 ```
 
-When either queue limit is full, LogBrew drops the incoming event and keeps earlier context unchanged. `queue_overflow` means the event-count limit was reached, `queue_bytes_overflow` means the compact event-byte limit was reached, and `event_too_large` means one event could not fit a request body by itself. Drop callbacks are advisory and must not interrupt application logging. The default queue remains memory-only; automatic delivery reduces normal flush work, while the app still calls `shutdown()` at its owned graceful-lifecycle boundary.
+When either queue limit is full, LogBrew drops the incoming event and keeps earlier context unchanged. `queue_overflow` means the event-count limit was reached, `queue_bytes_overflow` means the compact event-byte limit was reached, and `event_too_large` means one event could not fit a request body by itself.
 
-Server runtimes can supply an explicit synchronous `eventStore` when they need restart recovery. The core loads and revalidates compact records during client creation, persists each accepted event before adding it to memory, persists an accepted prefix before removing it from memory, and closes the store only after successful shutdown. `purgePendingEvents()` clears both layers only while no flush or shutdown is queued or active. Implementations must provide synchronous `load`, `append`, `acknowledge`, `purge`, and `close` methods; use the encrypted `persistentQueue` adapter in `@logbrew/node` instead of writing a filesystem adapter from scratch.
+A fatal crash report (an issue with `fatal` or `critical` level) is the exception. When it does not fit, LogBrew removes the oldest queued events that are not fatal until it fits, and counts each removed event as `evicted_for_fatal`. It never removes events in a batch that is being sent or retried, and it removes nothing if the fatal still cannot fit. So a full queue drops a fatal only when everything else in it is fatal or in a batch that is being sent; that fatal is counted as `queue_overflow` or `queue_bytes_overflow`. A fatal larger than one request body is still dropped as `event_too_large`. Drop callbacks are advisory and must not interrupt application logging. The default queue remains memory-only; automatic delivery reduces normal flush work, while the app still calls `shutdown()` at its owned graceful-lifecycle boundary.
+
+LogBrew sees these drops too. The next request body carries the per-reason counts since the last accepted report, such as `"client_report":{"dropped_events":{"queue_overflow":3}}`. The client clears exactly those counts after a `2xx` response for that request; drops recorded while it is in flight, and counts in a failed or retried request, wait for the next one. A report never sends a request of its own, is left out when nothing was dropped, lists at most 16 reasons, caps each count at 1,000,000,000, and fits inside `maxBatchBytes`. Counts live in memory only, so drops not yet reported are lost when the process restarts.
+
+Server runtimes can supply an explicit synchronous `eventStore` when they need restart recovery. The core loads and revalidates compact records during client creation, persists each accepted event before adding it to memory, persists an accepted prefix before removing it from memory, and closes the store only after successful shutdown. `purgePendingEvents()` clears both layers only while no flush or shutdown is queued or active. Implementations must provide synchronous `load`, `append`, `acknowledge`, `purge`, and `close` methods; use the encrypted `persistentQueue` adapter in `@logbrew/node` instead of writing a filesystem adapter from scratch. A store can also provide a synchronous `remove(indexes)` that deletes the records at those ascending `load()` positions. Fatal eviction needs it: a full queue on a store without `remove` drops the fatal as before.
 
 Flush requests are also bounded. Core and Node default to 100 events and 256 KiB of exact UTF-8 JSON per request. The browser factory defaults to 64 KiB so its normal request body cannot exceed the existing keepalive transport ceiling. A flush sends one queue snapshot in order, splitting it on either limit. The response reports total `attempts` and acknowledged `batches`. Existing clients that only set `maxQueueSize` keep working; the byte and batch settings are additive and are also accepted by the Node and browser client factories.
 

package/core.cjs

@@ -91,6 +91,8 @@ const DEFAULT_MAX_BATCH_BYTES = 256 * 1024;
 const DEFAULT_DELIVERY_INTERVAL_MS = 5000;
 const MAX_DELIVERY_INTERVAL_MS = 60 * 1000;
 const DEFAULT_DELIVERY_QUEUE_THRESHOLD = 50;
+const MAX_CLIENT_REPORT_REASONS = 16;
+const MAX_CLIENT_REPORT_COUNT = 1_000_000_000;
 const DELIVERY_HEALTH_SCHEMA_VERSION = 1;
 const EVENT_QUEUE_FACTORY = Symbol.for("@logbrew/sdk.eventQueueFactory");
 const CONTEXT_PROVIDER = Symbol.for("@logbrew/sdk.contextProvider");
@@ -366,9 +368,12 @@ class LogBrewClient {
     this.droppedEventCount = 0;
     this.droppedEventsByReason = {
       event_too_large: 0,
+      evicted_for_fatal: 0,
       queue_bytes_overflow: 0,
       queue_overflow: 0
     };
+    // In memory only: counts not yet acknowledged by a 2xx are lost on restart.
+    this.unreportedDropsByReason = {};
     this.lastDropReason = "none";
     this.storage = resolvedEventStore ? "persistent" : "memory";
     this.hydratedEventCount = recovered.events.length;
@@ -390,7 +395,7 @@ class LogBrewClient {
     this.lastAttemptAtUnixMs = 0;
     this.lastAcceptedAtUnixMs = 0;
     this.lastDroppedAtUnixMs = 0;
-    this.failedBatch = undefined;
+    this.unacknowledgedBatch = undefined;
     this.issueBreadcrumbs = [];
     this.issueBreadcrumbsTruncated = false;
     this.#scheduleAutomaticDelivery();
@@ -488,7 +493,7 @@ class LogBrewClient {
     }
     this.#clearDeliveryTimer();
     this.automaticFlushPending = false;
-    this.failedBatch = undefined;
+    this.unacknowledgedBatch = undefined;
     this.events.splice(0, this.events.length);
     this.serializedEvents.splice(0, this.serializedEvents.length);
     this.serializedEventBytes.splice(0, this.serializedEventBytes.length);
@@ -666,32 +671,84 @@ class LogBrewClient {
       this.#recordDroppedEvent(event, "event_too_large");
       return EVENT_ADMISSIONS.event_too_large;
     }
-    if (this.events.length >= this.maxQueueSize) {
-      this.#recordDroppedEvent(event, "queue_overflow");
-      return EVENT_ADMISSIONS.queue_overflow;
+    const fatal = isFatalEvent(event);
+    if (fatal) {
+      this.#synchronizeEventStore();
     }
-    if (this.queuedEventBytes + eventBytes > this.maxQueueBytes) {
-      this.#recordDroppedEvent(event, "queue_bytes_overflow");
-      return EVENT_ADMISSIONS.queue_bytes_overflow;
+    const overflow = this.#overflowReason(this.events.length, this.queuedEventBytes + eventBytes);
+    const evicted = overflow && fatal ? this.#evictForFatal(eventBytes) : [];
+    if (overflow && evicted.length === 0) {
+      this.#recordDroppedEvent(event, overflow);
+      return EVENT_ADMISSIONS[overflow];
     }
-    if (this.eventStore) {
-      requireSynchronousStoreResult("append", this.eventStore.append({
-        event: cloneEvent(event),
-        eventBytes,
-        serializedEvent
-      }));
+    try {
+      if (this.eventStore) {
+        requireSynchronousStoreResult("append", this.eventStore.append({
+          event: cloneEvent(event),
+          eventBytes,
+          serializedEvent
+        }));
+      }
+      this.events.push(event);
+      this.serializedEvents.push(serializedEvent);
+      this.serializedEventBytes.push(eventBytes);
+      this.queuedEventBytes += eventBytes;
+    } finally {
+      // Report evictions only after admission so drop callbacks cannot refill the freed room.
+      for (const evictedEvent of evicted) {
+        this.#recordDroppedEvent(evictedEvent, "evicted_for_fatal");
+      }
     }
-    this.events.push(event);
-    this.serializedEvents.push(serializedEvent);
-    this.serializedEventBytes.push(eventBytes);
-    this.queuedEventBytes += eventBytes;
     this.#scheduleAutomaticDelivery();
     return EVENT_ADMISSIONS.admitted;
   }
 
+  #overflowReason(count, bytes) {
+    if (count >= this.maxQueueSize) {
+      return "queue_overflow";
+    }
+    return bytes > this.maxQueueBytes ? "queue_bytes_overflow" : undefined;
+  }
+
+  // Evicts the oldest non-fatal events outside the unacknowledged batch, only when that makes room.
+  #evictForFatal(eventBytes) {
+    if (this.eventStore && typeof this.eventStore.remove !== "function") {
+      return [];
+    }
+    const indexes = [];
+    let count = this.events.length;
+    let bytes = this.queuedEventBytes + eventBytes;
+    for (
+      let index = this.unacknowledgedBatch?.eventsCount ?? 0;
+      index < this.events.length && this.#overflowReason(count, bytes);
+      index += 1
+    ) {
+      if (!isFatalEvent(this.events[index])) {
+        indexes.push(index);
+        count -= 1;
+        bytes -= this.serializedEventBytes[index];
+      }
+    }
+    if (this.#overflowReason(count, bytes)) {
+      return [];
+    }
+    if (this.eventStore) {
+      requireSynchronousStoreResult("remove", this.eventStore.remove(indexes));
+    }
+    const evicted = indexes.map((index) => this.events[index]);
+    for (let position = indexes.length - 1; position >= 0; position -= 1) {
+      const index = indexes[position];
+      this.events.splice(index, 1);
+      this.serializedEvents.splice(index, 1);
+      this.queuedEventBytes -= this.serializedEventBytes.splice(index, 1)[0];
+    }
+    return evicted;
+  }
+
   #recordDroppedEvent(event, reason) {
     this.droppedEventCount = incrementBounded(this.droppedEventCount);
     this.droppedEventsByReason[reason] = incrementBounded(this.droppedEventsByReason[reason]);
+    this.unreportedDropsByReason[reason] = Math.min(MAX_CLIENT_REPORT_COUNT, (this.unreportedDropsByReason[reason] ?? 0) + 1);
     this.lastDropReason = reason;
     this.lastDroppedAtUnixMs = nextBoundedTimestamp(this.lastDroppedAtUnixMs);
     if (!this.onEventDropped) {
@@ -849,15 +906,17 @@ class LogBrewClient {
     let batches = 0;
     let statusCode = 204;
     while (remainingEvents > 0) {
-      const batch = this.failedBatch ?? this.#nextBatch(Math.min(remainingEvents, this.maxBatchEvents));
-      let response;
-      try {
-        response = await this.#sendBatch(transport, batch.body);
-      } catch (error) {
-        this.failedBatch ??= Object.freeze({ body: batch.body, eventsCount: batch.eventsCount });
-        throw error;
+      // Eviction for a fatal event can shrink the snapshot while a batch is in flight.
+      this.unacknowledgedBatch ??= Object.freeze(
+        this.#nextBatch(Math.min(remainingEvents, this.events.length, this.maxBatchEvents))
+      );
+      const batch = this.unacknowledgedBatch;
+      const response = await this.#sendBatch(transport, batch.body);
+      this.unacknowledgedBatch = undefined;
+      // Clear only what this body carried; drops recorded while it was in flight stay pending.
+      for (const [reason, count] of Object.entries(batch.droppedEvents ?? {})) {
+        this.unreportedDropsByReason[reason] -= count;
       }
-      this.failedBatch = undefined;
       this.#acknowledge(batch.eventsCount);
       this.acceptedBatchCount = incrementBounded(this.acceptedBatchCount);
       remainingEvents -= batch.eventsCount;
@@ -870,10 +929,13 @@ class LogBrewClient {
   }
 
   #synchronizeEventStore() {
-    if (!this.eventStore) return;
+    if (!this.eventStore) {
+      return;
+    }
     const recovered = loadStoredEvents(this);
-    if (!isOrderedSubsequence(this.serializedEvents, recovered.serializedEvents))
+    if (!isOrderedSubsequence(this.serializedEvents, recovered.serializedEvents)) {
       throw new SdkError("persistence_error", "event store lost a queued record");
+    }
     this.events = recovered.events;
     this.serializedEvents = recovered.serializedEvents;
     this.serializedEventBytes = recovered.serializedEventBytes;
@@ -881,7 +943,25 @@ class LogBrewClient {
   }
 
   #nextBatch(maxEvents) {
-    let bodyBytes = this.batchPrefixBytes + this.batchSuffixBytes;
+    const reported = Object.entries(this.unreportedDropsByReason).filter(([, count]) => count > 0);
+    const droppedEvents = reported.length > 0
+      ? Object.fromEntries(reported.slice(0, MAX_CLIENT_REPORT_REASONS))
+      : undefined;
+    const reportSuffix = droppedEvents && `],"client_report":${JSON.stringify({ dropped_events: droppedEvents })}}`;
+    const reportedEvents = reportSuffix ? this.#batchEventCount(maxEvents, utf8ByteLength(reportSuffix)) : 0;
+    const eventsCount = reportedEvents || this.#batchEventCount(maxEvents, this.batchSuffixBytes);
+    if (eventsCount === 0) {
+      throw new SdkError("transport_error", "queued event cannot fit the configured batch byte limit");
+    }
+    return {
+      body: `${this.batchPrefix}${this.serializedEvents.slice(0, eventsCount).join(",")}${reportedEvents ? reportSuffix : this.batchSuffix}`,
+      droppedEvents: reportedEvents ? droppedEvents : undefined,
+      eventsCount
+    };
+  }
+
+  #batchEventCount(maxEvents, suffixBytes) {
+    let bodyBytes = this.batchPrefixBytes + suffixBytes;
     let eventsCount = 0;
     const eventIds = new Set();
     for (let index = 0; index < maxEvents; index += 1) {
@@ -897,13 +977,7 @@ class LogBrewClient {
       bodyBytes = nextBodyBytes;
       eventsCount += 1;
     }
-    if (eventsCount === 0) {
-      throw new SdkError("transport_error", "queued event cannot fit the configured batch byte limit");
-    }
-    return {
-      body: `${this.batchPrefix}${this.serializedEvents.slice(0, eventsCount).join(",")}${this.batchSuffix}`,
-      eventsCount
-    };
+    return eventsCount;
   }
 
   #acknowledge(eventsCount) {
@@ -1078,6 +1152,11 @@ function utf8ByteLength(value) {
   return bytes;
 }
 
+// Fatal input severity is stored as critical; a critical issue is a crash report.
+function isFatalEvent(event) {
+  return event.type === "issue" && event.attributes.level === "critical";
+}
+
 function incrementBounded(value) {
   return value >= Number.MAX_SAFE_INTEGER ? Number.MAX_SAFE_INTEGER : value + 1;
 }

package/index.d.ts

@@ -749,13 +749,15 @@ export type EventStore = {
   load(): StoredEvent[];
   append(record: StoredEvent): void;
   acknowledge(count: number): void;
+  /** Removes the records at ascending `load()` indexes; without it, fatal issues cannot evict queued events. */
+  remove?(indexes: number[]): void;
   purge(): void;
   close(): void;
 };
 
 /** Queue drop notification emitted when a bounded in-memory queue is full. */
 export type DroppedEvent = {
-  reason: "event_too_large" | "queue_bytes_overflow" | "queue_overflow";
+  reason: "event_too_large" | "evicted_for_fatal" | "queue_bytes_overflow" | "queue_overflow";
   eventType: Event["type"];
   eventId: string;
   droppedEvents: number;
@@ -794,10 +796,11 @@ export type DeliveryHealthSnapshot = Readonly<{
   droppedEvents: number;
   droppedByReason: Readonly<{
     event_too_large: number;
+    evicted_for_fatal: number;
     queue_bytes_overflow: number;
     queue_overflow: number;
   }>;
-  lastDropReason: "none" | "event_too_large" | "queue_bytes_overflow" | "queue_overflow";
+  lastDropReason: "none" | "event_too_large" | "evicted_for_fatal" | "queue_bytes_overflow" | "queue_overflow";
   scheduled: boolean;
   inFlight: boolean;
   coalesced: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@logbrew/sdk",
-  "version": "0.1.28",
+  "version": "0.1.29",
   "description": "Public LogBrew JavaScript SDK for building, validating, and flushing event batches.",
   "type": "module",
   "main": "./index.cjs",