@logbrew/sdk 0.1.38 → 0.1.39

package/event-pipeline.cjs

@@ -0,0 +1,325 @@
+const { BatchSender } = require("./batch-sender.cjs");
+const { addBounded, incrementBounded, nextBoundedTimestamp } = require("./bounded-number.cjs");
+const { DropLedger } = require("./client-report.cjs");
+const { cloneEvent } = require("./event-clone.cjs");
+const { EventQueue, isFatalEvent } = require("./event-queue.cjs");
+const { isClientReportStore } = require("./event-store.cjs");
+const { utf8ByteLength } = require("./utf8.cjs");
+const { isCallable } = require("./validation.cjs");
+
+/**
+ * @import { BatchTransport } from "./batch-sender.cjs"
+ * @import { DropReason } from "./client-report.cjs"
+ * @import { SdkEvent } from "./event-validation.cjs"
+ * @import { Batch, EventQueueConfig, OverflowReason } from "./event-queue.cjs"
+ * @import { ClientReportStore } from "./event-store.cjs"
+ * @import { HealthState } from "./delivery-health.cjs"
+ * @import { Callable, DeepReadonly } from "./validation.cjs"
+ * @typedef {Readonly<{ status: string, reason?: DropReason }>} EventAdmission
+ * @typedef {Readonly<{ eventFilter: () => unknown, onEventDropped: () => unknown, receiver: unknown }>} PipelineHooks
+ * @typedef {Readonly<{ statusCode: number, attempts: number, batches: number }>} FlushResult
+ * @typedef {"idle" | "empty" | "accepted" | "failed"} DeliveryOutcome
+ * @typedef {EventQueueConfig & Readonly<{
+ *   apiKey: () => string,
+ *   clientReportStore: ClientReportStore | undefined,
+ *   maxBatchEvents: number,
+ *   maxRetries: number,
+ *   hooks: PipelineHooks
+ * }>} PipelineConfig
+ */
+
+/**
+ * @param {DropReason} reason
+ * @returns {EventAdmission}
+ */
+function droppedAdmission(reason) {
+  return Object.freeze({
+    status: "dropped",
+
+    reason
+  });
+}
+
+const EVENT_ADMISSIONS = Object.freeze({
+  admitted: Object.freeze({ status: "admitted" }),
+
+  filtered: Object.freeze({ status: "filtered" }),
+
+  unavailable: Object.freeze({ status: "unavailable" }),
+
+  event_too_large: droppedAdmission("event_too_large"),
+
+  queue_overflow: droppedAdmission("queue_overflow"),
+
+  queue_bytes_overflow: droppedAdmission("queue_bytes_overflow")
+});
+const NO_CONTENT = 204;
+const NO_EVENTS = 0;
+const NO_OPERATIONS = 0;
+const STEP = 1;
+
+/**
+ * @param {Readonly<Promise<unknown>>} operation
+ * @returns {Promise<boolean>}
+ */
+async function settled(operation) {
+  try {
+    await operation;
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * @param {Callable} callback
+ * @param {unknown} receiver
+ * @param {unknown} argument
+ * @returns {boolean}
+ */
+function notify(callback, receiver, argument) {
+  try {
+    Reflect.apply(callback, receiver, [argument]);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+class EventPipeline {
+  /** @type {Promise<boolean>} */
+  operationTail = Promise.resolve(true);
+
+  pendingOperations = NO_OPERATIONS;
+
+  deliveryInFlight = false;
+
+  stats = {
+    acceptedBatchCount: NO_EVENTS,
+    acceptedEventCount: NO_EVENTS,
+    failedFlushCount: NO_EVENTS,
+    lastAcceptedAtUnixMs: NO_EVENTS,
+    successfulFlushCount: NO_EVENTS
+  };
+
+  /** @type {DeliveryOutcome} */
+  lastDeliveryOutcome = "idle";
+
+  /**
+   * @param {PipelineConfig} config
+   */
+  constructor(config) {
+    this.hooks = config.hooks;
+    this.maxBatchEvents = config.maxBatchEvents;
+    this.queue = new EventQueue(config);
+    const { eventStore } = this.queue;
+    this.ledger = new DropLedger(config.clientReportStore ?? (isClientReportStore(eventStore) ? eventStore : undefined));
+    this.sender = new BatchSender(config);
+  }
+
+  /**
+   * @returns {Omit<HealthState, "automaticDelivery" | "lifecycle" | "scheduled" | "coalesced" | "pausedReason" | "consecutiveFailures" | "retryDelayMs">}
+   */
+  health() {
+    const { ledger, queue, sender, stats } = this;
+    return {
+      acceptedEvents: stats.acceptedEventCount,
+      attempts: sender.deliveryAttemptCount,
+      batches: stats.acceptedBatchCount,
+      droppedByReason: ledger.droppedEventsByReason,
+      droppedEvents: ledger.droppedEventCount,
+      failures: stats.failedFlushCount,
+      flushes: stats.successfulFlushCount,
+      hydratedBytes: queue.hydrated.bytes,
+      hydratedEvents: queue.hydrated.events,
+      inFlight: this.deliveryInFlight,
+      lastAcceptedAtUnixMs: stats.lastAcceptedAtUnixMs,
+      lastAttemptAtUnixMs: sender.lastAttemptAtUnixMs,
+      lastDropReason: ledger.lastDropReason,
+      lastDroppedAtUnixMs: ledger.lastDroppedAtUnixMs,
+      lastOutcome: this.lastDeliveryOutcome,
+      lastStatusClass: sender.lastStatusClass,
+      pendingOperations: this.pendingOperations,
+      queueBytes: queue.queuedEventBytes,
+      queueEvents: queue.events.length,
+      storage: queue.eventStore === undefined ? "memory" : "persistent"
+    };
+  }
+
+  /**
+   * @param {DeepReadonly<SdkEvent>} event
+   * @returns {EventAdmission}
+   */
+  admit(event) {
+    const eventFilter = this.hooks.eventFilter();
+    if (isCallable(eventFilter) && Reflect.apply(eventFilter, this.hooks.receiver, [cloneEvent(event)]) === false) {
+      return EVENT_ADMISSIONS.filtered;
+    }
+    const serializedEvent = JSON.stringify(event);
+    const eventBytes = utf8ByteLength(serializedEvent);
+    const { queue } = this;
+    if (queue.limits.batchPrefixBytes + eventBytes + queue.limits.batchSuffixBytes > queue.limits.maxBatchBytes) {
+      this.recordDroppedEvent(event, "event_too_large");
+      return EVENT_ADMISSIONS.event_too_large;
+    }
+    return this.enqueue(event, serializedEvent, eventBytes);
+  }
+
+  /**
+   * @param {DeepReadonly<SdkEvent>} event
+   * @param {number} eventBytes
+   * @returns {Readonly<{ evicted: DeepReadonly<SdkEvent>[], overflow: OverflowReason | undefined }>}
+   */
+  makeRoom(event, eventBytes) {
+    const { queue } = this;
+    const fatal = isFatalEvent(event);
+    if (fatal) {
+      queue.synchronize();
+    }
+    const overflow = queue.overflowReason(queue.events.length, queue.queuedEventBytes + eventBytes);
+    return { evicted: overflow !== undefined && fatal ? queue.evictForFatal(eventBytes) : [], overflow };
+  }
+
+  /**
+   * @param {DeepReadonly<SdkEvent>} event
+   * @param {string} serializedEvent
+   * @param {number} eventBytes
+   * @returns {EventAdmission}
+   */
+  enqueue(event, serializedEvent, eventBytes) {
+    const { evicted, overflow } = this.makeRoom(event, eventBytes);
+    if (overflow !== undefined && evicted.length === NO_EVENTS) {
+      this.recordDroppedEvent(event, overflow);
+      return EVENT_ADMISSIONS[overflow];
+    }
+    try {
+      this.queue.append(event, serializedEvent, eventBytes);
+    } finally {
+      for (const evictedEvent of evicted) {
+        this.recordDroppedEvent(evictedEvent, "evicted_for_fatal");
+      }
+    }
+    return EVENT_ADMISSIONS.admitted;
+  }
+
+  /**
+   * @param {DeepReadonly<SdkEvent>} event
+   * @param {DropReason} reason
+   * @returns {boolean}
+   */
+  recordDroppedEvent(event, reason) {
+    this.ledger.record(reason);
+    const onEventDropped = this.hooks.onEventDropped();
+    return isCallable(onEventDropped) && notify(onEventDropped, this.hooks.receiver, {
+      droppedEvents: this.ledger.droppedEventCount,
+      eventId: event.id,
+      eventType: event.type,
+      reason
+    });
+  }
+
+  /**
+   * @template T
+   * @param {() => Promise<T>} operation
+   * @returns {Promise<T>}
+   */
+  async runSerialized(operation) {
+    this.pendingOperations = incrementBounded(this.pendingOperations);
+    const result = this.afterTail(operation);
+    this.operationTail = settled(result);
+    return result;
+  }
+
+  /**
+   * @template T
+   * @param {() => Promise<T>} operation
+   * @returns {Promise<T>}
+   */
+  async afterTail(operation) {
+    await this.operationTail;
+    try {
+      return await operation();
+    } finally {
+      this.pendingOperations = Math.max(NO_OPERATIONS, this.pendingOperations - STEP);
+    }
+  }
+
+  /**
+   * @param {BatchTransport} transport
+   * @returns {Promise<FlushResult>}
+   */
+  async flushWithHealth(transport) {
+    this.deliveryInFlight = true;
+    try {
+      const response = await this.flushSnapshot(transport);
+      this.stats.successfulFlushCount = incrementBounded(this.stats.successfulFlushCount);
+      this.lastDeliveryOutcome = response.batches === NO_EVENTS ? "empty" : "accepted";
+      return response;
+    } catch (error) {
+      this.stats.failedFlushCount = incrementBounded(this.stats.failedFlushCount);
+      this.lastDeliveryOutcome = "failed";
+      throw error;
+    } finally {
+      this.deliveryInFlight = false;
+    }
+  }
+
+  /**
+   * @param {BatchTransport} transport
+   * @returns {Promise<FlushResult>}
+   */
+  async flushSnapshot(transport) {
+    this.queue.synchronize();
+    const totals = {
+      statusCode: NO_CONTENT,
+
+      attempts: 0,
+
+      batches: 0
+    };
+    for (let remainingEvents = this.queue.events.length; remainingEvents > NO_EVENTS;) {
+      const batch = this.currentBatch(remainingEvents);
+      const response = await this.sender.send(transport, batch.body);
+      remainingEvents -= this.settleBatch(batch);
+      totals.attempts += response.attempts;
+      totals.batches += STEP;
+      totals.statusCode = response.statusCode;
+    }
+    return totals;
+  }
+
+  /**
+   * @param {number} remainingEvents
+   * @returns {Batch}
+   */
+  currentBatch(remainingEvents) {
+    const { queue } = this;
+    const batch = queue.unacknowledgedBatch ?? Object.freeze(queue.nextBatch(
+      Math.min(remainingEvents, queue.events.length, this.maxBatchEvents),
+      this.ledger.pendingReport()
+    ));
+    queue.unacknowledgedBatch = batch;
+    return batch;
+  }
+
+  /**
+   * @param {Batch} batch
+   * @returns {number}
+   */
+  settleBatch(batch) {
+    this.queue.unacknowledgedBatch = undefined;
+    if (batch.carriesClientReport) {
+      this.ledger.acknowledgeReport();
+    }
+    this.queue.acknowledge(batch.eventsCount);
+    this.stats.acceptedEventCount = addBounded(this.stats.acceptedEventCount, batch.eventsCount);
+    this.stats.lastAcceptedAtUnixMs = nextBoundedTimestamp(Math.max(
+      this.stats.lastAcceptedAtUnixMs,
+      this.sender.lastAttemptAtUnixMs
+    ));
+    this.stats.acceptedBatchCount = incrementBounded(this.stats.acceptedBatchCount);
+    return batch.eventsCount;
+  }
+}
+
+module.exports = { EVENT_ADMISSIONS, EventPipeline };

package/event-queue.cjs

@@ -0,0 +1,306 @@
+const { cloneEvent } = require("./event-clone.cjs");
+const { createEventStoreFromQueueFactory, loadStoredEvents } = require("./event-store.cjs");
+const { SdkError } = require("./sdk-error.cjs");
+const { isOrderedSubsequence, requireSynchronousStoreResult } = require("./stored-events.cjs");
+const { utf8ByteLength } = require("./utf8.cjs");
+
+/**
+ * @import { ClientReport } from "./client-report.cjs"
+ * @import { SdkEvent } from "./event-validation.cjs"
+ * @import { EventStore, StoreLimits } from "./event-store.cjs"
+ * @typedef {ReturnType<typeof loadStoredEvents>} RecoveredEvents
+ * @import { Callable, DeepReadonly } from "./validation.cjs"
+ * @typedef {Readonly<{ body: string, carriesClientReport: boolean, eventsCount: number }>} Batch
+ * @typedef {"queue_overflow" | "queue_bytes_overflow"} OverflowReason
+ * @typedef {Readonly<{
+ *   eventStore: EventStore | undefined,
+ *   eventQueueFactory: Callable | undefined,
+ *   sdk: Readonly<{ name: string, language: string, version: string }>,
+ *   maxBatchBytes: number,
+ *   maxQueueBytes: number,
+ *   maxQueueSize: number
+ * }>} EventQueueConfig
+ */
+
+const BATCH_SUFFIX = "]}";
+const BATCH_SUFFIX_BYTES = 2;
+const PREVIEW_INDENT = 2;
+const FIRST_INDEX = 0;
+const SEPARATOR_BYTES = 1;
+const NO_BYTES = 0;
+const NO_EVENTS = 0;
+const SINGLE = 1;
+const PREVIOUS = 1;
+
+/**
+ * @param {DeepReadonly<SdkEvent> | undefined} event
+ * @returns {boolean}
+ */
+function isFatalEvent(event) {
+  return event?.type === "issue" && event.attributes["level"] === "critical";
+}
+
+/**
+ * @param {EventStore} store
+ * @returns {boolean}
+ */
+function closeOwnedStore(store) {
+  try {
+    requireSynchronousStoreResult("close", store.close());
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * @param {EventQueueConfig} config
+ * @param {StoreLimits} limits
+ * @returns {Readonly<{ contents: RecoveredEvents, eventStore: EventStore | undefined }>}
+ */
+function openEventStore(config, limits) {
+  const { eventQueueFactory } = config;
+  const eventStore = eventQueueFactory === undefined
+    ? config.eventStore
+    : createEventStoreFromQueueFactory(eventQueueFactory, limits);
+  try {
+    return { contents: loadStoredEvents(eventStore, limits), eventStore };
+  } catch (error) {
+    if (eventQueueFactory !== undefined && eventStore !== undefined) {
+      closeOwnedStore(eventStore);
+    }
+    throw error;
+  }
+}
+
+class EventQueue {
+  /**
+   * @param {EventQueueConfig} config
+   */
+  constructor(config) {
+    this.sdk = config.sdk;
+    this.batchPrefix = `{"sdk":${JSON.stringify(config.sdk)},"events":[`;
+    /** @type {StoreLimits} */
+    this.limits = Object.freeze({
+      batchPrefixBytes: utf8ByteLength(this.batchPrefix),
+      batchSuffixBytes: BATCH_SUFFIX_BYTES,
+      maxBatchBytes: config.maxBatchBytes,
+      maxQueueBytes: config.maxQueueBytes,
+      maxQueueSize: config.maxQueueSize
+    });
+    const { contents, eventStore } = openEventStore(config, this.limits);
+    this.eventStore = eventStore;
+    this.contents = contents;
+    this.hydrated = Object.freeze({ bytes: contents.queuedEventBytes, events: contents.events.length });
+    /** @type {Batch | undefined} */
+    this.unacknowledgedBatch = undefined;
+  }
+
+  /**
+   * @returns {DeepReadonly<SdkEvent>[]}
+   */
+  get events() {
+    return this.contents.events;
+  }
+
+  /**
+   * @returns {number}
+   */
+  get queuedEventBytes() {
+    return this.contents.queuedEventBytes;
+  }
+
+  /**
+   * @returns {void}
+   */
+  synchronize() {
+    if (this.eventStore === undefined) {
+      return;
+    }
+    const recovered = loadStoredEvents(this.eventStore, this.limits);
+    if (!isOrderedSubsequence(this.contents.serializedEvents, recovered.serializedEvents)) {
+      throw new SdkError("persistence_error", "event store lost a queued record");
+    }
+    this.contents = recovered;
+  }
+
+  /**
+   * @param {number} count
+   * @param {number} bytes
+   * @returns {OverflowReason | undefined}
+   */
+  overflowReason(count, bytes) {
+    if (count >= this.limits.maxQueueSize) {
+      return "queue_overflow";
+    }
+    return bytes > this.limits.maxQueueBytes ? "queue_bytes_overflow" : undefined;
+  }
+
+  /**
+   * @param {number} eventBytes
+   * @returns {number[] | undefined}
+   */
+  evictionIndexes(eventBytes) {
+    const indexes = [];
+    const { events, serializedEventBytes } = this.contents;
+    let count = events.length;
+    let bytes = this.contents.queuedEventBytes + eventBytes;
+    for (
+      let index = this.unacknowledgedBatch?.eventsCount ?? FIRST_INDEX;
+      index < events.length && this.overflowReason(count, bytes) !== undefined;
+      index += SINGLE
+    ) {
+      if (!isFatalEvent(events[index])) {
+        indexes.push(index);
+        count -= SINGLE;
+        bytes -= serializedEventBytes[index] ?? NO_BYTES;
+      }
+    }
+    return this.overflowReason(count, bytes) === undefined ? indexes : undefined;
+  }
+
+  /**
+   * @param {number} eventBytes
+   * @returns {DeepReadonly<SdkEvent>[]}
+   */
+  evictForFatal(eventBytes) {
+    const { eventStore } = this;
+    if (eventStore !== undefined && typeof eventStore.remove !== "function") {
+      return [];
+    }
+    const indexes = this.evictionIndexes(eventBytes);
+    if (indexes === undefined) {
+      return [];
+    }
+    if (eventStore !== undefined) {
+      requireSynchronousStoreResult("remove", eventStore.remove?.(indexes));
+    }
+    return this.removeIndexes(indexes);
+  }
+
+  /**
+   * @param {readonly number[]} indexes
+   * @returns {DeepReadonly<SdkEvent>[]}
+   */
+  removeIndexes(indexes) {
+    const evicted = indexes.map((index) => this.contents.events[index]).filter((event) => event !== undefined);
+    for (let position = indexes.length - PREVIOUS; position >= FIRST_INDEX; position -= SINGLE) {
+      this.removeAt(indexes[position] ?? FIRST_INDEX);
+    }
+    return evicted;
+  }
+
+  /**
+   * @param {number} index
+   * @returns {void}
+   */
+  removeAt(index) {
+    const { contents } = this;
+    contents.events.splice(index, SINGLE);
+    contents.serializedEvents.splice(index, SINGLE);
+    contents.queuedEventBytes -= contents.serializedEventBytes.splice(index, SINGLE)[FIRST_INDEX] ?? NO_BYTES;
+  }
+
+  /**
+   * @param {DeepReadonly<SdkEvent>} event
+   * @param {string} serializedEvent
+   * @param {number} eventBytes
+   * @returns {void}
+   */
+  append(event, serializedEvent, eventBytes) {
+    if (this.eventStore !== undefined) {
+      requireSynchronousStoreResult("append", this.eventStore.append({ event: cloneEvent(event), eventBytes, serializedEvent }));
+    }
+    const { contents } = this;
+    contents.events.push(event);
+    contents.serializedEvents.push(serializedEvent);
+    contents.serializedEventBytes.push(eventBytes);
+    contents.queuedEventBytes += eventBytes;
+  }
+
+  /**
+   * @param {number} eventsCount
+   * @returns {void}
+   */
+  acknowledge(eventsCount) {
+    if (this.eventStore !== undefined) {
+      requireSynchronousStoreResult("acknowledge", this.eventStore.acknowledge(eventsCount));
+    }
+    const { contents } = this;
+    const acknowledgedBytes = contents.serializedEventBytes.slice(FIRST_INDEX, eventsCount).reduce((sum, bytes) => sum + bytes, NO_BYTES);
+    contents.events.splice(FIRST_INDEX, eventsCount);
+    contents.serializedEvents.splice(FIRST_INDEX, eventsCount);
+    contents.serializedEventBytes.splice(FIRST_INDEX, eventsCount);
+    contents.queuedEventBytes -= acknowledgedBytes;
+  }
+
+  /**
+   * @returns {number}
+   */
+  purge() {
+    const { contents } = this;
+    const purgedEvents = contents.events.length;
+    if (this.eventStore !== undefined) {
+      requireSynchronousStoreResult("purge", this.eventStore.purge());
+    }
+    this.unacknowledgedBatch = undefined;
+    contents.events.splice(FIRST_INDEX);
+    contents.serializedEvents.splice(FIRST_INDEX);
+    contents.serializedEventBytes.splice(FIRST_INDEX);
+    contents.queuedEventBytes = 0;
+    return purgedEvents;
+  }
+
+  /**
+   * @returns {string}
+   */
+  previewJson() {
+    return JSON.stringify({
+      sdk: this.sdk,
+
+      events: this.contents.events
+    }, undefined, PREVIEW_INDENT);
+  }
+
+  /**
+   * @param {number} maxEvents
+   * @param {ClientReport | undefined} report
+   * @returns {Batch}
+   */
+  nextBatch(maxEvents, report) {
+    const reportSuffix = report === undefined ? undefined : `],"client_report":${JSON.stringify(report)}}`;
+    const reportedEvents = reportSuffix === undefined ? NO_EVENTS : this.batchEventCount(maxEvents, utf8ByteLength(reportSuffix));
+    const eventsCount = reportedEvents === NO_EVENTS ? this.batchEventCount(maxEvents, BATCH_SUFFIX_BYTES) : reportedEvents;
+    if (eventsCount === NO_EVENTS) {
+      throw new SdkError("transport_error", "queued event cannot fit the configured batch byte limit");
+    }
+    const suffix = reportedEvents === NO_EVENTS ? BATCH_SUFFIX : reportSuffix ?? BATCH_SUFFIX;
+    return {
+      body: `${this.batchPrefix}${this.contents.serializedEvents.slice(FIRST_INDEX, eventsCount).join(",")}${suffix}`,
+      carriesClientReport: reportedEvents > NO_EVENTS,
+      eventsCount
+    };
+  }
+
+  /**
+   * @param {number} maxEvents
+   * @param {number} suffixBytes
+   * @returns {number}
+   */
+  batchEventCount(maxEvents, suffixBytes) {
+    const { events, serializedEventBytes } = this.contents;
+    let bodyBytes = this.limits.batchPrefixBytes + suffixBytes;
+    const eventIds = new Set();
+    for (let index = FIRST_INDEX; index < maxEvents; index += SINGLE) {
+      const event = events[index];
+      bodyBytes += (index === FIRST_INDEX ? NO_BYTES : SEPARATOR_BYTES) + (serializedEventBytes[index] ?? NO_BYTES);
+      if (event === undefined || eventIds.has(event.id) || bodyBytes > this.limits.maxBatchBytes) {
+        return index;
+      }
+      eventIds.add(event.id);
+    }
+    return maxEvents;
+  }
+}
+
+module.exports = { EventQueue, isFatalEvent };