@babelqueue/kafka 1.1.3 → 1.3.0

package/dist/index.js

@@ -1,6 +1,266 @@
 // src/index.ts
 import { annotate, BabelQueueError, EnvelopeCodec, SCHEMA_VERSION, UnknownUrnError, UnknownUrnStrategy } from "@babelqueue/core";
 
+// src/envelope-shape.ts
+var MAX_DEPTH = 1e4;
+var INT64_MIN = -(2n ** 63n);
+var INT64_MAX = 2n ** 63n - 1n;
+var META = {
+  id: "string",
+  queue: "string",
+  lang: "string",
+  schema_version: "integer",
+  created_at: "integer"
+};
+var DEAD_LETTER = {
+  reason: "string",
+  error: "string",
+  exception: "string",
+  original_queue: "string",
+  lang: "string",
+  failed_at: "integer",
+  attempts: "integer"
+};
+var TOP = {
+  job: "string",
+  trace_id: "string",
+  attempts: "integer",
+  data: "data",
+  meta: "object",
+  dead_letter: "object"
+};
+var Reject = class extends Error {
+};
+function reject(message) {
+  throw new Reject(message);
+}
+var isWs = (c) => c === " " || c === "	" || c === "\n" || c === "\r";
+var isDigit = (c) => c !== void 0 && c >= "0" && c <= "9";
+var Scanner = class {
+  constructor(s) {
+    this.s = s;
+  }
+  s;
+  i = 0;
+  /** Containers currently open around the cursor (Go counts nesting over the whole document, not per subtree). */
+  depth = 0;
+  /** Why the body is undecodable, or `null` when Go's typed decode accepts it. */
+  check() {
+    try {
+      this.ws();
+      if (this.s.startsWith("null", this.i)) {
+        this.i += 4;
+      } else if (this.s[this.i] === "{") {
+        this.members(TOP, "");
+      } else {
+        reject("expected a JSON object.");
+      }
+      this.ws();
+      if (this.i < this.s.length) reject("unexpected trailing data after the JSON value.");
+      return null;
+    } catch (error) {
+      if (error instanceof Reject) return error.message;
+      throw error;
+    }
+  }
+  ws() {
+    while (isWs(this.s[this.i])) this.i++;
+  }
+  /** Reads an object whose members are checked against `kinds`; the cursor sits on its `{`. */
+  members(kinds, path) {
+    this.i++;
+    this.depth++;
+    this.ws();
+    if (this.s[this.i] === "}") {
+      this.i++;
+      this.depth--;
+      return;
+    }
+    for (; ; ) {
+      this.ws();
+      if (this.s[this.i] !== '"') reject("expected an object key.");
+      const key = this.string();
+      this.ws();
+      if (this.s[this.i] !== ":") reject("expected ':' after an object key.");
+      this.i++;
+      this.ws();
+      const kind = Object.prototype.hasOwnProperty.call(kinds, key) ? kinds[key] : void 0;
+      if (kind === void 0) {
+        this.skip(false);
+      } else {
+        this.typed(kind, path + key, key);
+      }
+      this.ws();
+      const c = this.s[this.i++];
+      if (c === "}") {
+        this.depth--;
+        return;
+      }
+      if (c !== ",") reject("expected ',' or '}' in an object.");
+    }
+  }
+  typed(kind, name, lower) {
+    const c = this.s[this.i];
+    if (this.s.startsWith("null", this.i)) {
+      this.i += 4;
+      return;
+    }
+    switch (kind) {
+      case "string":
+        if (c !== '"') reject(`'${name}' must be a string.`);
+        this.string();
+        return;
+      case "integer": {
+        if (c !== "-" && !isDigit(c)) reject(`'${name}' must be an integer.`);
+        const token = this.number();
+        if (!/^-?[0-9]+$/.test(token)) reject(`'${name}' must be an integer.`);
+        const n = BigInt(token);
+        if (n < INT64_MIN || n > INT64_MAX) reject(`'${name}' does not fit an int64.`);
+        return;
+      }
+      case "data":
+        if (c !== "{") reject(`'${name}' must be a JSON object.`);
+        this.skip(true);
+        return;
+      case "object":
+        if (c !== "{") reject(`'${name}' must be a JSON object.`);
+        this.members(lower === "meta" ? META : DEAD_LETTER, name + ".");
+        return;
+    }
+  }
+  /** Reads a string literal and returns its decoded value; the cursor sits on the opening quote. */
+  string() {
+    const start = this.i++;
+    for (; ; ) {
+      const c = this.s[this.i];
+      if (c === void 0) reject("unterminated string.");
+      if (c === '"') break;
+      if (c < " ") reject("control character in a string.");
+      if (c === "\\") {
+        const e = this.s[this.i + 1];
+        if (e === "u") {
+          if (!/^[0-9a-fA-F]{4}$/.test(this.s.slice(this.i + 2, this.i + 6))) reject("invalid \\u escape.");
+          this.i += 6;
+          continue;
+        }
+        if (e === void 0 || !'"\\/bfnrt'.includes(e)) reject("invalid escape in a string.");
+        this.i += 2;
+        continue;
+      }
+      this.i++;
+    }
+    this.i++;
+    return JSON.parse(this.s.slice(start, this.i));
+  }
+  /** Reads a number literal per the JSON grammar and returns its text. */
+  number() {
+    const start = this.i;
+    if (this.s[this.i] === "-") this.i++;
+    if (this.s[this.i] === "0") {
+      this.i++;
+    } else if (isDigit(this.s[this.i])) {
+      while (isDigit(this.s[this.i])) this.i++;
+    } else {
+      reject("invalid number.");
+    }
+    if (this.s[this.i] === ".") {
+      this.i++;
+      if (!isDigit(this.s[this.i])) reject("invalid number.");
+      while (isDigit(this.s[this.i])) this.i++;
+    }
+    if (this.s[this.i] === "e" || this.s[this.i] === "E") {
+      this.i++;
+      if (this.s[this.i] === "+" || this.s[this.i] === "-") this.i++;
+      if (!isDigit(this.s[this.i])) reject("invalid number.");
+      while (isDigit(this.s[this.i])) this.i++;
+    }
+    return this.s.slice(start, this.i);
+  }
+  /** Skips one value iteratively (no recursion, so depth cannot overflow the stack); optionally requires finite numbers. */
+  skip(finite) {
+    const stack = [];
+    let expectValue = true;
+    for (; ; ) {
+      this.ws();
+      const c = this.s[this.i];
+      if (expectValue) {
+        if (c === "{" || c === "[") {
+          if (this.depth + stack.length >= MAX_DEPTH) reject("exceeded the maximum nesting depth.");
+          stack.push(c);
+          this.i++;
+          this.ws();
+          if (this.s[this.i] === (c === "{" ? "}" : "]")) {
+            this.i++;
+            stack.pop();
+            expectValue = false;
+          } else if (c === "{") {
+            this.key();
+          }
+          continue;
+        }
+        if (c === '"') {
+          this.string();
+        } else if (c === "-" || isDigit(c)) {
+          const token = this.number();
+          if (finite && !Number.isFinite(Number(token))) reject("a number in 'data' is not a finite double.");
+        } else if (this.s.startsWith("true", this.i)) {
+          this.i += 4;
+        } else if (this.s.startsWith("false", this.i)) {
+          this.i += 5;
+        } else if (this.s.startsWith("null", this.i)) {
+          this.i += 4;
+        } else {
+          reject("invalid JSON value.");
+        }
+        expectValue = false;
+      }
+      if (stack.length === 0) return;
+      const top = stack[stack.length - 1];
+      this.ws();
+      const d = this.s[this.i++];
+      if (d === ",") {
+        if (top === "{") this.key();
+        expectValue = true;
+      } else if (d === (top === "{" ? "}" : "]")) {
+        stack.pop();
+        if (stack.length === 0) return;
+      } else {
+        reject("expected ',' or the end of a container.");
+      }
+    }
+  }
+  /** Reads `"key" :` inside a skipped object. */
+  key() {
+    this.ws();
+    if (this.s[this.i] !== '"') reject("expected an object key.");
+    this.string();
+    this.ws();
+    if (this.s[this.i] !== ":") reject("expected ':' after an object key.");
+    this.i++;
+  }
+};
+function envelopeViolation(raw) {
+  return new Scanner(raw).check();
+}
+function normalizedDecimal(token) {
+  const m = /^(-?)([0-9]+)(?:\.([0-9]+))?(?:[eE]([+-]?[0-9]+))?$/.exec(token);
+  if (!m) return token;
+  const fraction = m[3] ?? "";
+  let digits = (m[2] + fraction).replace(/^0+/, "");
+  if (digits === "") return "0";
+  let exponent = Number(m[4] ?? "0") - fraction.length;
+  const trimmed = digits.replace(/0+$/, "");
+  exponent += digits.length - trimmed.length;
+  digits = trimmed;
+  return `${m[1]}${digits}e${exponent}`;
+}
+function isLossyNumberToken(token) {
+  const n = Number(token);
+  if (!Number.isFinite(n)) return true;
+  if (/^-?[0-9]+$/.test(token) && Number.isSafeInteger(n)) return false;
+  return normalizedDecimal(token) !== normalizedDecimal(String(n));
+}
+
 // src/headers.ts
 function sanitizeHeaders(headers) {
   const out = {};
@@ -58,6 +318,9 @@ function headerInt(headers, key, fallback) {
   const n = Number.parseInt(s.trim(), 10);
   return Number.isNaN(n) ? fallback : n;
 }
+function isThenable(value) {
+  return value !== null && (typeof value === "object" || typeof value === "function") && typeof value.then === "function";
+}
 var BLANK_ASCII = /^[ \t\n\v\f\r]*$/;
 function unsupportedSchemaVersion(message) {
   const value = headerString(message.headers, "bq-schema-version");
@@ -148,6 +411,180 @@ var KafkaPublisher = class _KafkaPublisher {
     });
   }
 };
+var KafkaRedriveIO = class {
+  constructor(options) {
+    this.options = options;
+    const consumer = options.consumer;
+    if (consumer) {
+      this.rewind = async (from) => {
+        if (from.topic === void 0 || from.partition === void 0 || from.offset === void 0) {
+          throw new Error("kafka redrive: cannot rewind a record without topic, partition and offset");
+        }
+        const pending = new Set(this.inFlight);
+        if (!this.acknowledged.has(from)) pending.add(from);
+        const first = /* @__PURE__ */ new Map();
+        for (const m of pending) {
+          if (m.topic === void 0 || m.partition === void 0 || m.offset === void 0) continue;
+          const key = `${m.topic}\0${m.partition}`;
+          const offset = BigInt(m.offset);
+          const seen = first.get(key);
+          if (!seen || offset < seen.offset) first.set(key, { topic: m.topic, partition: m.partition, offset });
+        }
+        this.inFlight.clear();
+        const failures = [];
+        for (const { topic, partition, offset } of first.values()) {
+          try {
+            await consumer.seek({ topic, partition, offset: offset.toString() });
+          } catch (err) {
+            failures.push(err);
+          }
+        }
+        if (failures.length === 1) throw failures[0];
+        if (failures.length > 1) throw new AggregateError(failures, "kafka redrive: seeking several partitions back failed");
+      };
+    }
+  }
+  options;
+  /** Seek the consumer back to every unacknowledged record (`from` included unless it was acknowledged already); defined only when a `consumer` was supplied. */
+  rewind;
+  /** Records handed out by {@link KafkaRedriveIO.pop} that carry a position and are not acknowledged yet. */
+  inFlight = /* @__PURE__ */ new Set();
+  /** Records handed out by {@link KafkaRedriveIO.pop} that were acknowledged since; a rewind never re-reads them. */
+  acknowledged = /* @__PURE__ */ new WeakSet();
+  async pop(queue) {
+    const message = await this.options.pop(queue);
+    if (message && this.rewind && message.topic !== void 0 && message.partition !== void 0 && message.offset !== void 0) {
+      const tracked = {
+        body: message.body,
+        key: message.key,
+        value: message.value,
+        headers: message.headers,
+        topic: message.topic,
+        partition: message.partition,
+        offset: message.offset,
+        ack: async () => {
+          await message.ack();
+          this.inFlight.delete(tracked);
+          this.acknowledged.add(tracked);
+        }
+      };
+      this.inFlight.add(tracked);
+      return tracked;
+    }
+    return message;
+  }
+  /** Append an already-encoded body to `queue` as a keyless record. */
+  async publish(queue, body) {
+    await this.send(queue, body, null);
+  }
+  /**
+   * Append `body` to `queue` on behalf of the reserved record `from`. Two modes (Go/Java/.NET parity):
+   *
+   * - **Restore** — `queue` is the topic `from` was consumed from and `body` is the body `pop` returned for
+   *   it (redrive writing an unchanged DLQ record back). The original record is re-written verbatim: its
+   *   value bytes (`from.value`), headers (`from.headers`) and key, with a fresh timestamp so the restored
+   *   copy does not age out of the DLQ early (never `meta.created_at`). A message that carries no `value`
+   *   re-writes `body`; one that carries no `headers` falls back to the `bq-*` projection of a decodable body.
+   *   Needs `from.topic` to be set; without it a restore is indistinguishable from a republish.
+   *   Header values the consumer decoded as `null` (a header with a -1 length) are not re-written: the
+   *   restore copy goes through `copyHeaders`, which drops them (a deliberate bound; the Go reference
+   *   writes them as they are).
+   * - **Republish** — any other queue/body (a redrive to the work topic): the `bq-*` header projection and
+   *   the `meta.created_at` timestamp, plus the key of `from` byte-for-byte (§6.5).
+   */
+  async publishFrom(queue, body, from) {
+    if (from.topic !== void 0 && queue === from.topic && body === from.body) {
+      await this.restoreVerbatim(queue, body, from);
+      return;
+    }
+    await this.send(queue, body, from.key ?? null);
+  }
+  async restoreVerbatim(queue, body, from) {
+    const record = {
+      // A tombstone stays a tombstone; KafkaJS accepts a null value although the structural type does not.
+      value: from.value === void 0 ? body : from.value,
+      timestamp: String(Date.now())
+    };
+    if (from.headers !== void 0) {
+      record.headers = copyHeaders(from.headers);
+    } else {
+      const decoded = EnvelopeCodec.decode(body);
+      if (EnvelopeCodec.accepts(decoded)) record.headers = kafkaHeaders(decoded);
+    }
+    if (from.key !== void 0 && from.key !== null) record.key = Buffer.from(from.key);
+    await this.options.producer.send({ topic: queue, messages: [record] });
+  }
+  async send(queue, body, key) {
+    const record = { value: body };
+    const decoded = EnvelopeCodec.decode(body);
+    if (EnvelopeCodec.accepts(decoded)) {
+      record.headers = kafkaHeaders(decoded);
+      record.timestamp = String(decoded.meta.created_at);
+    }
+    if (key !== null) record.key = Buffer.from(key);
+    await this.options.producer.send({ topic: queue, messages: [record] });
+  }
+};
+function decodeFailure(raw) {
+  const reason = envelopeViolation(raw);
+  return reason === null ? null : new BabelQueueError(`Undecodable BabelQueue body: ${reason}`);
+}
+function hasLossyNumber(raw) {
+  let i = 0;
+  while (i < raw.length) {
+    const c = raw[i];
+    if (c === '"') {
+      i++;
+      while (i < raw.length && raw[i] !== '"') i += raw[i] === "\\" ? 2 : 1;
+      i++;
+    } else if (c === "-" || c >= "0" && c <= "9") {
+      let j = i + 1;
+      while (j < raw.length && /[0-9.eE+-]/.test(raw[j])) j++;
+      if (isLossyNumberToken(raw.slice(i, j))) return true;
+      i = j;
+    } else {
+      i++;
+    }
+  }
+  return false;
+}
+function poisonRecord(raw, cause, originalQueue, attempts, keepMembers = true) {
+  let record = { raw };
+  try {
+    if (!keepMembers) throw new Error("carry the body verbatim");
+    const parsed = JSON.parse(raw);
+    if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) && !hasLossyNumber(raw)) {
+      record = parsed;
+    }
+  } catch {
+  }
+  delete record.dead_letter;
+  record.dead_letter = annotate({ attempts }, "poison", originalQueue, {
+    attempts,
+    error: cause instanceof Error ? cause.message : null,
+    exception: cause instanceof Error ? cause.name : null
+  }).dead_letter;
+  return record;
+}
+function poisonHeaders(record, originalQueue, attempts) {
+  const headers = {};
+  const str = (v) => typeof v === "string" && v !== "" ? v : void 0;
+  const meta = record.meta !== null && typeof record.meta === "object" ? record.meta : {};
+  const job = str(record.job) ?? str(record.urn);
+  if (job) headers["bq-job"] = job;
+  const trace = str(record.trace_id);
+  if (trace) headers["bq-trace-id"] = trace;
+  const id = str(meta.id);
+  if (id) headers["bq-message-id"] = id;
+  if (meta.schema_version != null && typeof meta.schema_version !== "object") {
+    headers["bq-schema-version"] = String(meta.schema_version);
+  }
+  const lang = str(meta.lang);
+  if (lang) headers["bq-source-lang"] = lang;
+  headers["bq-attempts"] = String(attempts);
+  headers["bq-original-topic"] = originalQueue;
+  return headers;
+}
 var KafkaConsumer = class {
   constructor(consumer, handlers, options = {}) {
     this.consumer = consumer;
@@ -170,29 +607,28 @@ var KafkaConsumer = class {
     const { message } = payload;
     const unsupported = unsupportedSchemaVersion(message);
     if (unsupported !== void 0) {
-      this.report(
-        new BabelQueueError(`Rejected a record from Kafka with unsupported bq-schema-version "${unsupported}" (supported: ${SCHEMA_VERSION}).`),
-        null,
-        payload
+      const rejected = new BabelQueueError(
+        `Rejected a record from Kafka with unsupported bq-schema-version "${unsupported}" (supported: ${SCHEMA_VERSION}).`
       );
-      await this.deadLetterRaw(payload);
-      await this.commit(payload);
+      try {
+        await this.deadLetterRaw(payload);
+        await this.commit(payload);
+      } finally {
+        this.report(rejected, null, payload);
+      }
       return;
     }
     const raw = message.value ? message.value.toString("utf8") : "";
+    const undecodable = decodeFailure(raw);
+    if (undecodable) {
+      await this.settlePoison(payload, raw, undecodable, null);
+      return;
+    }
     let envelope;
     try {
       envelope = this.reconcile(EnvelopeCodec.decode(raw), message);
     } catch (decodeError) {
-      this.report(decodeError, null, payload);
-      await this.deadLetterRaw(payload);
-      await this.commit(payload);
-      return;
-    }
-    if (!EnvelopeCodec.accepts(envelope)) {
-      this.report(new BabelQueueError("Rejected a non-conformant BabelQueue envelope from Kafka."), envelope, payload);
-      await this.deadLetterRaw(payload);
-      await this.commit(payload);
+      await this.settlePoison(payload, raw, decodeError, null);
       return;
     }
     const urn = headerString(message.headers, "bq-job") ?? EnvelopeCodec.urn(envelope);
@@ -205,9 +641,12 @@ var KafkaConsumer = class {
       await handler(envelope, message, headersOf(message));
       await this.commit(payload);
     } catch (error) {
-      this.report(error, envelope, payload);
-      await this.retryOrDeadLetter(payload, envelope, error);
-      await this.commit(payload);
+      try {
+        await this.retryOrDeadLetter(payload, envelope, error);
+        await this.commit(payload);
+      } finally {
+        this.report(error, envelope, payload);
+      }
     }
   }
   reconcile(envelope, message) {
@@ -272,6 +711,32 @@ var KafkaConsumer = class {
     headers["bq-original-topic"] = original;
     await this.produce(dlq, annotated, headers, payload.message.key);
   }
+  /** Dead-letter a poison record (wrapped), commit it, then report: settle first so `onError` cannot bypass it. */
+  async settlePoison(payload, raw, cause, envelope) {
+    try {
+      await this.deadLetterPoison(payload, raw, cause);
+      await this.commit(payload);
+    } finally {
+      this.report(cause, envelope, payload);
+    }
+  }
+  async deadLetterPoison(payload, raw, cause) {
+    const dlq = this.options.retryTopics?.dlqTopic ?? null;
+    if (!dlq) return;
+    this.requireProducer();
+    const original = this.originalTopic(payload);
+    const attempts = Math.max(headerInt(payload.message.headers, "bq-attempts", 0), 0);
+    let record = poisonRecord(raw, cause, original, attempts);
+    let json;
+    try {
+      json = JSON.stringify(record);
+    } catch {
+      record = poisonRecord(raw, cause, original, attempts, false);
+      json = JSON.stringify(record);
+    }
+    const headers = poisonHeaders(record, original, attempts);
+    await this.produceRaw(dlq, json, headers, payload.message.key);
+  }
   async deadLetterRaw(payload) {
     const dlq = this.options.retryTopics?.dlqTopic ?? null;
     if (!dlq) return;
@@ -292,8 +757,11 @@ var KafkaConsumer = class {
    * guaranteed.
    */
   async produce(topic, envelope, headers, key) {
+    await this.produceRaw(topic, EnvelopeCodec.encode(envelope), headers, key);
+  }
+  async produceRaw(topic, value, headers, key) {
     this.requireProducer();
-    const record = { value: EnvelopeCodec.encode(envelope), headers, timestamp: String(Date.now()) };
+    const record = { value, headers, timestamp: String(Date.now()) };
     if (key !== void 0 && key !== null) record.key = Buffer.from(key);
     await this.options.producer.send({ topic, messages: [record] });
   }
@@ -310,13 +778,26 @@ var KafkaConsumer = class {
       throw new BabelQueueError("This Kafka consumer needs a producer to republish (retry/DLQ).");
     }
   }
+  /**
+   * Hand a failure to the user's `onError` callback. Callers invoke this only AFTER the record has been
+   * settled (dead-lettered / retried and committed). A callback that throws synchronously, or returns a
+   * promise that later rejects (an `async` function is assignable to the `void` callback type), is
+   * swallowed here so user code can never bypass settlement or stop the consumer.
+   */
   report(error, envelope, payload) {
-    this.options.onError?.(error, envelope, payload);
+    try {
+      const returned = this.options.onError?.(error, envelope, payload);
+      if (isThenable(returned)) {
+        returned.then(void 0, () => void 0);
+      }
+    } catch {
+    }
   }
 };
 export {
   KafkaConsumer,
   KafkaPublisher,
+  KafkaRedriveIO,
   RetryTopics,
   headersOf,
   kafkaHeaders