@reventlessdev/reventless-local 3.0.0-alpha.316 → 3.0.0-alpha.318

package/src/adapter/LocalFeed.res.mjs

@@ -0,0 +1,419 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Nodenet from "node:net";
+import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
+import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as Logger$ReventlessCore from "@reventlessdev/reventless-core/src/util/Logger.res.mjs";
+
+let log = Logger$ReventlessCore.fromEnv();
+
+function settingOf(value) {
+  let v = Stdlib_Option.filter(Stdlib_Option.map(value, prim => prim.trim()), v => v !== "");
+  if (v === undefined) {
+    return "Ephemeral";
+  }
+  switch (v) {
+    case "false" :
+    case "none" :
+    case "off" :
+      return "Off";
+    default:
+      let p = Stdlib_Int.fromString(v, undefined);
+      if (p !== undefined && p >= 1024 && p <= 65535) {
+        return {
+          TAG: "Fixed",
+          _0: p
+        };
+      } else {
+        return "Ephemeral";
+      }
+  }
+}
+
+function levelOfString(s) {
+  let match = s.toLowerCase();
+  switch (match) {
+    case "debug" :
+      return "Debug";
+    case "error" :
+      return "Error";
+    case "info" :
+      return "Info";
+    case "warn" :
+    case "warning" :
+      return "Warn";
+    default:
+      return;
+  }
+}
+
+function parseFilter(line) {
+  let d;
+  try {
+    d = Stdlib_JSON.Decode.object(JSON.parse(line));
+  } catch (exn) {
+    return;
+  }
+  if (d === undefined) {
+    return;
+  }
+  let strings = k => Stdlib_Option.map(Stdlib_Option.flatMap(d[k], Stdlib_JSON.Decode.array), a => Stdlib_Array.filterMap(a, Stdlib_JSON.Decode.string));
+  let string = k => Stdlib_Option.flatMap(d[k], Stdlib_JSON.Decode.string);
+  return {
+    plugins: strings("plugins"),
+    level: Stdlib_Option.flatMap(string("level"), levelOfString),
+    components: strings("components"),
+    eventTypes: strings("eventTypes"),
+    correlationId: string("correlationId")
+  };
+}
+
+function matches(honours, filter, facets) {
+  let oneOf = (wanted, field, value) => {
+    if (wanted !== undefined && honours.includes(field)) {
+      return Stdlib_Option.mapOr(value, false, v => wanted.includes(v));
+    } else {
+      return true;
+    }
+  };
+  let tmp = false;
+  if (oneOf(filter.plugins, "Plugins", facets.plugin) && oneOf(filter.components, "Components", facets.comp) && oneOf(filter.eventTypes, "EventTypes", facets.eventType)) {
+    let id = filter.correlationId;
+    tmp = id !== undefined && honours.includes("CorrelationId") ? Primitive_object.equal(facets.correlationId, id) : true;
+  }
+  if (!tmp) {
+    return false;
+  }
+  let min = filter.level;
+  if (min !== undefined && honours.includes("Level")) {
+    return Stdlib_Option.mapOr(facets.level, false, l => Logger$ReventlessCore.levelToInt(l) >= Logger$ReventlessCore.levelToInt(min));
+  } else {
+    return true;
+  }
+}
+
+function skippedLine(count) {
+  return JSON.stringify(Object.fromEntries([
+    [
+      "kind",
+      "skipped"
+    ],
+    [
+      "count",
+      count
+    ]
+  ]));
+}
+
+function socketSink(socket) {
+  return {
+    write: line => socket.write(line + "\n"),
+    onceDrain: f => {
+      socket.once("drain", f);
+    },
+    buffered: () => Stdlib_Option.getOr(Primitive_option.fromNullable(socket.writableLength), 0),
+    close: () => {
+      socket.destroy();
+    }
+  };
+}
+
+function make(name, honours, backlog, maxLinesOpt, maxBytesOpt, filterWaitMsOpt, param) {
+  let maxLines = maxLinesOpt !== undefined ? maxLinesOpt : 10000;
+  let maxBytes = maxBytesOpt !== undefined ? maxBytesOpt : 8388608;
+  let filterWaitMs = filterWaitMsOpt !== undefined ? filterWaitMsOpt : 100;
+  return {
+    name: name,
+    honours: honours,
+    maxLines: maxLines,
+    maxBytes: maxBytes,
+    filterWaitMs: filterWaitMs,
+    backlog: backlog,
+    listeners: [],
+    server: undefined,
+    port: undefined
+  };
+}
+
+function port(feed) {
+  return feed.port;
+}
+
+function isLive(l) {
+  let match = l.phase;
+  return match !== "Waiting";
+}
+
+function hasReaders(feed) {
+  return feed.listeners.some(isLive);
+}
+
+function queuedLines(l) {
+  return l.queue.length - l.head | 0;
+}
+
+function queuedBytes(feed) {
+  return Stdlib_Array.reduce(feed.listeners, 0, (sum, l) => sum + l.queuedBytes | 0);
+}
+
+function heldBytes(feed) {
+  return Stdlib_Array.reduce(feed.listeners, 0, (sum, l) => (sum + l.queuedBytes | 0) + l.sink.buffered() | 0);
+}
+
+function drop(feed, l) {
+  Stdlib_Option.forEach(l.timer, prim => {
+    clearTimeout(prim);
+  });
+  feed.listeners = feed.listeners.filter(other => other !== l);
+}
+
+function dequeue(l) {
+  if (queuedLines(l) === 0) {
+    return;
+  }
+  let line = l.queue[l.head];
+  l.head = l.head + 1 | 0;
+  l.queuedBytes = l.queuedBytes - line.length | 0;
+  if (l.head > 1024 && (l.head << 1) > l.queue.length) {
+    l.queue = l.queue.slice(l.head, l.queue.length);
+    l.head = 0;
+  }
+  return line;
+}
+
+function pump(feed, l) {
+  let $$continue = true;
+  while ($$continue && !l.blocked) {
+    let next;
+    if (l.skipped > 0) {
+      let line = skippedLine(l.skipped);
+      l.skipped = 0;
+      next = line;
+    } else {
+      next = dequeue(l);
+    }
+    if (next !== undefined) {
+      let exit = 0;
+      let val;
+      try {
+        val = l.sink.write(next);
+        exit = 1;
+      } catch (exn) {
+        drop(feed, l);
+        $$continue = false;
+      }
+      if (exit === 1) {
+        if (val) {
+          
+        } else {
+          l.blocked = true;
+          l.sink.onceDrain(() => {
+            l.blocked = false;
+            pump(feed, l);
+          });
+        }
+      }
+    } else {
+      $$continue = false;
+    }
+  };
+}
+
+function enqueue(feed, l, line) {
+  l.queue.push(line);
+  l.queuedBytes = l.queuedBytes + line.length | 0;
+  while (queuedLines(l) > 0 && (queuedLines(l) > feed.maxLines || l.queuedBytes > feed.maxBytes)) {
+    dequeue(l);
+    l.skipped = l.skipped + 1 | 0;
+  };
+}
+
+function offer(feed, l, item) {
+  if (isLive(l) && matches(feed.honours, l.filter, item.facets)) {
+    enqueue(feed, l, item.line);
+    return pump(feed, l);
+  }
+}
+
+function publish(feed, item) {
+  feed.listeners.forEach(l => offer(feed, l, item));
+}
+
+function broadcast(feed, line) {
+  publish(feed, {
+    line: line,
+    facets: {}
+  });
+}
+
+function start_(feed, l, filter) {
+  if (!isLive(l)) {
+    Stdlib_Option.forEach(l.timer, prim => {
+      clearTimeout(prim);
+    });
+    l.timer = undefined;
+    Stdlib_Option.forEach(filter, f => {
+      l.filter = f;
+    });
+    l.phase = "Live";
+    Stdlib_Option.forEach(feed.backlog, backlog => {
+      backlog().forEach(item => {
+        if (matches(feed.honours, l.filter, item.facets)) {
+          return enqueue(feed, l, item.line);
+        }
+      });
+    });
+    return pump(feed, l);
+  }
+}
+
+function newListener(sink, phase) {
+  return {
+    sink: sink,
+    phase: phase,
+    filter: {},
+    queue: [],
+    head: 0,
+    queuedBytes: 0,
+    skipped: 0,
+    blocked: false,
+    inbound: "",
+    timer: undefined
+  };
+}
+
+function accept(feed, socket) {
+  let l = newListener(socketSink(socket), "Waiting");
+  socket.setEncoding("utf8");
+  socket.on("error", param => drop(feed, l));
+  socket.on("close", () => drop(feed, l));
+  socket.on("data", chunk => {
+    if (isLive(l)) {
+      return;
+    }
+    l.inbound = l.inbound + chunk;
+    let i = l.inbound.indexOf("\n");
+    if (i === -1) {
+      if (l.inbound.length > 65536) {
+        l.inbound = "";
+        return start_(feed, l, undefined);
+      } else {
+        return;
+      }
+    }
+    let line = l.inbound.slice(0, i);
+    l.inbound = "";
+    let f = parseFilter(line);
+    if (f !== undefined) {
+      return start_(feed, l, f);
+    } else {
+      log.warn(feed.name, undefined, `ignored a filter line that is not a JSON object`);
+      return start_(feed, l, undefined);
+    }
+  });
+  l.timer = Primitive_option.some(setTimeout(() => start_(feed, l, undefined), feed.filterWaitMs));
+  feed.listeners = feed.listeners.concat([l]);
+}
+
+function start(feed, setting, onBoundOpt, param) {
+  let onBound = onBoundOpt !== undefined ? onBoundOpt : param => {};
+  let match = feed.server;
+  if (typeof setting !== "object" && setting === "Off") {
+    return;
+  }
+  if (match !== undefined) {
+    return;
+  }
+  let requested;
+  requested = typeof setting !== "object" ? 0 : setting._0;
+  let s = Nodenet.createServer(socket => accept(feed, socket));
+  s.on("error", err => {
+    log.warn(feed.name, undefined, `could not listen: ` + Stdlib_Option.getOr(Stdlib_JsExn.message(err), "unknown error"));
+    feed.server = undefined;
+    feed.port = undefined;
+  });
+  s.unref();
+  s.listen(requested, "127.0.0.1", () => {
+    let addr = s.address();
+    if (addr == null) {
+      return;
+    }
+    let bound = addr.port;
+    feed.port = bound;
+    log.info(feed.name, undefined, `listening on 127.0.0.1:` + bound.toString());
+    onBound(bound);
+  });
+  feed.server = Primitive_option.some(s);
+}
+
+function resetForTests(feed) {
+  feed.listeners.forEach(l => Stdlib_Option.forEach(l.timer, prim => {
+    clearTimeout(prim);
+  }));
+  feed.listeners = [];
+  feed.server = undefined;
+  feed.port = undefined;
+}
+
+function addConnectionForTests(feed, socket) {
+  feed.listeners = feed.listeners.concat([newListener(socketSink(socket), "Live")]);
+}
+
+function subscribe(feed, filter, sink) {
+  let l = newListener(sink, "Waiting");
+  feed.listeners = feed.listeners.concat([l]);
+  start_(feed, l, filter);
+  return () => drop(feed, l);
+}
+
+function stopForTests(feed) {
+  let s = feed.server;
+  if (s !== undefined) {
+    let s$1 = Primitive_option.valFromOption(s);
+    feed.listeners.forEach(l => l.sink.close());
+    resetForTests(feed);
+    return new Promise((resolve, _reject) => {
+      s$1.close(() => resolve());
+    });
+  }
+  resetForTests(feed);
+  return Promise.resolve();
+}
+
+export {
+  log,
+  settingOf,
+  levelOfString,
+  parseFilter,
+  matches,
+  skippedLine,
+  socketSink,
+  make,
+  port,
+  isLive,
+  hasReaders,
+  queuedLines,
+  queuedBytes,
+  heldBytes,
+  drop,
+  dequeue,
+  pump,
+  enqueue,
+  offer,
+  publish,
+  broadcast,
+  start_,
+  newListener,
+  accept,
+  start,
+  resetForTests,
+  addConnectionForTests,
+  subscribe,
+  stopForTests,
+}
+/* log Not a pure module */

package/src/adapter/LocalLogFeed.res

@@ -0,0 +1,217 @@
+// The local platform's recent log records, and the feed that serves them to a
+// tool that did not start the platform and so cannot read its stdout.
+//
+// Two rings keep what was logged: the most recent records, and, separately, the
+// most recent warnings and errors, so a problem outlives a busy minute. Both are
+// bounded by count and by bytes. `REVENTLESS_LOG_RING` and
+// `REVENTLESS_LOG_PROBLEM_RING` set them, as `<count>` or `<count>,<bytes>`;
+// `0` keeps nothing.
+//
+// The feed is a `LocalFeed` on a loopback socket whose port the registry entry
+// names as `logPort`. On connect it sends the rings, filtered, then each record
+// as it is logged. A line is the record's JSON line, the one stdout prints with
+// `REVENTLESS_LOG_FORMAT=json`. `REVENTLESS_LOG_FEED=off|<port>` controls it the
+// way `REVENTLESS_EVENT_TAP` controls the tap.
+
+module Logger = ReventlessCore.Logger
+
+// ─── Rings ───────────────────────────────────────────────────────────────────
+
+type entry = {
+  // Arrival order across both rings, so a record kept by both is sent once.
+  seq: int,
+  record: Logger.record,
+  item: LocalFeed.item,
+}
+
+type ring = {
+  mutable maxCount: int,
+  mutable maxBytes: int,
+  mutable entries: array<entry>,
+  mutable head: int,
+  mutable bytes: int,
+}
+
+let makeRing = (~maxCount: int, ~maxBytes: int): ring => {
+  maxCount,
+  maxBytes,
+  entries: [],
+  head: 0,
+  bytes: 0,
+}
+
+let size = (r: ring) => r.entries->Array.length - r.head
+
+let entryBytes = (e: entry) => e.item.line->String.length
+
+let push = (r: ring, e: entry) =>
+  if r.maxCount > 0 && r.maxBytes > 0 {
+    r.entries->Array.push(e)
+    r.bytes = r.bytes + entryBytes(e)
+    while size(r) > 0 && (size(r) > r.maxCount || r.bytes > r.maxBytes) {
+      r.bytes = r.bytes - entryBytes(r.entries->Array.getUnsafe(r.head))
+      r.head = r.head + 1
+    }
+
+    // Compact once the dropped prefix is most of the array.
+    if r.head > 1024 && r.head * 2 > r.entries->Array.length {
+      r.entries = r.entries->Array.slice(~start=r.head, ~end=r.entries->Array.length)
+      r.head = 0
+    }
+  }
+
+let toArray = (r: ring): array<entry> =>
+  r.entries->Array.slice(~start=r.head, ~end=r.entries->Array.length)
+
+let ringBytes = (r: ring) => r.bytes
+
+let clear = (r: ring) => {
+  r.entries = []
+  r.head = 0
+  r.bytes = 0
+}
+
+let ringEnvVar = "REVENTLESS_LOG_RING"
+let problemRingEnvVar = "REVENTLESS_LOG_PROBLEM_RING"
+
+/** `<count>` or `<count>,<bytes>`; anything else keeps the defaults. */
+let boundsOf = (value: option<string>, ~count: int, ~bytes: int): (int, int) =>
+  switch value->Option.map(v => v->String.trim->String.split(",")) {
+  | Some([c]) => (c->String.trim->Int.fromString->Option.getOr(count), bytes)
+  | Some([c, b]) => (
+      c->String.trim->Int.fromString->Option.getOr(count),
+      b->String.trim->Int.fromString->Option.getOr(bytes),
+    )
+  | _ => (count, bytes)
+  }
+
+let recent = makeRing(~maxCount=5_000, ~maxBytes=8 * 1024 * 1024)
+let problems = makeRing(~maxCount=1_000, ~maxBytes=2 * 1024 * 1024)
+
+let isProblem = (level: Logger.level) =>
+  switch level {
+  | Warn | Error => true
+  | Debug | Info => false
+  }
+
+/** What both rings hold, oldest first, each record once. */
+let snapshot = (): array<entry> => {
+  let seen = Set.make()
+  recent
+  ->toArray
+  ->Array.concat(problems->toArray)
+  ->Array.filter(e =>
+    if seen->Set.has(e.seq) {
+      false
+    } else {
+      seen->Set.add(e.seq)
+      true
+    }
+  )
+  ->Array.toSorted((a, b) => (a.seq - b.seq)->Int.toFloat)
+}
+
+// ─── Records as feed lines ───────────────────────────────────────────────────
+
+let facetsOf = (r: Logger.record): LocalFeed.facets => {
+  plugin: ?r.plugin,
+  level: r.level,
+  comp: ?r.comp,
+  correlationId: ?(r.annotations->Dict.get("correlationId")),
+}
+
+let itemOf = (r: Logger.record): LocalFeed.item => {
+  line: r->Logger.recordToJson->JSON.stringify,
+  facets: facetsOf(r),
+}
+
+/** The filter fields a log line can be narrowed by. Event types are about events
+    and are ignored here. */
+let honours: array<LocalFeed.filterField> = [Plugins, Level, Components, CorrelationId]
+
+/** The recent records a filter selects, oldest first, at most `limit` (the most
+    recent ones). */
+let recentRecords = (~filter: LocalFeed.filter, ~limit: option<int>=?): array<Logger.record> => {
+  let selected =
+    snapshot()
+    ->Array.filter(e => LocalFeed.matches(~honours, filter, e.item.facets))
+    ->Array.map(e => e.record)
+  switch limit {
+  | Some(n) if n >= 0 && selected->Array.length > n =>
+    selected->Array.slice(~start=selected->Array.length - n, ~end=selected->Array.length)
+  | _ => selected
+  }
+}
+
+// ─── The feed ────────────────────────────────────────────────────────────────
+
+let envVar = "REVENTLESS_LOG_FEED"
+
+let feed = LocalFeed.make(
+  ~name="LogFeed",
+  ~honours,
+  ~backlog=() => snapshot()->Array.map(e => e.item),
+  ~maxBytes=16 * 1024 * 1024,
+  (),
+)
+
+let port = () => feed->LocalFeed.port
+let hasReaders = () => feed->LocalFeed.hasReaders
+
+let seq = ref(0)
+let removeSink: ref<option<unit => unit>> = ref(None)
+
+let onRecord = (record: Logger.record) => {
+  seq := seq.contents + 1
+  let entry = {seq: seq.contents, record, item: itemOf(record)}
+  push(recent, entry)
+  if isProblem(record.level) {
+    push(problems, entry)
+  }
+  if hasReaders() {
+    feed->LocalFeed.publish(entry.item)
+  }
+}
+
+/** Keeps a warning or an error that was never logged — the platform's own note of a
+    rejected command, say — in the problem ring and sends it to the feed's
+    listeners, without printing it. A record below warn is ignored. */
+let keepProblem = (record: Logger.record) =>
+  if isProblem(record.level) {
+    seq := seq.contents + 1
+    let entry = {seq: seq.contents, record, item: itemOf(record)}
+    push(problems, entry)
+    if hasReaders() {
+      feed->LocalFeed.publish(entry.item)
+    }
+  }
+
+/** Starts keeping records, once per process: sizes the rings from the
+    environment and registers the sink. */
+let install = (~env=NodeProcess.env) => {
+  let (c, b) = boundsOf(env->Dict.get(ringEnvVar), ~count=5_000, ~bytes=8 * 1024 * 1024)
+  recent.maxCount = c
+  recent.maxBytes = b
+  let (pc, pb) = boundsOf(env->Dict.get(problemRingEnvVar), ~count=1_000, ~bytes=2 * 1024 * 1024)
+  problems.maxCount = pc
+  problems.maxBytes = pb
+  switch removeSink.contents {
+  | Some(_) => ()
+  | None => removeSink := Some(Logger.addSink(onRecord))
+  }
+}
+
+/** Binds the feed's socket and reports the port it got, once it is listening. */
+let start = (~env=NodeProcess.env, ~onBound: int => unit=_ => (), ()): unit =>
+  feed->LocalFeed.start(~setting=LocalFeed.settingOf(env->Dict.get(envVar)), ~onBound, ())
+
+/** Test seam: removes the sink and empties the rings. */
+let resetForTests = () => {
+  removeSink.contents->Option.forEach(remove => remove())
+  removeSink := None
+  clear(recent)
+  clear(problems)
+  feed->LocalFeed.resetForTests
+}
+
+let stopForTests = () => feed->LocalFeed.stopForTests