@rotorsoft/act 0.6.1 → 0.6.3

package/dist/index.js

@@ -197,71 +197,84 @@ async function sleep(ms) {
 
 // src/adapters/InMemoryStore.ts
 var InMemoryStream = class {
-  stream;
-  source;
-  at = -1;
-  retry = -1;
-  blocked = false;
-  error = "";
-  leased_at = void 0;
-  leased_by = void 0;
-  leased_until = void 0;
   constructor(stream, source) {
     this.stream = stream;
     this.source = source;
   }
+  _at = -1;
+  _retry = -1;
+  _blocked = false;
+  _error = "";
+  _leased_by = void 0;
+  _leased_until = void 0;
   get is_avaliable() {
-    return !this.blocked && (!this.leased_until || this.leased_until <= /* @__PURE__ */ new Date());
+    return !this._blocked && (!this._leased_until || this._leased_until <= /* @__PURE__ */ new Date());
+  }
+  get at() {
+    return this._at;
   }
   /**
    * Attempt to lease this stream for processing.
-   * @param at - The end-of-lease watermark.
-   * @param by - The lease holder.
+   * @param lease - The lease request.
    * @param millis - Lease duration in milliseconds.
    * @returns The granted lease or undefined if blocked.
    */
-  lease(at, by, millis) {
-    if (this.is_avaliable && at > this.at) {
-      this.leased_at = at;
-      this.leased_by = by;
-      this.leased_until = new Date(Date.now() + millis);
-      millis > 0 && (this.retry = this.retry + 1);
+  lease(lease, millis) {
+    if (this.is_avaliable) {
+      if (millis > 0) {
+        this._leased_by = lease.by;
+        this._leased_until = new Date(Date.now() + millis);
+        this._retry = this._retry + 1;
+      }
       return {
         stream: this.stream,
         source: this.source,
-        at,
-        by,
-        retry: this.retry
+        at: lease.at,
+        by: lease.by,
+        retry: this._retry,
+        lagging: lease.lagging
       };
     }
   }
   /**
    * Acknowledge completion of processing for this stream.
-   * @param at - Last processed watermark.
-   * @param by - Lease holder that processed the watermark.
+   * @param lease - The lease request.
    */
-  ack(at, by) {
-    if (this.leased_by === by && at >= this.at) {
-      this.leased_at = void 0;
-      this.leased_by = void 0;
-      this.leased_until = void 0;
-      this.at = at;
-      this.retry = -1;
-      return true;
+  ack(lease) {
+    if (this._leased_by === lease.by) {
+      this._leased_by = void 0;
+      this._leased_until = void 0;
+      this._at = lease.at;
+      this._retry = -1;
+      return {
+        stream: this.stream,
+        source: this.source,
+        at: this._at,
+        by: lease.by,
+        retry: this._retry,
+        lagging: lease.lagging
+      };
     }
-    return false;
   }
   /**
    * Block a stream for processing after failing to process and reaching max retries with blocking enabled.
+   * @param lease - The lease request.
    * @param error Blocked error message.
    */
-  block(by, error) {
-    if (this.leased_by === by) {
-      this.blocked = true;
-      this.error = error;
-      return true;
+  block(lease, error) {
+    if (this._leased_by === lease.by) {
+      this._blocked = true;
+      this._error = error;
+      return {
+        stream: this.stream,
+        source: this.source,
+        at: this._at,
+        by: this._leased_by,
+        retry: this._retry,
+        error: this._error,
+        lagging: lease.lagging
+      };
     }
-    return false;
   }
 };
 var InMemoryStore = class {
@@ -293,6 +306,15 @@ var InMemoryStore = class {
     this._events.length = 0;
     this._streams = /* @__PURE__ */ new Map();
   }
+  in_query(query, e) {
+    if (query.stream && !RegExp(`^${query.stream}$`).test(e.stream))
+      return false;
+    if (query.names && !query.names.includes(e.name)) return false;
+    if (query.correlation && e.meta?.correlation !== query.correlation)
+      return false;
+    if (e.name === SNAP_EVENT && !query.with_snaps) return false;
+    return true;
+  }
   /**
    * Query events in the store, optionally filtered by query options.
    * @param callback - Function to call for each event.
@@ -301,30 +323,32 @@ var InMemoryStore = class {
    */
   async query(callback, query) {
     await sleep();
-    const {
-      stream,
-      names,
-      before,
-      after = -1,
-      limit,
-      created_before,
-      created_after,
-      correlation,
-      with_snaps = false
-    } = query || {};
-    let i = after + 1, count = 0;
-    while (i < this._events.length) {
-      const e = this._events[i++];
-      if (stream && !RegExp(`^${stream}$`).test(e.stream)) continue;
-      if (names && !names.includes(e.name)) continue;
-      if (correlation && e.meta?.correlation !== correlation) continue;
-      if (created_after && e.created <= created_after) continue;
-      if (e.name === SNAP_EVENT && !with_snaps) continue;
-      if (before && e.id >= before) break;
-      if (created_before && e.created >= created_before) break;
-      callback(e);
-      count++;
-      if (limit && count >= limit) break;
+    let count = 0;
+    if (query?.backward) {
+      let i = (query?.before || this._events.length) - 1;
+      while (i >= 0) {
+        const e = this._events[i--];
+        if (query && !this.in_query(query, e)) continue;
+        if (query?.created_before && e.created >= query.created_before)
+          continue;
+        if (query.after && e.id <= query.after) break;
+        if (query.created_after && e.created <= query.created_after) break;
+        callback(e);
+        count++;
+        if (query?.limit && count >= query.limit) break;
+      }
+    } else {
+      let i = (query?.after ?? -1) + 1;
+      while (i < this._events.length) {
+        const e = this._events[i++];
+        if (query && !this.in_query(query, e)) continue;
+        if (query?.created_after && e.created <= query.created_after) continue;
+        if (query?.before && e.id >= query.before) break;
+        if (query?.created_before && e.created >= query.created_before) break;
+        callback(e);
+        count++;
+        if (query?.limit && count >= query.limit) break;
+      }
     }
     return count;
   }
@@ -366,13 +390,25 @@ var InMemoryStore = class {
   }
   /**
    * Polls the store for unblocked streams needing processing, ordered by lease watermark ascending.
-   * @param limit - Maximum number of streams to poll.
-   * @param descending - Whether to poll streams in descending order (aka poll the most advanced first).
+   * @param lagging - Max number of streams to poll in ascending order.
+   * @param leading - Max number of streams to poll in descending order.
    * @returns The polled streams.
    */
-  async poll(limit, descending = false) {
+  async poll(lagging, leading) {
     await sleep();
-    return [...this._streams.values()].filter((s) => s.is_avaliable).sort((a, b) => descending ? b.at - a.at : a.at - b.at).slice(0, limit).map(({ stream, source, at }) => ({ stream, source, at }));
+    const a = [...this._streams.values()].filter((s) => s.is_avaliable).sort((a2, b2) => a2.at - b2.at).slice(0, lagging).map(({ stream, source, at }) => ({
+      stream,
+      source,
+      at,
+      lagging: true
+    }));
+    const b = [...this._streams.values()].filter((s) => s.is_avaliable).sort((a2, b2) => b2.at - a2.at).slice(0, leading).map(({ stream, source, at }) => ({
+      stream,
+      source,
+      at,
+      lagging: false
+    }));
+    return [...a, ...b];
   }
   /**
    * Lease streams for processing (e.g., for distributed consumers).
@@ -382,10 +418,11 @@ var InMemoryStore = class {
    */
   async lease(leases, millis) {
     await sleep();
-    return leases.map(({ stream, at, by, source }) => {
-      const found = this._streams.get(stream) || // store new correlations
-      this._streams.set(stream, new InMemoryStream(stream, source)).get(stream);
-      return found.lease(at, by, millis);
+    return leases.map((l) => {
+      if (!this._streams.has(l.stream)) {
+        this._streams.set(l.stream, new InMemoryStream(l.stream, l.source));
+      }
+      return this._streams.get(l.stream)?.lease(l, millis);
     }).filter((l) => !!l);
   }
   /**
@@ -394,9 +431,7 @@ var InMemoryStore = class {
    */
   async ack(leases) {
     await sleep();
-    return leases.filter(
-      (lease) => this._streams.get(lease.stream)?.ack(lease.at, lease.by)
-    );
+    return leases.map((l) => this._streams.get(l.stream)?.ack(l)).filter((l) => !!l);
   }
   /**
    * Block a stream for processing after failing to process and reaching max retries with blocking enabled.
@@ -405,9 +440,7 @@ var InMemoryStore = class {
    */
   async block(leases) {
     await sleep();
-    return leases.filter(
-      (lease) => this._streams.get(lease.stream)?.block(lease.by, lease.error)
-    );
+    return leases.map((l) => this._streams.get(l.stream)?.block(l, l.error)).filter((l) => !!l);
   }
 };
 
@@ -667,6 +700,7 @@ var Act = class {
   }
   _emitter = new EventEmitter();
   _drain_locked = false;
+  _drain_lag2lead_ratio = 0.5;
   _correlation_interval = void 0;
   emit(event, args) {
     return this._emitter.emit(event, args);
@@ -769,7 +803,7 @@ var Act = class {
    * @returns The lease with results
    */
   async handle(lease, payloads) {
-    if (payloads.length === 0) return { lease, at: lease.at };
+    if (payloads.length === 0) return { lease, handled: 0, at: lease.at };
     const stream = lease.stream;
     let at = payloads.at(0).event.id, handled = 0;
     lease.retry > 0 && logger.warn(`Retrying ${stream}@${at} (${lease.retry}).`);
@@ -785,6 +819,7 @@ var Act = class {
         block && logger.error(`Blocking ${stream} after ${lease.retry} retries.`);
         return {
           lease,
+          handled,
           at,
           // only report error when nothing was handled
           error: handled === 0 ? error.message : void 0,
@@ -792,7 +827,7 @@ var Act = class {
         };
       }
     }
-    return { lease, at };
+    return { lease, handled, at };
   }
   /**
    * Drains and processes events from the store, triggering reactions and updating state.
@@ -807,42 +842,36 @@ var Act = class {
   async drain({
     streamLimit = 10,
     eventLimit = 10,
-    leaseMillis = 1e4,
-    descending = false
+    leaseMillis = 1e4
   } = {}) {
     if (!this._drain_locked) {
       try {
         this._drain_locked = true;
-        const polled = await store().poll(streamLimit, descending);
+        const lagging = Math.ceil(streamLimit * this._drain_lag2lead_ratio);
+        const leading = streamLimit - lagging;
+        const polled = await store().poll(lagging, leading);
         const fetched = await Promise.all(
-          polled.map(async ({ stream, source, at }) => {
+          polled.map(async ({ stream, source, at, lagging: lagging2 }) => {
             const events = await this.query_array({
               stream: source,
               after: at,
               limit: eventLimit
             });
-            return { stream, source, events };
+            return { stream, source, at, lagging: lagging2, events };
           })
         );
-        fetched.length && tracer.fetched(fetched);
-        const [last_at, count] = fetched.reduce(
-          ([last_at2, count2], { events }) => [
-            Math.max(last_at2, events.at(-1)?.id || 0),
-            count2 + events.length
-          ],
-          [0, 0]
-        );
-        if (count > 0) {
+        if (fetched.length) {
+          tracer.fetched(fetched);
           const leases = /* @__PURE__ */ new Map();
-          fetched.forEach(({ stream, events }) => {
+          const fetch_window_at = fetched.reduce(
+            (max, { at, events }) => Math.max(max, events.at(-1)?.id || at),
+            0
+          );
+          fetched.forEach(({ stream, lagging: lagging2, events }) => {
             const payloads = events.flatMap((event) => {
-              const register = this.registry.events[event.name];
-              if (!register) return [];
+              const register = this.registry.events[event.name] || [];
               return [...register.reactions.values()].filter((reaction) => {
-                const resolved = typeof reaction.resolver === "function" ? (
-                  // @ts-expect-error index by key
-                  reaction.resolver(event)
-                ) : reaction.resolver;
+                const resolved = typeof reaction.resolver === "function" ? reaction.resolver(event) : reaction.resolver;
                 return resolved && resolved.target === stream;
               }).map((reaction) => ({ ...reaction, event }));
             });
@@ -850,43 +879,51 @@ var Act = class {
               lease: {
                 stream,
                 by: randomUUID2(),
-                at: events.at(-1)?.id || last_at,
-                // move the lease watermark forward when no events found in window
-                retry: 0
+                at: events.at(-1)?.id || fetch_window_at,
+                // ff when no matching events
+                retry: 0,
+                lagging: lagging2
               },
               // @ts-expect-error indexed by key
               payloads
             });
           });
-          if (leases.size) {
-            const leased = await store().lease(
-              [...leases.values()].map((l) => l.lease),
-              leaseMillis
-            );
-            if (leased.length) {
-              tracer.leased(leased);
-              const handled = await Promise.all(
-                leased.map(
-                  (lease) => this.handle(lease, leases.get(lease.stream).payloads)
-                )
-              );
-              const acked = await store().ack(
-                handled.filter(({ error }) => !error).map(({ at, lease }) => ({ ...lease, at }))
-              );
-              if (acked.length) {
-                tracer.acked(acked);
-                this.emit("acked", acked);
-              }
-              const blocked = await store().block(
-                handled.filter(({ block }) => block).map(({ lease, error }) => ({ ...lease, error }))
-              );
-              if (blocked.length) {
-                tracer.blocked(blocked);
-                this.emit("blocked", blocked);
-              }
-              return { leased, acked, blocked };
-            }
+          const leased = await store().lease(
+            [...leases.values()].map(({ lease }) => lease),
+            leaseMillis
+          );
+          tracer.leased(leased);
+          const handled = await Promise.all(
+            leased.map(
+              (lease) => this.handle(lease, leases.get(lease.stream).payloads)
+            )
+          );
+          const [lagging_handled, leading_handled] = handled.reduce(
+            ([lagging_handled2, leading_handled2], { lease, handled: handled2 }) => [
+              lagging_handled2 + (lease.lagging ? handled2 : 0),
+              leading_handled2 + (lease.lagging ? 0 : handled2)
+            ],
+            [0, 0]
+          );
+          const lagging_avg = lagging > 0 ? lagging_handled / lagging : 0;
+          const leading_avg = leading > 0 ? leading_handled / leading : 0;
+          const total = lagging_avg + leading_avg;
+          this._drain_lag2lead_ratio = total > 0 ? Math.max(0.2, Math.min(0.8, lagging_avg / total)) : 0.5;
+          const acked = await store().ack(
+            handled.filter(({ error }) => !error).map(({ at, lease }) => ({ ...lease, at }))
+          );
+          if (acked.length) {
+            tracer.acked(acked);
+            this.emit("acked", acked);
+          }
+          const blocked = await store().block(
+            handled.filter(({ block }) => block).map(({ lease, error }) => ({ ...lease, error }))
+          );
+          if (blocked.length) {
+            tracer.blocked(blocked);
+            this.emit("blocked", blocked);
           }
+          return { fetched, leased, acked, blocked };
         }
       } catch (error) {
         logger.error(error);
@@ -894,7 +931,7 @@ var Act = class {
         this._drain_locked = false;
       }
     }
-    return { leased: [], acked: [], blocked: [] };
+    return { fetched: [], leased: [], acked: [], blocked: [] };
   }
   /**
    * Correlates streams using reaction resolvers.
@@ -922,6 +959,7 @@ var Act = class {
         by: randomUUID2(),
         at: 0,
         retry: 0,
+        lagging: true,
         payloads
       }));
       const leased = await store().lease(leases, 0);