@zlink-systems/framework 0.10.0

package/src/runtime/actors/post-commit-actor-binder.ts

@@ -0,0 +1,70 @@
+import type { ActorRef } from '../../contracts';
+import { ZLinkStateLane } from '../execution/state-lane';
+import { ZLinkActorRetryDelay } from './actor-retry-delay';
+
+export interface ZLinkPostCommitActorBinderOptions {
+  readonly bind: (actorRef: ActorRef, force: boolean) => Promise<void>;
+  readonly reportError?: (error: unknown) => void;
+  readonly signal?: AbortSignal | (() => AbortSignal | undefined);
+}
+
+export class ZLinkPostCommitActorBinder {
+  private readonly lane = new ZLinkStateLane();
+  private readonly desiredRefs = new Map<string, ActorRef>();
+  private readonly tasks = new Map<string, Promise<void>>();
+
+  constructor(private readonly options: ZLinkPostCommitActorBinderOptions) {}
+
+  bindEventually(actorRef: ActorRef): void {
+    void this.bind(actorRef);
+  }
+
+  bind(actorRef: ActorRef): Promise<void> {
+    this.desiredRefs.set(actorRef.actorId, actorRef);
+    const existing = this.tasks.get(actorRef.actorId);
+    if (existing !== undefined) return existing;
+    let task!: Promise<void>;
+    task = this.run(actorRef.actorId)
+      .finally(async () => await this.lane.run(() => {
+        if (this.tasks.get(actorRef.actorId) === task) {
+          this.tasks.delete(actorRef.actorId);
+        }
+      }));
+    this.tasks.set(actorRef.actorId, task);
+    return task;
+  }
+
+  private async run(actorId: string): Promise<void> {
+    const retryDelay = new ZLinkActorRetryDelay();
+    for (;;) {
+      const signal = this.signal();
+      if (signal?.aborted === true) return;
+      const actorRef = this.desiredRefs.get(actorId);
+      if (actorRef === undefined) {
+        return;
+      }
+      try {
+        await this.options.bind(actorRef, true);
+        if (await this.lane.run(() => this.completeBindCore(actorId, actorRef))) {
+          return;
+        }
+        retryDelay.reset();
+      } catch (error) {
+        this.options.reportError?.(error);
+        if (!await retryDelay.wait(signal)) return;
+      }
+    }
+  }
+
+  private signal(): AbortSignal | undefined {
+    return typeof this.options.signal === 'function'
+      ? this.options.signal()
+      : this.options.signal;
+  }
+
+  private completeBindCore(actorId: string, actorRef: ActorRef): boolean {
+    if (this.desiredRefs.get(actorId) !== actorRef) return false;
+    this.desiredRefs.delete(actorId);
+    return true;
+  }
+}

package/src/runtime/actors/post-commit-actor-location.ts

@@ -0,0 +1,114 @@
+import type { RoutingId } from '../../contracts';
+import { ZLinkStateLane } from '../execution/state-lane';
+import type { ZLinkLocationLifecycle } from '../locations';
+import { ZLinkActorRetryDelay } from './actor-retry-delay';
+
+export interface ZLinkPostCommitActorLocationOptions {
+  readonly lifecycle: ZLinkLocationLifecycle;
+  readonly reportError?: (error: unknown) => void;
+  readonly signal?: AbortSignal;
+}
+
+interface ActorOperationQueue {
+  readonly operations: Array<(() => Promise<void>) | undefined>;
+  head: number;
+  count: number;
+}
+
+export class ZLinkPostCommitActorLocation {
+  private readonly lane = new ZLinkStateLane();
+  private readonly queues = new Map<string, ActorOperationQueue>();
+  private readonly tasks = new Map<string, Promise<void>>();
+
+  constructor(private readonly options: ZLinkPostCommitActorLocationOptions) {}
+
+  joinedEventually(
+    actorType: string,
+    actorId: string,
+    meshName: string,
+    spotId: RoutingId,
+    spotGeneration: bigint,
+    membershipEpoch: bigint,
+    ownerNodeGeneration: bigint
+  ): void {
+    this.enqueue(actorId, () => this.options.lifecycle.notifyActorJoinedSpot(
+      actorType,
+      actorId,
+      meshName,
+      spotId,
+      spotGeneration,
+      membershipEpoch,
+      ownerNodeGeneration
+    ));
+  }
+
+  leftEventually(
+    actorType: string,
+    actorId: string,
+    entrySpotId: RoutingId,
+    entrySpotGeneration: bigint,
+    membershipEpoch: bigint,
+    ownerNodeGeneration: bigint
+  ): void {
+    this.enqueue(actorId, () => this.options.lifecycle.notifyActorLeftSpot(
+      actorType,
+      actorId,
+      entrySpotId,
+      entrySpotGeneration,
+      membershipEpoch,
+      ownerNodeGeneration
+    ));
+  }
+
+  private enqueue(actorId: string, operation: () => Promise<void>): void {
+    const queue = this.queues.get(actorId) ?? { operations: [], head: 0, count: 0 };
+    queue.operations.push(operation);
+    queue.count += 1;
+    this.queues.set(actorId, queue);
+    if (this.tasks.has(actorId)) {
+      return;
+    }
+    let task!: Promise<void>;
+    task = this.run(actorId).finally(async () => await this.lane.run(() => {
+      if (this.tasks.get(actorId) === task) {
+        this.tasks.delete(actorId);
+      }
+    }));
+    this.tasks.set(actorId, task);
+  }
+
+  private async run(actorId: string): Promise<void> {
+    const retryDelay = new ZLinkActorRetryDelay();
+    while (this.options.signal?.aborted !== true) {
+      const queue = this.queues.get(actorId);
+      const operation = queue?.operations[queue.head];
+      if (queue === undefined || operation === undefined) {
+        this.queues.delete(actorId);
+        return;
+      }
+      try {
+        await operation();
+        await this.lane.run(() => this.completeCore(queue));
+        retryDelay.reset();
+      } catch (error) {
+        this.options.reportError?.(error);
+        if (!await retryDelay.wait(this.options.signal)) {
+          return;
+        }
+      }
+    }
+  }
+
+  private completeCore(queue: ActorOperationQueue): void {
+    queue.operations[queue.head] = undefined;
+    queue.head += 1;
+    queue.count -= 1;
+    if (queue.count === 0) {
+      queue.operations.length = 0;
+      queue.head = 0;
+    } else if (queue.head >= 1024 && queue.head * 2 >= queue.operations.length) {
+      queue.operations.splice(0, queue.head);
+      queue.head = 0;
+    }
+  }
+}

package/src/runtime/actors/spot-actor-dispatch.ts

@@ -0,0 +1,362 @@
+import { ZLinkFrameworkInternalErrorKind, createInternalFrameworkException  } from '../framework-errors-internal';
+import type {
+  Type,
+  ZLinkActor,
+  ZLinkActorHandlerRegistry,
+  ZLinkSpot,
+  ZLinkSpotActorJoinResult,
+  ZLinkMessageContext,
+  ZLinkSpotActorRequestHandler,
+  ZLinkSpotActorSendHandler
+} from '../../contracts';
+import type { ZLinkProviderResolver } from '../../contracts/Common/ZLinkProviderResolver';
+import {
+  ZLinkFrameworkException,
+  ZLinkMessageMetadataEmpty
+} from '../../contracts';
+import type { Message } from '../../contracts/Common/Message';
+import { ZLinkConfigurationException } from '../configuration';
+import { readZLinkDecoratorMetadata } from '../../contracts/Handlers/Attributes';
+import { wrapFrameworkPayloadMessage } from '../messaging/payload-codec';
+import type { ZLinkMessageSerializer } from '../../contracts';
+import { actorJoinIdentity } from './actor-lifecycle-snapshot';
+import { runActorHandlerWithDeferredJoins } from './actor-join-deferred-scope';
+import { runWithLifecycleHandler } from '../handlers/handler-instance-scope';
+import type { ZLinkSerialWorkOptions } from '../execution/serial-execution-queue';
+
+export enum ZLinkActorPacketKind {
+  Send = 'send',
+  Request = 'request'
+}
+
+export interface ZLinkActorPacketDescriptor {
+  readonly kind: ZLinkActorPacketKind;
+  readonly packetName: string;
+  readonly actorType: Type<ZLinkActor>;
+  readonly handlerType: Type;
+}
+
+export class ZLinkSpotActorHandlerRegistryRuntime implements ZLinkActorHandlerRegistry {
+  private readonly packets = new Map<string, ZLinkActorPacketDescriptor>();
+
+  addHandler<THandler>(handlerType: Type<THandler>, packetName?: string): this {
+    const metadata = readZLinkDecoratorMetadata(handlerType).find((entry) =>
+      entry.kind === 'spotActorSend' || entry.kind === 'spotActorRequest');
+    const resolvedPacketName = packetName ?? metadata?.packetName;
+    if (metadata === undefined || resolvedPacketName === undefined) {
+      throw new ZLinkConfigurationException(
+        `Actor packet handler '${handlerType.name}' must declare a packet name.`
+      );
+    }
+    return this.addPacket({
+      kind: metadata.kind === 'spotActorSend'
+        ? ZLinkActorPacketKind.Send
+        : ZLinkActorPacketKind.Request,
+      packetName: resolvedPacketName,
+      actorType: Object as unknown as Type<ZLinkActor>,
+      handlerType
+    });
+  }
+
+  addPacket(descriptor: ZLinkActorPacketDescriptor): this {
+    const key = packetKey(descriptor.kind, descriptor.actorType, descriptor.packetName);
+    if (this.packets.has(key)) {
+      throw new ZLinkConfigurationException(
+        `Actor packet '${descriptor.packetName}' for '${descriptor.actorType.name}' is already registered.`
+      );
+    }
+    this.packets.set(key, descriptor);
+    return this;
+  }
+
+  resolvePacket(
+    kind: ZLinkActorPacketKind,
+    actor: ZLinkActor,
+    packetName: string
+  ): ZLinkActorPacketDescriptor | undefined {
+    const actorType = actor.constructor as Type<ZLinkActor>;
+    const exact = this.packets.get(packetKey(kind, actorType, packetName));
+    if (exact !== undefined) {
+      return exact;
+    }
+    const wildcard = this.packets.get(packetKey(kind, Object as unknown as Type<ZLinkActor>, packetName));
+    if (wildcard !== undefined) {
+      return wildcard;
+    }
+
+    for (const descriptor of this.packets.values()) {
+      if (
+        descriptor.kind === kind
+        && descriptor.packetName === packetName
+        && actor instanceof descriptor.actorType
+      ) {
+        return descriptor;
+      }
+    }
+    return undefined;
+  }
+
+}
+
+export interface ZLinkSpotActorDispatcherOptions {
+  readonly registry: ZLinkSpotActorHandlerRegistryRuntime;
+  readonly spot: ZLinkSpot;
+  readonly providerResolver?: ZLinkProviderResolver;
+  readonly serial?: {
+    execute<T>(operation: () => Promise<T> | T, workOptions?: ZLinkSerialWorkOptions): Promise<T>;
+  };
+  readonly serialWorkOptions?: ZLinkSerialWorkOptions;
+  readonly messageSerializers?: ReadonlyMap<string, ZLinkMessageSerializer>;
+  readonly onAdmitted?: () => void;
+  readonly onHandlerStart?: () => void;
+}
+
+interface ZLinkSpotActorReplyOptionsSnapshot {
+  readonly metadata: ReadonlyMap<string, string>;
+  readonly compressPayload: boolean;
+}
+
+export class ZLinkSpotActorDispatcher {
+  constructor(private readonly options: ZLinkSpotActorDispatcherOptions) {}
+
+  dispatchSend<TMessage>(
+    actor: ZLinkActor,
+    packetName: string,
+    message: TMessage,
+    context: Partial<ZLinkMessageContext> = {}
+  ): Promise<void> {
+    return this.dispatchSendDecoded(actor, packetName, () => message, context);
+  }
+
+  dispatchSendDecoded<TMessage>(
+    actor: ZLinkActor,
+    packetName: string,
+    decode: () => TMessage,
+    context: Partial<ZLinkMessageContext> = {}
+  ): Promise<void> {
+    return this.execute(async () => {
+      const descriptor = this.requirePacket(ZLinkActorPacketKind.Send, actor, packetName);
+      const message = decode();
+      this.options.onHandlerStart?.();
+      await this.invokeHandler<
+        ZLinkSpotActorSendHandler<ZLinkSpot, ZLinkActor, TMessage>,
+        void
+      >(actor, descriptor, (handler) =>
+        runActorHandlerWithDeferredJoins(() =>
+          handler.handle(
+            this.options.spot,
+            actor,
+            this.createContext(packetName, context),
+            message
+          )));
+    });
+  }
+
+  dispatchRequest<TRequest, TReply>(
+    actor: ZLinkActor,
+    packetName: string,
+    request: TRequest,
+    context: Partial<ZLinkMessageContext> = {}
+  ): Promise<TReply> {
+    return this.dispatchRequestThenDecoded<TRequest, TReply, TReply>(
+      actor,
+      packetName,
+      () => request,
+      context,
+      (reply) => reply
+    );
+  }
+
+  dispatchRequestDecoded<TRequest, TReply>(
+    actor: ZLinkActor,
+    packetName: string,
+    decode: () => TRequest,
+    context: Partial<ZLinkMessageContext> = {}
+  ): Promise<TReply> {
+    return this.dispatchRequestThenDecoded<TRequest, TReply, TReply>(
+      actor,
+      packetName,
+      decode,
+      context,
+      (reply) => reply
+    );
+  }
+
+  dispatchRequestThen<TRequest, TReply, TResult>(
+    actor: ZLinkActor,
+    packetName: string,
+    request: TRequest,
+    context: Partial<ZLinkMessageContext> = {},
+    afterReply: (reply: TReply, options: ZLinkSpotActorReplyOptionsSnapshot) => Promise<TResult> | TResult,
+    beforeReply?: (reply: TReply, options: ZLinkSpotActorReplyOptionsSnapshot) => Promise<void> | void
+  ): Promise<TResult> {
+    return this.dispatchRequestThenDecoded(
+      actor,
+      packetName,
+      () => request,
+      context,
+      afterReply,
+      beforeReply
+    );
+  }
+
+  dispatchRequestThenDecoded<TRequest, TReply, TResult>(
+    actor: ZLinkActor,
+    packetName: string,
+    decode: () => TRequest,
+    context: Partial<ZLinkMessageContext> = {},
+    afterReply: (reply: TReply, options: ZLinkSpotActorReplyOptionsSnapshot) => Promise<TResult> | TResult,
+    beforeReply?: (reply: TReply, options: ZLinkSpotActorReplyOptionsSnapshot) => Promise<void> | void
+  ): Promise<TResult> {
+    return this.execute(async () => {
+      const descriptor = this.requirePacket(ZLinkActorPacketKind.Request, actor, packetName);
+      const request = decode();
+      this.options.onHandlerStart?.();
+      return await this.invokeHandler<
+        ZLinkSpotActorRequestHandler<ZLinkSpot, ZLinkActor, TRequest, TReply>,
+        TResult
+      >(actor, descriptor, (handler) =>
+        runActorHandlerWithDeferredJoins(
+          () => handler.handle(
+            this.options.spot,
+            actor,
+            this.createContext(packetName, context),
+            request
+          ),
+          (reply) => afterReply(reply, {
+            metadata: new Map<string, string>(),
+            compressPayload: false
+          }),
+          beforeReply === undefined
+            ? undefined
+            : (reply) => beforeReply(reply, {
+                metadata: new Map<string, string>(),
+                compressPayload: false
+              })
+        ));
+    });
+  }
+
+  admitActorJoin(
+    actor: ZLinkActor,
+    request: Message,
+    commit: () => Promise<void> | void
+  ): Promise<ZLinkSpotActorJoinResult> {
+    return this.evaluateActorJoin(actor, request).then(async (result) => {
+      if (result.accepted) {
+        await this.commitActorJoin(actor, commit);
+      }
+      return result;
+    });
+  }
+
+  evaluateActorJoin(
+    actor: ZLinkActor,
+    request: Message,
+    contentType = 'application/json'
+  ): Promise<ZLinkSpotActorJoinResult> {
+    return this.execute(async () => {
+      const payload = wrapFrameworkPayloadMessage(
+        request,
+        this.options.messageSerializers,
+        contentType
+      );
+      const onActorJoin = (this.options.spot as Partial<ZLinkSpot>).onActorJoin;
+      return onActorJoin === undefined
+        ? { accepted: false }
+        : await onActorJoin.call(this.options.spot, actorJoinIdentity(actor), payload);
+    });
+  }
+
+  commitActorJoin(
+    actor: ZLinkActor,
+    commit: () => Promise<void> | void
+  ): Promise<void> {
+    return this.execute(async () => {
+      await commit();
+      await this.options.spot.onJoinedActor(actor);
+    });
+  }
+
+  notifyJoinActor(actor: ZLinkActor): Promise<void> {
+    return this.execute(() => this.options.spot.onJoinedActor(actor));
+  }
+
+  notifyLeaveActor(actor: ZLinkActor): Promise<void> {
+    return this.execute(() => this.options.spot.onLeaveActor(actor));
+  }
+
+  notifyDisconnectActor(actor: ZLinkActor): Promise<void> {
+    return this.execute(() => this.options.spot.onDisconnectActor?.(actor));
+  }
+
+  private requirePacket(
+    kind: ZLinkActorPacketKind,
+    actor: ZLinkActor,
+    packetName: string
+  ): ZLinkActorPacketDescriptor {
+    const descriptor = this.options.registry.resolvePacket(kind, actor, packetName);
+    if (descriptor === undefined) {
+      throw actorDispatchHandlerNotFound(`No Spot actor ${kind} handler is registered for '${packetName}'.`);
+    }
+    this.ensureActorType(descriptor, actor);
+    return descriptor;
+  }
+
+  private ensureActorType(descriptor: ZLinkActorPacketDescriptor, actor: ZLinkActor): void {
+    if (descriptor.actorType === (Object as unknown as Type<ZLinkActor>)) {
+      return;
+    }
+    if (actor instanceof descriptor.actorType) {
+      return;
+    }
+    throw actorDispatchHandlerNotFound(
+      `Actor handler '${descriptor.handlerType.name}' expects actor '${descriptor.actorType.name}'.`
+    );
+  }
+
+  private async invokeHandler<THandler, TResult>(
+    actor: ZLinkActor,
+    descriptor: Pick<ZLinkActorPacketDescriptor, 'handlerType'>,
+    callback: (handler: THandler) => Promise<TResult>
+  ): Promise<TResult> {
+    return await runWithLifecycleHandler(
+      actor,
+      descriptor.handlerType,
+      this.options.providerResolver,
+      (resolved) => callback(resolved as THandler)
+    );
+  }
+
+  private execute<T>(operation: () => Promise<T> | T): Promise<T> {
+    const pending = this.options.serial?.execute(operation, this.options.serialWorkOptions)
+      ?? Promise.resolve().then(operation);
+    this.options.onAdmitted?.();
+    return pending;
+  }
+
+  private createContext(
+    packetName: string,
+    context: Partial<ZLinkMessageContext>
+  ): ZLinkMessageContext {
+    return {
+      meshName: context.meshName,
+      channelName: context.channelName,
+      packetName,
+      contentType: context.contentType,
+      metadata: context.metadata ?? ZLinkMessageMetadataEmpty,
+      correlationId: context.correlationId
+    };
+  }
+}
+
+function packetKey(kind: ZLinkActorPacketKind, actorType: Type<ZLinkActor>, packetName: string): string {
+  return `${kind}:${actorType.name}:${packetName}`;
+}
+
+function actorDispatchHandlerNotFound(message: string): ZLinkFrameworkException {
+  return createInternalFrameworkException(
+    ZLinkFrameworkInternalErrorKind.ActorDispatchHandlerNotFound,
+    message
+  );
+}

package/src/runtime/actors/transferred-actor-rollback.ts

@@ -0,0 +1,123 @@
+import type { ZLinkActor } from '../../contracts';
+import type {
+  ZLinkBackendActorRef,
+  ZLinkBackendMeshNode
+} from '../backend/contracts';
+import { closeMeshCompletion } from '../backend';
+import { RequestResult } from '../backend/runtime-values';
+import type { ZLinkActorManagerOptions } from './actor-runtime-contracts';
+import type { ZLinkActorRuntimeState } from './actor-runtime-state';
+import { ZLinkActorRetryDelay } from './actor-retry-delay';
+
+type ZLinkTransferredActorRollbackOptions = Pick<
+  ZLinkActorManagerOptions,
+  | 'actorDestroyedCleanup'
+  | 'nativeActorNode'
+  | 'nativeActorNodeProvider'
+  | 'nativeActorCompletionTableProvider'
+  | 'shutdownSignal'
+  | 'metrics'
+>;
+
+export class ZLinkTransferredActorRollbackCoordinator {
+  private readonly tasks = new Map<string, Promise<void>>();
+
+  constructor(
+    private readonly states: Map<string, ZLinkActorRuntimeState>,
+    private readonly options: ZLinkTransferredActorRollbackOptions
+  ) {}
+
+  async rollback(actor: ZLinkActor, signal?: AbortSignal): Promise<void> {
+    const state = this.states.get(actor.context.actorId);
+    if (state?.actor !== actor) {
+      return;
+    }
+
+    state.clearJoinedSpot();
+    if (!state.isMoving) {
+      state.beginMove();
+    }
+
+    const actorRef = state.nativeActorRef;
+    const node = this.options.nativeActorNode ?? this.options.nativeActorNodeProvider?.();
+    try {
+      if (node !== undefined && actorRef !== undefined) {
+        await this.destroyNativeActor(node, actorRef, signal);
+      }
+    } catch (error) {
+      this.schedule(actor, state, node, actorRef);
+      throw error;
+    }
+    this.complete(actor, state);
+  }
+
+  private schedule(
+    actor: ZLinkActor,
+    state: ZLinkActorRuntimeState,
+    node: ZLinkBackendMeshNode | undefined,
+    actorRef: ZLinkBackendActorRef | undefined
+  ): void {
+    if (this.tasks.has(actor.context.actorId)) {
+      return;
+    }
+    const task = this.retry(actor, state, node, actorRef)
+      .finally(() => this.tasks.delete(actor.context.actorId));
+    this.tasks.set(actor.context.actorId, task);
+  }
+
+  private async retry(
+    actor: ZLinkActor,
+    state: ZLinkActorRuntimeState,
+    node: ZLinkBackendMeshNode | undefined,
+    actorRef: ZLinkBackendActorRef | undefined
+  ): Promise<void> {
+    const retryDelay = new ZLinkActorRetryDelay();
+    while (this.states.get(actor.context.actorId) === state && this.options.shutdownSignal?.aborted !== true) {
+      if (!await retryDelay.wait(this.options.shutdownSignal)) return;
+      try {
+        if (node !== undefined && actorRef !== undefined) {
+          await this.destroyNativeActor(node, actorRef);
+        }
+        this.complete(actor, state);
+        return;
+      } catch {
+        // The tombstone prevents dispatch while the next retry is pending.
+      }
+    }
+  }
+
+  private async destroyNativeActor(
+    node: ZLinkBackendMeshNode,
+    actorRef: ZLinkBackendActorRef,
+    signal?: AbortSignal
+  ): Promise<void> {
+    const completions = this.options.nativeActorCompletionTableProvider?.();
+    if (completions === undefined) {
+      throw new Error('Actor rollback requires a running MeshNode completion table.');
+    }
+    const completion = await completions.submit(() => node.destroyActor({
+      nodeRid: actorRef.nodeRid,
+      actorId: actorRef.actorId,
+      generation: actorRef.generation
+    }), signal);
+    try {
+      if (completion.terminalResult !== RequestResult.Ok) {
+        throw new Error(
+          `Actor rollback destroy failed with '${completion.terminalResult}' (errno ${completion.failureErrno}).`
+        );
+      }
+    } finally {
+      closeMeshCompletion(completion);
+    }
+  }
+
+  private complete(actor: ZLinkActor, state: ZLinkActorRuntimeState): void {
+    if (this.states.get(actor.context.actorId) !== state) {
+      return;
+    }
+    this.options.actorDestroyedCleanup?.(actor.context.actorId);
+    state.clearAfterDestroy();
+    this.states.delete(actor.context.actorId);
+    this.options.metrics?.change('zlink.actor.count', -1);
+  }
+}