@tokamakdev/plugin 0.1.0-beta.107 → 0.1.0-beta.114

package/src/index.ts

@@ -1,173 +1,78 @@
-interface NativeTransport {
-  postMessage(message: string): void;
-  onmessage: ((event: { data: string }) => void) | null;
+import { capabilitiesOf, nativeConnection, type CallOptions, type NativeConnection, type Overflow, type TokamakContext } from "./context.js";
+import { byteStream, type JsonStream } from "./streams.js";
+import { decodeJson, encodeJson } from "./codec.js";
+import { NativeReference } from "./resources.js";
+
+export type { TokamakContext } from "./worker.js";
+export type { CallOptions, NativePlatform, Overflow } from "./context.js";
+export { capabilitiesOf } from "./context.js";
+export type { JsonStream } from "./streams.js";
+export type { JsonValue } from "./codec.js";
+export { decodeJson, encodeJson } from "./codec.js";
+export { NativeReference } from "./resources.js";
+
+/** Whether the app is on screen. */
+export type LifecycleStage = "foreground" | "background";
+
+/**
+ * The app's stage now, which can differ from the last lifecycle event delivered. Rejects with
+ * `NotSupportedError` without a native host.
+ */
+export async function lifecycleStage(ctx: TokamakContext): Promise<LifecycleStage> {
+  return decodeJson<LifecycleStage>(await ctx[nativeConnection].call("tokamak", "lifecycleStage", encodeJson(null)));
 }
 
-interface NativeError {
-  name: string;
-  message: string;
-}
-
-interface NativeResponse {
-  session: string;
-  id: number;
-  value?: unknown;
-  error?: NativeError;
-  done: boolean;
-}
-
-interface PendingCall {
-  kind: "call";
-  resolve(value: unknown): void;
-  reject(error: DOMException): void;
-}
-
-interface PendingSubscription {
-  kind: "subscription";
-  next(value: unknown): void;
-  error(error: DOMException): void;
-}
-
-type Pending = PendingCall | PendingSubscription;
-
-/** Calls `method` of `plugin` from the Worker, resolving with its value. */
-type WorkerCall = (plugin: string, method: string, arguments_: unknown) => Promise<unknown>;
-
-/** Subscribes to `method` of `plugin` from the Worker, returning the function that unsubscribes. */
-type WorkerListen = (
-  plugin: string,
-  method: string,
-  arguments_: unknown,
-  next: (value: unknown) => void,
-  error: (error: DOMException) => void,
-) => () => void;
-
-declare global {
-  /** The page's native transport, which the shells define. */
-  var __tokamakNative: NativeTransport | undefined;
-  /** The Worker's native calls, which the runtime and `tok dev` define. */
-  var __tokamakNativeCall: WorkerCall | undefined;
-  var __tokamakNativeListen: WorkerListen | undefined;
-}
-
-let nextRequestId = 1;
-const session = `${String(Date.now())}-${String(Math.random())}`;
-let connected: NativeTransport | undefined;
-const pending = new Map<number, Pending>();
-
-/** The base class of a plugin's API, which the page and the Worker call alike. */
+/** Native endpoint calls owned by the receiving invocation. */
 export abstract class Plugin {
-  protected constructor(private readonly pluginId: string) {}
+  private readonly connection: NativeConnection;
 
-  protected get hasNativeTransport(): boolean {
-    return pageTransport() !== undefined || globalThis.__tokamakNativeCall !== undefined;
+  protected constructor(private readonly ctx: TokamakContext, private readonly pluginId: string) {
+    const connection = Reflect.get(ctx, nativeConnection) as NativeConnection | undefined;
+    if (!connection) throw new TypeError("A Tokamak invocation context is required");
+    this.connection = connection;
   }
 
-  protected call<T>(method: string, arguments_: unknown = null): Promise<T> {
-    const page = pageTransport();
-    if (page) {
-      return new Promise<T>((resolve, reject) => {
-        send(page, this.pluginId, method, arguments_, { kind: "call", resolve, reject });
-      });
-    }
-    const call = globalThis.__tokamakNativeCall;
-    if (call) return call(this.pluginId, method, arguments_) as Promise<T>;
-    return Promise.reject(unavailable());
+  protected async call<T>(method: string, value: unknown = null, options?: CallOptions): Promise<T> {
+    return decodeJson<T>(await this.callBytes(method, encodeJson(value), options));
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-parameters -- T types the native values, as in call.
-  protected listen<T>(
-    method: string,
-    next: (value: T) => void,
-    error: (error: DOMException) => void,
-    arguments_: unknown = null,
-  ): () => void {
-    const page = pageTransport();
-    if (page) {
-      const id = send(page, this.pluginId, method, arguments_, {
-        kind: "subscription",
-        next,
-        error,
-      });
-      return () => {
-        if (!pending.delete(id)) return;
-        page.postMessage(JSON.stringify({ type: "cancel", session, id }));
-      };
-    }
-    const listen = globalThis.__tokamakNativeListen;
-    if (listen) return listen(this.pluginId, method, arguments_, next as (value: unknown) => void, error);
-    throw unavailable();
+  /**
+   * Whether each operation has an implementation on the platform the app runs on. It does not
+   * report permission or promise that a call succeeds.
+   */
+  get capabilities(): Readonly<Record<string, boolean>> {
+    return capabilitiesOf(this.ctx, this.pluginId);
   }
-}
-
-function unavailable(): DOMException {
-  return new DOMException("Native plugin transport is unavailable", "NotSupportedError");
-}
-
-/** The page's native transport, connected on first use. */
-function pageTransport(): NativeTransport | undefined {
-  const transport = globalThis.__tokamakNative;
-  if (!transport || typeof transport.postMessage !== "function") return undefined;
-  if (connected !== transport) connect(transport);
-  return transport;
-}
 
-function connect(transport: NativeTransport): void {
-  if (!connected) {
-    const page: { addEventListener?: (type: string, listener: () => void) => void } = globalThis;
-    page.addEventListener?.("pagehide", cancelSubscriptions);
+  protected callBytes(method: string, payload: Uint8Array, options?: CallOptions): Promise<Uint8Array> {
+    return this.connection.call(this.pluginId, method, payload, options);
   }
-  connected = transport;
-  transport.onmessage = ({ data }) => {
-    receive(JSON.parse(data) as NativeResponse);
-  };
-  transport.postMessage(JSON.stringify({ type: "reset", session }));
-}
 
-/** Posts a call or subscription, keeping `request` pending unless posting throws. */
-function send(
-  transport: NativeTransport,
-  plugin: string,
-  method: string,
-  arguments_: unknown,
-  request: Pending,
-): number {
-  const id = nextRequestId++;
-  const type = request.kind === "call" ? "call" : "subscribe";
-  pending.set(id, request);
-  try {
-    transport.postMessage(
-      JSON.stringify({ type, session, id, plugin, method, arguments: arguments_ }),
-    );
-  } catch (error) {
-    pending.delete(id);
-    throw error;
+  /** Retains the native object that the subscription `method` holds open, without copying it. */
+  protected resource(method: string, value: unknown = null): Promise<NativeReference> {
+    return NativeReference.acquire(this.ctx, this.pluginId, method, encodeJson(value));
   }
-  return id;
-}
 
-function receive(response: NativeResponse): void {
-  if (response.session !== session) return;
-  const request = pending.get(response.id);
-  if (!request) return;
-  if (response.done) pending.delete(response.id);
-
-  if (response.error) {
-    const error = new DOMException(response.error.message, response.error.name);
-    if (request.kind === "call") request.reject(error);
-    else request.error(error);
-    return;
+  /** The ID that passes `reference` to this plugin's native endpoint, which finds the same object. */
+  protected reference(reference: NativeReference): string {
+    return reference.idFor(this.ctx);
   }
 
-  if (request.kind === "call") request.resolve(response.value);
-  else request.next(response.value);
-}
-
-function cancelSubscriptions(): void {
-  if (!connected) return;
-  for (const [id, request] of pending) {
-    if (request.kind !== "subscription") continue;
-    pending.delete(id);
-    connected.postMessage(JSON.stringify({ type: "cancel", session, id }));
+  /**
+   * A stream of newline-delimited JSON values, consumed with decodeValues. `overflow` says what
+   * happens to values a slow reader has not taken: ending the stream, or dropping the oldest.
+   */
+  protected stream<T>(method: string, value: unknown = null, overflow: Overflow = "error"): JsonStream<T> {
+    return byteStream((next, error, close) => this.connection.listen(this.pluginId, method, encodeJson(value), {
+      next(payload, done) {
+        const line = new Uint8Array(payload.byteLength + 1);
+        line.set(payload);
+        line[payload.byteLength] = 10;
+        next(line);
+        if (done) close();
+      },
+      error,
+      overflow,
+    }), overflow);
   }
 }

package/src/native-protocol.d.mts

@@ -0,0 +1,24 @@
+export interface MessageHeader {
+  type: string;
+  id?: number;
+  plugin?: string;
+  method?: string;
+  name?: string;
+  overflow?: string;
+  failed?: boolean;
+  done?: boolean;
+}
+export interface NativeMessage {
+  header: MessageHeader;
+  payload: Uint8Array;
+}
+export function encodeMessage(header: MessageHeader, payload?: Uint8Array): Uint8Array<ArrayBuffer>;
+export function decodeMessage(bytes: Uint8Array): NativeMessage;
+export function messages(body: AsyncIterable<Uint8Array> | Iterable<Uint8Array> | null): AsyncGenerator<NativeMessage>;
+/** A failed endpoint payload. */
+export interface NativeFailure {
+  name: string;
+  message: string;
+  cause?: unknown;
+}
+export function nativeError(failure: NativeFailure): DOMException;

package/src/native-protocol.mjs

@@ -0,0 +1,77 @@
+const encoder = new TextEncoder();
+const decoder = new TextDecoder("utf-8", { fatal: true });
+
+/** A length-delimited routing header followed by an opaque endpoint payload. */
+export function encodeMessage(header, payload = new Uint8Array()) {
+  const metadata = encoder.encode(JSON.stringify(header));
+  const length = 4 + metadata.byteLength + payload.byteLength;
+  if (length > 0xffffffff) throw new RangeError("Native message exceeds 4 GiB");
+  const result = new Uint8Array(4 + length);
+  const view = new DataView(result.buffer);
+  view.setUint32(0, length, true);
+  view.setUint32(4, metadata.byteLength, true);
+  result.set(metadata, 8);
+  result.set(payload, 8 + metadata.byteLength);
+  return result;
+}
+
+export function decodeMessage(bytes) {
+  if (bytes.byteLength < 8) throw new TypeError("Incomplete native message");
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const length = view.getUint32(0, true);
+  const metadata = view.getUint32(4, true);
+  if (length !== bytes.byteLength - 4 || metadata > length - 4) {
+    throw new TypeError("Invalid native message length");
+  }
+  const header = JSON.parse(decoder.decode(bytes.subarray(8, 8 + metadata)));
+  if (header === null || typeof header !== "object" || Array.isArray(header) || typeof header.type !== "string" ||
+      (header.id !== undefined && (!Number.isSafeInteger(header.id) || header.id < 0)) ||
+      ["plugin", "method", "name", "overflow"].some(key => header[key] !== undefined && typeof header[key] !== "string") ||
+      ["failed", "done"].some(key => header[key] !== undefined && typeof header[key] !== "boolean")) {
+    throw new TypeError("Invalid native routing header");
+  }
+  return { header, payload: bytes.subarray(8 + metadata) };
+}
+
+/** Reads messages independently of HTTP chunk boundaries. */
+export async function* messages(body) {
+  let prefix = new Uint8Array(4);
+  let used = 0;
+  let remaining = 0;
+  let parts = [];
+  for await (const chunk of body ?? []) {
+    let offset = 0;
+    while (offset < chunk.byteLength) {
+      if (used < 4) {
+        const count = Math.min(4 - used, chunk.byteLength - offset);
+        prefix.set(chunk.subarray(offset, offset + count), used);
+        used += count;
+        offset += count;
+        if (used < 4) continue;
+        remaining = new DataView(prefix.buffer).getUint32(0, true);
+        if (remaining < 4) throw new TypeError("Invalid native message length");
+        parts.push(prefix);
+      }
+      const count = Math.min(remaining, chunk.byteLength - offset);
+      parts.push(chunk.subarray(offset, offset + count));
+      remaining -= count;
+      offset += count;
+      if (remaining) continue;
+      const frame = new Uint8Array(parts.reduce((length, part) => length + part.byteLength, 0));
+      let position = 0;
+      for (const part of parts) { frame.set(part, position); position += part.byteLength; }
+      yield decodeMessage(frame);
+      prefix = new Uint8Array(4);
+      used = 0;
+      parts = [];
+    }
+  }
+  if (used) throw new TypeError("Incomplete native message");
+}
+
+/** The error a failed endpoint payload describes, with its details as the `cause`. */
+export function nativeError({ name, message, cause }) {
+  const error = new DOMException(message, name);
+  if (cause !== undefined) Object.defineProperty(error, "cause", { value: cause, writable: true, configurable: true });
+  return error;
+}

package/src/registry.ts

@@ -1,6 +1,14 @@
+import { decodeJson, encodeJson } from "./codec.js";
+
 /** A registered listener. */
 export type RegisteredListener = (event: unknown, env: unknown, ctx: unknown) => unknown;
 
+/** How an event's data and reply travel as bytes. An empty reply is no reply. */
+export interface EventCodec<T, Reply> {
+  decode(payload: Uint8Array): T;
+  encode(reply: Reply): Uint8Array;
+}
+
 /** What a dispatch returns. */
 export interface Dispatched {
   /** The first value, in registration order, that a listener returned other than `undefined`. */
@@ -9,8 +17,18 @@ export interface Dispatched {
   listened: string[];
 }
 
+const json: EventCodec<unknown, unknown> = {
+  decode: decodeJson,
+  encode: (reply) => reply === undefined ? new Uint8Array() : encodeJson(reply),
+};
+
 /** Every listener, by event name, in registration order. */
 const listeners = new Map<string, RegisteredListener[]>();
+const codecs = new Map<string, EventCodec<unknown, unknown>>();
+
+export function listenedEvents(): string[] {
+  return [...listeners.keys()];
+}
 
 export function register(name: string, listener: RegisteredListener): void {
   const registered = listeners.get(name);
@@ -18,6 +36,15 @@ export function register(name: string, listener: RegisteredListener): void {
   else listeners.set(name, [listener]);
 }
 
+export function setCodec(name: string, codec: EventCodec<unknown, unknown>): void {
+  codecs.set(name, codec);
+}
+
+/** The codec of the event `name`: the one its definition gave, or JSON. */
+export function eventCodec(name: string): EventCodec<unknown, unknown> {
+  return codecs.get(name) ?? json;
+}
+
 /**
  * Runs the listeners of the event `name` and waits for them to settle. A listener that throws is
  * reported with `reportError`, and the others still run.
@@ -36,5 +63,5 @@ export async function dispatch(
     if (result.status === "rejected") reportError(result.reason);
     else if (reply === undefined) reply = result.value;
   }
-  return { reply, listened: [...listeners.keys()] };
+  return { reply, listened: listenedEvents() };
 }

package/src/resources.ts

@@ -0,0 +1,101 @@
+import { nativeConnection, type TokamakContext } from "./context.js";
+import { decodeJson, encodeJson } from "./codec.js";
+
+// Workers have FinalizationRegistry from compatibility date 2025-05-05 unless `disable_weak_ref` is set.
+const collected = typeof FinalizationRegistry === "function"
+  ? new FinalizationRegistry<() => void>((release) => { release(); })
+  : undefined;
+
+/** The invocation that owns `reference`. */
+export let ownerOf: (reference: NativeReference) => TokamakContext;
+
+/**
+ * A native object that stays in native memory. The receiving invocation owns it until `close`,
+ * the invocation's end, or garbage collection where the runtime has `FinalizationRegistry`.
+ */
+export class NativeReference {
+  readonly #ctx: TokamakContext;
+  readonly #id: string;
+  #release: (() => void) | undefined;
+
+  static {
+    ownerOf = (reference) => reference.#ctx;
+  }
+
+  private constructor(ctx: TokamakContext, id: string, release: () => void) {
+    this.#ctx = ctx;
+    this.#id = id;
+    this.#release = release;
+    collected?.register(this, release, this);
+  }
+
+  /**
+   * Retains the object that the native subscription `method` of `plugin` holds open: its first
+   * value is the object's ID, and later values go to `next`.
+   */
+  static async acquire(
+    ctx: TokamakContext,
+    plugin: string,
+    method: string,
+    payload: Uint8Array,
+    next: (value: Uint8Array) => void = () => {},
+  ): Promise<NativeReference> {
+    let release = () => {};
+    try {
+      // The subscription's listeners never hold the reference, so garbage collection can release it.
+      const id = await new Promise<Uint8Array>((resolve, reject) => {
+        let settled = false;
+        release = ctx[nativeConnection].listen(plugin, method, payload, {
+          next(value, done) {
+            if (settled) {
+              if (!done) next(value);
+              return;
+            }
+            settled = true;
+            if (done) reject(new DOMException("The native object was not retained", "InvalidStateError"));
+            else resolve(value);
+          },
+          // A later failure ends the subscription: the released object fails where it is used.
+          error(error) {
+            if (settled) return;
+            settled = true;
+            reject(error);
+          },
+        });
+      });
+      return new NativeReference(ctx, decodeJson<string>(id), release);
+    } catch (error) {
+      release();
+      throw error;
+    }
+  }
+
+  /** The object retained as `id` in the scope of the invocation `ctx`, which ends with it. */
+  static scoped(ctx: TokamakContext, id: string): NativeReference {
+    return new NativeReference(ctx, id, () => {
+      // A failed release changes nothing: the invocation's end has released the object.
+      ctx[nativeConnection].call("tokamak", "resource.release", encodeJson(id)).catch(() => undefined);
+    });
+  }
+
+  /** Copies the current contents of a native byte buffer. Other native objects reject with `TypeError`. */
+  async bytes(): Promise<Uint8Array> {
+    return this.#ctx[nativeConnection].call("tokamak", "resource.read", encodeJson(this.idFor(this.#ctx)));
+  }
+
+  /** Releases the native object. Later calls do nothing. */
+  close(): void {
+    const release = this.#release;
+    if (!release) return;
+    this.#release = undefined;
+    collected?.unregister(this);
+    release();
+  }
+
+  /** The ID a plugin's native endpoint uses to find the object, within the invocation `ctx` that owns it. */
+  idFor(ctx: TokamakContext): string {
+    if (!this.#release) throw new DOMException("The native reference is closed", "InvalidStateError");
+    if (ctx !== this.#ctx) throw new DOMException("The native reference belongs to another invocation", "InvalidStateError");
+    return this.#id;
+  }
+}

package/src/rpc/binding.mjs

@@ -0,0 +1,109 @@
+import { RpcTarget, WorkerEntrypoint } from 'cloudflare:workers';
+import { invoke } from './client.mjs';
+import { serveRpc } from './endpoint.mjs';
+
+const original = Symbol.for('tokamak.dev.service');
+
+class Port extends RpcTarget {
+  #socket;
+
+  constructor(socket) { super(); this.#socket = socket; }
+  receive(frame) { this.#socket.dispatchEvent(new MessageEvent('message', { data: frame })); }
+  [Symbol.dispose]() { this.#socket.close(); }
+}
+
+/** Workerd owns these ports for the lifetime of the calling invocation. */
+class Socket extends EventTarget {
+  #peer;
+  readyState = 0;
+
+  connect(peer) {
+    if (this.readyState === 3) { peer[Symbol.dispose](); return; }
+    this.#peer = peer;
+    this.readyState = 1;
+    this.dispatchEvent(new Event('open'));
+  }
+  fail(error) { this.dispatchEvent(new ErrorEvent('error', { error })); this.close(); }
+  send(frame) { return this.#peer.receive(frame); }
+  close() {
+    if (this.readyState === 3) return;
+    this.readyState = 3;
+    this.#peer?.[Symbol.dispose]();
+    this.dispatchEvent(new Event('close'));
+  }
+}
+
+/** Open the receiving invocation using workerd's internal RPC ownership. */
+export function createRpcPort(peer, createWorker, methods, ctx) {
+  const socket = new Socket();
+  socket.connect(peer.dup());
+  ctx.waitUntil(serveRpc(socket, createWorker, methods, ctx));
+  return new Port(socket);
+}
+
+/** Enforce Tokamak's RPC contract over a development worker's service binding. */
+export function createBinding(service, methods) {
+  const binding = {
+    ...Object.fromEntries(methods.map(method => [method, async (...args) => {
+      const socket = new Socket();
+      service.__tokamakRpc(new Port(socket)).then(peer => socket.connect(peer), error => socket.fail(error));
+      return invoke(socket, method, args);
+    }])),
+    async fetch(input, init) {
+      const response = await service.fetch(input, init);
+      if (response.status === 101) {
+        response.webSocket?.accept();
+        response.webSocket?.close(1000);
+        throw new TypeError('Service binding WebSocket upgrades are not supported');
+      }
+      return response;
+    },
+  };
+  Object.defineProperty(binding, original, { value: service });
+  return binding;
+}
+
+/** Install generated local bindings before the application's modules are evaluated. */
+export function installBindings(env, bindings) {
+  for (const [name, methods] of Object.entries(bindings)) {
+    const binding = env[name];
+    const service = Object.getOwnPropertyDescriptor(binding, original)?.value ?? binding;
+    env[name] = createBinding(service, methods);
+  }
+  return env;
+}
+
+function instance(Type, ctx, env) {
+  return typeof Type === 'function' ? new Type(ctx, env) : Type;
+}
+
+/** Route development RPC through the shared contract and preserve standard Worker handlers. */
+export function developmentEntrypoint(Type, methods, bindings) {
+  class Entry extends WorkerEntrypoint {
+    __tokamakRpc(peer) {
+      const env = installBindings(this.env, bindings);
+      return createRpcPort(peer, ctx => instance(Type, ctx, env), methods, this.ctx);
+    }
+  }
+  for (const name of ['fetch', 'queue', 'scheduled', 'email', 'tail', 'trace', 'tailStream', 'test']) {
+    if (typeof (typeof Type === 'function' ? Type.prototype : Type)?.[name] !== 'function') continue;
+    Object.defineProperty(Entry.prototype, name, {
+      value(event) {
+        const env = installBindings(this.env, bindings);
+        const worker = instance(Type, this.ctx, env);
+        return worker[name](event, env, this.ctx);
+      },
+    });
+  }
+  return Entry;
+}
+
+/** Install local bindings for host event dispatch. */
+export function developmentEvents(Type, bindings) {
+  return class extends Type {
+    dispatch(name, data) {
+      installBindings(this.env, bindings);
+      return super.dispatch(name, data);
+    }
+  };
+}