@nmtjs/protocol 0.6.0

package/dist/server/transport.js

@@ -0,0 +1,7 @@
+import { kTransportPlugin } from "./constants.js";
+export const createTransport = (name, init)=>({
+        name,
+        init,
+        [kTransportPlugin]: true
+    });
+export const isTransportPlugin = (plugin)=>kTransportPlugin in plugin;

package/dist/server/transport.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../lib/server/transport.ts"],"sourcesContent":["import type { Async } from '@nmtjs/common'\nimport type { BasePlugin, PluginContext } from '@nmtjs/core'\nimport type { ServerMessageType } from '../common/enums.ts'\nimport type { Connection } from './connection.ts'\nimport { kTransportPlugin } from './constants.ts'\nimport type { Protocol } from './protocol.ts'\nimport type { ProtocolRegistry } from './registry.ts'\n\nexport interface Transport<T = unknown> {\n  start: () => Promise<void>\n  stop: () => Promise<void>\n  send: (\n    connection: Connection<T>,\n    messageType: ServerMessageType,\n    buffer: ArrayBuffer,\n  ) => Async<void>\n}\n\nexport interface TransportPluginContext extends PluginContext {\n  protocol: Protocol\n  registry: ProtocolRegistry\n}\n\nexport interface TransportPlugin<Type = unknown, Options = unknown>\n  extends BasePlugin<Transport<Type>, Options, TransportPluginContext> {\n  [kTransportPlugin]: any\n}\n\nexport const createTransport = <Type = unknown, Options = unknown>(\n  name: string,\n  init: TransportPlugin<Type, Options>['init'],\n): TransportPlugin<Type, Options> => ({ name, init, [kTransportPlugin]: true })\n\nexport const isTransportPlugin = (\n  plugin: BasePlugin<any, any, any>,\n): plugin is TransportPlugin => kTransportPlugin in plugin\n"],"names":["kTransportPlugin","createTransport","name","init","isTransportPlugin","plugin"],"mappings":"AAIA,SAASA,gBAAgB,QAAQ,iBAAgB;AAwBjD,OAAO,MAAMC,kBAAkB,CAC7BC,MACAC,OACoC,CAAA;QAAED;QAAMC;QAAM,CAACH,iBAAiB,EAAE;IAAK,CAAA,EAAE;AAE/E,OAAO,MAAMI,oBAAoB,CAC/BC,SAC8BL,oBAAoBK,OAAM"}

package/dist/server/utils.js

@@ -0,0 +1,10 @@
+export const getFormat = (format, { acceptType, contentType })=>{
+    const encoder = contentType ? format.supportsEncoder(contentType) : undefined;
+    if (!encoder) throw new Error('Unsupported content-type');
+    const decoder = acceptType ? format.supportsDecoder(acceptType) : undefined;
+    if (!decoder) throw new Error('Unsupported accept');
+    return {
+        encoder,
+        decoder
+    };
+};

package/dist/server/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../lib/server/utils.ts"],"sourcesContent":["import type { Format } from './format.ts'\n\nexport type ResolveFormatParams = {\n  contentType?: string | null\n  acceptType?: string | null\n}\n\nexport const getFormat = (\n  format: Format,\n  { acceptType, contentType }: ResolveFormatParams,\n) => {\n  const encoder = contentType ? format.supportsEncoder(contentType) : undefined\n  if (!encoder) throw new Error('Unsupported content-type')\n\n  const decoder = acceptType ? format.supportsDecoder(acceptType) : undefined\n  if (!decoder) throw new Error('Unsupported accept')\n\n  return {\n    encoder,\n    decoder,\n  }\n}\n"],"names":["getFormat","format","acceptType","contentType","encoder","supportsEncoder","undefined","Error","decoder","supportsDecoder"],"mappings":"AAOA,OAAO,MAAMA,YAAY,CACvBC,QACA,EAAEC,UAAU,EAAEC,WAAW,EAAuB;IAEhD,MAAMC,UAAUD,cAAcF,OAAOI,eAAe,CAACF,eAAeG;IACpE,IAAI,CAACF,SAAS,MAAM,IAAIG,MAAM;IAE9B,MAAMC,UAAUN,aAAaD,OAAOQ,eAAe,CAACP,cAAcI;IAClE,IAAI,CAACE,SAAS,MAAM,IAAID,MAAM;IAE9B,OAAO;QACLH;QACAI;IACF;AACF,EAAC"}

package/lib/client/events.ts

@@ -0,0 +1,66 @@
+import type { Callback } from '@nmtjs/common'
+
+export type EventMap = { [K: string]: any[] }
+
+export function untilAborted(signal: AbortSignal) {
+  return new Promise((_, reject) => {
+    const handler = () => reject(new Error('aborted'))
+    const options = { once: true }
+    signal.addEventListener('abort', handler, options)
+  })
+}
+
+export function onAbort(signal: AbortSignal, listener: () => void) {
+  signal.addEventListener('abort', listener, { once: true })
+  return () => signal.removeEventListener('abort', listener)
+}
+
+/**
+ * Very simple node-like event emitter wrapper around EventTarget
+ *
+ * @todo add errors and promise rejections handling
+ */
+export class EventEmitter<
+  Events extends EventMap = EventMap,
+  EventNames extends Extract<keyof Events, string> = Extract<
+    keyof Events,
+    string
+  >,
+> {
+  #target = new EventTarget()
+  #listeners = new Map<Callback, Callback>()
+
+  on<E extends EventNames>(
+    event: E | (Object & string),
+    listener: (...args: Events[E]) => void,
+    options?: AddEventListenerOptions,
+  ) {
+    const wrapper = (event) => listener(...event.detail)
+    this.#listeners.set(listener, wrapper)
+    this.#target.addEventListener(event, wrapper, options)
+    return () => this.#target.removeEventListener(event, wrapper)
+  }
+
+  once<E extends EventNames>(
+    event: E | (Object & string),
+    listener: (...args: Events[E]) => void,
+    options?: AddEventListenerOptions,
+  ) {
+    return this.on(event, listener, { ...options, once: true })
+  }
+
+  off(event: EventNames | (Object & string), listener: Callback) {
+    const wrapper = this.#listeners.get(listener)
+    if (wrapper) this.#target.removeEventListener(event, wrapper)
+  }
+
+  emit<E extends EventNames | (Object & string)>(
+    event: E,
+    ...args: E extends EventEmitter ? Events[E] : any[]
+  ) {
+    return this.#target.dispatchEvent(new CustomEvent(event, { detail: args }))
+  }
+}
+
+export const once = (ee: EventEmitter, event: string) =>
+  new Promise((resolve) => ee.once(event, resolve))

package/lib/client/format.ts

@@ -0,0 +1,22 @@
+import type {
+  BaseClientDecoder,
+  BaseClientEncoder,
+  DecodeRPCContext,
+  EncodeRPCContext,
+  ProtocolRPC,
+  ProtocolRPCResponse,
+} from '../common/types.ts'
+
+export abstract class BaseClientFormat
+  implements BaseClientDecoder, BaseClientEncoder
+{
+  abstract contentType: string
+
+  abstract encode(data: any): ArrayBuffer
+  abstract encodeRPC(rpc: ProtocolRPC, context: EncodeRPCContext): ArrayBuffer
+  abstract decode(buffer: ArrayBuffer): any
+  abstract decodeRPC(
+    buffer: ArrayBuffer,
+    context: DecodeRPCContext,
+  ): ProtocolRPCResponse
+}

package/lib/client/index.ts

@@ -0,0 +1,4 @@
+export * from './protocol.ts'
+export * from './events.ts'
+export * from './stream.ts'
+export * from './format.ts'

package/lib/client/protocol.ts

@@ -0,0 +1,440 @@
+import { type InteractivePromise, createPromise } from '@nmtjs/common'
+import { concat, decodeNumber, encodeNumber } from '../common/binary.ts'
+import type { ProtocolBlobMetadata } from '../common/blob.ts'
+import { ClientMessageType, ServerMessageType } from '../common/enums.ts'
+import type { ProtocolRPC } from '../common/types.ts'
+import { EventEmitter } from './events.ts'
+import type { BaseClientFormat } from './format.ts'
+import {
+  ProtocolClientBlobStream,
+  ProtocolServerBlobStream,
+  ProtocolServerStream,
+} from './stream.ts'
+
+export class ProtocolError extends Error {
+  code: string
+  data?: any
+
+  constructor(code: string, message?: string, data?: any) {
+    super(message)
+    this.code = code
+    this.data = data
+  }
+
+  get message() {
+    return `${this.code} ${super.message}`
+  }
+
+  toString() {
+    return `${this.code} ${this.message}`
+  }
+
+  toJSON() {
+    return {
+      code: this.code,
+      message: this.message,
+      data: this.data,
+    }
+  }
+}
+
+export class ProtocolClientStreams {
+  readonly #collection = new Map<number, ProtocolClientBlobStream>()
+
+  get(streamId: number) {
+    const stream = this.#collection.get(streamId)
+    if (!stream) throw new Error('Stream not found')
+    return stream
+  }
+
+  add(
+    source: ReadableStream,
+    streamId: number,
+    metadata: ProtocolBlobMetadata,
+  ) {
+    const stream = new ProtocolClientBlobStream(source, streamId, metadata)
+    this.#collection.set(streamId, stream)
+    return stream
+  }
+
+  remove(streamId: number) {
+    this.#collection.delete(streamId)
+  }
+
+  abort(streamId: number) {
+    const stream = this.get(streamId)
+    stream.abort()
+    this.remove(streamId)
+  }
+
+  pull(streamId: number, size: number) {
+    const stream = this.get(streamId)
+    return stream.read(size)
+  }
+
+  end(streamId: number) {
+    const stream = this.get(streamId)
+    stream.end()
+    this.remove(streamId)
+  }
+}
+
+export class ProtocolServerStreams {
+  readonly #collection = new Map<number, ProtocolServerStream>()
+
+  has(streamId: number) {
+    return this.#collection.has(streamId)
+  }
+
+  get(streamId: number) {
+    const stream = this.#collection.get(streamId)
+    if (!stream) throw new Error('Stream not found')
+    return stream
+  }
+
+  add(streamId: number, stream: ProtocolServerStream) {
+    this.#collection.set(streamId, stream)
+    return stream
+  }
+
+  remove(streamId: number) {
+    this.#collection.delete(streamId)
+  }
+
+  abort(streamId: number) {
+    if (this.has(streamId)) {
+      const stream = this.get(streamId)
+      stream.abort()
+      this.remove(streamId)
+    }
+  }
+
+  async push(streamId: number, chunk: ArrayBuffer) {
+    const stream = this.get(streamId)
+    return await stream.push(chunk)
+  }
+
+  end(streamId: number) {
+    const stream = this.get(streamId)
+    stream.end()
+    this.remove(streamId)
+  }
+}
+
+export interface ProtocolTransport
+  extends EventEmitter<{
+    [K in `${ServerMessageType}`]: [ArrayBuffer]
+  }> {
+  connect(
+    auth: any,
+    contentType: BaseClientFormat['contentType'],
+  ): Promise<void>
+  disconnect(): Promise<void>
+  send(messageType: ClientMessageType, buffer: ArrayBuffer): Promise<void>
+}
+
+export class ProtocolBaseTransformer {
+  encodeRPC(namespace: string, procedure: string, payload: any) {
+    return payload
+  }
+  decodeRPC(namespace: string, procedure: string, payload: any) {
+    return payload
+  }
+  decodeRPCChunk(namespace: string, procedure: string, payload: any) {
+    return payload
+  }
+  decodeEvent(namespace: string, event: string, payload: any) {
+    return payload
+  }
+}
+
+export type ProtocolClientCall = InteractivePromise<any> &
+  Pick<ProtocolRPC, 'namespace' | 'procedure'>
+
+export abstract class ProtocolBaseClient<
+  T extends Record<string, Record<string, any>>,
+> extends EventEmitter<
+  {
+    [N in keyof T]: {
+      [E in keyof T[N] as `${Extract<N, string>}/${Extract<E, string>}`]: [
+        payload: T[N][E],
+      ]
+    }
+  }[keyof T]
+> {
+  readonly #clientStreams: ProtocolClientStreams
+  readonly #serverStreams: ProtocolServerStreams
+  readonly #serverRPCStreams: ProtocolServerStreams
+  readonly #serverRPCStreamCalls = new Map<
+    number,
+    Pick<ProtocolRPC, 'namespace' | 'procedure'>
+  >()
+  readonly #calls = new Map<number, ProtocolClientCall>()
+
+  #callId = 0
+  #streamId = 0
+
+  constructor(
+    protected readonly transport: ProtocolTransport,
+    protected readonly format: BaseClientFormat,
+    protected readonly transformer: ProtocolBaseTransformer = new ProtocolBaseTransformer(),
+  ) {
+    super()
+
+    this.#clientStreams = new ProtocolClientStreams()
+    this.#serverStreams = new ProtocolServerStreams()
+    this.#serverRPCStreams = new ProtocolServerStreams()
+
+    this.transport.on(`${ServerMessageType.Event}`, (buffer) => {
+      const [namespace, event, payload] = this.format.decode(buffer)
+      const name = `${namespace}/${event}`
+      const transformed = this.transformer.decodeEvent(
+        namespace,
+        event,
+        payload,
+      )
+      this.emit(name, transformed)
+    })
+
+    this.transport.on(`${ServerMessageType.RpcResponse}`, (buffer) => {
+      const { call, error, payload } = this.#handleResponse(buffer)
+      if (error) call.reject(error)
+      else call.resolve(payload)
+    })
+
+    this.transport.on(`${ServerMessageType.RpcStreamResponse}`, (buffer) => {
+      const { call, response, payload, error } = this.#handleResponse(buffer)
+      if (error) return call.reject(error)
+      console.log('Creating RPC stream', response)
+      const stream = new ProtocolServerStream()
+      this.#serverRPCStreams.add(response.callId, stream)
+      this.#serverRPCStreamCalls.set(response.callId, {
+        namespace: call.namespace,
+        procedure: call.procedure,
+      })
+      call.resolve([payload, stream])
+    })
+
+    this.transport.on(`${ServerMessageType.RpcStreamChunk}`, async (buffer) => {
+      const callId = decodeNumber(buffer, 'Uint32')
+      console.log('RPC stream chunk', callId)
+
+      const chunk = buffer.slice(Uint32Array.BYTES_PER_ELEMENT)
+      if (chunk.byteLength === 0) {
+        this.#serverRPCStreams.end(callId)
+        this.#serverRPCStreamCalls.delete(callId)
+      } else {
+        const call = this.#serverRPCStreamCalls.get(callId)
+        console.log('RPC stream call', call)
+        if (call) {
+          const payload = this.format.decode(chunk)
+          console.log('RPC stream payload', payload)
+          try {
+            const transformed = this.transformer.decodeRPCChunk(
+              call.namespace,
+              call.procedure,
+              payload,
+            )
+            await this.#serverRPCStreams.push(callId, transformed)
+          } catch (error) {
+            this._send(
+              ClientMessageType.RpcStreamAbort,
+              encodeNumber(callId, 'Uint32'),
+            )
+            this.#serverRPCStreams.remove(callId)
+            this.#serverRPCStreamCalls.delete(callId)
+          }
+        }
+      }
+    })
+
+    this.transport.on(`${ServerMessageType.RpcStreamAbort}`, (buffer) => {
+      const callId = decodeNumber(buffer, 'Uint32')
+      console.log('RPC stream abort', callId)
+      const call = this.#calls.get(callId)
+      if (call) {
+        this.#serverStreams.end(callId)
+        this.#serverRPCStreams.abort(callId)
+      }
+    })
+
+    this.transport.on(
+      `${ServerMessageType.ServerStreamPush}`,
+      async (buffer) => {
+        const streamId = decodeNumber(buffer, 'Uint32')
+        const chunk = buffer.slice(Uint32Array.BYTES_PER_ELEMENT)
+        console.log('Server stream push', streamId, chunk.byteLength)
+        try {
+          await this.#serverStreams.push(streamId, chunk)
+          this._send(
+            ClientMessageType.ServerStreamPull,
+            encodeNumber(streamId, 'Uint32'),
+          )
+        } catch (error) {
+          this._send(
+            ClientMessageType.ServerStreamAbort,
+            encodeNumber(streamId, 'Uint32'),
+          )
+          this.#serverStreams.remove(streamId)
+        }
+      },
+    )
+
+    this.transport.on(`${ServerMessageType.ServerStreamEnd}`, (buffer) => {
+      const streamId = decodeNumber(buffer, 'Uint32')
+      console.log('Server stream end', streamId)
+      this.#serverStreams.end(streamId)
+    })
+
+    this.transport.on(`${ServerMessageType.ServerStreamAbort}`, (buffer) => {
+      const streamId = decodeNumber(buffer, 'Uint32')
+      console.log('Server stream abort', streamId)
+      this.#serverStreams.abort(streamId)
+    })
+
+    this.transport.on(`${ServerMessageType.ClientStreamAbort}`, (buffer) => {
+      const streamId = decodeNumber(buffer, 'Uint32')
+      console.log('Client stream abort', streamId)
+      this.#clientStreams.abort(streamId)
+    })
+
+    this.transport.on(
+      `${ServerMessageType.ClientStreamPull}`,
+      async (buffer) => {
+        const streamId = decodeNumber(buffer, 'Uint32')
+        console.log('Client stream pull', streamId)
+        const size = decodeNumber(
+          buffer,
+          'Uint32',
+          Uint32Array.BYTES_PER_ELEMENT,
+        )
+        const streamIdEncoded = encodeNumber(streamId, 'Uint32')
+        try {
+          const chunk = await this.#clientStreams.pull(streamId, size)
+          if (chunk) {
+            this._send(
+              ClientMessageType.ClientStreamPush,
+              concat(streamIdEncoded, chunk),
+            )
+          } else {
+            this._send(ClientMessageType.ClientStreamEnd, streamIdEncoded)
+            this.#clientStreams.end(streamId)
+          }
+        } catch (error) {
+          console.error(error)
+          this._send(ClientMessageType.ClientStreamAbort, streamIdEncoded)
+        }
+      },
+    )
+  }
+
+  async connect(auth: any) {
+    return await this.transport.connect(auth, this.format.contentType)
+  }
+
+  async disconnect() {
+    return await this.transport.disconnect()
+  }
+
+  protected async _send(messageType: ClientMessageType, buffer: ArrayBuffer) {
+    console.log(
+      'Client transport send',
+      ClientMessageType[messageType],
+      buffer.byteLength,
+    )
+    return await this.transport.send(messageType, buffer)
+  }
+
+  protected async _call(
+    namespace: string,
+    procedure: string,
+    payload: any,
+    options = {},
+  ) {
+    const callId = ++this.#callId
+    const call = Object.assign(createPromise(), {
+      namespace,
+      procedure,
+    })
+    const buffer = this.format.encodeRPC(
+      {
+        callId,
+        namespace,
+        procedure,
+        payload: this.transformer.encodeRPC(namespace, procedure, payload),
+      },
+      {
+        addStream: (blob) => {
+          const streamId = ++this.#streamId
+          const stream = this.#clientStreams.add(
+            blob.source,
+            streamId,
+            blob.metadata,
+          )
+          return stream
+        },
+        getStream: (id) => {
+          const stream = this.#clientStreams.get(id)
+          return stream
+        },
+      },
+    )
+
+    this.transport.send(ClientMessageType.Rpc, buffer).catch(console.error)
+
+    this.#calls.set(callId, call)
+
+    return call.promise
+  }
+
+  #handleResponse(buffer: ArrayBuffer) {
+    const callStreams: ProtocolServerBlobStream[] = []
+    const response = this.format.decodeRPC(buffer, {
+      addStream: (id, metadata) => {
+        console.log('Client transport blob stream', id, metadata)
+        const stream = new ProtocolServerBlobStream(id, metadata, () => {
+          this._send(
+            ClientMessageType.ServerStreamPull,
+            encodeNumber(id, 'Uint32'),
+          )
+        })
+        callStreams.push(stream)
+        this.#serverStreams.add(id, stream)
+        return stream
+      },
+      getStream: (id) => {
+        return this.#serverStreams.get(id)
+      },
+    })
+
+    console.log('Client transport response', response)
+
+    const call = this.#calls.get(response.callId)
+
+    if (call) {
+      this.#calls.delete(response.callId)
+
+      if (response.error) {
+        const error = new ProtocolError(
+          response.error.code,
+          response.error.message,
+          response.error.data,
+        )
+        return { call, response, error }
+      } else {
+        const payload = this.transformer.decodeRPC(
+          call.namespace,
+          call.procedure,
+          response.payload,
+        )
+        return { call, response, payload }
+      }
+    }
+
+    for (const stream of callStreams) {
+      this.#serverStreams.abort(stream.id)
+    }
+
+    throw new Error('Call not found')
+  }
+}

package/lib/client/stream.ts

@@ -0,0 +1,116 @@
+import { type Callback, defer } from '@nmtjs/common'
+import { encodeText } from '../common/binary.ts'
+import type { ProtocolBlobMetadata } from '../common/blob.ts'
+
+export class ProtocolClientBlobStream extends TransformStream<
+  any,
+  ArrayBuffer
+> {
+  #queue: ArrayBuffer
+  #reader: ReadableStreamDefaultReader
+
+  constructor(
+    readonly source: ReadableStream,
+    readonly id: number,
+    readonly metadata: ProtocolBlobMetadata,
+  ) {
+    super({
+      start: () => {
+        defer(() => source.pipeThrough(this))
+      },
+      transform: (chunk, controller) => {
+        if (chunk instanceof ArrayBuffer) {
+          controller.enqueue(chunk)
+        } else if (chunk instanceof Uint8Array) {
+          controller.enqueue(chunk.buffer)
+        } else if (typeof chunk === 'string') {
+          controller.enqueue(encodeText(chunk))
+        } else {
+          throw new Error(
+            'Invalid chunk data type. Expected ArrayBuffer, Uint8Array, or string.',
+          )
+        }
+      },
+    })
+
+    this.#queue = new ArrayBuffer(0)
+    this.#reader = this.readable.getReader()
+  }
+
+  async read(size: number) {
+    if (this.#queue.byteLength >= size) {
+      const chunk = this.#queue.slice(0, size)
+      const remaining = this.#queue.slice(size)
+      this.#queue = remaining
+      return chunk
+    } else {
+      const { done, value } = await this.#reader.read()
+      if (!done) {
+        const buffer = value as ArrayBuffer
+        const chunk = buffer.slice(0, size)
+        const remaining = buffer.slice(size)
+        this.#queue = remaining
+        return chunk
+      }
+      return null
+    }
+  }
+
+  abort(error = new Error('Stream aborted')) {
+    this.#reader.cancel(error)
+  }
+
+  async end() {
+    if (!this.writable.locked && (await this.writable.getWriter().closed)) {
+      await this.writable.close()
+    }
+  }
+}
+
+export class ProtocolServerStream<T = any> extends TransformStream<any, T> {
+  #writer: WritableStreamDefaultWriter
+
+  constructor(start?: Callback) {
+    super({ start })
+    this.#writer = this.writable.getWriter()
+  }
+
+  async *[Symbol.asyncIterator]() {
+    const reader = this.readable.getReader()
+    while (true) {
+      const { done, value } = await reader.read()
+      if (!done) yield value
+      else return void 0
+    }
+  }
+
+  async push(chunk: any) {
+    await this.#writer.write(chunk)
+  }
+
+  async end() {
+    await this.#writer.close()
+  }
+
+  abort(error = new Error('Stream aborted')) {
+    this.#writer.abort(error)
+  }
+}
+
+export class ProtocolServerBlobStream extends ProtocolServerStream<ArrayBuffer> {
+  constructor(
+    readonly id: number,
+    readonly metadata: ProtocolBlobMetadata,
+    start: Callback,
+  ) {
+    super(start)
+  }
+
+  push(chunk: ArrayBuffer) {
+    return super.push(chunk)
+  }
+
+  end() {
+    return super.end()
+  }
+}