@nmtjs/protocol 0.15.0-beta.3 → 0.15.0-beta.4

package/package.json

@@ -8,25 +8,26 @@
   },
   "dependencies": {
     "hookable": "6.0.0-rc.1",
-    "@nmtjs/common": "0.15.0-beta.3",
-    "@nmtjs/type": "0.15.0-beta.3",
-    "@nmtjs/contract": "0.15.0-beta.3"
+    "@nmtjs/common": "0.15.0-beta.4",
+    "@nmtjs/type": "0.15.0-beta.4",
+    "@nmtjs/contract": "0.15.0-beta.4"
   },
   "devDependencies": {
-    "@nmtjs/core": "0.15.0-beta.3"
+    "@nmtjs/core": "0.15.0-beta.4"
   },
   "peerDependencies": {
-    "@nmtjs/common": "0.15.0-beta.3",
-    "@nmtjs/core": "0.15.0-beta.3",
-    "@nmtjs/contract": "0.15.0-beta.3",
-    "@nmtjs/type": "0.15.0-beta.3"
+    "@nmtjs/common": "0.15.0-beta.4",
+    "@nmtjs/contract": "0.15.0-beta.4",
+    "@nmtjs/type": "0.15.0-beta.4",
+    "@nmtjs/core": "0.15.0-beta.4"
   },
   "files": [
     "dist",
+    "src",
     "LICENSE.md",
     "README.md"
   ],
-  "version": "0.15.0-beta.3",
+  "version": "0.15.0-beta.4",
   "scripts": {
     "clean-build": "rm -rf ./dist",
     "build": "tsc --declaration --sourcemap",

package/src/client/format.ts

@@ -0,0 +1,49 @@
+import type { ProtocolBlob } from '../common/blob.ts'
+import type { DecodeRPCContext } from '../common/types.ts'
+import type {
+  ProtocolClientBlobStream,
+  ProtocolServerBlobStream,
+} from './stream.ts'
+
+export interface EncodeRPCContext<T = any> {
+  addStream: (blob: ProtocolBlob) => T
+}
+
+export type ProtocolRPCEncode = ArrayBufferView
+
+export interface BaseClientDecoder {
+  decode(buffer: ArrayBufferView): unknown
+  decodeRPC(
+    buffer: ArrayBufferView,
+    context: DecodeRPCContext<
+      (options?: { signal?: AbortSignal }) => ProtocolServerBlobStream
+    >,
+  ): unknown
+}
+
+export interface BaseClientEncoder {
+  encode(data: unknown): ArrayBufferView
+  encodeRPC(
+    data: unknown,
+    context: EncodeRPCContext<ProtocolClientBlobStream>,
+  ): ProtocolRPCEncode
+}
+
+export abstract class BaseClientFormat
+  implements BaseClientDecoder, BaseClientEncoder
+{
+  abstract contentType: string
+
+  abstract encode(data: unknown): ArrayBufferView
+  abstract encodeRPC(
+    data: unknown,
+    context: EncodeRPCContext,
+  ): ProtocolRPCEncode
+  abstract decode(buffer: ArrayBufferView): unknown
+  abstract decodeRPC(
+    buffer: ArrayBufferView,
+    context: DecodeRPCContext<
+      (options?: { signal?: AbortSignal }) => ProtocolServerBlobStream
+    >,
+  ): unknown
+}

package/src/client/index.ts

@@ -0,0 +1,8 @@
+export * from './format.ts'
+export * from './protocol.ts'
+export * from './stream.ts'
+
+import { ProtocolVersion } from '../common/enums.ts'
+import { ProtocolVersion1 } from './versions/v1.ts'
+
+export const versions = { [ProtocolVersion.v1]: new ProtocolVersion1() }

package/src/client/protocol.ts

@@ -0,0 +1,107 @@
+import type { ProtocolBlob, ProtocolBlobMetadata } from '../common/blob.ts'
+import type {
+  ClientMessageType,
+  ProtocolVersion,
+  ServerMessageType,
+} from '../common/enums.ts'
+import type { BaseProtocolError, EncodeRPCStreams } from '../common/types.ts'
+import type { BaseClientDecoder, BaseClientEncoder } from './format.ts'
+import type {
+  ProtocolClientBlobStream,
+  ProtocolServerBlobStream,
+} from './stream.ts'
+import { concat } from '../common/binary.ts'
+
+export type MessageContext = {
+  decoder: BaseClientDecoder
+  encoder: BaseClientEncoder
+  addClientStream: (blob: ProtocolBlob) => ProtocolClientBlobStream
+  addServerStream: (
+    streamId: number,
+    metadata: ProtocolBlobMetadata,
+  ) => (options?: { signal?: AbortSignal }) => ProtocolServerBlobStream
+  transport: { send: (buffer: ArrayBufferView) => void }
+  streamId: () => number
+}
+
+export type ClientMessageTypePayload = {
+  [ClientMessageType.Rpc]: { callId: number; procedure: string; payload: any }
+  [ClientMessageType.RpcAbort]: { callId: number; reason?: string }
+  [ClientMessageType.RpcPull]: { callId: number }
+  [ClientMessageType.ClientStreamPush]: {
+    streamId: number
+    chunk: ArrayBufferView
+  }
+  [ClientMessageType.ClientStreamEnd]: { streamId: number }
+  [ClientMessageType.ClientStreamAbort]: { streamId: number; reason?: string }
+  [ClientMessageType.ServerStreamPull]: { streamId: number; size: number }
+  [ClientMessageType.ServerStreamAbort]: { streamId: number; reason?: string }
+}
+
+export type ServerMessageTypePayload = {
+  [ServerMessageType.RpcResponse]: {
+    callId: number
+    result?: any
+    error?: BaseProtocolError
+    streams?: EncodeRPCStreams
+  }
+  [ServerMessageType.RpcStreamResponse]: {
+    callId: number
+    error?: BaseProtocolError
+  }
+  [ServerMessageType.RpcStreamChunk]: { callId: number; chunk: ArrayBufferView }
+  [ServerMessageType.RpcStreamEnd]: { callId: number }
+  [ServerMessageType.RpcStreamAbort]: { callId: number; reason?: string }
+  [ServerMessageType.ServerStreamAbort]: { streamId: number; reason?: string }
+  [ServerMessageType.ServerStreamEnd]: { streamId: number }
+  [ServerMessageType.ServerStreamPush]: {
+    streamId: number
+    chunk: ArrayBufferView
+  }
+  [ServerMessageType.ClientStreamAbort]: { streamId: number; reason?: string }
+  [ServerMessageType.ClientStreamPull]: { streamId: number; size: number }
+}
+
+export abstract class ProtocolVersionInterface {
+  abstract version: ProtocolVersion
+  abstract decodeMessage(
+    context: MessageContext,
+    buffer: ArrayBufferView,
+  ): {
+    [K in keyof ServerMessageTypePayload]: {
+      type: K
+    } & ServerMessageTypePayload[K]
+  }[keyof ServerMessageTypePayload]
+  abstract encodeMessage<T extends ClientMessageType>(
+    context: MessageContext,
+    messageType: T,
+    payload: ClientMessageTypePayload[T],
+  ): any
+
+  protected encode(...chunks: (ArrayBuffer | ArrayBufferView)[]) {
+    return concat(...chunks)
+  }
+}
+
+export class ProtocolError extends Error implements BaseProtocolError {
+  code: string
+  data?: any
+
+  constructor(code: string, message?: string, data?: any) {
+    super(message)
+    this.code = code
+    this.data = data
+  }
+
+  get message() {
+    return `${this.code} ${super.message}`
+  }
+
+  toString() {
+    return `${this.code} ${this.message}`
+  }
+
+  toJSON() {
+    return { code: this.code, message: this.message, data: this.data }
+  }
+}

package/src/client/stream.ts

@@ -0,0 +1,222 @@
+import type { Callback, DuplexStreamOptions } from '@nmtjs/common'
+import { DuplexStream } from '@nmtjs/common'
+
+import type {
+  ProtocolBlobInterface,
+  ProtocolBlobMetadata,
+} from '../common/blob.ts'
+import { concat, decodeText, encodeText } from '../common/binary.ts'
+import { kBlobKey } from '../common/constants.ts'
+
+export class ProtocolClientBlobStream
+  extends DuplexStream<any, ArrayBufferView>
+  implements ProtocolBlobInterface
+{
+  readonly [kBlobKey] = true
+
+  #queue: Uint8Array
+  #reader: ReadableStreamDefaultReader
+  #sourceReader: ReadableStreamDefaultReader | null = null
+
+  constructor(
+    readonly source: ReadableStream,
+    readonly id: number,
+    readonly metadata: ProtocolBlobMetadata,
+  ) {
+    let sourceReader: ReadableStreamDefaultReader | null = null
+    super({
+      start: () => {
+        sourceReader = source.getReader()
+      },
+      pull: async (controller) => {
+        const { done, value } = await sourceReader!.read()
+        if (done) {
+          controller.close()
+          return
+        }
+        const chunk = value
+        controller.enqueue(
+          chunk instanceof Uint8Array
+            ? chunk
+            : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength),
+        )
+      },
+      transform: (chunk) => {
+        if (chunk instanceof ArrayBuffer) {
+          return new Uint8Array(chunk)
+        } else if (chunk instanceof Uint8Array) {
+          return chunk
+        } else if (typeof chunk === 'string') {
+          return encodeText(chunk)
+        } else {
+          throw new Error(
+            'Invalid chunk data type. Expected ArrayBuffer, Uint8Array, or string.',
+          )
+        }
+      },
+      cancel: (reason) => {
+        // Use reader.cancel() if reader exists (stream is locked), otherwise source.cancel()
+        if (sourceReader) {
+          sourceReader.cancel(reason)
+        } else {
+          source.cancel(reason)
+        }
+      },
+    })
+
+    this.#queue = new Uint8Array(0)
+    this.#reader = this.readable.getReader()
+    this.#sourceReader = sourceReader
+  }
+
+  async abort(reason = 'Stream aborted') {
+    await this.#reader.cancel(reason)
+    this.#reader.releaseLock()
+    this.#sourceReader?.releaseLock()
+  }
+
+  async end() {
+    // Release the reader lock when the stream is finished
+    this.#reader.releaseLock()
+    this.#sourceReader?.releaseLock()
+  }
+
+  async read(size: number) {
+    while (this.#queue.byteLength < size) {
+      const { done, value } = await this.#reader.read()
+      if (done) {
+        if (this.#queue.byteLength > 0) {
+          const chunk = this.#queue
+          this.#queue = new Uint8Array(0)
+          return chunk
+        }
+        return null
+      }
+      const buffer = value
+      this.#queue = concat(this.#queue, buffer)
+    }
+    const chunk = this.#queue.subarray(0, size)
+    this.#queue = this.#queue.subarray(size)
+    return chunk
+  }
+}
+
+export abstract class ProtocolServerStreamInterface<
+  O = unknown,
+> extends DuplexStream<O, ArrayBufferView> {
+  async *[Symbol.asyncIterator]() {
+    const reader = this.readable.getReader()
+    try {
+      while (true) {
+        const { done, value } = await reader.read()
+        if (!done) yield value
+        else break
+      }
+    } finally {
+      reader.releaseLock()
+    }
+  }
+}
+
+export class ProtocolServerStream<T = unknown>
+  extends ProtocolServerStreamInterface<T>
+  implements ProtocolServerStreamInterface<T> {}
+
+export class ProtocolServerRPCStream<
+  T = unknown,
+> extends ProtocolServerStream<T> {
+  createAsyncIterable(onDone: Callback) {
+    return {
+      [Symbol.asyncIterator]: () => {
+        const iterator = this[Symbol.asyncIterator]()
+        return {
+          async next() {
+            const result = await iterator.next()
+            if (result.done) onDone()
+            return result
+          },
+          async return(value) {
+            onDone()
+            return iterator.return?.(value) ?? { done: true, value }
+          },
+          async throw(error) {
+            onDone()
+            return iterator.throw?.(error) ?? Promise.reject(error)
+          },
+        }
+      },
+    }
+  }
+}
+
+export class ProtocolServerBlobStream
+  extends ProtocolServerStreamInterface<ArrayBufferView>
+  implements ProtocolBlobInterface, Blob
+{
+  readonly [kBlobKey] = true
+
+  constructor(
+    readonly metadata: ProtocolBlobMetadata,
+    options?: DuplexStreamOptions<ArrayBufferView, ArrayBufferView>,
+  ) {
+    super(options)
+  }
+
+  get size() {
+    return this.metadata.size || Number.NaN
+  }
+
+  get type() {
+    return this.metadata.type || 'application/octet-stream'
+  }
+
+  async text() {
+    const chunks: ArrayBufferView[] = []
+    for await (const chunk of this) chunks.push(chunk)
+    return decodeText(concat(...chunks))
+  }
+
+  async bytes() {
+    const chunks: ArrayBufferView[] = []
+    for await (const chunk of this) chunks.push(chunk)
+    return concat(...chunks)
+  }
+
+  async arrayBuffer() {
+    const bytes = await this.bytes()
+    return bytes.buffer
+  }
+
+  async json<T = unknown>() {
+    const text = await this.text()
+    return JSON.parse(text) as T
+  }
+
+  stream() {
+    const transform = new TransformStream<ArrayBufferView, Uint8Array>({
+      transform: (chunk, controller) => {
+        controller.enqueue(
+          chunk instanceof Uint8Array
+            ? chunk
+            : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength),
+        )
+      },
+    })
+    this.readable.pipeThrough(transform)
+    return transform.readable as ReadableStream<Uint8Array<ArrayBuffer>>
+  }
+
+  /**
+   * Throws an error
+   */
+  async formData(): Promise<FormData> {
+    throw new Error('Method not implemented.')
+  }
+
+  /**
+   * Throws an error
+   */
+  slice(): Blob {
+    throw new Error('Unable to slice')
+  }
+}

package/src/client/versions/v1.ts

@@ -0,0 +1,205 @@
+import type { BaseProtocolError } from '../../common/types.ts'
+import type { ClientMessageTypePayload, MessageContext } from '../protocol.ts'
+import {
+  decodeNumber,
+  decodeText,
+  encodeNumber,
+  encodeText,
+} from '../../common/binary.ts'
+import {
+  ClientMessageType,
+  MessageByteLength,
+  ProtocolVersion,
+  ServerMessageType,
+} from '../../common/enums.ts'
+import { ProtocolVersionInterface } from '../protocol.ts'
+
+export class ProtocolVersion1 extends ProtocolVersionInterface {
+  version = ProtocolVersion.v1
+
+  decodeMessage(context: MessageContext, buffer: Uint8Array) {
+    const messageType = decodeNumber(buffer, 'Uint8')
+    const payload = buffer.subarray(MessageByteLength.MessageType)
+    switch (messageType) {
+      // case ServerMessageType.Event: {
+      //   const { event, data } = context.decoder.decode(payload)
+      //   return { type: messageType, event, data }
+      // }
+      case ServerMessageType.RpcResponse: {
+        const callId = decodeNumber(payload, 'Uint32')
+        const isError = decodeNumber(payload, 'Uint8', MessageByteLength.CallId)
+        const dataPayload = payload.subarray(
+          MessageByteLength.CallId + MessageByteLength.MessageError,
+        )
+
+        if (isError) {
+          const error = context.decoder.decode(dataPayload) as BaseProtocolError
+          return { type: messageType, callId, error }
+        } else {
+          const result = context.decoder.decodeRPC(dataPayload, {
+            addStream: (streamId, metadata) => {
+              return context.addServerStream(streamId, metadata)
+            },
+          })
+          return { type: messageType, callId, result }
+        }
+      }
+      case ServerMessageType.RpcStreamResponse: {
+        const callId = decodeNumber(payload, 'Uint32')
+        const errorPayload = payload.subarray(MessageByteLength.CallId)
+        const error =
+          errorPayload.byteLength > 0
+            ? (context.decoder.decode(errorPayload) as
+                | BaseProtocolError
+                | undefined)
+            : undefined
+        return { type: messageType, callId, error }
+      }
+      case ServerMessageType.RpcStreamChunk: {
+        const callId = decodeNumber(payload, 'Uint32')
+        const chunk = payload.subarray(MessageByteLength.CallId)
+        return { type: messageType, callId, chunk }
+      }
+      case ServerMessageType.RpcStreamEnd: {
+        const callId = decodeNumber(payload, 'Uint32')
+        return { type: messageType, callId }
+      }
+      case ServerMessageType.RpcStreamAbort: {
+        const callId = decodeNumber(payload, 'Uint32')
+        const reasonPayload = payload.subarray(MessageByteLength.CallId)
+        const reason =
+          reasonPayload.byteLength > 0 ? decodeText(reasonPayload) : undefined
+        return { type: messageType, callId, reason }
+      }
+      case ServerMessageType.ClientStreamPull: {
+        const streamId = decodeNumber(payload, 'Uint32')
+        const size = decodeNumber(payload, 'Uint32', MessageByteLength.StreamId)
+        return { type: messageType, streamId, size }
+      }
+      case ServerMessageType.ClientStreamAbort: {
+        const streamId = decodeNumber(payload, 'Uint32')
+        const reasonPayload = payload.subarray(MessageByteLength.StreamId)
+        const reason =
+          reasonPayload.byteLength > 0 ? decodeText(reasonPayload) : undefined
+
+        return { type: messageType, streamId, reason }
+      }
+      case ServerMessageType.ServerStreamPush: {
+        const streamId = decodeNumber(payload, 'Uint32')
+        const chunk = payload.subarray(MessageByteLength.StreamId)
+        return { type: messageType, streamId, chunk }
+      }
+      case ServerMessageType.ServerStreamEnd: {
+        const streamId = decodeNumber(payload, 'Uint32')
+        const reasonPayload = payload.subarray(MessageByteLength.StreamId)
+        const reason =
+          reasonPayload.byteLength > 0 ? decodeText(reasonPayload) : undefined
+
+        return { type: messageType, streamId, reason }
+      }
+      case ServerMessageType.ServerStreamAbort: {
+        const streamId = decodeNumber(payload, 'Uint32')
+        const reasonPayload = payload.subarray(MessageByteLength.StreamId)
+        const reason =
+          reasonPayload.byteLength > 0 ? decodeText(reasonPayload) : undefined
+
+        return { type: messageType, streamId, reason }
+      }
+
+      default:
+        throw new Error(`Unsupported message type: ${messageType}`)
+    }
+  }
+
+  encodeMessage<T extends ClientMessageType>(
+    context: MessageContext,
+    messageType: T,
+    payload: ClientMessageTypePayload[T],
+  ) {
+    switch (messageType) {
+      case ClientMessageType.Rpc: {
+        const {
+          callId,
+          procedure,
+          payload: rpcPayload,
+        } = payload as ClientMessageTypePayload[ClientMessageType.Rpc]
+        const procedureBuffer = encodeText(procedure)
+        return this.encode(
+          encodeNumber(messageType, 'Uint8'),
+          encodeNumber(callId, 'Uint32'),
+          encodeNumber(procedureBuffer.byteLength, 'Uint16'),
+          procedureBuffer,
+          context.encoder.encodeRPC(rpcPayload, {
+            addStream: (blob) => context.addClientStream(blob),
+          }),
+        )
+      }
+      case ClientMessageType.RpcAbort: {
+        const { callId, reason } =
+          payload as ClientMessageTypePayload[ClientMessageType.RpcAbort]
+
+        return this.encode(
+          encodeNumber(messageType, 'Uint8'),
+          encodeNumber(callId, 'Uint32'),
+          reason ? encodeText(reason) : new Uint8Array(0),
+        )
+      }
+      case ClientMessageType.RpcPull: {
+        const { callId } =
+          payload as ClientMessageTypePayload[ClientMessageType.RpcPull]
+
+        return this.encode(
+          encodeNumber(messageType, 'Uint8'),
+          encodeNumber(callId, 'Uint32'),
+        )
+      }
+      case ClientMessageType.ClientStreamPush: {
+        const { streamId, chunk } =
+          payload as ClientMessageTypePayload[ClientMessageType.ClientStreamPush]
+        return this.encode(
+          encodeNumber(messageType, 'Uint8'),
+          encodeNumber(streamId, 'Uint32'),
+          chunk,
+        )
+      }
+      case ClientMessageType.ClientStreamEnd: {
+        const { streamId } =
+          payload as ClientMessageTypePayload[ClientMessageType.ClientStreamEnd]
+        return this.encode(
+          encodeNumber(messageType, 'Uint8'),
+          encodeNumber(streamId, 'Uint32'),
+        )
+      }
+      case ClientMessageType.ClientStreamAbort: {
+        const { streamId, reason } =
+          payload as ClientMessageTypePayload[ClientMessageType.ClientStreamAbort]
+        return this.encode(
+          encodeNumber(messageType, 'Uint8'),
+          encodeNumber(streamId, 'Uint32'),
+          reason ? encodeText(reason) : new Uint8Array(0),
+        )
+      }
+      case ClientMessageType.ServerStreamPull: {
+        const { streamId, size } =
+          payload as ClientMessageTypePayload[ClientMessageType.ServerStreamPull]
+        return this.encode(
+          encodeNumber(messageType, 'Uint8'),
+          encodeNumber(streamId, 'Uint32'),
+          encodeNumber(size, 'Uint32'),
+        )
+      }
+      case ClientMessageType.ServerStreamAbort: {
+        const { streamId, reason } =
+          payload as ClientMessageTypePayload[ClientMessageType.ServerStreamAbort]
+        return this.encode(
+          encodeNumber(messageType, 'Uint8'),
+          encodeNumber(streamId, 'Uint32'),
+          reason ? encodeText(reason) : new Uint8Array(0),
+        )
+      }
+
+      default:
+        throw new Error(`Unsupported message type: ${messageType}`)
+    }
+  }
+}

package/src/common/binary.ts

@@ -0,0 +1,70 @@
+// TODO: get rid of lib DOM somehow...
+/// <reference lib="dom" />
+
+const utf8decoder = new TextDecoder()
+const utf8encoder = new TextEncoder()
+
+export type BinaryTypes = {
+  Int8: number
+  Int16: number
+  Int32: number
+  Uint8: number
+  Uint16: number
+  Uint32: number
+  Float32: number
+  Float64: number
+  BigInt64: bigint
+  BigUint64: bigint
+}
+
+export const encodeNumber = <T extends keyof BinaryTypes>(
+  value: BinaryTypes[T],
+  type: T,
+  littleEndian = true,
+) => {
+  const bytesNeeded = globalThis[`${type}Array`].BYTES_PER_ELEMENT
+  const ab = new ArrayBuffer(bytesNeeded)
+  const dv = new DataView(ab)
+  dv[`set${type}`](0, value as never, littleEndian)
+  return ab
+}
+
+export const decodeNumber = <T extends keyof BinaryTypes>(
+  buffer: ArrayBuffer | ArrayBufferView,
+  type: T,
+  offset = 0,
+  littleEndian = true,
+): BinaryTypes[T] => {
+  const ab = buffer instanceof ArrayBuffer ? buffer : buffer.buffer
+  const baseOffset = buffer instanceof ArrayBuffer ? 0 : buffer.byteOffset
+  const view = new DataView(ab)
+  return view[`get${type}`](baseOffset + offset, littleEndian) as BinaryTypes[T]
+}
+
+export const encodeText = (text: string) => utf8encoder.encode(text)
+
+export const decodeText = (buffer: Parameters<typeof utf8decoder.decode>[0]) =>
+  utf8decoder.decode(buffer)
+
+export const concat = (...buffers: (ArrayBuffer | ArrayBufferView)[]) => {
+  let totalLength = 0
+  for (const buffer of buffers) totalLength += buffer.byteLength
+  const view = new Uint8Array(totalLength)
+  let offset = 0
+  for (const buffer of buffers) {
+    const chunk =
+      buffer instanceof ArrayBuffer
+        ? new Uint8Array(buffer)
+        : new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
+    view.set(chunk, offset)
+    offset += chunk.byteLength
+  }
+  return view
+}
+
+export const UTF8Transform = () =>
+  new TransformStream<string, Uint8Array>({
+    transform(chunk, controller) {
+      controller.enqueue(encodeText(chunk))
+    },
+  })