@libp2p/interface-compliance-tests 3.0.7-ea8a0637 → 3.0.7-eabf6f36

package/src/stream-muxer/close-test.ts

@@ -0,0 +1,442 @@
+/* eslint max-nested-callbacks: ["error", 8] */
+import { abortableSource } from 'abortable-iterator'
+import { expect } from 'aegir/chai'
+import delay from 'delay'
+import all from 'it-all'
+import drain from 'it-drain'
+import { duplexPair } from 'it-pair/duplex'
+import { pipe } from 'it-pipe'
+import { pbStream } from 'it-protobuf-stream'
+import toBuffer from 'it-to-buffer'
+import pDefer from 'p-defer'
+import { Uint8ArrayList } from 'uint8arraylist'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import { Message } from './fixtures/pb/message.js'
+import type { TestSetup } from '../index.js'
+import type { StreamMuxerFactory } from '@libp2p/interface/stream-muxer'
+
+function randomBuffer (): Uint8Array {
+  return uint8ArrayFromString(Math.random().toString())
+}
+
+const infiniteRandom = {
+  [Symbol.asyncIterator]: async function * () {
+    while (true) {
+      yield new Uint8ArrayList(randomBuffer())
+      await delay(50)
+    }
+  }
+}
+
+export default (common: TestSetup<StreamMuxerFactory>): void => {
+  describe('close', () => {
+    it('closing underlying socket closes streams', async () => {
+      let openedStreams = 0
+      const expectedStreams = 5
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+
+      // Listener is echo server :)
+      const listenerFactory = await common.setup()
+      const listener = listenerFactory.createStreamMuxer({
+        direction: 'inbound',
+        onIncomingStream: (stream) => {
+          openedStreams++
+          void pipe(stream, stream)
+        }
+      })
+
+      const p = duplexPair<Uint8Array>()
+      void pipe(p[0], dialer, p[0])
+      void pipe(p[1], listener, p[1])
+
+      const streams = await Promise.all(Array(expectedStreams).fill(0).map(async () => dialer.newStream()))
+
+      void Promise.all(
+        streams.map(async stream => {
+          await pipe(
+            infiniteRandom,
+            stream,
+            drain
+          )
+        })
+      )
+
+      expect(dialer.streams).to.have.lengthOf(expectedStreams)
+
+      // Pause, and then close the dialer
+      await delay(50)
+      await pipe([], dialer, drain)
+
+      expect(openedStreams).to.have.equal(expectedStreams)
+      expect(dialer.streams).to.have.lengthOf(0)
+    })
+
+    it('calling close closes streams', async () => {
+      let openedStreams = 0
+      const expectedStreams = 5
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+
+      // Listener is echo server :)
+      const listenerFactory = await common.setup()
+      const listener = listenerFactory.createStreamMuxer({
+        direction: 'inbound',
+        onIncomingStream: (stream) => {
+          openedStreams++
+          void pipe(stream, stream).catch(() => {})
+        }
+      })
+
+      const p = duplexPair<Uint8Array>()
+      void pipe(p[0], dialer, p[0])
+      void pipe(p[1], listener, p[1])
+
+      const streams = await Promise.all(Array(expectedStreams).fill(0).map(async () => dialer.newStream()))
+
+      void Promise.all(
+        streams.map(async stream => {
+          await pipe(
+            infiniteRandom,
+            stream,
+            drain
+          )
+        })
+      )
+
+      expect(dialer.streams, 'dialer - number of opened streams should match number of calls to newStream').to.have.lengthOf(expectedStreams)
+
+      // Pause, and then close the dialer
+      await delay(50)
+
+      await dialer.close()
+
+      expect(openedStreams, 'listener - number of opened streams should match number of calls to newStream').to.have.equal(expectedStreams)
+      expect(dialer.streams, 'all tracked streams should be deleted after the muxer has called close').to.have.lengthOf(0)
+    })
+
+    it('calling close with an error aborts streams', async () => {
+      let openedStreams = 0
+      const expectedStreams = 5
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+
+      // Listener is echo server :)
+      const listenerFactory = await common.setup()
+      const listener = listenerFactory.createStreamMuxer({
+        direction: 'inbound',
+        onIncomingStream: (stream) => {
+          openedStreams++
+          void pipe(stream, stream).catch(() => {})
+        }
+      })
+
+      const p = duplexPair<Uint8Array>()
+      void pipe(p[0], dialer, p[0])
+      void pipe(p[1], listener, p[1])
+
+      const streams = await Promise.all(Array(expectedStreams).fill(0).map(async () => dialer.newStream()))
+
+      const streamPipes = streams.map(async stream => {
+        await pipe(
+          infiniteRandom,
+          stream,
+          drain
+        )
+      })
+
+      expect(dialer.streams, 'dialer - number of opened streams should match number of calls to newStream').to.have.lengthOf(expectedStreams)
+
+      // Pause, and then close the dialer
+      await delay(50)
+
+      // close _with an error_
+      dialer.abort(new Error('Oh no!'))
+
+      const timeoutError = new Error('timeout')
+      for (const pipe of streamPipes) {
+        try {
+          await Promise.race([
+            pipe,
+            new Promise((_resolve, reject) => setTimeout(() => { reject(timeoutError) }, 20))
+          ])
+          expect.fail('stream pipe with infinite source should never return')
+        } catch (e) {
+          if (e === timeoutError) {
+            expect.fail('expected stream pipe to throw an error after muxer closed with error')
+          }
+        }
+      }
+
+      expect(openedStreams, 'listener - number of opened streams should match number of calls to newStream').to.have.equal(expectedStreams)
+      expect(dialer.streams, 'all tracked streams should be deleted after the muxer has called close').to.have.lengthOf(0)
+    })
+
+    it('calling newStream after close throws an error', async () => {
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+
+      await dialer.close()
+
+      try {
+        await dialer.newStream()
+        expect.fail('newStream should throw if called after close')
+      } catch (e) {
+        expect(dialer.streams, 'closed muxer should have no streams').to.have.lengthOf(0)
+      }
+    })
+
+    it('closing one of the muxed streams doesn\'t close others', async () => {
+      const p = duplexPair<Uint8Array>()
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+
+      // Listener is echo server :)
+      const listenerFactory = await common.setup()
+      const listener = listenerFactory.createStreamMuxer({
+        direction: 'inbound',
+        onIncomingStream: (stream) => {
+          void pipe(stream, stream).catch(() => {})
+        }
+      })
+
+      void pipe(p[0], dialer, p[0])
+      void pipe(p[1], listener, p[1])
+
+      const stream = await dialer.newStream()
+      const streams = await Promise.all(Array.from(Array(5), async () => dialer.newStream()))
+      let closed = false
+      const controllers: AbortController[] = []
+
+      const streamResults = streams.map(async stream => {
+        const controller = new AbortController()
+        controllers.push(controller)
+
+        try {
+          const abortableRand = abortableSource(infiniteRandom, controller.signal, { abortCode: 'ERR_TEST_ABORT' })
+          await pipe(abortableRand, stream, drain)
+        } catch (err: any) {
+          if (err.code !== 'ERR_TEST_ABORT') throw err
+        }
+
+        if (!closed) throw new Error('stream should not have ended yet!')
+      })
+
+      // Pause, and then send some data and close the first stream
+      await delay(50)
+      await pipe([new Uint8ArrayList(randomBuffer())], stream, drain)
+      closed = true
+
+      // Abort all the other streams later
+      await delay(50)
+      controllers.forEach(c => { c.abort() })
+
+      // These should now all resolve without error
+      await Promise.all(streamResults)
+    })
+
+    it('can close a stream for writing', async () => {
+      const deferred = pDefer<Error>()
+
+      const p = duplexPair<Uint8Array>()
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+      const data = [randomBuffer(), randomBuffer()]
+
+      const listenerFactory = await common.setup()
+      const listener = listenerFactory.createStreamMuxer({
+        direction: 'inbound',
+        onIncomingStream: (stream) => {
+          void Promise.resolve().then(async () => {
+            // Immediate close for write
+            await stream.closeWrite()
+
+            const results = await pipe(stream, async (source) => {
+              const data = []
+              for await (const chunk of source) {
+                data.push(chunk.slice())
+              }
+              return data
+            })
+            expect(results).to.eql(data)
+
+            try {
+              await stream.sink([new Uint8ArrayList(randomBuffer())])
+            } catch (err: any) {
+              deferred.resolve(err)
+            }
+
+            deferred.reject(new Error('should not support writing to closed writer'))
+          })
+        }
+      })
+
+      void pipe(p[0], dialer, p[0])
+      void pipe(p[1], listener, p[1])
+
+      const stream = await dialer.newStream()
+      await stream.sink(data)
+
+      const err = await deferred.promise
+      expect(err).to.have.property('code', 'ERR_SINK_INVALID_STATE')
+    })
+
+    it('can close a stream for reading', async () => {
+      const deferred = pDefer<Uint8ArrayList[]>()
+      const p = duplexPair<Uint8Array>()
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+      const data = [randomBuffer(), randomBuffer()].map(d => new Uint8ArrayList(d))
+      const expected = toBuffer(data.map(d => d.subarray()))
+
+      const listenerFactory = await common.setup()
+      const listener = listenerFactory.createStreamMuxer({
+        direction: 'inbound',
+        onIncomingStream: (stream) => {
+          void all(stream.source).then(deferred.resolve, deferred.reject)
+        }
+      })
+
+      void pipe(p[0], dialer, p[0])
+      void pipe(p[1], listener, p[1])
+
+      const stream = await dialer.newStream()
+      await stream.closeRead()
+
+      // Source should be done
+      void Promise.resolve().then(async () => {
+        expect(await stream.source.next()).to.have.property('done', true)
+        await stream.sink(data)
+      })
+
+      const results = await deferred.promise
+      expect(toBuffer(results.map(b => b.subarray()))).to.equalBytes(expected)
+    })
+
+    it('calls onStreamEnd for closed streams not previously written', async () => {
+      const deferred = pDefer()
+
+      const onStreamEnd = (): void => { deferred.resolve() }
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({
+        direction: 'outbound',
+        onStreamEnd
+      })
+
+      const stream = await dialer.newStream()
+
+      await stream.close()
+      await deferred.promise
+    })
+
+    it('calls onStreamEnd for read and write closed streams not previously written', async () => {
+      const deferred = pDefer()
+
+      const onStreamEnd = (): void => { deferred.resolve() }
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({
+        direction: 'outbound',
+        onStreamEnd
+      })
+
+      const stream = await dialer.newStream()
+
+      await stream.closeWrite()
+      await stream.closeRead()
+      await deferred.promise
+    })
+
+    it('should wait for all data to be sent when closing streams', async () => {
+      const deferred = pDefer<Message>()
+
+      const p = duplexPair<Uint8Array>()
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+
+      const listenerFactory = await common.setup()
+      const listener = listenerFactory.createStreamMuxer({
+        direction: 'inbound',
+        onIncomingStream: (stream) => {
+          const pb = pbStream(stream)
+
+          void pb.read(Message)
+            .then(async message => {
+              deferred.resolve(message)
+              await pb.unwrap().close()
+            })
+            .catch(err => {
+              deferred.reject(err)
+            })
+        }
+      })
+
+      void pipe(p[0], dialer, p[0])
+      void pipe(p[1], listener, p[1])
+
+      const message = {
+        message: 'hello world',
+        value: 5,
+        flag: true
+      }
+
+      const stream = await dialer.newStream()
+
+      const pb = pbStream(stream)
+      await pb.write(message, Message)
+      await pb.unwrap().close()
+
+      await expect(deferred.promise).to.eventually.deep.equal(message)
+    })
+    /*
+    it('should abort closing a stream with outstanding data to read', async () => {
+      const deferred = pDefer<Message>()
+
+      const p = duplexPair<Uint8Array>()
+      const dialerFactory = await common.setup()
+      const dialer = dialerFactory.createStreamMuxer({ direction: 'outbound' })
+
+      const listenerFactory = await common.setup()
+      const listener = listenerFactory.createStreamMuxer({
+        direction: 'inbound',
+        onIncomingStream: (stream) => {
+          const pb = pbStream(stream)
+
+          void pb.read(Message)
+            .then(async message => {
+              await pb.write(message, Message)
+              await pb.unwrap().close()
+              deferred.resolve(message)
+            })
+            .catch(err => {
+              deferred.reject(err)
+            })
+        }
+      })
+
+      void pipe(p[0], dialer, p[0])
+      void pipe(p[1], listener, p[1])
+
+      const message = {
+        message: 'hello world',
+        value: 5,
+        flag: true
+      }
+
+      const stream = await dialer.newStream()
+
+      const pb = pbStream(stream)
+      await pb.write(message, Message)
+
+      console.info('await write back')
+      await deferred.promise
+
+      // let message arrive
+      await delay(100)
+
+      // close should time out as message is never read
+      await expect(pb.unwrap().close()).to.eventually.be.rejected
+        .with.property('code', 'ERR_CLOSE_READ_ABORTED')
+    })
+    */
+  })
+}

package/src/stream-muxer/fixtures/pb/message.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+message Message {
+  string message = 1;
+  uint32 value = 2;
+  bool flag = 3;
+}

package/src/stream-muxer/fixtures/pb/message.ts

@@ -0,0 +1,87 @@
+/* eslint-disable import/export */
+/* eslint-disable complexity */
+/* eslint-disable @typescript-eslint/no-namespace */
+/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
+/* eslint-disable @typescript-eslint/no-empty-interface */
+
+import { encodeMessage, decodeMessage, message } from 'protons-runtime'
+import type { Codec } from 'protons-runtime'
+import type { Uint8ArrayList } from 'uint8arraylist'
+
+export interface Message {
+  message: string
+  value: number
+  flag: boolean
+}
+
+export namespace Message {
+  let _codec: Codec<Message>
+
+  export const codec = (): Codec<Message> => {
+    if (_codec == null) {
+      _codec = message<Message>((obj, w, opts = {}) => {
+        if (opts.lengthDelimited !== false) {
+          w.fork()
+        }
+
+        if ((obj.message != null && obj.message !== '')) {
+          w.uint32(10)
+          w.string(obj.message)
+        }
+
+        if ((obj.value != null && obj.value !== 0)) {
+          w.uint32(16)
+          w.uint32(obj.value)
+        }
+
+        if ((obj.flag != null && obj.flag !== false)) {
+          w.uint32(24)
+          w.bool(obj.flag)
+        }
+
+        if (opts.lengthDelimited !== false) {
+          w.ldelim()
+        }
+      }, (reader, length) => {
+        const obj: any = {
+          message: '',
+          value: 0,
+          flag: false
+        }
+
+        const end = length == null ? reader.len : reader.pos + length
+
+        while (reader.pos < end) {
+          const tag = reader.uint32()
+
+          switch (tag >>> 3) {
+            case 1:
+              obj.message = reader.string()
+              break
+            case 2:
+              obj.value = reader.uint32()
+              break
+            case 3:
+              obj.flag = reader.bool()
+              break
+            default:
+              reader.skipType(tag & 7)
+              break
+          }
+        }
+
+        return obj
+      })
+    }
+
+    return _codec
+  }
+
+  export const encode = (obj: Partial<Message>): Uint8Array => {
+    return encodeMessage(obj, Message.codec())
+  }
+
+  export const decode = (buf: Uint8Array | Uint8ArrayList): Message => {
+    return decodeMessage(buf, Message.codec())
+  }
+}

package/src/stream-muxer/index.ts

@@ -0,0 +1,15 @@
+import baseTest from './base-test.js'
+import closeTest from './close-test.js'
+import megaStressTest from './mega-stress-test.js'
+import stressTest from './stress-test.js'
+import type { TestSetup } from '../index.js'
+import type { StreamMuxerFactory } from '@libp2p/interface/stream-muxer'
+
+export default (common: TestSetup<StreamMuxerFactory>): void => {
+  describe('interface-stream-muxer', () => {
+    baseTest(common)
+    closeTest(common)
+    stressTest(common)
+    megaStressTest(common)
+  })
+}

package/src/stream-muxer/mega-stress-test.ts

@@ -0,0 +1,14 @@
+import spawn from './spawner.js'
+import type { TestSetup } from '../index.js'
+import type { StreamMuxer, StreamMuxerFactory, StreamMuxerInit } from '@libp2p/interface/stream-muxer'
+
+export default (common: TestSetup<StreamMuxerFactory>): void => {
+  const createMuxer = async (init?: StreamMuxerInit): Promise<StreamMuxer> => {
+    const factory = await common.setup()
+    return factory.createStreamMuxer(init)
+  }
+
+  describe.skip('mega stress test', function () {
+    it('10,000 streams with 10,000 msg', async () => { await spawn(createMuxer, 10000, 10000, 5000) })
+  })
+}

package/src/stream-muxer/spawner.ts

@@ -0,0 +1,55 @@
+import { expect } from 'aegir/chai'
+import all from 'it-all'
+import drain from 'it-drain'
+import { duplexPair } from 'it-pair/duplex'
+import { pipe } from 'it-pipe'
+import pLimit from 'p-limit'
+import { Uint8ArrayList } from 'uint8arraylist'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
+import type { StreamMuxer, StreamMuxerInit } from '@libp2p/interface/stream-muxer'
+
+export default async (createMuxer: (init?: StreamMuxerInit) => Promise<StreamMuxer>, nStreams: number, nMsg: number, limit?: number): Promise<void> => {
+  const [dialerSocket, listenerSocket] = duplexPair<Uint8Array>()
+
+  const msg = new Uint8ArrayList(uint8ArrayFromString('simple msg'))
+
+  const listener = await createMuxer({
+    direction: 'inbound',
+    onIncomingStream: (stream) => {
+      void pipe(
+        stream,
+        drain
+      ).then(async () => {
+        await stream.close()
+      })
+        .catch(err => { stream.abort(err) })
+    }
+  })
+  const dialer = await createMuxer({ direction: 'outbound' })
+
+  void pipe(listenerSocket, listener, listenerSocket)
+  void pipe(dialerSocket, dialer, dialerSocket)
+
+  const spawnStream = async (): Promise<void> => {
+    const stream = await dialer.newStream()
+    expect(stream).to.exist // eslint-disable-line
+
+    const res = await pipe(
+      (async function * () {
+        for (let i = 0; i < nMsg; i++) {
+          yield msg
+        }
+      }()),
+      stream,
+      async (source) => all(source)
+    )
+
+    expect(res).to.be.eql([])
+  }
+
+  const limiter = pLimit(limit ?? Infinity)
+
+  await Promise.all(
+    Array.from(Array(nStreams), async () => { await limiter(async () => { await spawnStream() }) })
+  )
+}

package/src/stream-muxer/stress-test.ts

@@ -0,0 +1,27 @@
+import spawn from './spawner.js'
+import type { TestSetup } from '../index.js'
+import type { StreamMuxerFactory, StreamMuxerInit, StreamMuxer } from '@libp2p/interface/stream-muxer'
+
+export default (common: TestSetup<StreamMuxerFactory>): void => {
+  const createMuxer = async (init?: StreamMuxerInit): Promise<StreamMuxer> => {
+    const factory = await common.setup()
+    return factory.createStreamMuxer(init)
+  }
+
+  describe('stress test', function () {
+    this.timeout(1600000)
+
+    it('1 stream with 1 msg', async () => { await spawn(createMuxer, 1, 1) })
+    it('1 stream with 10 msg', async () => { await spawn(createMuxer, 1, 10) })
+    it('1 stream with 100 msg', async () => { await spawn(createMuxer, 1, 100) })
+    it('10 streams with 1 msg', async () => { await spawn(createMuxer, 10, 1) })
+    it('10 streams with 10 msg', async () => { await spawn(createMuxer, 10, 10) })
+    it('10 streams with 100 msg', async () => { await spawn(createMuxer, 10, 100) })
+    it('100 streams with 1 msg', async () => { await spawn(createMuxer, 100, 1) })
+    it('100 streams with 10 msg', async () => { await spawn(createMuxer, 100, 10) })
+    it('100 streams with 100 msg', async () => { await spawn(createMuxer, 100, 100) })
+    it('1000 streams with 1 msg', async () => { await spawn(createMuxer, 1000, 1) })
+    it('1000 streams with 10 msg', async () => { await spawn(createMuxer, 1000, 10) })
+    it('1000 streams with 100 msg', async () => { await spawn(createMuxer, 1000, 100) })
+  })
+}