@sprucelabs/mercury-chunking-emitter 1.0.4

package/src/__tests__/behavioral/ChunkingEmitter.test.ts

@@ -0,0 +1,290 @@
+import { EventName } from '@sprucelabs/mercury-types'
+import { FieldDefinitions, buildSchema } from '@sprucelabs/schema'
+import { eventFaker, fake } from '@sprucelabs/spruce-test-fixtures'
+import { test, assert, errorAssert, generateId } from '@sprucelabs/test-utils'
+import { chunkFieldDefinition } from '../../chunkingEmitter/chunkFieldDefinition'
+import ChunkingEmitterImpl from '../../chunkingEmitter/ChunkingEmitter'
+import AbstractChunkingEmitterTest from '../support/AbstractChunkingEmitterTest'
+
+@fake.login()
+export default class ChunkingEmitterTest extends AbstractChunkingEmitterTest {
+	private static fqen2: EventName
+	private static allTargetAndPayloads: any[] = []
+	private static hitCount: number
+	protected static emitter: SpyEmitter
+	private static readonly itemFieldDefinition: FieldDefinitions = {
+		type: 'raw',
+		isArray: true,
+		options: {
+			valueType: 'any',
+		},
+	}
+
+	protected static async beforeEach() {
+		await super.beforeEach()
+
+		ChunkingEmitterImpl.Class = SpyEmitter
+
+		this.hitCount = 0
+		this.fqen = 'test.test::v2021_01_01' as EventName
+		this.fqen2 = 'test2.test3::v2022_02_02' as EventName
+
+		this.allTargetAndPayloads = []
+
+		this.mixinTestContract()
+
+		await this.resetEmitterWithChunkSize(1)
+
+		await this.fakedClient.on(this.fqen, (targetAndPayload) => {
+			this.hitCount++
+			this.allTargetAndPayloads.push(targetAndPayload)
+		})
+	}
+
+	@test()
+	protected static async throwsWithMissing() {
+		//@ts-ignore
+		const err = await assert.doesThrowAsync(() => ChunkingEmitterImpl.Emitter())
+		errorAssert.assertError(err, 'MISSING_PARAMETERS', {
+			parameters: ['client'],
+		})
+	}
+
+	@test()
+	protected static async emitThrowsWithMissing() {
+		//@ts-ignore
+		const err = await assert.doesThrowAsync(() => this.emitter.emit())
+		errorAssert.assertError(err, 'MISSING_PARAMETERS', {
+			parameters: ['eventName', 'items', 'payloadKey'],
+		})
+	}
+
+	@test()
+	protected static async emitsNothingIfPassedNoItems() {
+		await this.emitWithItems([])
+		assert.isFalse(this.wasEventEmitted)
+	}
+
+	@test()
+	protected static async emitsThePassedEvent() {
+		await this.emitWithItems([{ id: '1' }])
+		assert.isTrue(this.wasEventEmitted)
+	}
+
+	@test('passes one item to payload', [{ id: '1' }])
+	@test('passes two items to payload', [{ id: '1' }, { id: '2' }])
+	protected static async passesThePayloadThroughUsingTheKeyPassed(
+		items: Record<string, any>[]
+	) {
+		await this.resetEmitterWithChunkSize(10)
+		await this.emitWithItems(items)
+		this.assertLastEmittedItemsEqual(items)
+	}
+
+	@test()
+	protected static async canPassToDifferentPayloadKey() {
+		this.payloadKey = 'items2'
+		const item = await this.emitWithOneItem()
+		assert.isEqualDeep(this.lastEmittedPayload?.items2, [item])
+	}
+
+	@test()
+	protected static async emitsNameOfEventPassed() {
+		this.fqen = this.fqen2
+
+		let wasHit = false
+		await eventFaker.on(this.fqen2, () => {
+			wasHit = true
+		})
+
+		await this.emitWithOneItem()
+		assert.isTrue(wasHit)
+	}
+
+	@test('emits 2 chunks', 2)
+	@test('emits 3 chunks', 3)
+	protected static async emitsCorrectNumberOfChunks(total: number) {
+		const items = await this.emitWithTotalItems(total)
+		this.assertTotalEmits(total)
+
+		const expected = items.map((item) => [item])
+		assert.isEqualDeep(
+			this.allTargetAndPayloads.map((t) => t.payload.items),
+			expected
+		)
+	}
+
+	@test()
+	protected static async doesNotStopIfOneFails() {
+		let hitCount = 0
+		await eventFaker.on(this.fqen, () => {
+			hitCount++
+			if (hitCount === 2) {
+				throw new Error('Failed')
+			}
+		})
+
+		await this.emitWithTotalItems(10)
+		assert.isEqual(hitCount, 10)
+		this.assertTotalErrors(1)
+	}
+
+	@test()
+	protected static async accuratelyTracksTotalErrors() {
+		await this.makeEventThrow()
+		await this.assertEveryEmitErrors()
+		await this.assertEveryEmitErrors()
+	}
+
+	@test()
+	protected static async includesChuckingPlacementInPayloadForOneItem() {
+		await this.emitWithOneItem()
+		assert.isEqualDeep(this.lastEmittedPayload?.chunk, {
+			total: 1,
+			current: 0,
+		})
+	}
+
+	@test()
+	protected static async includesChuckingPlacementInPayloadForTwoItems() {
+		await this.emitWithTotalItems(2)
+		const expected = [
+			{ total: 2, current: 0 },
+			{ total: 2, current: 1 },
+		]
+		assert.isEqualDeep(
+			this.allTargetAndPayloads.map((t) => t.payload.chunk),
+			expected
+		)
+	}
+
+	@test()
+	protected static async includesTargetInEmit() {
+		this.fqen = generateId() as EventName
+
+		const payloadSchema = buildSchema({
+			id: generateId(),
+			fields: {
+				items: this.itemFieldDefinition,
+				chunk: chunkFieldDefinition(),
+			},
+		})
+
+		const targetSchema = buildSchema({
+			id: generateId(),
+			fields: {
+				personId: {
+					type: 'id',
+				},
+			},
+		})
+
+		this.mixinEventSignatures({
+			[this.fqen]: this.buildSignature(payloadSchema, targetSchema),
+		})
+
+		let passedTarget: Record<string, any> | undefined
+
+		//@ts-ignore
+		await eventFaker.on(this.fqen, ({ target }) => {
+			passedTarget = target
+		})
+
+		const target = { personId: generateId() }
+		await this.emitWithOneItem(target)
+		assert.isEqualDeep(passedTarget, target)
+	}
+
+	@test()
+	protected static async defaultsToChunkSizeOfTen() {
+		this.emitter = (await ChunkingEmitterImpl.Emitter({
+			client: this.fakedClient,
+		})) as SpyEmitter
+
+		assert.isEqualDeep(this.emitter.getChunkSize(), 10)
+	}
+
+	private static async makeEventThrow() {
+		await eventFaker.on(this.fqen, () => {
+			throw new Error('Failed')
+		})
+	}
+
+	private static assertTotalErrors(expected: number) {
+		assert.isEqual(this.emitter.getTotalErrors(), expected)
+	}
+
+	private static assertTotalEmits(total: number) {
+		assert.isEqual(this.hitCount, total)
+	}
+
+	private static async emitWithTotalItems(total: number) {
+		const items = new Array(total).fill(0).map(() => this.generateItem())
+		await this.emitWithItems(items)
+		return items
+	}
+
+	private static assertLastEmittedItemsEqual(items: Record<string, any>[]) {
+		assert.isEqualDeep(this.lastEmittedPayload?.items, items)
+	}
+
+	private static async resetEmitterWithChunkSize(chunkSize: number) {
+		this.emitter = (await ChunkingEmitterImpl.Emitter({
+			client: this.fakedClient,
+			chunkSize,
+		})) as SpyEmitter
+	}
+
+	private static async emitWithOneItem(target?: Record<string, any>) {
+		const items = [this.generateItem()]
+		await this.emitWithItems(items, target)
+		return items[0]
+	}
+
+	private static get lastEmittedPayload() {
+		return this.lastTargetAndPayload?.payload
+	}
+
+	private static mixinTestContract() {
+		const eventSignatures = {
+			[this.fqen]: this.eventSignature,
+			[this.fqen2]: this.eventSignature,
+		}
+		this.mixinEventSignatures(eventSignatures)
+	}
+
+	private static get eventSignature() {
+		const payload = buildSchema({
+			id: generateId(),
+			fields: {
+				items: this.itemFieldDefinition,
+				items2: this.itemFieldDefinition,
+				chunk: chunkFieldDefinition(),
+			},
+		})
+
+		return this.buildSignature(payload)
+	}
+
+	private static async assertEveryEmitErrors() {
+		await this.emitWithTotalItems(10)
+		this.assertTotalErrors(10)
+	}
+
+	private static get lastTargetAndPayload() {
+		return this.allTargetAndPayloads[this.allTargetAndPayloads.length - 1]
+	}
+
+	private static get wasEventEmitted() {
+		return this.hitCount > 0
+	}
+}
+
+class SpyEmitter extends ChunkingEmitterImpl {
+	public constructor(options: any) {
+		super(options)
+	}
+	public getChunkSize() {
+		return this.chunkSize
+	}
+}

package/src/__tests__/behavioral/TestingChunkingEmitter.test.ts

@@ -0,0 +1,163 @@
+import { EventName } from '@sprucelabs/mercury-types'
+import {
+	FieldDefinitions,
+	SchemaFieldsByName,
+	buildSchema,
+} from '@sprucelabs/schema'
+import { fake } from '@sprucelabs/spruce-test-fixtures'
+import { test, assert, generateId } from '@sprucelabs/test-utils'
+import { chunkFieldDefinition } from '../../chunkingEmitter/chunkFieldDefinition'
+import ChunkingEmitterImpl from '../../chunkingEmitter/ChunkingEmitter'
+import MockChunkingEmitter from '../../chunkingEmitter/MockChunkingEmitter'
+import AbstractChunkingEmitterTest from '../support/AbstractChunkingEmitterTest'
+
+@fake.login()
+export default class TestingChunkingEmitterTest extends AbstractChunkingEmitterTest {
+	protected static emitter: MockChunkingEmitter
+	private static readonly itemsField: FieldDefinitions = {
+		type: 'raw',
+		isArray: true,
+		options: {
+			valueType: 'any',
+		},
+	}
+
+	protected static async beforeEach(): Promise<void> {
+		await super.beforeEach()
+		ChunkingEmitterImpl.Class = MockChunkingEmitter
+		this.fqen = generateId() as EventName
+		this.emitter = (await ChunkingEmitterImpl.Emitter({
+			client: this.fakedClient,
+		})) as MockChunkingEmitter
+	}
+
+	@test()
+	protected static async canSetSpyClass() {
+		assert.isInstanceOf(this.emitter, MockChunkingEmitter)
+	}
+
+	@test()
+	protected static async knowsIfWasHit() {
+		assert.doesThrow(() => this.emitter.assertDidEmit())
+		await this.emit()
+		this.emitter.assertDidEmit()
+	}
+
+	@test()
+	protected static async knowsEventNamePassedToIt() {
+		await this.emit()
+		assert.doesThrow(() =>
+			this.emitter.assertDidEmitEventNamed(generateId() as EventName)
+		)
+		this.emitter.assertDidEmitEventNamed(this.fqen)
+	}
+
+	@test()
+	protected static async knowsItemsEmitted() {
+		const items = [this.generateItem(), this.generateItem()]
+		this.assertDidEmitItemsThrows(items)
+		await this.emitWithItems(items)
+		this.emitter.assertEmittedItems(items)
+		this.assertDidEmitItemsThrows([])
+		this.assertDidEmitItemsThrows([this.generateItem()])
+	}
+
+	@test()
+	protected static async canResetFactoryClassReference() {
+		ChunkingEmitterImpl.reset()
+		assert.isFalsy(ChunkingEmitterImpl.Class)
+	}
+
+	@test()
+	protected static async canAssertPayloadKey() {
+		this.payloadKey = generateId()
+		await this.emit()
+		assert.doesThrow(() => this.emitter.assertDidEmitPayloadKey('wrong'))
+		this.emitter.assertDidEmitPayloadKey(this.payloadKey)
+	}
+
+	@test()
+	protected static async throwsIfEventSignatureIsBad() {
+		const sigWithoutChunking = this.buildSignatureWithPayloadFields({
+			items: this.itemsField,
+		})
+
+		const sigWithChunking = this.buildSignatureWithPayloadFields({
+			items: this.itemsField,
+			chunk: chunkFieldDefinition(),
+		})
+
+		const fqenWithChunk = generateId() as EventName
+		this.mixinEventSignatures({
+			[this.fqen]: sigWithoutChunking,
+			[fqenWithChunk]: sigWithChunking,
+		})
+
+		const fqen = this.fqen
+		this.assertSignatureDoesNotConformToChunk(fqen)
+		this.assertSignatureDoesConfirmToChunk(fqenWithChunk)
+	}
+
+	@test()
+	protected static async throwsIfChunkIsThereButBad() {
+		this.assertChunkFieldThrowsWithBadMatch({
+			type: 'number',
+		})
+		this.assertChunkFieldThrowsWithBadMatch({
+			type: 'schema',
+			options: {
+				schema: buildSchema({
+					id: generateId(),
+					fields: {
+						test: {
+							type: 'number',
+						},
+					},
+				}),
+			},
+		})
+	}
+
+	private static assertChunkFieldThrowsWithBadMatch(
+		chunkSig: FieldDefinitions
+	) {
+		const fqen = generateId() as EventName
+		const sig = this.buildSignatureWithPayloadFields({
+			items: this.itemsField,
+			chunk: chunkSig,
+		})
+
+		this.mixinEventSignatures({
+			[fqen]: sig,
+		})
+
+		this.assertSignatureDoesNotConformToChunk(fqen)
+	}
+
+	private static assertSignatureDoesConfirmToChunk(fqenWithChunk: EventName) {
+		this.emitter.assertEventHonorsChunkingSignature(fqenWithChunk)
+	}
+
+	private static assertSignatureDoesNotConformToChunk(fqen: EventName) {
+		assert.doesThrow(() =>
+			this.emitter.assertEventHonorsChunkingSignature(fqen)
+		)
+	}
+
+	private static buildSignatureWithPayloadFields(fields: SchemaFieldsByName) {
+		return this.buildSignature(
+			buildSchema({
+				id: generateId(),
+				fields,
+			})
+		)
+	}
+
+	private static assertDidEmitItemsThrows(items: { id: string }[]) {
+		assert.doesThrow(() => this.emitter.assertEmittedItems(items))
+	}
+
+	private static async emit() {
+		return this.emitWithItems([this.generateItem()])
+	}
+}

package/src/__tests__/support/AbstractChunkingEmitterTest.ts

@@ -0,0 +1,54 @@
+import { MercuryTestClient } from '@sprucelabs/mercury-client'
+import { EventName, EventSignaturesByName } from '@sprucelabs/mercury-types'
+import { Schema } from '@sprucelabs/schema'
+import { buildEmitTargetAndPayloadSchema } from '@sprucelabs/spruce-event-utils'
+import { AbstractSpruceFixtureTest } from '@sprucelabs/spruce-test-fixtures'
+import { generateId } from '@sprucelabs/test-utils'
+import { ChunkingEmitter } from '../../chunkingEmitter/ChunkingEmitter'
+
+export default abstract class AbstractChunkingEmitterTest extends AbstractSpruceFixtureTest {
+	protected static emitter: ChunkingEmitter
+	protected static fqen: EventName
+	protected static payloadKey: string
+
+	protected static async beforeEach() {
+		await super.beforeEach()
+		this.payloadKey = 'items'
+	}
+
+	protected static mixinEventSignatures(
+		eventSignatures: EventSignaturesByName
+	) {
+		const client = this.fakedClient as MercuryTestClient
+		client.mixinContract({
+			eventSignatures,
+		})
+	}
+
+	protected static buildSignature(payload: Schema, target?: Schema) {
+		return {
+			isGlobal: true,
+			emitPayloadSchema: buildEmitTargetAndPayloadSchema({
+				eventName: this.fqen,
+				payloadSchema: payload,
+				targetSchema: target,
+			}),
+		}
+	}
+
+	protected static async emitWithItems(
+		items: Record<string, unknown>[],
+		target?: Record<string, any>
+	) {
+		await this.emitter.emit({
+			eventName: this.fqen,
+			items,
+			payloadKey: this.payloadKey,
+			target,
+		})
+	}
+
+	protected static generateItem() {
+		return { id: generateId() }
+	}
+}

package/src/chunkingEmitter/ChunkingEmitter.ts

@@ -0,0 +1,107 @@
+import { MercuryClient } from '@sprucelabs/mercury-client'
+import { EventName } from '@sprucelabs/mercury-types'
+import { assertOptions } from '@sprucelabs/schema'
+import { buildLog } from '@sprucelabs/spruce-skill-utils'
+
+export default class ChunkingEmitterImpl {
+	private client: MercuryClient
+	protected chunkSize: number
+	private log = buildLog('ChunkingEmitter')
+	public static Class?: new (options: ChunkingEmitterOptions) => ChunkingEmitter
+	private totalErrors = 0
+
+	protected constructor(options: ChunkingEmitterOptions) {
+		const { client, chunkSize } = assertOptions(options, ['client'])
+		this.client = client
+		this.chunkSize = chunkSize ?? 10
+	}
+
+	public static async Emitter(options: ChunkingEmitterOptions) {
+		assertOptions(options, ['client'])
+		return new (this.Class ?? this)(options)
+	}
+
+	public async emit(options: ChunkingEmitterEmitOptions) {
+		const { eventName, items, payloadKey, target } = assertOptions(options, [
+			'eventName',
+			'items',
+			'payloadKey',
+		])
+
+		this.totalErrors = 0
+		const chunks = this.splitItemsIntoChunks(items)
+		let current = 0
+
+		for (const chunk of chunks) {
+			try {
+				let targetAndPayload: Record<string, any> = {
+					payload: {
+						[payloadKey]: chunk,
+						chunk: {
+							current: current++,
+							total: chunks.length,
+						},
+					},
+				}
+
+				if (target) {
+					targetAndPayload.target = target
+				}
+
+				await this.client.emitAndFlattenResponses(
+					eventName as EventName,
+					targetAndPayload
+				)
+			} catch (err: any) {
+				this.log.error('Failed to emit chunk', err)
+				this.totalErrors++
+			}
+		}
+	}
+
+	public static reset() {
+		this.Class = undefined
+	}
+
+	private splitItemsIntoChunks(items: Record<string, any>[]) {
+		const chunks: Record<string, any>[][] = []
+
+		let index = 0
+
+		for (const item of items) {
+			if (!chunks[index]) {
+				chunks[index] = []
+			}
+
+			if (chunks[index].length === this.chunkSize) {
+				index++
+				chunks[index] = []
+			}
+
+			chunks[index].push(item)
+		}
+
+		return chunks
+	}
+
+	public getTotalErrors(): number {
+		return this.totalErrors
+	}
+}
+
+interface ChunkingEmitterOptions {
+	client: MercuryClient
+	chunkSize?: number
+}
+
+type ChunkingEmitterEmitOptions = {
+	eventName: EventName
+	items: Record<string, any>[]
+	payloadKey: string
+	target?: Record<string, any>
+}
+
+export interface ChunkingEmitter {
+	emit(options: ChunkingEmitterEmitOptions): Promise<void>
+	getTotalErrors(): number
+}

package/src/chunkingEmitter/MockChunkingEmitter.ts

@@ -0,0 +1,84 @@
+import { MercuryTestClient } from '@sprucelabs/mercury-client'
+import { EventName } from '@sprucelabs/mercury-types'
+import { assert } from '@sprucelabs/test-utils'
+import { chunkFieldDefinition } from './chunkFieldDefinition'
+import { ChunkingEmitter } from './ChunkingEmitter'
+
+export default class MockChunkingEmitter implements ChunkingEmitter {
+	private didEmit = false
+	private emittedEventName?: EventName
+	private emittedItems?: Record<string, unknown>[]
+	private emittedPayloadKey?: string
+
+	public async emit(options: {
+		eventName: EventName
+		items: Record<string, unknown>[]
+		payloadKey: string
+	}): Promise<void> {
+		const { eventName, items, payloadKey } = options
+		this.didEmit = true
+		this.emittedEventName = eventName
+		this.emittedItems = items
+		this.emittedPayloadKey = payloadKey
+	}
+
+	public assertDidEmitPayloadKey(payloadKey: string) {
+		this.assertDidEmit()
+		assert.isEqual(
+			this.emittedPayloadKey,
+			payloadKey,
+			`I expected chunkingEmitter.emit() with payloadKey '${payloadKey}'! But you emitted ${
+				this.emittedPayloadKey || 'nothing'
+			}`
+		)
+	}
+
+	public getTotalErrors(): number {
+		return 0
+	}
+
+	public assertEmittedItems(items: Record<string, any>[]) {
+		this.assertDidEmit()
+		assert.isEqualDeep(
+			this.emittedItems,
+			items,
+			'You did not chunkingEmitter.emit(...) the expected items!!'
+		)
+	}
+
+	public assertDidEmit() {
+		assert.isTrue(
+			this.didEmit,
+			`You did not ever call chunkingEmitter.emit(...). That is your next step!`
+		)
+	}
+
+	public assertDidEmitEventNamed(fqen: EventName) {
+		this.assertDidEmit()
+		assert.isEqual(
+			this.emittedEventName,
+			fqen,
+			`I expected chunkingEmitter to emit '${fqen}'! But you emitted ${
+				this.emittedEventName || 'nothing'
+			}`
+		)
+	}
+
+	public assertEventHonorsChunkingSignature(fqen: EventName) {
+		const emitter = MercuryTestClient.getInternalEmitter()
+		const contract = emitter.getContract()
+		const payload = contract.eventSignatures[fqen]?.emitPayloadSchema?.fields
+		// @ts-ignore
+		const chunkField = payload?.payload?.options?.schema?.fields?.chunk
+		assert.isTruthy(
+			chunkField,
+			'Your event does not conform to the chunking signature. Please add a chunk field to your event payload. Add a field to your payload called chunk: chunkFieldDefinition()'
+		)
+
+		assert.isEqualDeep(
+			chunkField,
+			chunkFieldDefinition(),
+			'Your chunk field is there but not properly formed. Use chunkFieldDefinition() to define it.'
+		)
+	}
+}

package/src/chunkingEmitter/chunkFieldDefinition.ts

@@ -0,0 +1,12 @@
+import { FieldDefinitions } from '@sprucelabs/schema'
+import chunkPagingSchema from '#spruce/schemas/chunkingEmitter/v2023_10_21/chunkPaging.schema'
+
+export function chunkFieldDefinition() {
+	return {
+		type: 'schema',
+		isRequired: true,
+		options: {
+			schema: chunkPagingSchema,
+		},
+	} as FieldDefinitions
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+import { ChunkingEmitter } from './chunkingEmitter/ChunkingEmitter'
+export default ChunkingEmitter
+export { default as MockChunkingEmitter } from './chunkingEmitter/MockChunkingEmitter'