@luxonis/depthai-viewer-common 0.0.1

package/src/dai-connection/connection.ts

@@ -0,0 +1,402 @@
+import type { Pipeline } from '@luxonis/depthai-pipeline-lib';
+import { type PublicChannelData, VisualizerConnection } from '@luxonis/visualizer-protobuf';
+// TODO: Outdated import. Replace after updating visualizer-protobuf
+import type { VisualizerConnectionArgs } from '@luxonis/visualizer-protobuf/dist/lib/src/connection/connection.js';
+import { DAI_SERVICE_NAME } from '../constants/connection.js';
+import type { Topic } from '../hooks/connection.js';
+import type { DecoderWorkerInput, DecoderWorkerOutput } from './decoder.worker.js';
+import _ from 'lodash';
+import { StreamTypes } from '../constants/streams.js';
+
+export type DAIConnectionInitArgs = {
+	type: 'ws' | 'webrtc';
+	connectionUrl: string;
+	clientId: string;
+	token: string;
+	automaticTokenRefresh?: (token: string) => void;
+} & {
+	enabledStreams: string[];
+};
+
+type DaiConnectionEventMap = {
+	open: () => void;
+	error: (error: MessageEvent<unknown> | ErrorEvent) => void;
+	close: () => void;
+	pipeline: (pipeline: Pipeline | null) => void;
+	topics: (topics: Topic[]) => void;
+	disconnect: (error: ErrorEvent) => void;
+};
+
+export type DAIService = keyof typeof DAI_SERVICE_NAME;
+
+export class DAIConnection {
+	alive = true;
+	connection?: VisualizerConnection;
+
+	// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+	protected listeners: Partial<Record<keyof DaiConnectionEventMap, any[]>> = {};
+	// biome-ignore lint/suspicious/noExplicitAny: Typed only on a usage level
+	protected serviceListeners: Record<string, (data: any) => void> = {};
+
+	protected enabledTopics: string[] = [];
+	protected previouslyEnabledTopics: string[] = [];
+
+	protected decoderWorker!: Worker;
+	protected allowedServices: Set<DAIService> = new Set();
+	protected activeServices: Set<DAIService> = new Set([]);
+	protected serviceInterval: NodeJS.Timeout | null = null;
+
+	protected topicsTouched = false;
+	protected channels: PublicChannelData[] = [];
+	protected groups: Record<string, string> = {};
+	protected topics: Topic[] = [];
+	protected pipeline: Pipeline | null = null;
+
+	protected recreateDecoderWorker() {
+		this.decoderWorker?.terminate();
+		this.decoderWorker = new Worker(new URL('./decoder.worker.js', import.meta.url), {
+			type: 'module',
+			/* @vite-ignore */
+			basePath: window.__basePath,
+		} as WorkerOptions);
+
+		this.decoderWorker.addEventListener('error', (event) =>
+			console.error('Decoder Worker Error:', event.message, event),
+		);
+		this.decoderWorker.addEventListener('messageerror', (event) =>
+			console.error('Decoder Worker Error (message):', event.data, event),
+		);
+
+		this.decoderWorker.addEventListener(
+			'message',
+			({ data: { kind, data, name } }: MessageEvent<DecoderWorkerOutput>) => {
+				if (kind === 'pipeline') {
+					if (JSON.stringify(this.pipeline) !== JSON.stringify(data)) {
+						this.pipeline = data;
+						for (const listener of this.listeners.pipeline ?? []) {
+							listener(data);
+						}
+					}
+				} else if (name) {
+					this.serviceListeners[name]?.(data);
+				} else {
+					this.serviceListeners[data.name]?.(data);
+				}
+			},
+		);
+	}
+
+	protected reportTopics(type?: 'ws' | 'webrtc') {
+		if (!this.groups || !this.channels || !this.alive) {
+			return;
+		}
+
+		const topicList: (Omit<Topic, 'annotations' | 'extraAnnotations'> & {
+			group: string | null;
+		})[] = [];
+		const annotationGroups: Record<string, string[]> = {};
+		const annotations: string[] = [];
+
+		for (const channel of this.channels) {
+			const group = this.groups[channel.topic];
+
+			if (channel.kind === 'annotations') {
+				annotations.push(channel.topic);
+				if (group) {
+					annotationGroups[group] ??= [];
+					annotationGroups[group].push(channel.topic);
+				}
+
+				continue;
+			}
+
+			const shouldOpenDefaultStream =
+				(this.enabledTopics.length === 0 ||
+					(this.enabledTopics.length !== 0 &&
+						this.enabledTopics.includes(StreamTypes.NeuralNetwork))) &&
+				!this.topicsTouched;
+
+			if (shouldOpenDefaultStream) {
+				// If the connection is over WebSockets we want to open RAW stream as default stream
+				if (
+					type === 'ws' &&
+					_.isEqual(channel.topic.toLowerCase(), StreamTypes.Raw.toLowerCase())
+				) {
+					this.topicsTouched = true;
+					this.enabledTopics.push(channel.topic);
+
+					if (!this.enabledTopics.includes(StreamTypes.NeuralNetwork)) {
+						this.enabledTopics.push(StreamTypes.NeuralNetwork);
+					}
+					// If the connection is over WebRTC we want to open ENCODED stream as default stream
+				} else if (
+					type === 'webrtc' &&
+					_.isEqual(channel.topic.toLowerCase(), StreamTypes.Encoded.toLowerCase())
+				) {
+					this.topicsTouched = true;
+					this.enabledTopics.push(channel.topic);
+
+					if (!this.enabledTopics.includes(StreamTypes.NeuralNetwork)) {
+						this.enabledTopics.push(StreamTypes.NeuralNetwork);
+					}
+				}
+			}
+
+			topicList.push({
+				name: channel.topic,
+				kind: channel.kind,
+				enabled: this.enabledTopics.includes(channel.topic),
+				group: this.groups[channel.topic] ?? null,
+			});
+		}
+
+		const fullAnnotationGroups = Object.fromEntries(
+			Object.entries(annotationGroups).map(([group, topics]) => [
+				group,
+				{
+					topics,
+					extra: annotations.filter((annotation) => !topics.includes(annotation)),
+				},
+			]),
+		);
+
+		const updatedTopics = topicList.map<Topic>((topic) => ({
+			...topic,
+			annotations: fullAnnotationGroups[topic.group ?? '']?.topics ?? annotations,
+			extraAnnotations: fullAnnotationGroups[topic.group ?? '']?.extra ?? [],
+		}));
+
+		if (JSON.stringify(this.topics) !== JSON.stringify(updatedTopics)) {
+			const [oldTopics, newTopics] = updatedTopics.reduce<[Set<string>, Set<string>]>(
+				(sets, topic) => {
+					const set = sets[Number(topic.enabled)];
+					for (const name of [topic.name, ...topic.annotations, ...topic.extraAnnotations]) {
+						set.add(name);
+					}
+					return sets;
+				},
+				[new Set(), new Set()],
+			);
+
+			for (const annotation of newTopics) {
+				oldTopics.delete(annotation);
+			}
+
+			// TODO: Rewrite lib to control enabled annotations
+			// This setup subscribes to all annotations regardless of the enabled state
+			if (oldTopics.size !== 0) {
+				this.connection?.unsubscribe({ topics: [...oldTopics] });
+			}
+
+			if (newTopics.size !== 0) {
+				this.connection?.subscribe({ topics: [...newTopics] });
+			}
+
+			this.topics = updatedTopics;
+			for (const listener of this.listeners.topics ?? []) {
+				listener(updatedTopics);
+			}
+		}
+	}
+
+	protected pollServices() {
+		if (this.serviceInterval) {
+			clearInterval(this.serviceInterval);
+		}
+		this.serviceInterval = setInterval(() => {
+			for (const service of this.activeServices) {
+				this.fetchService(service);
+			}
+		}, 3_000);
+	}
+
+	init({ enabledStreams, ...args }: DAIConnectionInitArgs) {
+		if (enabledStreams && enabledStreams.length > 0) {
+			console.debug('Initializing connection with predefined streams', enabledStreams);
+			this.enabledTopics = enabledStreams;
+		}
+
+		this.recreateDecoderWorker();
+		this.setupConnection(args);
+	}
+
+	setupConnection({
+		type,
+		connectionUrl,
+		clientId,
+		token,
+		automaticTokenRefresh,
+	}: Pick<
+		DAIConnectionInitArgs,
+		'type' | 'clientId' | 'connectionUrl' | 'token' | 'automaticTokenRefresh'
+	>) {
+		// TODO: Enable Luxonis ICE Servers
+		this.connection = new VisualizerConnection({
+			type: type as never,
+			data: (type === 'ws'
+				? ({ connectionUrl } as (VisualizerConnectionArgs & {
+						type: 'ws';
+					})['data'])
+				: ({
+						config: {
+							clientId,
+							applicationIdentifier: '', // TODO:
+							authPayload: token,
+							signalingConnectionRetries: 5,
+							useLuxonisIceServers: true,
+						},
+						automaticTokenRefresh,
+					} as (VisualizerConnectionArgs & {
+						type: 'webrtc';
+					})['data'])) as never,
+		});
+
+		this.connection.on('open', () => {
+			if (!this.alive) return;
+			for (const listener of this.listeners.open ?? []) {
+				listener();
+			}
+		});
+
+		this.connection.on('close', () => {
+			if (!this.alive) return;
+			this.dispose();
+		});
+
+		this.connection.on('disconnect', (error) => {
+			for (const listener of this.listeners.disconnect ?? []) {
+				listener(error);
+			}
+		});
+
+		this.connection.on('services', (message) => {
+			if (!this.alive) return;
+			this.allowedServices.clear();
+			for (const service of message.data) {
+				this.allowedServices.add(service as DAIService);
+				if (
+					service === DAI_SERVICE_NAME.getDevices ||
+					service === DAI_SERVICE_NAME.pipelineSchema ||
+					service === DAI_SERVICE_NAME.topicGroups
+				) {
+					this.fetchService(service as DAIService);
+				}
+			}
+			this.pollServices();
+		});
+
+		this.connection.on('channels', (message) => {
+			if (!this.alive) return;
+			this.channels = message.data;
+			this.reportTopics(type);
+
+			// If no default topic is enabled we want to enable at least one topic
+			if (this.enabledTopics.length === 0 && this.channels.at(0)?.topic) {
+				console.warn(
+					`No default topic is enabled - using ${this.channels.at(0)?.topic} as default topic.`,
+				);
+
+				this.toggleTopic(this.channels.at(0)!.topic);
+			}
+		});
+	}
+
+	async fetchService(
+		service: DAIService,
+		// biome-ignore lint/suspicious/noExplicitAny: Can't possibly be typed, we don't know request specifics
+		body?: Record<string, any> | string | number,
+	) {
+		if (!this.alive) {
+			console.warn(`DAI: fetchService called while not alive for service "${service}"`);
+			return;
+		}
+
+		const canPoll = this.allowedServices.has(service);
+
+		if (canPoll) {
+			const data = await this.connection?.request({
+				name: DAI_SERVICE_NAME[service],
+				timeout: 60_000,
+				// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+				body: body as Record<string, any>,
+			});
+
+			if (data) {
+				this.decoderWorker.postMessage(
+					{
+						kind: service === 'pipelineSchema' ? 'pipeline' : 'json',
+						name: service,
+						data,
+					} as DecoderWorkerInput,
+					[data.buffer],
+				);
+			}
+		} else {
+			console.warn(
+				`DAI: fetchService called for service "${service}" while it's not allowed to poll`,
+			);
+		}
+	}
+
+	setActiveServices(services: DAIService[]) {
+		this.activeServices = new Set(services);
+	}
+
+	dispose() {
+		this.alive = false;
+		this.decoderWorker.terminate();
+		if (this.serviceInterval) {
+			clearInterval(this.serviceInterval);
+		}
+		this.connection?.close();
+		for (const listener of this.listeners.close ?? []) {
+			listener();
+		}
+	}
+
+	handleKeyPress(key: string) {
+		void this.connection?.request({
+			name: DAI_SERVICE_NAME.keyPressed,
+			body: { key },
+			ignoreResponse: true,
+		});
+	}
+
+	toggleTopic(topic: string) {
+		this.topicsTouched = true;
+		const enabledIndex = this.enabledTopics.indexOf(topic);
+
+		if (enabledIndex === -1) {
+			this.enabledTopics.push(topic);
+		} else {
+			this.enabledTopics.splice(enabledIndex, 1);
+		}
+
+		this.reportTopics();
+	}
+
+	stopTopics() {
+		this.topicsTouched = false;
+		this.previouslyEnabledTopics = this.enabledTopics;
+
+		this.enabledTopics = [];
+		this.reportTopics();
+	}
+
+	restartTopics() {
+		this.topicsTouched = true;
+		this.enabledTopics = this.previouslyEnabledTopics;
+		this.previouslyEnabledTopics = [];
+
+		this.reportTopics();
+	}
+
+	on<T extends keyof DaiConnectionEventMap>(event: T, callback: DaiConnectionEventMap[T]) {
+		this.listeners[event] ??= [];
+		this.listeners[event].push(callback);
+	}
+
+	setOnService<T>(service: string, callback: (data: T) => void) {
+		this.serviceListeners[service] = callback;
+	}
+}

package/src/dai-connection/decoder.worker.ts

@@ -0,0 +1,50 @@
+import {
+	type Pipeline,
+	parsePipeline,
+} from '../../../../node_modules/@luxonis/depthai-pipeline-lib/dist/src/services/pipeline.js';
+
+function decodeService(kind: 'pipeline' | 'json', response: DataView | null) {
+	if (!response) {
+		return null;
+	}
+
+	try {
+		const decoder = new TextDecoder('utf-8');
+		const decodedPayload = decoder.decode(response);
+		let parsed = JSON.parse(decodedPayload);
+
+		if (kind === 'pipeline') {
+			parsed = parsePipeline(parsed);
+		}
+
+		return parsed;
+	} catch (error) {
+		console.error('Error decoding JSON:', error);
+		return null;
+	}
+}
+
+export type DecoderWorkerInput = {
+	kind: 'pipeline' | 'json';
+	name?: string;
+	data: DataView;
+};
+
+export type DecoderWorkerOutput = { name?: string } & (
+	| {
+			kind: 'pipeline';
+			data: Pipeline;
+	  }
+	| {
+			kind: 'json';
+			data: Record<string, string>;
+	  }
+);
+
+self.addEventListener(
+	'message',
+	({ data: { kind, data, name } }: MessageEvent<DecoderWorkerInput>) => {
+		const decoded = decodeService(kind, data);
+		self.postMessage({ kind, name, data: decoded } as DecoderWorkerOutput);
+	},
+);

package/src/hooks/connection.ts

@@ -0,0 +1,191 @@
+import { useToast } from '@luxonis/common-fe-components';
+import type { Pipeline } from '@luxonis/depthai-pipeline-lib';
+import type { VisualizerConnection } from '@luxonis/visualizer-protobuf';
+import React from 'react';
+import { StreamTypes } from '../constants/streams.js';
+import { DAIConnection, type DAIService } from '../dai-connection/connection.js';
+import { useFlatSearch } from './search.js';
+
+export type Topic = {
+	name: string;
+	enabled: boolean;
+	kind: 'pointCloud' | 'image';
+	annotations: string[];
+	extraAnnotations: string[];
+};
+
+export type ConnectionData = {
+	connected: boolean;
+	daiConnection: DAIConnection | null;
+	connections: VisualizerConnection[];
+	pipeline: Pipeline | null | undefined;
+	isPipelineLoading: boolean;
+	setIsPipelineLoading: (isLoading: boolean) => void;
+	topics: Topic[];
+	toggleTopic: (topic: string) => void;
+};
+
+export type UseCreateConnectionArgs = {
+	type: 'webrtc' | 'ws';
+	connectionUrl: string;
+	token: string;
+	clientId: string;
+	activeServices: DAIService[];
+	onNewConnection?: (connection: DAIConnection) => void;
+};
+
+export function useCreateConnection({
+	type,
+	activeServices,
+	onNewConnection,
+	...params
+}: UseCreateConnectionArgs): ConnectionData {
+	const { updateSearch, search } = useFlatSearch({
+		defaultInit: { streams: [], cid: '', t: '' },
+		arrayFields: ['streams'],
+	});
+	const [connected, setConnected] = React.useState(false);
+	const [topics, setTopics] = React.useState<Topic[]>([]);
+	const [pipeline, setPipeline] = React.useState<Pipeline | null | undefined>(undefined);
+	const [isPipelineLoading, setIsPipelineLoading] = React.useState(false);
+
+	const [connection, setConnection] = React.useState<DAIConnection | null>(null);
+
+	const { toast } = useToast();
+	const handleGetCameraIntrinsics = () => {
+		if (connection) {
+			const getCameraIntrinsics = localStorage.getItem('neuralCameraIntrinsics');
+			if (!getCameraIntrinsics) {
+				connection.setOnService('getCameraIntrinsics', (data) => {
+					// @ts-ignore - The responce is not typed
+					localStorage.setItem('neuralCameraIntrinsics', JSON.stringify(data.data));
+				});
+				connection.fetchService('getCameraIntrinsics');
+			}
+		} else {
+			console.warn('Failed to get camera intrinsics, no connection');
+		}
+	};
+	handleGetCameraIntrinsics();
+
+	const refreshConnection = () => {
+		const newConnection = new DAIConnection();
+
+		newConnection.on('error', console.error);
+		newConnection.on('open', () => {
+			setConnected(true);
+			handleGetCameraIntrinsics();
+			setIsPipelineLoading(true);
+		});
+		newConnection.on('close', () => {
+			setConnected(false);
+			setTopics([]);
+			setPipeline(undefined);
+			setIsPipelineLoading(false);
+		});
+		newConnection.on('topics', setTopics);
+		newConnection.on('pipeline', (pipeline) => {
+			setPipeline(pipeline);
+			setIsPipelineLoading(false);
+		});
+		newConnection.on('disconnect', (_error) => {
+			toast({
+				description:
+					'The link to the application frontend is not valid. You need to open the frontend from Luxonis Hub again.',
+				colorVariant: 'error',
+				duration: 'long',
+				name: 'connection-error',
+			});
+			setConnected(false);
+			setTopics([]);
+			setPipeline(undefined);
+		});
+		onNewConnection?.(newConnection);
+
+		if (newConnection.alive) {
+			newConnection.init({
+				type,
+				...params,
+				automaticTokenRefresh: (token) => {
+					updateSearch('t', token);
+				},
+				enabledStreams: search.streams as string[],
+			});
+
+			setConnection(newConnection);
+		}
+	};
+
+	// biome-ignore lint/correctness/useExhaustiveDependencies: `params` will always change on re-renders
+	React.useEffect(() => {
+		if (!connected) {
+			refreshConnection();
+		} else {
+			handleGetCameraIntrinsics();
+		}
+	}, [params.clientId, params.connectionUrl, params.token, onNewConnection, connected]);
+
+	React.useEffect(() => {
+		connection?.setActiveServices(activeServices);
+	}, [connection, activeServices]);
+
+	const connections = React.useMemo(
+		() => (connection?.connection ? [connection.connection] : []),
+		[connection],
+	);
+
+	React.useEffect(() => {
+		if (connected) {
+			const callback = (event: KeyboardEvent) => connection?.handleKeyPress(event.key);
+			window.addEventListener('keydown', callback);
+			return () => window.removeEventListener('keydown', callback);
+		}
+	}, [connected, connection]);
+
+	React.useEffect(() => {
+		if (topics.length > 0) {
+			if (
+				topics.find((topic) => topic.name === StreamTypes.PointCloud)?.enabled &&
+				topics.find((t) => t.name === StreamTypes.Encoded)?.enabled === false
+			) {
+				connection?.toggleTopic(StreamTypes.Encoded);
+			}
+			updateSearch(
+				'streams',
+				topics.filter((topic) => topic.enabled).map((topic) => topic.name),
+			);
+		}
+	}, [topics, updateSearch]);
+
+	const toggleTopic = React.useCallback(
+		(topicName: string, isRawSteamEnabledManually?: boolean): void => {
+			if (
+				topicName === StreamTypes.PointCloud &&
+				topics.find((topic) => topic.name === topicName)?.enabled === false &&
+				topics.find((t) => t.name === StreamTypes.Encoded)?.enabled === false
+			) {
+				connection?.toggleTopic(StreamTypes.Encoded);
+			}
+			if (
+				topicName === StreamTypes.PointCloud &&
+				isRawSteamEnabledManually === false &&
+				topics.find((n) => n.name === 'Point Cloud')?.enabled
+			) {
+				connection?.toggleTopic(StreamTypes.Encoded);
+			}
+			connection?.toggleTopic(topicName);
+		},
+		[connection],
+	);
+
+	return {
+		connected,
+		daiConnection: connection,
+		connections,
+		pipeline,
+		isPipelineLoading,
+		setIsPipelineLoading,
+		topics,
+		toggleTopic,
+	};
+}

package/src/hooks/navigation.ts

@@ -0,0 +1,25 @@
+import React from 'react';
+import { useLocation } from 'react-router-dom';
+import { parseUrl as defaultParseUrl } from '../utils/url.js';
+
+export function useNavigation(parseUrl?: (pathname: string) => { basePath: string; page: string }) {
+	const location = useLocation();
+
+	// biome-ignore lint/correctness/useExhaustiveDependencies: <explanation>
+	const { page, basePath } = React.useMemo(
+		() => (parseUrl ?? defaultParseUrl)(location.pathname),
+		[location.pathname],
+	);
+
+	const makePath = React.useCallback(
+		(page: string, options: { noSearch?: boolean; rootPage?: string } = {}) => {
+			const { noSearch = false, rootPage = 'streams' } = options;
+			const target = page === rootPage ? '' : page;
+
+			return `/${basePath}${basePath ? '/' : ''}${target}${noSearch ? '' : location.search}`;
+		},
+		[basePath, location.search],
+	);
+
+	return { basePath, page, makePath };
+}

package/src/hooks/search.ts

@@ -0,0 +1,45 @@
+import React from 'react';
+import { useSearchParams } from 'react-router-dom';
+
+export type UseFlatSearchArgs = {
+	defaultInit?: Record<string, string | string[]>;
+	arrayFields?: string[];
+};
+
+export function useFlatSearch({ defaultInit = {}, arrayFields }: UseFlatSearchArgs) {
+	const [search, setSearch] = useSearchParams(defaultInit);
+
+	const flatSearch = React.useMemo(
+		() => ({
+			...defaultInit,
+			...Object.fromEntries(
+				[...search.entries()].map(([key, value]) => [
+					key,
+					arrayFields?.includes(key) ? (value.length === 0 ? [] : value.split(',')) : value,
+				]),
+			),
+		}),
+		[arrayFields, defaultInit, search],
+	);
+
+	const updateSearch = React.useCallback(
+		(key: string, value: string | number | string[] | number[] | null) => {
+			setSearch((current) => {
+				if (Array.isArray(value)) {
+					current.set(key, value.join(','));
+				} else if (value !== null) {
+					current.set(key, value.toString());
+				} else {
+					current.delete(key);
+				}
+				return current;
+			});
+		},
+		[setSearch],
+	);
+
+	return {
+		search: flatSearch,
+		updateSearch: updateSearch,
+	};
+}