atom.io 0.18.0 → 0.18.1

package/realtime-server/src/ipc-sockets/parent-socket.ts

@@ -0,0 +1,185 @@
+import { IMPLICIT, Subject } from "atom.io/internal"
+import { parseJson, stringifyJson } from "atom.io/json"
+import type { Json } from "atom.io/json"
+
+import { SetRTX } from "atom.io/transceivers/set-rtx"
+import { CustomSocket } from "./custom-socket"
+import type { EventBuffer, Events } from "./custom-socket"
+
+export class SubjectSocket<
+	I extends Events,
+	O extends Events,
+> extends CustomSocket<I, O> {
+	public in: Subject<[string, ...Json.Serializable[]]>
+	public out: Subject<[string, ...Json.Serializable[]]>
+	public id = `no_id_retrieved`
+	public disposalFunctions: (() => void)[] = []
+
+	public constructor(id: string) {
+		super((...args) => {
+			this.out.next(args as any)
+			return this
+		})
+		this.id = id
+		this.in = new Subject()
+		this.out = new Subject()
+		this.in.subscribe(`socket`, (event) => {
+			this.handleEvent(...(event as [string, ...I[keyof I]]))
+		})
+	}
+
+	public dispose(): void {
+		for (const dispose of this.disposalFunctions) {
+			dispose()
+		}
+	}
+}
+
+export class ParentSocket<
+	I extends Events & {
+		[id in string as `user:${id}`]: [string, ...Json.Serializable[]]
+	} & {
+		/* eslint-disable quotes */
+		"user-joins": [string]
+		"user-leaves": [string]
+		/* eslint-enable quotes */
+	},
+	O extends Events & {
+		[id in string as `relay:${id}`]: [string, ...Json.Serializable[]]
+	},
+> extends CustomSocket<I, O> {
+	protected incompleteData = ``
+	protected unprocessedEvents: string[] = []
+	protected relays: Map<string, SubjectSocket<any, any>>
+	protected relayServices: ((
+		socket: SubjectSocket<any, any>,
+	) => (() => void) | void)[]
+	protected process: NodeJS.Process
+
+	public id = `#####`
+
+	protected log(...args: any[]): void {
+		this.process.stderr.write(
+			stringifyJson(
+				args.map((arg) =>
+					arg instanceof SetRTX
+						? `{ ${arg.toJSON().members.join(` | `)} }`
+						: arg,
+				),
+			) + `\x03`,
+		)
+	}
+	public logger = {
+		info: (...args: any[]): void => this.log(`i`, ...args),
+		warn: (...args: any[]): void => this.log(`w`, ...args),
+		error: (...args: any[]): void => this.log(`e`, ...args),
+	}
+
+	public constructor() {
+		super((event, ...args) => {
+			const stringifiedEvent = JSON.stringify([event, ...args])
+			this.process.stdout.write(stringifiedEvent + `\x03`)
+			return this
+		})
+		this.process = process
+		this.process.stdin.resume()
+		this.relays = new Map()
+		this.relayServices = []
+
+		this.process.stdin.on(
+			`data`,
+			<Event extends keyof I>(buffer: EventBuffer<string, I[Event]>) => {
+				const chunk = buffer.toString()
+				this.unprocessedEvents.push(...chunk.split(`\x03`))
+				const newInput = this.unprocessedEvents.shift()
+				this.incompleteData += newInput || ``
+
+				try {
+					const parsedEvent = parseJson(this.incompleteData)
+					this.logger.info(`🎰`, `received`, parsedEvent)
+					this.handleEvent(...(parsedEvent as [string, ...I[keyof I]]))
+					while (this.unprocessedEvents.length > 0) {
+						const event = this.unprocessedEvents.shift()
+						if (event) {
+							if (this.unprocessedEvents.length === 0) {
+								this.incompleteData = event
+							}
+							const parsedEvent = parseJson(event)
+							this.handleEvent(...(parsedEvent as [string, ...I[keyof I]]))
+						}
+					}
+					this.incompleteData = ``
+				} catch (thrown) {
+					if (thrown instanceof Error) {
+						this.logger.error(`❗`, thrown.message, thrown.cause, thrown.stack)
+					}
+				}
+			},
+		)
+
+		this.on(`exit`, () => {
+			process.exit(0)
+		})
+		process.on(`exit`, () => {
+			this.logger.info(`🔥`, this.id, `exited`)
+			process.exit(0)
+		})
+		process.on(`end`, () => {
+			this.logger.info(`🔥`, this.id, `ended`)
+			process.exit(0)
+		})
+		process.on(`SIGTERM`, () => {
+			this.logger.error(`🔥`, this.id, `terminated`)
+			process.exit(0)
+		})
+		process.on(`SIGINT`, () => {
+			this.logger.error(`🔥`, this.id, `interrupted`)
+			process.exit(0)
+		})
+
+		if (process.pid) {
+			this.id = process.pid?.toString()
+		}
+
+		this.on(`user-joins`, (username) => {
+			this.logger.info(`👤`, `user`, username, `joined`)
+			const relay = new SubjectSocket(`user:${username}`)
+			this.relays.set(username, relay)
+			this.logger.info(
+				`🔗`,
+				`attaching services:`,
+				`[${[...this.relayServices.keys()].join(`, `)}]`,
+			)
+			for (const attachServices of this.relayServices) {
+				const cleanup = attachServices(relay)
+				if (cleanup) {
+					relay.disposalFunctions.push(cleanup)
+				}
+			}
+			this.on(`user:${username}`, (...data) => {
+				relay.in.next(data)
+			})
+			relay.out.subscribe(`socket`, (data) => {
+				this.emit(...(data as [string, ...O[keyof O]]))
+			})
+		})
+
+		this.on(`user-leaves`, (username) => {
+			const relay = this.relays.get(username)
+			this.off(`relay:${username}`)
+			if (relay) {
+				relay.dispose()
+				this.relays.delete(username)
+			}
+		})
+
+		process.stdout.write(`✨`)
+	}
+
+	public relay(
+		attachServices: (socket: SubjectSocket<any, any>) => (() => void) | void,
+	): void {
+		this.logger.info(`🔗`, `running relay method`)
+		this.relayServices.push(attachServices)
+	}
+}

package/realtime-server/src/realtime-continuity-synchronizer.ts

@@ -4,8 +4,8 @@ import {
 	actUponStore,
 	findInStore,
 	getFromStore,
+	getJsonToken,
 	isRootStore,
-	setIntoStore,
 	subscribeToState,
 	subscribeToTransaction,
 } from "atom.io/internal"
@@ -14,9 +14,9 @@ import type { ContinuityToken } from "atom.io/realtime"
 
 import type { ServerConfig, Socket } from "."
 import { socketAtoms, usersOfSockets } from "."
-import { redactedPerspectiveUpdateSelectors } from "./realtime-server-stores"
+
 import {
-	completeUpdateAtoms,
+	redactTransactionUpdateContent,
 	userUnacknowledgedQueues,
 } from "./realtime-server-stores"
 
@@ -88,69 +88,148 @@ export function realtimeContinuitySynchronizer({
 		)
 		const unsubscribeFunctions: (() => void)[] = []
 
+		const revealPerspectives = (): (() => void) => {
+			const unsubscribeFunctions: (() => void)[] = []
+			for (const perspective of continuity.perspectives) {
+				const { viewAtoms } = perspective
+				const userViewState = findInStore(viewAtoms, userKey, store)
+				const unsubscribe = subscribeToState(
+					userViewState,
+					({ oldValue, newValue }) => {
+						const oldKeys = oldValue.map((token) => token.key)
+						const newKeys = newValue.map((token) => token.key)
+						const concealed = oldValue.filter(
+							(token) => !newKeys.includes(token.key),
+						)
+						const revealed = newValue
+							.filter((token) => !oldKeys.includes(token.key))
+							.flatMap((token) => {
+								const resourceToken =
+									token.type === `mutable_atom` ? getJsonToken(token) : token
+								const resource = getFromStore(resourceToken, store)
+								return [resourceToken, resource]
+							})
+						store.logger.info(
+							`👁`,
+							`atom`,
+							perspective.resourceAtoms.key,
+							`${userKey} has a new perspective`,
+							{ oldKeys, newKeys, revealed, concealed },
+						)
+						if (revealed.length > 0) {
+							socket?.emit(`reveal:${continuityKey}`, revealed)
+						}
+						if (concealed.length > 0) {
+							socket?.emit(`conceal:${continuityKey}`, concealed)
+						}
+					},
+					`sync-continuity:${continuityKey}:${userKey}:perspective:${perspective.resourceAtoms.key}`,
+					store,
+				)
+				unsubscribeFunctions.push(unsubscribe)
+			}
+			return () => {
+				for (const unsubscribe of unsubscribeFunctions) unsubscribe()
+			}
+		}
+		const unsubscribeFromPerspectives = revealPerspectives()
+
 		const sendInitialPayload = () => {
 			const initialPayload: Json.Serializable[] = []
 			for (const atom of continuity.globals) {
-				initialPayload.push(atom, getFromStore(atom, store))
+				const resourceToken =
+					atom.type === `mutable_atom` ? getJsonToken(atom) : atom
+				initialPayload.push(resourceToken, getFromStore(atom, store))
 			}
-			for (const { perspectiveAtoms } of continuity.perspectives) {
-				const perspectiveTokensState = findInStore(
-					perspectiveAtoms,
-					userKey,
-					store,
-				)
-				const perspectiveTokens = getFromStore(perspectiveTokensState, store)
-				for (const perspectiveToken of perspectiveTokens) {
-					const resource = getFromStore(perspectiveToken, store)
-					initialPayload.push(perspectiveToken, resource)
+			for (const perspective of continuity.perspectives) {
+				const { viewAtoms, resourceAtoms } = perspective
+				const userViewState = findInStore(viewAtoms, userKey, store)
+				const userView = getFromStore(userViewState, store)
+				store.logger.info(`👁`, `atom`, resourceAtoms.key, `${userKey} can see`, {
+					viewAtoms,
+					resourceAtoms,
+					userView,
+				})
+				for (const visibleToken of userView) {
+					const resourceToken =
+						visibleToken.type === `mutable_atom`
+							? getJsonToken(visibleToken)
+							: visibleToken
+					const resource = getFromStore(resourceToken, store)
+
+					initialPayload.push(resourceToken, resource)
 				}
 			}
 
 			const epoch = isRootStore(store)
 				? store.transactionMeta.epoch.get(continuityKey) ?? null
 				: null
+
 			socket?.emit(`continuity-init:${continuityKey}`, epoch, initialPayload)
 
 			for (const transaction of continuity.actions) {
 				const unsubscribeFromTransaction = subscribeToTransaction(
 					transaction,
 					(update) => {
-						// store.logger.info(`userId`, userKey)
-						const updateState = findInStore(
-							completeUpdateAtoms,
-							update.id,
-							store,
-						)
-						setIntoStore(updateState, update, store)
-						const redactedUpdateKey = {
-							userId: userKey,
-							syncGroupKey: continuityKey,
-							updateId: update.id,
-						}
-						const redactedUpdateState = findInStore(
-							redactedPerspectiveUpdateSelectors,
-							redactedUpdateKey,
-							store,
-						)
+						try {
+							const visibleKeys = continuity.globals
+								.map((atom) => atom.key)
+								.concat(
+									continuity.perspectives.flatMap((perspective) => {
+										const { viewAtoms } = perspective
+										const userPerspectiveTokenState = findInStore(
+											viewAtoms,
+											userKey,
+											store,
+										)
+										const visibleTokens = getFromStore(
+											userPerspectiveTokenState,
+											store,
+										)
+										return visibleTokens.map((token) => {
+											const key =
+												token.type === `mutable_atom`
+													? `*` + token.key
+													: token.key
+											return key
+										})
+									}),
+								)
+							const redactedUpdates = redactTransactionUpdateContent(
+								visibleKeys,
+								update.updates,
+							)
+							const redactedUpdate = {
+								...update,
+								updates: redactedUpdates,
+							}
+							// setIntoStore(
+							// 	userUnacknowledgedQueue,
+							// 	(updates) => {
+							// 		if (redactedUpdate) {
+							// 			updates.push(redactedUpdate)
+							// 			updates.sort((a, b) => a.epoch - b.epoch)
+							// 		}
+							// 		return updates
+							// 	},
+							// 	store,
+							// )
 
-						const redactedUpdate = getFromStore(redactedUpdateState, store)
-
-						setIntoStore(
-							userUnacknowledgedQueue,
-							(updates) => {
-								if (redactedUpdate) {
-									updates.push(redactedUpdate)
-									updates.sort((a, b) => a.epoch - b.epoch)
-								}
-								return updates
-							},
-							store,
-						)
-
-						socket?.emit(
-							`tx-new:${continuityKey}`,
-							redactedUpdate as Json.Serializable,
-						)
+							socket?.emit(
+								`tx-new:${continuityKey}`,
+								redactedUpdate as Json.Serializable,
+							)
+						} catch (thrown) {
+							if (thrown instanceof Error) {
+								store.logger.error(
+									`❌`,
+									`continuity`,
+									continuityKey,
+									`failed to send update from transaction ${transaction.key} to ${userKey}`,
+									thrown.message,
+								)
+							}
+						}
 					},
 					`sync-continuity:${continuityKey}:${userKey}`,
 					store,
@@ -164,17 +243,30 @@ export function realtimeContinuitySynchronizer({
 		const fillTransactionRequest = (
 			update: Pick<AtomIO.TransactionUpdate<JsonIO>, `id` | `key` | `params`>,
 		) => {
+			store.logger.info(`🛎️`, `continuity`, continuityKey, `received`, update)
 			const transactionKey = update.key
 			const updateId = update.id
 			const performanceKey = `tx-run:${transactionKey}:${updateId}`
 			const performanceKeyStart = `${performanceKey}:start`
 			const performanceKeyEnd = `${performanceKey}:end`
 			performance.mark(performanceKeyStart)
-			actUponStore(
-				{ type: `transaction`, key: transactionKey },
-				updateId,
-				store,
-			)(...update.params)
+			try {
+				actUponStore(
+					{ type: `transaction`, key: transactionKey },
+					updateId,
+					store,
+				)(...update.params)
+			} catch (thrown) {
+				if (thrown instanceof Error) {
+					store.logger.error(
+						`❌`,
+						`continuity`,
+						continuityKey,
+						`failed to run transaction ${transactionKey} with update ${updateId}`,
+						thrown.message,
+					)
+				}
+			}
 			performance.mark(performanceKeyEnd)
 			const metric = performance.measure(
 				performanceKey,
@@ -188,58 +280,95 @@ export function realtimeContinuitySynchronizer({
 				updateId,
 				metric.duration,
 			)
-		}
-		socket.off(`tx-run:${continuityKey}`, fillTransactionRequest)
-		socket.on(`tx-run:${continuityKey}`, fillTransactionRequest)
 
-		let i = 1
-		let next = 1
-		const retry = setInterval(() => {
-			const toEmit = userUnacknowledgedUpdates[0]
-			store.logger.info(
-				`🔄`,
-				`continuity`,
-				continuityKey,
-				`${store.config.name} retrying ${userKey} (${i}/${next})`,
-				socket?.id,
-				userUnacknowledgedUpdates,
-			)
-			if (toEmit && i === next) {
-				socket?.emit(`tx-new:${continuityKey}`, toEmit as Json.Serializable)
-				next *= 2
-			}
+			const valuesOfCardsViewKey = `valuesOfCardsView("${userKey}")`
+			const rootsOfCardValueView =
+				store.selectorAtoms.getRelatedKeys(valuesOfCardsViewKey)
+			const myCardValueView = store.valueMap.get(valuesOfCardsViewKey)
 
-			i++
-		}, 250)
-		const trackClientAcknowledgement = (epoch: number) => {
 			store.logger.info(
-				`👍`,
+				`👁`,
 				`continuity`,
 				continuityKey,
-				`${userKey} acknowledged epoch ${epoch}`,
+				`seeing ${userKey} card values`,
+				{
+					valuesOfCardsViewKey,
+					rootsOfCardValueView,
+					myCardValueView,
+				},
 			)
-			i = 1
-			next = 1
-			const isUnacknowledged = userUnacknowledgedUpdates[0]?.epoch === epoch
-
-			if (isUnacknowledged) {
-				setIntoStore(
-					userUnacknowledgedQueue,
-					(updates) => {
-						updates.shift()
-						return updates
-					},
-					store,
-				)
-			}
 		}
-		socket.off(`ack:${continuityKey}`, trackClientAcknowledgement)
-		socket.on(`ack:${continuityKey}`, trackClientAcknowledgement)
+		socket.off(`tx-run:${continuityKey}`, fillTransactionRequest)
+		socket.on(`tx-run:${continuityKey}`, fillTransactionRequest)
+
+		// let i = 0
+		// let n = 1
+		// let retryTimeout: NodeJS.Timeout | undefined
+		// const trackClientAcknowledgement = (epoch: number) => {
+		// 	store.logger.info(
+		// 		`👍`,
+		// 		`continuity`,
+		// 		continuityKey,
+		// 		`${userKey} acknowledged epoch ${epoch}`,
+		// 	)
+		// 	const isUnacknowledged = userUnacknowledgedUpdates[0]?.epoch === epoch
+		// 	if (isUnacknowledged) {
+		// 		setIntoStore(
+		// 			userUnacknowledgedQueue,
+		// 			(updates) => {
+		// 				updates.shift()
+		// 				return updates
+		// 			},
+		// 			store,
+		// 		)
+		// 	}
+		// }
+		// subscribeToState(
+		// 	userUnacknowledgedQueue,
+		// 	({ newValue }) => {
+		// 		if (newValue.length === 0) {
+		// 			clearInterval(retryTimeout)
+		// 			socket?.off(`ack:${continuityKey}`, trackClientAcknowledgement)
+		// 			retryTimeout = undefined
+		// 		}
+		// 		if (newValue.length > 0) {
+		// 			if (retryTimeout) {
+		// 				return
+		// 			}
+
+		// 			socket?.on(`ack:${continuityKey}`, trackClientAcknowledgement)
+
+		// 			retryTimeout = setInterval(() => {
+		// 				i++
+		// 				if (i === n) {
+		// 					n += i
+		// 					const toEmit = newValue[0]
+		// 					if (!toEmit) return
+		// 					store.logger.info(
+		// 						`🔄`,
+		// 						`continuity`,
+		// 						continuityKey,
+		// 						`${store.config.name} retrying ${userKey}`,
+		// 						socket?.id,
+		// 						newValue,
+		// 					)
+		// 					socket?.emit(
+		// 						`tx-new:${continuityKey}`,
+		// 						toEmit as Json.Serializable,
+		// 					)
+		// 				}
+		// 			}, 250)
+		// 		}
+		// 	},
+		// 	`sync-continuity:${continuityKey}:${userKey}`,
+		// 	store,
+		// )
 
 		return () => {
-			clearInterval(retry)
+			// clearInterval(retryTimeout)
 			for (const unsubscribe of unsubscribeFunctions) unsubscribe()
-			socket?.off(`ack:${continuityKey}`, trackClientAcknowledgement)
+			// socket?.off(`ack:${continuityKey}`, trackClientAcknowledgement)
+			unsubscribeFromPerspectives()
 			socket?.off(`get:${continuityKey}`, sendInitialPayload)
 			socket?.off(`tx-run:${continuityKey}`, fillTransactionRequest)
 		}

package/realtime-server/src/realtime-server-stores/index.ts

@@ -1,4 +1,5 @@
 export * from "./realtime-continuity-store"
-export * from "./server-room-store"
+export * from "./server-room-external-actions"
+export * from "./server-room-external-store"
 export * from "./server-sync-store"
 export * from "./server-user-store"

package/realtime-server/src/realtime-server-stores/realtime-continuity-store.ts

@@ -1,8 +1,9 @@
 import { selectorFamily } from "atom.io"
 import type { TransactionUpdate } from "atom.io"
+import { IMPLICIT, getJsonToken, getUpdateToken } from "atom.io/internal"
 import type { JsonIO } from "atom.io/json"
 import { SyncGroup } from "atom.io/realtime"
-import { completeUpdateAtoms } from "atom.io/realtime-server"
+// import { completeUpdateAtoms } from "atom.io/realtime-server"
 
 const redactorAtoms = selectorFamily<
 	(update: TransactionUpdate<any>) => TransactionUpdate<any>,
@@ -20,14 +21,21 @@ const redactorAtoms = selectorFamily<
 			}
 
 			const userPerspectiveTokens = syncGroup.perspectives.flatMap(
-				({ perspectiveAtoms, resourceAtoms }) => {
-					const userPerspectiveToken = find(perspectiveAtoms, userId)
+				({ viewAtoms }) => {
+					const userPerspectiveToken = find(viewAtoms, userId)
 					const userPerspective = get(userPerspectiveToken)
-					const visibleTokens = [...userPerspective].map((subKey) => {
-						const resourceToken = find(resourceAtoms, subKey)
-						return resourceToken.key
+					const visibleTokens = [...userPerspective].map((token) => {
+						return token.type === `mutable_atom`
+							? getUpdateToken(token).key
+							: token.key
 					})
-
+					IMPLICIT.STORE.logger.info(
+						`🔭`,
+						`continuity`,
+						syncGroupKey,
+						`${userId} can see ${visibleTokens.length} tokens in ${viewAtoms.key}`,
+						visibleTokens,
+					)
 					return visibleTokens
 				},
 			)
@@ -36,6 +44,14 @@ const redactorAtoms = selectorFamily<
 				visible: string[],
 				transactionUpdate: TransactionUpdate<any>,
 			): TransactionUpdate<any> => {
+				IMPLICIT.STORE.logger.info(
+					`🖌`,
+					`continuity`,
+					syncGroupKey,
+					`redacting updates from ${transactionUpdate.epoch}:${transactionUpdate.key}:${transactionUpdate.id}`,
+					visible,
+					transactionUpdate.updates,
+				)
 				const updates = transactionUpdate.updates
 					.filter((update) => {
 						if (`newValue` in update) {
@@ -57,8 +73,10 @@ const redactorAtoms = selectorFamily<
 			}
 			const filter: (updates: TransactionUpdate<any>) => TransactionUpdate<any> =
 				(update) => {
-					const visibleKeys: string[] = syncGroup.globals.map(
-						(atomToken) => atomToken.key,
+					const visibleKeys: string[] = syncGroup.globals.map((atomToken) =>
+						atomToken.type === `mutable_atom`
+							? getUpdateToken(atomToken).key
+							: atomToken.key,
 					)
 					visibleKeys.push(...userPerspectiveTokens)
 					return filterTransactionUpdate(visibleKeys, update)
@@ -66,25 +84,26 @@ const redactorAtoms = selectorFamily<
 			return filter
 		},
 })
-export const redactedPerspectiveUpdateSelectors = selectorFamily<
-	Pick<
-		TransactionUpdate<JsonIO>,
-		`epoch` | `id` | `key` | `output` | `updates`
-	> | null,
-	{ userId: string; syncGroupKey: string; updateId: string }
->({
-	key: `redactedPerspectiveUpdate`,
-	get:
-		({ userId, syncGroupKey, updateId }) =>
-		({ get, find }) => {
-			const updateState = find(completeUpdateAtoms, updateId)
-			const update = get(updateState)
-			const redactorKey = { userId, syncGroupKey }
-			const redactorState = find(redactorAtoms, redactorKey)
-			const redact = get(redactorState)
-			if (update) {
-				return redact(update)
-			}
-			return null
-		},
-})
+// export const occludedUpdateSelectors = selectorFamily<
+// 	Pick<
+// 		TransactionUpdate<JsonIO>,
+// 		`epoch` | `id` | `key` | `output` | `updates`
+// 	> | null,
+// 	{ userId: string; syncGroupKey: string; updateId: string }
+// >({
+// 	key: `occludedUpdate`,
+// 	get:
+// 		({ userId, syncGroupKey, updateId }) =>
+// 		({ get, find }) => {
+// 			const updateState = find(completeUpdateAtoms, updateId)
+// 			const update = get(updateState)
+// 			const redactorKey = { userId, syncGroupKey }
+// 			const redactorState = find(redactorAtoms, redactorKey)
+// 			const redact = get(redactorState)
+// 			if (update) {
+// 				// return redact(update)
+// 				return update
+// 			}
+// 			return null
+// 		},
+// })