@wovin/core 0.0.0-ciao-mobx-955482e8

package/src/thread/utils.ts

@@ -0,0 +1,146 @@
+import { Logger } from 'besonders-logger'
+import { curry, debounce, partial, pull, uniq, uniqBy, uniqWith } from 'lodash-es'
+
+import { CID } from 'multiformats'
+import { ensureTsPvAndFinalizeApplogs, joinThreads } from '../applog/applog-helpers.ts'
+import { compareApplogsByEnAt, compareApplogsByTs, objEqualByKeys } from '../applog/applog-utils.ts'
+import { Applog, ApplogForInsert, CidString, EntityID } from '../applog/datom-types.ts'
+import { Thread } from './basic.ts'
+import { MappedThread } from './mapped.ts'
+import { ThreadInMemory, WriteableThread } from './writeable.ts'
+
+const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
+
+export function entityCount(thread: Thread) {
+	return allEntityIDs(thread).size // at some point: index size?
+}
+export function allEntityIDs(thread: Thread) {
+	return accumulateLogsToSet(thread, log => log.en)
+}
+
+function accumulateLogsToSet<ACC>(
+	threadOrLogs: Thread | Applog[],
+	callback: (log: Applog, acc: Set<ACC>) => ACC | undefined,
+) {
+	const logs = threadOrLogs instanceof Thread ? threadOrLogs.applogs : threadOrLogs
+	const set = new Set<ACC>()
+	for (const log of logs) {
+		set.add(callback(log, set))
+	}
+	return set
+}
+
+export function debounceWrites(
+	thread: Thread,
+	wait = 700,
+	removeDuplicatesWith = compareApplogsByEnAt,
+) {
+	if (thread.readOnly) throw ERROR(`[debounceWrites] but readOnly thread`, thread.name)
+	let insertQueue: Array<Applog | ApplogForInsert> = []
+
+	const debouncedCommit = debounce(() => {
+		WARN(`Debounce tail`, { thread, mappedThread, insertQueue })
+		const toInsert = ensureTsPvAndFinalizeApplogs(
+			// ? uniq, sure - but which one is used? (update: seems the first one, so reverse)
+			uniqWith(insertQueue.reverse(), removeDuplicatesWith),
+			thread,
+		)
+		thread.insertRaw(toInsert)
+		insertQueue.splice(0, insertQueue.length) // clear queue
+	}, wait)
+	const handleInsert = (applogs: Applog[] | ApplogForInsert[]) => {
+		DEBUG(`Debounce input:`, applogs, { thread, mappedThread, insertQueue })
+		insertQueue.push(...applogs)
+		debouncedCommit()
+		return null // don't insert anything
+	}
+
+	const mappedThread = MappedThread.mapWrites(
+		thread,
+		`Debounce(${wait})`,
+		handleInsert,
+	)
+	return mappedThread
+}
+
+/**
+ * @param deduplicateHoldItemsWith called with (heldLog, realLog), if it returns true, the held log is skipped
+ * @param onFirstWrite called with held logs about to be inserted, if it returns an array, those logs will be inserted instead
+ */
+export function holdTillFirstWrite(
+	thread: Thread,
+	applogsToHold: ApplogForInsert[],
+	opts: {
+		deduplicateHoldItemsWith?: typeof compareApplogsByEnAt
+		onFirstWrite: (heldLogsToInsert: Applog[]) => Applog[] | undefined
+	},
+) {
+	DEBUG(`[holdTillFirstWrite] holding logs:`, { applogsToHold })
+	if (thread.readOnly) throw ERROR(`[holdTillFirstWrite] but readOnly thread`, thread.nameAndSizeUntracked)
+	const heldLogs = ensureTsPvAndFinalizeApplogs([...applogsToHold], thread)
+	const heldThread = ThreadInMemory.fromArray(heldLogs)
+
+	let hasInserted = false
+	const handleInsert = (realApplogs: Applog[] | ApplogForInsert[]) => {
+		if (hasInserted) return realApplogs
+		hasInserted = true
+
+		let toInsert = [...heldLogs]
+		// heldLogs.splice(0, heldLogs.length) // ? joinThread could take care of this
+		if (opts.deduplicateHoldItemsWith) {
+			toInsert = toInsert.filter(heldLog =>
+				// some duplicate? so don't insert
+				!realApplogs.some(realLog => opts.deduplicateHoldItemsWith(heldLog, realLog))
+			)
+		}
+		if (opts.onFirstWrite) {
+			const callbackResult = opts.onFirstWrite(toInsert)
+			if (callbackResult !== undefined) {
+				toInsert = callbackResult
+			}
+		}
+		DEBUG(`[holdTillFirstWrite] adding hold logs:`, { applogsToHold, heldLogs, toInsert, realApplogs })
+		return [...toInsert, ...realApplogs]
+	}
+	const joinedThread = joinThreads([thread, heldThread])
+	return new MappedThread(
+		`HoldTillFirstWrite[${applogsToHold.length}]<${thread.nameAndSizeUntracked}>`,
+		joinedThread,
+		thread.filters,
+		// @ts-expect-error ? what's the proper way
+		joinedThread.applogs,
+		null,
+		(applogs) => {
+			const logsToInsert = handleInsert(applogs)
+			thread.insert(logsToInsert)
+			return null
+		},
+	)
+}
+
+export function getAgents(thread: Thread) {
+	return uniq(thread.map(l => l.ag))
+}
+
+/**
+ * Returns applogs NOT contained in the exclusion set.
+ * Uses CID-based comparison (consistent with removeDuplicateAppLogs).
+ */
+export const excludeApplogsContainedIn = (
+	applogs: readonly Applog[],
+	exclude: Thread | readonly Applog[] | Set<CID | CidString>,
+): Applog[] => {
+	const excludeCids: Set<string> = exclude instanceof Set
+		? new Set([...exclude].map(c => c.toString()))
+		: exclude instanceof Thread
+			? new Set(exclude.applogsCids)
+			: new Set(exclude.map(a => a.cid))
+
+	return applogs.filter(applog => {
+		if (!applog.cid) {
+			ERROR(`applog with missing CID`, applog)
+			throw new Error(`applog with missing CID`)
+		}
+		return !excludeCids.has(applog.cid)
+	})
+}

package/src/thread/writeable.ts

@@ -0,0 +1,163 @@
+import { Logger } from 'besonders-logger'
+import { ensureTsPvAndFinalizeApplogs } from '../applog/applog-helpers.ts'
+import { isTsBefore, removeDuplicateAppLogs, sortApplogsByTs } from '../applog/applog-utils.ts'
+import { type Applog, ApplogForInsert, CidString, isValidApplog } from '../applog/datom-types.ts'
+import { Thread } from './basic.ts'
+
+const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
+
+export abstract class WriteableThread extends Thread {
+	constructor(
+		name: string,
+		// parents: Thread | readonly Thread[] | null,  // ? would this ever be a thing
+		applogs: Applog[] = [],
+		filters: readonly string[],
+	) {
+		super(name, null, filters, applogs)
+	}
+
+	public purge(cidsToPurge: CidString[]) {
+		const beforeCount = this.applogs.length
+		// HACK setting readonly
+		this._applogs = this.applogs.filter(log => !(cidsToPurge.includes(log.cid)))
+		return beforeCount - this.applogs.length
+	}
+	public insert(appLogsToInsert: ApplogForInsert[]) {
+		DEBUG(`[WriteableThread.insert] ENTER - ${appLogsToInsert.length} applogs for thread "${this.name}"`)
+		DEBUG(`[WriteableThread.insert] About to call ensureTsPvAndFinalizeApplogs`)
+		const mapped = ensureTsPvAndFinalizeApplogs(appLogsToInsert, this)
+		DEBUG(`[WriteableThread.insert] ensureTsPvAndFinalizeApplogs completed, mapped=${mapped.length} applogs`)
+		DEBUG(`[WriteableThread.insert] About to call insertRaw`)
+		const result = this.insertRaw(mapped)
+		DEBUG(`[WriteableThread.insert] insertRaw completed`)
+		return result
+	}
+
+	/**
+	 * Insert only applogs not already in this thread.
+	 * @param byRef If true, compares by reference; if false, compares by CID (default)
+	 * @returns The applogs that were actually inserted
+	 */
+	public insertMissing(appLogsToInsert: readonly Applog[], byRef = false): Applog[] {
+		const missing = appLogsToInsert.filter(log => !this.hasApplog(log, byRef))
+		if (missing.length === 0) {
+			VERBOSE(`[insertMissing] no missing applogs`)
+			return []
+		}
+		return this.insertRaw(missing) ?? []
+	}
+
+	/**
+	 * Insert raw applogs directly into the thread.
+	 *
+	 * STRICT VALIDATION: This method throws errors for:
+	 * - Duplicate applogs in input array (programming error)
+	 * - Invalid applogs (missing required fields)
+	 * - Applogs already in thread (programming error)
+	 *
+	 * For external imports where duplicates are expected, use removeDuplicateAppLogs(..., 'cleanup')
+	 * before calling this method.
+	 *
+	 * @param appLogsToInsert Must be deduplicated and validated. Needs to be mutable because it will be sorted
+	 * 	(and if you need to clone it, so do it when you need to) - this is weird as TS is slathering type safety onto ducks
+	 * @throws Error if validation fails
+	 */
+	public insertRaw(appLogsToInsert: Applog[]) {
+		DEBUG(`[WriteableThread.insertRaw] ENTER - ${appLogsToInsert.length} applogs for thread "${this.name}"`)
+		DEBUG(`[WriteableThread.insertRaw] About to deduplicate`)
+		const deduplicated = removeDuplicateAppLogs(appLogsToInsert, 'safety')
+		if (deduplicated.length !== appLogsToInsert.length) {
+			throw ERROR(`[insertRaw] duplicate applogs passed: ${appLogsToInsert.length - deduplicated.length}`, {
+				appLogsToInsert,
+				deduplicated,
+			})
+		}
+		DEBUG(`[WriteableThread.insertRaw] Deduplication done`)
+
+		DEBUG(`[WriteableThread.insertRaw] About to validate`)
+		const bogus = appLogsToInsert.filter(log => !isValidApplog(log))
+		if (bogus.length) {
+			throw ERROR(`[insertRaw] bogus applogs passed: ${bogus.length}`, { bogus })
+		}
+		DEBUG(`[WriteableThread.insertRaw] Validation done`)
+
+		DEBUG(`[WriteableThread.insertRaw] About to check for existing`)
+		const existing = appLogsToInsert.filter(log => this.hasApplog(log, false))
+		if (existing.length) {
+			throw ERROR(`[insertRaw] already existing applogs passed: ${existing.length}`, { existing })
+		}
+		DEBUG(`[WriteableThread.insertRaw] Existing check done`)
+
+		if (!appLogsToInsert.length) {
+			WARN('[insertRaw] skipping empty insert empty logs array')
+			return
+		}
+
+		;(!this.hasParents && !(this instanceof ThreadInMemory) ? LOG : DEBUG)(
+			'Inserting:',
+			appLogsToInsert.length === 1 ? appLogsToInsert[0] : appLogsToInsert,
+			{ ds: this },
+		)
+		DEBUG(`[WriteableThread.insertRaw] About to sort applogs`)
+		sortApplogsByTs(appLogsToInsert)
+		const sortNeeded = this._applogs.length && isTsBefore(appLogsToInsert[0], this._applogs[this._applogs.length - 1])
+		DEBUG(`[WriteableThread.insertRaw] About to push to _applogs array`)
+		// Chunked push: spread is fast but causes stack overflow with 100k+ items
+		// Chunking keeps us under the limit while using optimized push()
+		const CHUNK_SIZE = 50_000
+		for (let i = 0; i < appLogsToInsert.length; i += CHUNK_SIZE) {
+			this._applogs.push(...appLogsToInsert.slice(i, i + CHUNK_SIZE))
+		}
+		if (sortNeeded) {
+			DEBUG(`[WriteableThread.insertRaw] About to sort _applogs (sortNeeded=true)`)
+			sortApplogsByTs(this._applogs)
+		}
+		DEBUG(`[WriteableThread.insertRaw] About to notify subscribers`)
+		this.notifySubscribers({ added: appLogsToInsert, removed: null })
+		DEBUG(`[WriteableThread.insertRaw] Subscribers notified`)
+
+		// ? persist sync
+		DEBUG(`[WriteableThread.insertRaw] About to call persist (void - not awaited)`)
+		void this.persist(appLogsToInsert)
+		DEBUG(`[WriteableThread.insertRaw] EXIT - returning ${appLogsToInsert.length} applogs`)
+		return appLogsToInsert
+	}
+
+	get readOnly() {
+		return false
+	}
+
+	protected abstract persist(logs: Applog[]): Promise<void>
+}
+export class ThreadInMemory extends WriteableThread {
+	static empty(name?: string) {
+		return ThreadInMemory.fromArray([], name ?? 'empty in-memory', false)
+	}
+	static fromArray(applogs: Applog[], name?: string, readOnly = false) {
+		return new ThreadInMemory(name ?? 'in-memory', applogs, [], readOnly)
+	}
+	static fromReadOnlyArray(applogs: readonly Applog[], name?: string) {
+		// @ts-expect-error readonly conditional drama
+		return new ThreadInMemory(name ?? 'in-memory', applogs, [], true) 
+	}
+
+	constructor(
+		name: string,
+		applogs: Applog[],
+		filters: readonly string[],
+		readonly _readOnly: boolean,
+	) {
+		super(name, applogs, filters)
+	}
+
+	get readOnly() {
+		return this._readOnly
+	}
+
+	protected async persist(logs: Applog[]) {
+		VERBOSE(`[InMem.persist] no persist for`, logs)
+		if (this.readOnly) {
+			throw ERROR(`[persist] called for readOnly thread`)
+		}
+	}
+}

package/src/thread.ts

@@ -0,0 +1,5 @@
+export * from './thread/basic.ts'
+export * from './thread/filters.ts'
+export * from './thread/mapped.ts'
+export * from './thread/writeable.ts'
+export * from './thread/utils.ts'

package/src/types/typescript-utils.ts

@@ -0,0 +1,64 @@
+import { Type as T } from '@sinclair/typebox'
+// import { Thread } from '../thread/basic.ts'
+
+// ! circular import
+// export const ThreadTB = TypeSystem.Type<Thread>('Thread', (options, value) => {
+// 	return value instanceof Thread
+// })()
+const { String: StringTB, Optional: OptionalTB, Boolean: BooleanTB, Object: ObjectTB, Number: NumberTB } = T
+export const Str: typeof StringTB = StringTB.bind(T)
+export const Num: typeof NumberTB = NumberTB.bind(T)
+export const Obj: typeof ObjectTB = ObjectTB.bind(T)
+export const Opt: typeof OptionalTB = OptionalTB.bind(T)
+export const Bool: typeof BooleanTB = BooleanTB.bind(T)
+
+export const STR: ReturnType<typeof T.String> = Str()
+export const NUM: ReturnType<typeof T.Number> = Num()
+export const BOOL: ReturnType<typeof T.Boolean> = Bool()
+
+export type GenericObject = Record<string, any>
+export type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>
+export type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>
+export type PromiseType<T extends Promise<any>> = T extends Promise<infer U> ? U : never
+
+export type CoerceToString<T> = T extends string ? T : never
+
+/**
+ * Define nominal type of U based on type of T. Similar to Opaque types in Flow
+ */
+export type Tagged<T, Tag> = T & { tag?: Tag }
+
+// export interface Service {
+// 	endpoint?: URL
+// 	token: string
+// 	rateLimiter?: RateLimiter
+// 	fetch?: typeof _fetch
+// }
+
+// https://stackoverflow.com/a/76276541
+export type LastElementOf<T extends readonly unknown[]> = T extends readonly [...unknown[], infer Last] ? Last : never
+
+export function checkParityTB() {
+	/* Most examples are constantly calling Type.*() - needed to check if its really needed
+    https://github.com/sinclairzx81/typebox/issues/587#issuecomment-1712457623
+     */
+	const s1 = Str()
+	const s2 = Str()
+	const n1 = Num()
+	const n2 = Num()
+	console.log({ s1, s2, n1, n2 })
+}
+
+/** solidjs Setter requires returning something, which I often don't */
+export type GenericSetter<T> = (newValue: T) => void
+
+export type ArrayOrSingle<T> = T | T[] | readonly T[]
+export function arrayIfSingle<T>(arrOrSingle: ArrayOrSingle<T>): readonly T[] {
+	if (Array.isArray(arrOrSingle)) return arrOrSingle
+	else return [arrOrSingle] as readonly T[]
+}
+export type ArrayType<T> = T extends (infer U)[] ? U : T
+export type ArrayElementType<T> = T extends readonly (infer U)[] ? Extract<U, string> : never
+export type ConstructorType<T, S = {}> = {
+	new(...args: any[]): T
+} & S

package/src/types.ts

@@ -0,0 +1 @@
+export * from './types/typescript-utils.ts'

package/src/utils/debug-name.ts

@@ -0,0 +1,54 @@
+import stringify from 'safe-stable-stringify'
+import type { Applog, DatalogQueryPattern } from '../applog/datom-types.ts'
+import type { Thread } from '../thread/basic.ts'
+
+export const createDebugName = ({ caller, thread, pattern, args }: {
+	caller?: string
+	thread?: Thread | Applog[]
+	pattern?: DatalogQueryPattern | DatalogQueryPattern[] | string
+	args?: any
+}) => {
+	args = args || pattern
+	const str = `${(!Array.isArray(thread) && thread?.name) ? thread.name + ' | ' : ''}`
+		+ `${caller ?? 'caller?'}`
+		+ `${args ? `{${typeof args === 'string' ? args : stringify(args)}}` : ''}`
+	return str
+}
+
+export const createDebugNameObj = (args: Parameters<typeof createDebugName>[0]) => {
+	return { name: createDebugName(args) } as const
+}
+
+export function prettifyThreadName(input: string): string {
+	let depth = 0
+	let result = ''
+	let insideCurlyBraces = 0
+
+	for (let i = 0; i < input.length; i++) {
+		const char = input[i]
+
+		if (char === '(') {
+			result += char + '\n' + '\t'.repeat(++depth)
+		} else if (char === ')') {
+			result += '\n' + '\t'.repeat(--depth) + char
+		} else if (char === ',' && insideCurlyBraces === 0) {
+			result += char + '\n' + '\t'.repeat(depth)
+		} else if (char === '{' && insideCurlyBraces === 0) {
+			insideCurlyBraces++
+			result += char + '\n' + '\t'.repeat(depth + 1)
+		} else if (char === '}' && insideCurlyBraces === 1) {
+			insideCurlyBraces--
+			result += '\n' + '\t'.repeat(depth) + char
+		} else if (char === '{' && insideCurlyBraces > 0) {
+			insideCurlyBraces++
+			result += char
+		} else if (char === '}' && insideCurlyBraces > 1) {
+			insideCurlyBraces--
+			result += char
+		} else {
+			result += char
+		}
+	}
+
+	return result
+}

package/src/utils.ts

@@ -0,0 +1,4 @@
+import { Applog, ApplogEncNoCid } from './applog/datom-types.ts'
+
+export const isTruthy = (l: Applog | ApplogEncNoCid): boolean => !!l
+export const keepTruthy = (arr: readonly any[]): any[] => arr.filter(isTruthy)