@abraca/dabra 1.8.2 → 2.0.0

package/src/DocUtils.ts

@@ -0,0 +1,89 @@
+/**
+ * Shared utilities for the DocumentManager ORM layer.
+ *
+ * These functions were previously duplicated across `mcp/src/utils.ts`,
+ * `mcp/src/server.ts`, `cli/src/connection.ts`, and `mcp/src/tools/tree.ts`.
+ */
+import * as Y from "yjs";
+
+/**
+ * Wait for a provider's `synced` event with a timeout.
+ * Resolves immediately if the provider is already synced.
+ */
+export function waitForSync(
+	provider: {
+		isSynced?: boolean;
+		on(event: string, cb: () => void): void;
+		off(event: string, cb: () => void): void;
+	},
+	timeoutMs = 15000,
+): Promise<void> {
+	// Already synced — resolve immediately.
+	if (provider.isSynced) return Promise.resolve();
+
+	return new Promise<void>((resolve, reject) => {
+		const timer = setTimeout(() => {
+			provider.off("synced", handler);
+			reject(new Error(`Sync timed out after ${timeoutMs}ms`));
+		}, timeoutMs);
+
+		function handler() {
+			clearTimeout(timer);
+			resolve();
+		}
+
+		provider.on("synced", handler);
+	});
+}
+
+/**
+ * Wrap a promise with a timeout.
+ */
+export function withTimeout<T>(
+	promise: Promise<T>,
+	timeoutMs: number,
+	message?: string,
+): Promise<T> {
+	return new Promise<T>((resolve, reject) => {
+		const timer = setTimeout(
+			() =>
+				reject(
+					new Error(
+						message ?? `Operation timed out after ${timeoutMs}ms`,
+					),
+				),
+			timeoutMs,
+		);
+		promise.then(
+			(val) => {
+				clearTimeout(timer);
+				resolve(val);
+			},
+			(err) => {
+				clearTimeout(timer);
+				reject(err);
+			},
+		);
+	});
+}
+
+/**
+ * Normalize a document ID so the hub/root doc ID is treated as the tree root
+ * (null). This lets callers pass the hub doc_id from list_spaces as
+ * parentId/rootId and get the expected root-level results instead of an empty
+ * set.
+ */
+export function normalizeRootId(
+	id: string | null | undefined,
+	rootDocId: string | null,
+): string | null {
+	if (id == null) return null;
+	return id === rootDocId ? null : id;
+}
+
+/**
+ * Safely read a tree map value, converting Y.Map to plain object if needed.
+ */
+export function toPlain(val: unknown): unknown {
+	return val instanceof Y.Map ? val.toJSON() : val;
+}

package/src/DocumentManager.ts

@@ -0,0 +1,319 @@
+/**
+ * DocumentManager — high-level ORM entry point for Abracadabra CRDT documents.
+ *
+ * Consolidates the shared connection lifecycle, authentication, space discovery,
+ * provider caching, and Y.Map access patterns that were previously duplicated
+ * across `mcp/src/server.ts` and `cli/src/connection.ts`.
+ *
+ * Usage:
+ * ```ts
+ * const dm = new DocumentManager({
+ *   url: 'https://my-server.example.com',
+ *   name: 'My Agent',
+ * })
+ * // Authenticate before connecting (consumer handles crypto):
+ * await dm.client.loginWithKey(publicKey, signFn)
+ * await dm.connect()
+ *
+ * // Use the ORM:
+ * const docs = dm.tree.childrenOf(null)        // root-level docs
+ * const content = await dm.content.read(docId)  // markdown + children
+ * dm.meta.update(docId, { color: '#ff0000' })   // update metadata
+ * ```
+ */
+import * as Y from "yjs";
+import { AbracadabraProvider } from "./AbracadabraProvider.ts";
+import { AbracadabraClient } from "./AbracadabraClient.ts";
+import type { AbracadabraWS } from "./AbracadabraWS.ts";
+import type { ServerInfo, DocumentMeta } from "./types.ts";
+import { Kind, SERVER_ROOT_ID } from "./types.ts";
+import { TreeManager } from "./TreeManager.ts";
+import { ContentManager } from "./ContentManager.ts";
+import { MetaManager } from "./MetaManager.ts";
+import { waitForSync } from "./DocUtils.ts";
+
+export interface DocumentManagerConfig {
+	/** Server base URL (http or https). */
+	url: string;
+	/** Display name for awareness (cursor labels, presence). */
+	name?: string;
+	/** Cursor / awareness color. */
+	color?: string;
+	/** Invite code for first-time registration. */
+	inviteCode?: string;
+	/** Suppress stderr logging. */
+	quiet?: boolean;
+	/** Disable IndexedDB offline persistence. */
+	disableOfflineStore?: boolean;
+	/** Custom fetch for Node.js environments. */
+	fetch?: typeof globalThis.fetch;
+	/**
+	 * Pre-configured AbracadabraClient instance.
+	 * When provided, the `url` and `fetch` fields are ignored — the client's
+	 * configuration is used instead. This is useful when the consumer has
+	 * already authenticated or configured the client externally.
+	 */
+	client?: AbracadabraClient;
+	/**
+	 * Shared WebSocket connection. Required in Node.js environments — pass an
+	 * AbracadabraWS constructed with WebSocketPolyfill set to the `ws` package.
+	 * When omitted, each provider creates its own connection from client.wsUrl.
+	 * The caller owns this instance and must destroy it after dm.destroy().
+	 */
+	websocketProvider?: AbracadabraWS;
+	/**
+	 * Known root document ID. When provided, connect() skips server discovery
+	 * and connects directly to this document. Useful in tests or CLIs where
+	 * the entry-point docId is already known.
+	 */
+	rootDocId?: string;
+}
+
+interface CachedProvider {
+	provider: AbracadabraProvider;
+	lastAccessed: number;
+}
+
+export class DocumentManager {
+	readonly client: AbracadabraClient;
+
+	/** Tree CRUD operations. */
+	readonly tree: TreeManager;
+	/** Document content read/write. */
+	readonly content: ContentManager;
+	/** Document metadata read/write. */
+	readonly meta: MetaManager;
+
+	private _config: DocumentManagerConfig;
+	private _serverInfo: ServerInfo | null = null;
+	private _rootDocId: string | null = null;
+	private _spaces: DocumentMeta[] = [];
+	private _rootDoc: Y.Doc | null = null;
+	private _rootProvider: AbracadabraProvider | null = null;
+	private childCache = new Map<string, CachedProvider>();
+
+	constructor(config: DocumentManagerConfig) {
+		this._config = config;
+		this.client =
+			config.client ??
+			new AbracadabraClient({
+				url: config.url,
+				persistAuth: false,
+				fetch: config.fetch,
+			});
+
+		this.tree = new TreeManager(this);
+		this.content = new ContentManager(this);
+		this.meta = new MetaManager(this);
+	}
+
+	// ── Accessors ─────────────────────────────────────────────────────────────
+
+	get displayName(): string {
+		return this._config.name || "DocumentManager";
+	}
+
+	get displayColor(): string {
+		return this._config.color || "hsl(45, 90%, 55%)";
+	}
+
+	get serverInfo(): ServerInfo | null {
+		return this._serverInfo;
+	}
+
+	get rootDocId(): string | null {
+		return this._rootDocId;
+	}
+
+	get rootDocument(): Y.Doc | null {
+		return this._rootDoc;
+	}
+
+	get rootProvider(): AbracadabraProvider | null {
+		return this._rootProvider;
+	}
+
+	/**
+	 * Spaces visible to the caller — direct children of the server root with
+	 * `kind === "space"`. Populated by {@link connect}.
+	 */
+	get spaces(): DocumentMeta[] {
+		return this._spaces;
+	}
+
+	get isConnected(): boolean {
+		return this._rootProvider?.isConnected ?? false;
+	}
+
+	// ── Y.Map access ──────────────────────────────────────────────────────────
+
+	/** Get the root doc-tree Y.Map. */
+	getTreeMap(): Y.Map<any> | null {
+		return this._rootDoc?.getMap("doc-tree") ?? null;
+	}
+
+	/** Get the root doc-trash Y.Map. */
+	getTrashMap(): Y.Map<any> | null {
+		return this._rootDoc?.getMap("doc-trash") ?? null;
+	}
+
+	/** Get the root space-plugins Y.Map. */
+	getPluginsMap(): Y.Map<any> | null {
+		return this._rootDoc?.getMap("space-plugins") ?? null;
+	}
+
+	// ── Lifecycle ─────────────────────────────────────────────────────────────
+
+	/**
+	 * Connect to the server: discover the entry-point document, sync the root
+	 * Y.Doc, and set awareness.
+	 *
+	 * **Authentication must be done before calling connect().**
+	 * Call `dm.client.loginWithKey(publicKey, signFn)` or `dm.client.login()`
+	 * to authenticate first.
+	 *
+	 * When `rootDocId` is set in the config, server discovery is skipped and
+	 * the manager connects directly to that document.
+	 */
+	async connect(): Promise<void> {
+		// Step 1: Discover server info (used for awareness colour, name, etc.).
+		this._serverInfo = await this.client.serverInfo();
+
+		// Step 2: Pick an entry-point doc.
+		//
+		// In the new model the dashboard's notion of "open the hub" maps to
+		// "open the first Space". A Space is a top-level doc with
+		// `kind === "space"`. If the caller pinned a `rootDocId`, honour it
+		// directly — useful in tests/CLIs where the entry doc is known.
+		let initialDocId: string | null = this._config.rootDocId ?? null;
+
+		if (!initialDocId) {
+			const roots = await this.client.listChildren();
+			this._spaces = roots.filter((d) => d.kind === Kind.Space);
+			const first = this._spaces[0] ?? roots[0];
+			if (first) {
+				initialDocId = first.id;
+				this.log(`Entry document: ${first.label ?? first.id} (${first.id})`);
+			}
+		}
+
+		if (!initialDocId) {
+			throw new Error(
+				`No entry point found: server has no top-level documents under ${SERVER_ROOT_ID}. Create a Space first.`,
+			);
+		}
+
+		this._rootDocId = initialDocId;
+
+		// Step 3: Connect provider and sync
+		await this._connectToRoot(initialDocId);
+		this.log("Connected and synced");
+	}
+
+	/** Switch active space to a different root document. */
+	async switchSpace(docId: string): Promise<void> {
+		// Destroy existing child providers
+		for (const [, cached] of this.childCache) {
+			cached.provider.destroy();
+		}
+		this.childCache.clear();
+
+		// Destroy current root provider (not the wsp — caller owns it)
+		if (this._rootProvider) {
+			this._rootProvider.destroy();
+			this._rootProvider = null;
+		}
+		this._rootDoc = null;
+
+		await this._connectToRoot(docId);
+		this._rootDocId = docId;
+		this.log(`Switched to space ${docId}`);
+	}
+
+	/** Create, sync, and set awareness on a root provider for the given docId. */
+	private async _connectToRoot(docId: string): Promise<void> {
+		const doc = new Y.Doc({ guid: docId });
+		const wsp = this._config.websocketProvider;
+		const provider = new AbracadabraProvider({
+			name: docId,
+			document: doc,
+			client: this.client,
+			disableOfflineStore: this._config.disableOfflineStore ?? true,
+			subdocLoading: "lazy",
+			...(wsp ? { websocketProvider: wsp } : {}),
+		});
+
+		// When a shared websocketProvider is supplied, manageSocket=false so
+		// the base constructor does NOT call attach(). We must do it ourselves.
+		if (wsp) provider.attach();
+
+		await waitForSync(provider);
+
+		provider.awareness?.setLocalStateField("user", {
+			name: this.displayName,
+			color: this.displayColor,
+		});
+		provider.awareness?.setLocalStateField("status", null);
+
+		this._rootDoc = doc;
+		this._rootProvider = provider;
+	}
+
+	/** Graceful shutdown. */
+	async destroy(): Promise<void> {
+		for (const [, cached] of this.childCache) {
+			cached.provider.destroy();
+		}
+		this.childCache.clear();
+
+		if (this._rootProvider) {
+			this._rootProvider.awareness?.setLocalStateField(
+				"status",
+				null,
+			);
+			this._rootProvider.destroy();
+			this._rootProvider = null;
+		}
+		this._rootDoc = null;
+
+		this.log("Disconnected");
+	}
+
+	// ── Provider cache ────────────────────────────────────────────────────────
+
+	/** Get or create a child provider for a document (synced). */
+	async getChildProvider(docId: string): Promise<AbracadabraProvider> {
+		const cached = this.childCache.get(docId);
+		if (cached) {
+			cached.lastAccessed = Date.now();
+			return cached.provider;
+		}
+
+		if (!this._rootProvider) {
+			throw new Error("Not connected. Call connect() first.");
+		}
+
+		const childProvider = await this._rootProvider.loadChild(docId);
+		await waitForSync(childProvider);
+
+		childProvider.awareness?.setLocalStateField("user", {
+			name: this.displayName,
+			color: this.displayColor,
+		});
+
+		this.childCache.set(docId, {
+			provider: childProvider,
+			lastAccessed: Date.now(),
+		});
+
+		return childProvider;
+	}
+
+	// ── Internal ──────────────────────────────────────────────────────────────
+
+	private log(msg: string): void {
+		if (!this._config.quiet) {
+			console.error(`[abracadabra] ${msg}`);
+		}
+	}
+}

package/src/E2EAbracadabraProvider.ts

@@ -16,6 +16,19 @@
  *   are fetched on subsequent connects.
  * - After sync, a fresh encrypted snapshot is saved.
  *
+ * Client-side compaction:
+ * - After `compactionThreshold` encrypted updates have been applied in this
+ *   session (local + remote), and the doc has been quiescent for
+ *   `compactionQuiescenceMs`, the provider merges the whole Y.Doc, encrypts it,
+ *   and sends `snapshot:compact` — the server atomically replaces the per-doc
+ *   update log with that single compacted blob. The server acknowledges by
+ *   broadcasting `snapshot:compacted`, which emits the `"compacted"` event.
+ * - Requires Owner or above (server silently drops non-Owner requests).
+ * - Callers that want a final compaction before teardown should
+ *   `await provider.compactNow()` before `destroy()`. `destroy()` does not
+ *   compact (it'd race with the socket teardown) — any pending debounce is
+ *   cancelled.
+ *
  * Key availability limitation: if the user's WebAuthn key is not in
  * DocKeyManager's in-memory cache and there is no network, E2E docs show
  * empty — the key fetch requires either a cached in-memory key or network.
@@ -30,15 +43,49 @@ import type { AbracadabraClient } from "./AbracadabraClient.ts";
 import { UpdateMessage } from "./OutgoingMessages/UpdateMessage.ts";
 import { encryptField, decryptField } from "./EncryptedY.ts";
 import { E2EOfflineStore } from "./E2EOfflineStore.ts";
+import type { onCompactedParameters } from "./types.ts";
 
 function fromBase64(b64: string): Uint8Array {
 	return Uint8Array.from(atob(b64), (c) => c.charCodeAt(0));
 }
 
+function toBase64(bytes: Uint8Array): string {
+	let bin = "";
+	for (let i = 0; i < bytes.length; i += 0x8000) {
+		bin += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+	}
+	return btoa(bin);
+}
+
 export interface E2EAbracadabraProviderConfiguration extends AbracadabraProviderConfiguration {
 	docKeyManager: DocKeyManager;
 	keystore: CryptoIdentityKeystore;
 	client: AbracadabraClient;
+	/**
+	 * Enable client-side compaction: after `compactionThreshold` E2E updates,
+	 * the provider merges the whole doc, re-encrypts it, and asks the server to
+	 * atomically replace its update log via the `snapshot:compact` stateless
+	 * protocol. Requires the user to have a role that `can_manage` the doc
+	 * (Owner or above); otherwise the server silently rejects.
+	 *
+	 * Default: true.
+	 */
+	compactionEnabled?: boolean;
+	/**
+	 * Number of E2E updates applied in this session before compaction fires.
+	 * Default: 50. Ignored when `compactionEnabled === false`.
+	 */
+	compactionThreshold?: number;
+	/**
+	 * Quiescence delay: once the threshold is crossed, wait this many ms with
+	 * no further updates before firing compaction. Debounces active editing
+	 * and reduces the chance of racing an in-flight remote update that hasn't
+	 * reached us yet (the server does not currently validate state-vector
+	 * coverage, so a compaction fired mid-flight could drop an update).
+	 *
+	 * Default: 2000 ms.
+	 */
+	compactionQuiescenceMs?: number;
 }
 
 export class E2EAbracadabraProvider extends AbracadabraProvider {
@@ -52,6 +99,18 @@ export class E2EAbracadabraProvider extends AbracadabraProvider {
 	private e2eStore: E2EOfflineStore | null = null;
 	private readonly e2eServerOrigin: string | undefined;
 
+	// ── Client-side compaction ────────────────────────────────────────────────
+	private readonly compactionEnabled: boolean;
+	private readonly compactionThreshold: number;
+	private readonly compactionQuiescenceMs: number;
+	private updatesSinceCompaction = 0;
+	private compactionInFlight = false;
+	/** Cleared on `snapshot:compacted` or on 30s timeout. */
+	private compactionInFlightTimeout: ReturnType<typeof setTimeout> | null = null;
+	/** Quiescence debounce: reset on every update, fires the actual compaction. */
+	private compactionDebounceTimer: ReturnType<typeof setTimeout> | null = null;
+	private destroyed = false;
+
 	constructor(configuration: E2EAbracadabraProviderConfiguration) {
 		// Disable the parent's offline store — E2E uses its own encrypted store.
 		super({ ...configuration, disableOfflineStore: true });
@@ -59,6 +118,10 @@ export class E2EAbracadabraProvider extends AbracadabraProvider {
 		this.keystore = configuration.keystore;
 		this.e2eClient = configuration.client;
 
+		this.compactionEnabled = configuration.compactionEnabled !== false;
+		this.compactionThreshold = Math.max(1, configuration.compactionThreshold ?? 50);
+		this.compactionQuiescenceMs = Math.max(0, configuration.compactionQuiescenceMs ?? 2000);
+
 		// Derive server origin for E2EOfflineStore namespacing
 		this.e2eServerOrigin = E2EAbracadabraProvider.deriveServerOrigin(
 			configuration,
@@ -89,6 +152,14 @@ export class E2EAbracadabraProvider extends AbracadabraProvider {
 
 	/** Handle stateless messages including e2e_ready and e2e_update. */
 	override receiveStateless(payload: string): void {
+		// `snapshot:compacted {"doc_id":"...","by":"..."}` is a space-separated
+		// stateless frame — not valid JSON as a whole — so handle it before the
+		// JSON.parse fallthrough.
+		if (payload.startsWith("snapshot:compacted ")) {
+			this._handleCompactedBroadcast(payload.slice("snapshot:compacted ".length));
+			return;
+		}
+
 		let parsed: unknown;
 		try {
 			parsed = JSON.parse(payload);
@@ -165,6 +236,7 @@ export class E2EAbracadabraProvider extends AbracadabraProvider {
 			const plaintext = await decryptField(encryptedData, key);
 			Y.applyUpdate(this.document, plaintext, this);
 			this.lastSeq = Math.max(this.lastSeq, seq);
+			this._noteUpdateApplied();
 		} catch (e) {
 			console.error("[E2EAbracadabraProvider] decryption failed for seq", seq, e);
 		}
@@ -190,9 +262,126 @@ export class E2EAbracadabraProvider extends AbracadabraProvider {
 			update: encrypted,
 			documentName: this.configuration.name,
 		});
+		this._noteUpdateApplied();
+	}
+
+	// ── Compaction ────────────────────────────────────────────────────────────
+
+	/**
+	 * Force an immediate compaction attempt, bypassing the threshold and
+	 * quiescence debounce. Resolves once the `snapshot:compact` frame has been
+	 * sent (or rejected locally for missing prerequisites — destroyed, no
+	 * doc key, or already in-flight). The server acknowledges via a
+	 * `snapshot:compacted` broadcast, which emits the `"compacted"` event.
+	 */
+	async compactNow(): Promise<void> {
+		if (this.compactionDebounceTimer) {
+			clearTimeout(this.compactionDebounceTimer);
+			this.compactionDebounceTimer = null;
+		}
+		await this._performCompaction();
+	}
+
+	private _noteUpdateApplied(): void {
+		if (!this.compactionEnabled || this.destroyed) return;
+		this.updatesSinceCompaction += 1;
+		if (this.updatesSinceCompaction < this.compactionThreshold) return;
+
+		// Debounce: reset on every update so compaction only fires after a
+		// quiescent window. This avoids racing concurrent remote updates that
+		// are still propagating through the server.
+		if (this.compactionDebounceTimer) clearTimeout(this.compactionDebounceTimer);
+		this.compactionDebounceTimer = setTimeout(() => {
+			this.compactionDebounceTimer = null;
+			this._performCompaction().catch((e) => {
+				console.error("[E2EAbracadabraProvider] compaction failed:", e);
+			});
+		}, this.compactionQuiescenceMs);
+	}
+
+	private async _performCompaction(): Promise<void> {
+		if (this.destroyed) return;
+		// Single-flight: server handles concurrent compactions, but we don't
+		// want to pile up outbound frames.
+		if (this.compactionInFlight) return;
+		// Nothing to compact — skip unless explicitly forced after threshold.
+		if (this.updatesSinceCompaction === 0) return;
+
+		// Compaction replaces the entire server-side update log with our state.
+		// Only run after the initial sync — otherwise we'd overwrite the log
+		// with a partial state.
+		if (!this.synced) return;
+
+		const key = await this.ensureDocKey();
+		if (!key) return;
+		if (this.destroyed) return; // key fetch may be async
+
+		this.compactionInFlight = true;
+		try {
+			const stateVector = Y.encodeStateVector(this.document);
+			const fullState = Y.encodeStateAsUpdate(this.document);
+			const encrypted = await encryptField(fullState, key);
+			if (this.destroyed) {
+				this.compactionInFlight = false;
+				return;
+			}
+
+			const payload = `snapshot:compact ${JSON.stringify({
+				state_vector: toBase64(stateVector),
+				compacted: toBase64(encrypted),
+			})}`;
+			this.sendStateless(payload);
+
+			// Release in-flight on server broadcast; if nothing arrives in 30s
+			// (e.g. permissions rejected, connection dropped) release anyway.
+			this.compactionInFlightTimeout = setTimeout(() => {
+				this.compactionInFlight = false;
+				this.compactionInFlightTimeout = null;
+			}, 30_000);
+		} catch (e) {
+			this.compactionInFlight = false;
+			throw e;
+		}
+	}
+
+	private _handleCompactedBroadcast(jsonStr: string): void {
+		let parsed: { doc_id?: string; by?: string } = {};
+		try {
+			parsed = JSON.parse(jsonStr);
+		} catch {
+			/* malformed — still clear in-flight below */
+		}
+
+		// Only act on broadcasts for our own doc (server scopes this already,
+		// but be defensive).
+		if (parsed.doc_id && parsed.doc_id !== this.configuration.name) return;
+
+		if (this.compactionInFlightTimeout) {
+			clearTimeout(this.compactionInFlightTimeout);
+			this.compactionInFlightTimeout = null;
+		}
+		this.compactionInFlight = false;
+		this.updatesSinceCompaction = 0;
+
+		const event: onCompactedParameters = {
+			docId: parsed.doc_id ?? this.configuration.name,
+			by: parsed.by,
+		};
+		this.emit("compacted", event);
 	}
 
 	override destroy(): void {
+		if (this.destroyed) return;
+		this.destroyed = true;
+
+		if (this.compactionDebounceTimer) {
+			clearTimeout(this.compactionDebounceTimer);
+			this.compactionDebounceTimer = null;
+		}
+		if (this.compactionInFlightTimeout) {
+			clearTimeout(this.compactionInFlightTimeout);
+			this.compactionInFlightTimeout = null;
+		}
 		this.e2eStore?.destroy();
 		this.e2eStore = null;
 		super.destroy();