@tiddlywiki/mws 0.0.21 → 0.1.1

package/plugins/client/src/multiwikiclientadaptor.ts

@@ -1,5 +1,5 @@
 /*\
-title: $:/plugins/tiddlywiki/tiddlyweb/tiddlywebadaptor.js
+title: $:/plugins/mws/client/tiddlywebadaptor.js
 type: application/javascript
 module-type: syncadaptor
 
@@ -18,23 +18,192 @@ previous operation to complete before sending a new one.
 
 // the blank line is important, and so is the following use strict
 "use strict";
-import type { Logger } from "$:/core/modules/utils/logger.js";
-import type { Syncer, Tiddler, ITiddlyWiki } from "tiddlywiki";
+
+import type { ServerEventsMap } from '@tiddlywiki/events';
+import type { ZodRoute, WikiStatusRoutes, WikiRecipeRoutes } from '@tiddlywiki/mws';
+import type { zod } from '@tiddlywiki/server';
+import type { Syncer, Tiddler, TiddlerFields } from 'tiddlywiki';
+declare global { const fflate: typeof import("./fflate"); }
+declare const self: never;
+
+// registerZodRoutes(parent, new WikiStatusRoutes(), Object.keys({
+// 	handleGetAllBagStates: true,
+// 	handleGetBagState: true,
+// 	handleGetRecipeStatus: true,
+// 	handleGetRecipeEvents: true,
+// 	handleGetBags: true,
+// } satisfies RouterKeyMap<WikiStatusRoutes, true>));
+
+// registerZodRoutes(parent, new WikiRecipeRoutes(), Object.keys({
+// 	handleDeleteRecipeTiddler: true,
+// 	handleLoadRecipeTiddler: true,
+// 	handleSaveRecipeTiddler: true,
+// 	rpcDeleteRecipeTiddlerList: true,
+// 	rpcLoadRecipeTiddlerList: true,
+// 	rpcSaveRecipeTiddlerList: true,
+// } satisfies RouterKeyMap<WikiRecipeRoutes, true>));
+
+declare class Logger {
+	constructor(componentName: any, options: any);
+	componentName: any;
+	colour: any;
+	enable: any;
+	save: any;
+	saveLimit: any;
+	saveBufferLogger: this;
+	buffer: string;
+	alertCount: number;
+	setSaveBuffer(logger: any): void;
+	log(...args: any[]): any;
+	getBuffer(): string;
+	table(value: any): void;
+	alert(...args: any[]): void;
+	clearAlerts(): void;
+}
+
+type WikiStatusTypes = {
+	[K in keyof WikiStatusRoutes]: WikiStatusRoutes[K] extends ZodRoute<infer M, infer B, infer P, infer Q, infer T, infer R>
+	? { M: M, B: B, P: P, Q: Q, T: T, R: R }
+	: never
+}
+type WikiRecipeTypes = {
+	[K in keyof WikiRecipeRoutes]: WikiRecipeRoutes[K] extends ZodRoute<infer M, infer B, infer P, infer Q, infer T, infer R>
+	? { M: M, B: B, P: P, Q: Q, T: T, R: R }
+	: never
+}
+interface RouteTypes extends WikiStatusTypes, WikiRecipeTypes {
+}
 
 declare module 'tiddlywiki' {
-	export interface Syncer {
+	export interface Syncer<AD> {
 		wiki: Wiki;
 		logger: Logger;
-		tiddlerInfo: Record<string, { bag: string; revision: string }>;
+		tiddlerInfo: Record<string, {
+			changeCount: number,
+			adaptorInfo: AD,
+			revision: string,
+			timestampLastSaved: Date
+		}>;
 		enqueueLoadTiddler(title: string): void;
 		storeTiddler(tiddler: Tiddler): void;
 		processTaskQueue(): void;
+		syncFromServer(): void;
 	}
 	interface ITiddlyWiki {
 		browserStorage: any;
 	}
 }
 
+type ServerStatusCallback = (
+	err: any,
+	/** 
+	 * $:/status/IsLoggedIn mostly appears alongside the username 
+	 * or other login-conditional behavior. 
+	 */
+	isLoggedIn?: boolean,
+	/**
+	 * $:/status/UserName is still used for things like drafts even if the 
+	 * user isn't logged in, although the username is less likely to be shown 
+	 * to the user. 
+	 */
+	username?: string,
+	/** 
+	 * $:/status/IsReadOnly puts the UI in readonly mode, 
+	 * but does not prevent automatic changes from attempting to save. 
+	 */
+	isReadOnly?: boolean,
+	/** 
+	 * $:/status/IsAnonymous does not appear anywhere in the TW5 repo! 
+	 * So it has no apparent purpose. 
+	 */
+	isAnonymous?: boolean
+) => void
+
+interface SyncAdaptor<AD> {
+	name?: string;
+
+	isReady?(): boolean;
+
+	getStatus?(
+		cb: ServerStatusCallback
+	): void;
+
+	getSkinnyTiddlers?(
+		cb: (err: any, tiddlerFields: Record<string, string>[]) => void
+	): void;
+	getUpdatedTiddlers?(
+		syncer: Syncer<AD>,
+		cb: (
+			err: any,
+			/** Arrays of titles that have been modified or deleted */
+			updates?: { modifications: string[], deletions: string[] }
+		) => void
+	): void;
+
+	/** 
+	 * used to override the default Syncer getTiddlerRevision behavior
+	 * of returning the revision field
+	 * 
+	 */
+	getTiddlerRevision?(title: string): string;
+	/** 
+	 * used to get the adapter info from a tiddler in situations
+	 * other than the saveTiddler callback
+	 */
+	getTiddlerInfo(tiddler: Tiddler): AD | undefined;
+
+	saveTiddler(
+		tiddler: any,
+		cb: (
+			err: any,
+			adaptorInfo?: AD,
+			revision?: string
+		) => void,
+		extra: { tiddlerInfo: SyncerTiddlerInfo<AD> }
+	): void;
+
+	saveTiddlers?(options: {
+		syncer: Syncer<AD>,
+		tiddlers: Tiddler[],
+		onNext: (title: string, adaptorInfo: any, revision: string) => void,
+		onDone: () => void,
+		onError: (err: Error) => void
+	}): void;
+
+	loadTiddlers?(options: {
+		syncer: Syncer<AD>,
+		titles: string[],
+		onNext: (tiddlerFields: TiddlerFields) => void,
+		onDone: () => void,
+		onError: (err: Error) => void
+	}): void;
+
+	deleteTiddlers?(options: {
+		syncer: Syncer<AD>,
+		titles: string[],
+		onNext: (title: string) => void,
+		onDone: () => void,
+		onError: (err: Error) => void
+	}): void;
+
+	setLoggerSaveBuffer?: (loggerForSaving: Logger) => void;
+	displayLoginPrompt?(syncer: Syncer<AD>): void;
+	login?(username: string, password: string, cb: (err: any) => void): void;
+	logout?(cb: (err: any) => void): any;
+}
+interface SyncerTiddlerInfo<AD> {
+	/** this comes from the wiki changeCount record */
+	changeCount: number;
+	/** Adapter info returned by the sync adapter */
+	adaptorInfo: AD;
+	/** Revision return by the sync adapter */
+	revision: string;
+	/** Timestamp set in the callback of the previous save */
+	timestampLastSaved: Date;
+}
+
+declare const $tw: any;
+
 declare const exports: {
 	adaptorClass: typeof MultiWikiClientAdaptor;
 };
@@ -46,48 +215,65 @@ var CONFIG_HOST_TIDDLER = "$:/config/multiwikiclient/host",
 	REVISION_STATE_TIDDLER = "$:/state/multiwikiclient/tiddlers/revision",
 	CONNECTION_STATE_TIDDLER = "$:/state/multiwikiclient/connection",
 	INCOMING_UPDATES_FILTER_TIDDLER = "$:/config/multiwikiclient/incoming-updates-filter",
-	ENABLE_SSE_TIDDLER = "$:/config/multiwikiclient/use-server-sent-events";
+	ENABLE_SSE_TIDDLER = "$:/config/multiwikiclient/use-server-sent-events",
+	IS_DEV_MODE_TIDDLER = `$:/state/multiwikiclient/dev-mode`,
+	ENABLE_GZIP_STREAM_TIDDLER = `$:/state/multiwikiclient/gzip-stream`;
+
 
 var SERVER_NOT_CONNECTED = "NOT CONNECTED",
 	SERVER_CONNECTING_SSE = "CONNECTING SSE",
 	SERVER_CONNECTED_SSE = "CONNECTED SSE",
 	SERVER_POLLING = "SERVER POLLING";
 
-class MultiWikiClientAdaptor {
-	wiki;
-	host;
-	recipe;
-	useServerSentEvents;
-	last_known_tiddler_id;
-	outstandingRequests;
-	lastRecordedUpdate;
-	logger;
-	isLoggedIn;
-	isReadOnly;
-	logoutIsAvailable;
-	incomingUpdatesFilterFn;
-	serverUpdateConnectionStatus!: string;
+interface MWSAdaptorInfo {
+	bag: string
+	revision: string
+	title: string
+}
+
+
+class MultiWikiClientAdaptor implements SyncAdaptor<MWSAdaptorInfo> {
+	private wiki;
+	private host;
+	private recipe;
+	private useServerSentEvents;
+	private last_known_revision_id;
+	private outstandingRequests;
+	private lastRecordedUpdate;
+	private logger;
+	private isLoggedIn;
+	private isReadOnly;
+	private offline;
+	private username;
+	private incomingUpdatesFilterFn;
+	private serverUpdateConnectionStatus!: string;
+	private isDevMode: boolean;
+	private useGzipStream: boolean;
+	private triggerPoll: (() => void) | null = null;
 
 	name = "multiwikiclient";
-	supportsLazyLoading = true;
+	private supportsLazyLoading = true;
 	constructor(options: { wiki: any }) {
 		this.wiki = options.wiki;
 		this.host = this.getHost();
 		this.recipe = this.wiki.getTiddlerText("$:/config/multiwikiclient/recipe");
 		this.useServerSentEvents = this.wiki.getTiddlerText(ENABLE_SSE_TIDDLER) === "yes";
-		this.last_known_tiddler_id = $tw.utils.parseNumber(this.wiki.getTiddlerText("$:/state/multiwikiclient/recipe/last_tiddler_id", "0"));
+		this.isDevMode = this.wiki.getTiddlerText(IS_DEV_MODE_TIDDLER) === "yes";
+		this.useGzipStream = this.wiki.getTiddlerText(ENABLE_GZIP_STREAM_TIDDLER) === "yes";
+		this.last_known_revision_id = this.wiki.getTiddlerText("$:/state/multiwikiclient/recipe/last_revision_id", "0")
 		this.outstandingRequests = Object.create(null); // Hashmap by title of outstanding request object: {type: "PUT"|"GET"|"DELETE"}
-		this.lastRecordedUpdate = Object.create(null); // Hashmap by title of last recorded update via SSE: {type: "update"|"detetion", tiddler_id:}
+		this.lastRecordedUpdate = Object.create(null); // Hashmap by title of last recorded update via SSE: {type: "update"|"detetion", revision_id:}
 		this.logger = new $tw.utils.Logger("MultiWikiClientAdaptor");
 		this.isLoggedIn = false;
 		this.isReadOnly = false;
-		this.logoutIsAvailable = true;
+		this.offline = false;
+		this.username = "";
 		// Compile the dirty tiddler filter
 		this.incomingUpdatesFilterFn = this.wiki.compileFilter(this.wiki.getTiddlerText(INCOMING_UPDATES_FILTER_TIDDLER));
-		this.setUpdateConnectionStatus(SERVER_NOT_CONNECTED);
+		this.setConnectionStatus(SERVER_NOT_CONNECTED);
 	}
 
-	setUpdateConnectionStatus(status: string) {
+	private setConnectionStatus(status: string) {
 		this.serverUpdateConnectionStatus = status;
 		this.wiki.addTiddler({
 			title: CONNECTION_STATE_TIDDLER,
@@ -100,20 +286,29 @@ class MultiWikiClientAdaptor {
 	isReady() {
 		return true;
 	}
-	getHost() {
+	subscribe(callback: () => void) {
+		// TODO: not working at the moment for some reason
+		console.log("Subscribing to server updates");
+		this.triggerPoll = callback;
+		this.connectRecipeEvents();
+	}
+	private getHost() {
 		var text = this.wiki.getTiddlerText(CONFIG_HOST_TIDDLER, DEFAULT_HOST_TIDDLER), substitutions = [
 			{ name: "protocol", value: document.location.protocol },
 			{ name: "host", value: document.location.host },
 			{ name: "pathname", value: document.location.pathname }
 		];
 		for (var t = 0; t < substitutions.length; t++) {
-			var s = substitutions[t];
+			var s = substitutions[t]!;
 			text = $tw.utils.replaceString(text, new RegExp("\\$" + s.name + "\\$", "mg"), s.value);
 		}
 		return text;
 	}
-	getTiddlerInfo(tiddler: Tiddler) {
-		var title = tiddler.fields.title, revision = this.wiki.extractTiddlerDataItem(REVISION_STATE_TIDDLER, title), bag = this.wiki.extractTiddlerDataItem(BAG_STATE_TIDDLER, title);
+
+	getTiddlerInfo(tiddler: Tiddler): MWSAdaptorInfo | undefined {
+		var title = tiddler.fields.title,
+			revision = this.wiki.extractTiddlerDataItem(REVISION_STATE_TIDDLER, title),
+			bag = this.wiki.extractTiddlerDataItem(BAG_STATE_TIDDLER, title);
 		if (revision && bag) {
 			return {
 				title: title,
@@ -124,354 +319,374 @@ class MultiWikiClientAdaptor {
 			return undefined;
 		}
 	}
-	getTiddlerBag(title: string) {
+	private getTiddlerBag(title: string) {
 		return this.wiki.extractTiddlerDataItem(BAG_STATE_TIDDLER, title);
 	}
 	getTiddlerRevision(title: string) {
 		return this.wiki.extractTiddlerDataItem(REVISION_STATE_TIDDLER, title);
 	}
-	setTiddlerInfo(title: string, revision: string, bag: string) {
-		this.wiki.setText(BAG_STATE_TIDDLER, null, title, bag, { suppressTimestamp: true });
-		this.wiki.setText(REVISION_STATE_TIDDLER, null, title, revision, { suppressTimestamp: true });
+	private setTiddlerInfo({ title, revision_id, bag_name }: { title: string; revision_id: string; bag_name: string; }) {
+		this.wiki.setText(BAG_STATE_TIDDLER, null, title, bag_name, { suppressTimestamp: true });
+		this.wiki.setText(REVISION_STATE_TIDDLER, null, title, revision_id, { suppressTimestamp: true });
 	}
-	removeTiddlerInfo(title: string) {
+	private removeTiddlerInfo({ title, revision_id, bag_id }: { title: string; revision_id: string; bag_id: string; }) {
 		this.wiki.setText(BAG_STATE_TIDDLER, null, title, undefined, { suppressTimestamp: true });
 		this.wiki.setText(REVISION_STATE_TIDDLER, null, title, undefined, { suppressTimestamp: true });
 	}
 
-	httpRequest<RT extends "text" | "arraybuffer" | "json">(options: {
-		/** url to retrieve (must not contain `?` if GET or HEAD) */
-		url: string;
-		/** hashmap of headers to send */
-		headers?: Record<string, string>;
-		/** request method: GET, PUT, POST etc */
-		type?: string;
-		/** optional function invoked with (lengthComputable,loaded,total) */
-		progress?: (lengthComputable: boolean, loaded: number, total: number) => void;
-		/** name of the property to return as first argument of callback */
-		returnProp?: string;
-		responseType?: RT;
-		useDefaultHeaders?: boolean;
-		/** urlencoded string or hashmap of data to send. If type is GET or HEAD, this is appended to the URL as a query string */
-		data?: object | string;
-	}) {
-		type ResponseErr = [false, any, undefined];
-		type ResponseOk = [true, undefined, {
-			data:
-			"json" extends RT ? any :
-			"text" extends RT ? string :
-			"arraybuffer" extends RT ? ArrayBuffer :
-			unknown;
-			headers: Record<string, string>
-		}];
-		return (new Promise<ResponseErr | ResponseOk>((resolve) => {
-			$tw.utils.httpRequest({
-				...options,
-				responseType: options.responseType === "json" ? "text" : options.responseType,
-				callback: (err: any, data: any, request: XMLHttpRequest) => {
-					if (err) return resolve([false, err || new Error("Unknown error"), undefined]);
-
-					// Create a map of header names to values
-
-					const headers = {} as any;
-					request.getAllResponseHeaders()?.trim().split(/[\r\n]+/).forEach((line) => {
-						const parts = line.split(": ");
-						const header = parts.shift()?.toLowerCase();
-						const value = parts.join(": ");
-						if (header) headers[header] = value;
-					});
-					// Resolve the promise with the response data and headers
-					resolve([true, undefined, {
-						headers,
-						data: options.responseType === "json" ? $tw.utils.parseJSONSafe(data, () => undefined) : data,
-					}]);
-				},
-			});
-		}));
-	}
+	error: string | null = null;
 	/*
 	Get the current status of the server connection
 	*/
-	async getStatus(callback: (
-		err: any,
-		isLoggedIn?: boolean,
-		username?: string,
-		isReadOnly?: boolean,
-		isAnonymous?: boolean,
-	) => void) {
-		interface UserAuthStatus {
-			isAdmin: boolean;
-			user_id: number;
-			username: string;
-			isLoggedIn: boolean;
-			isReadOnly: boolean;
-			allowAnonReads: boolean;
-			allowAnonWrites: boolean;
-		}
+	async getStatus(callback: ServerStatusCallback) {
 
-		const [ok, error, data] = await this.httpRequest({
-			url: this.host + "recipes/" + this.recipe + "/status",
-			type: "GET",
-			responseType: "json",
-			headers: {
-				'Content-Type': 'application/json',
-				"X-Requested-With": "TiddlyWiki"
-			},
+		const [ok, error, data] = await this.recipeRequest({
+			key: "handleGetRecipeStatus",
+			method: "GET",
+			url: "/status",
 		});
-		if (!ok) {
-			this.logger.log("Error getting status", error);
-			if (callback) callback(error);
-			return;
+
+		if (!ok && data?.status === 0) {
+			this.error = "The webpage is forbidden from contacting the server."
+			this.isLoggedIn = false;
+			this.isReadOnly = true;
+			this.username = "(offline)";
+			this.offline = true;
+		} else if (ok) {
+			this.error = null;
+			const status = data.responseJSON;
+			this.isReadOnly = status?.isReadOnly ?? true;
+			this.isLoggedIn = status?.isLoggedIn ?? false;
+			this.username = status?.username ?? "(anon)";
+			this.offline = false;
+		} else {
+			this.error = error.message ?? `${error}`;
 		}
-		/** @type {Partial<UserAuthStatus>} */
-		const status = data.data;
 		if (callback) {
 			callback(
 				// Error
-				null,
+				this.error,
 				// Is logged in
-				status.isLoggedIn ?? false,
+				this.isLoggedIn,
 				// Username
-				status.username ?? "(anon)",
+				this.username,
 				// Is read only
-				status.isReadOnly ?? true,
+				this.isReadOnly,
 				// Is anonymous
-				!status.isLoggedIn,
+				// no idea what this means, always return false
+				false,
 			);
 		}
 	}
 	/*
 	Get details of changed tiddlers from the server
 	*/
-	getUpdatedTiddlers(syncer: Syncer, callback: (err: any, changes?: { modifications: string[]; deletions: string[] }) => void) {
-		if (!this.useServerSentEvents) {
-			this.pollServer({
-				callback: function (err, changes) {
-					callback(null, changes);
-				}
-			});
-			return;
-		}
+	async getUpdatedTiddlers(syncer: Syncer<MWSAdaptorInfo>, callback: (err: any, changes?: { modifications: string[]; deletions: string[] }) => void) {
+		if (this.offline) return callback(null);
+		if (!this.bags || this.serverUpdateConnectionStatus === SERVER_NOT_CONNECTED) await this.pollServer();
+		const { modifications, deletions, last_revision } = this.updateBags();
+		this.last_known_revision_id = last_revision;
+		callback(null, { modifications, deletions });
+	}
 
-		var self = this;
-		// Do nothing if there's already a connection in progress.
-		if (this.serverUpdateConnectionStatus !== SERVER_NOT_CONNECTED) {
-			return callback(null, {
-				modifications: [],
-				deletions: []
-			});
-		}
-		// Try to connect a server stream
-		this.setUpdateConnectionStatus(SERVER_CONNECTING_SSE);
-		this.connectServerStream({
-			syncer: syncer,
-			onerror: function (err) {
-				self.logger.log("Error connecting SSE stream", err);
-				// If the stream didn't work, try polling
-				self.setUpdateConnectionStatus(SERVER_POLLING);
-				self.pollServer({
-					callback: function (err, changes) {
-						self.setUpdateConnectionStatus(SERVER_NOT_CONNECTED);
-						callback(null, changes);
+
+	connectRecipeEvents() {
+		const event = new EventSource(this.host + "recipe/" + encodeURIComponent(this.recipe) + "/events?first-event-id=" + encodeURIComponent(this.last_known_revision_id));
+		this.setConnectionStatus(SERVER_CONNECTING_SSE);
+		let syncTimeout: NodeJS.Timeout | null = null;
+		const debounceSyncFromServer = () => {
+			if (syncTimeout) {
+				clearTimeout(syncTimeout);
+			}
+			syncTimeout = setTimeout(() => {
+				syncTimeout = null;
+				this.triggerPoll?.();
+			}, 100);
+		};
+
+		const onUpdate = (e: MessageEvent) => {
+			if (!this.bags) return console.log("No bags loaded, cannot process updates");
+			const updates: {
+				bag: {
+					bag_name: PrismaField<"Bags", "bag_name">;
+					tiddlers: {
+						revision_id: PrismaField<"Tiddlers", "revision_id">;
+						title: PrismaField<"Tiddlers", "title">;
+						is_deleted: PrismaField<"Tiddlers", "is_deleted">;
+					}[];
+				};
+			}[] = JSON.parse(e.data);
+			const existBags = new Map(this.bags.map(b => [b.bag_name, b.tiddlers] as const));
+			updates.forEach(e => {
+				const existBag = existBags.get(e.bag.bag_name);
+				if (!existBag) return;
+				const existTids = new Map(existBag.map(t => [t.title, t] as const));
+				e.bag.tiddlers.forEach(tid => {
+					const existTid = existTids.get(tid.title);
+					if (existTid) {
+						Object.assign(existTid, tid)
+					} else {
+						existBag.push(tid);
 					}
 				});
-			},
-			onopen: function () {
-				self.setUpdateConnectionStatus(SERVER_CONNECTED_SSE);
-				// The syncer is expecting a callback but we don't have any data to send
-				callback(null, {
-					modifications: [],
-					deletions: []
-				});
+			});
+			debounceSyncFromServer();
+		};
+		// event.addEventListener("tiddler.save", onEvent);
+		// event.addEventListener("tiddler.delete", onEvent);
+		event.addEventListener("tiddler.since-last-event", onUpdate);
+		event.addEventListener("test", console.log);
+		event.addEventListener("open", () => {
+			this.setConnectionStatus(SERVER_CONNECTED_SSE);
+			console.log("Connected to server events");
+		});
+		event.addEventListener("error", (e) => {
+			if (event.readyState === EventSource.CLOSED) {
+				this.setConnectionStatus(SERVER_NOT_CONNECTED);
+				console.error("Server events connection closed");
+			} else if (event.readyState === EventSource.CONNECTING) {
+				this.setConnectionStatus(SERVER_CONNECTING_SSE);
+				console.warn("Connecting to server events");
+			} else {
+				console.error("Error in server events connection", e);
 			}
 		});
-
 	}
-	/*
-	Attempt to establish an SSE stream with the server and transfer tiddler changes. Options include:
-  
-	syncer: reference to syncer object used for storing data
-	onopen: invoked when the stream is successfully opened
-	onerror: invoked if there is an error
-	*/
-	connectServerStream(options: {
-		syncer: Syncer;
-		onopen: (event: Event) => void;
-		onerror: (event: Event) => void;
-	}) {
-		var self = this;
-		const eventSource = new EventSource("/recipes/" + this.recipe + "/events?last_known_tiddler_id=" + this.last_known_tiddler_id);
-		eventSource.onerror = function (event) {
-			if (options.onerror) {
-				options.onerror(event);
-			}
-		};
-		eventSource.onopen = function (event) {
-			if (options.onopen) {
-				options.onopen(event);
-			}
+
+	bags?: {
+		position: number & {
+			__prisma_table: "Recipe_bags";
+			__prisma_field: "position";
 		};
-		eventSource.addEventListener("change", function (event) {
-
-			const data: {
-				title: string;
-				tiddler_id: number;
-				is_deleted: boolean;
-				bag_name: string;
-				tiddler: any;
-			} = $tw.utils.parseJSONSafe(event.data);
-			if (!data) return;
-
-			console.log("SSE data", data);
-			// Update last seen tiddler_id
-			if (data.tiddler_id > self.last_known_tiddler_id) {
-				self.last_known_tiddler_id = data.tiddler_id;
-			}
-			// Record the last update to this tiddler
-			self.lastRecordedUpdate[data.title] = {
-				type: data.is_deleted ? "deletion" : "update",
-				tiddler_id: data.tiddler_id
+		bag_id: string;
+		bag_name: string;
+		tiddlers: {
+			title: string;
+			revision_id: string;
+			is_deleted: boolean;
+		}[];
+	}[]
+
+	private async pollServer() {
+		type t1 = RouteTypes["handleGetBags"];
+		const [ok, err, result] = await this.recipeRequest({
+			key: "handleGetBags",
+			url: "/bags",
+			method: "GET",
+		});
+		if (!ok) throw err;
+		if (!result.responseJSON) throw new Error("no result returned");
+		type t2 = RouteTypes["handleGetBagState"];
+		this.bags = await Promise.all(result.responseJSON.map(e => this.recipeRequest({
+			key: "handleGetBagState",
+			url: "/bags/" + encodeURIComponent(e.bag_name) + "/state",
+			method: "GET",
+			queryParams: { include_deleted: "yes" },
+		}).then(([ok, err, f]) => {
+			if (!ok) throw err;
+			if (!f.responseJSON) throw new Error("no result returned");
+			return {
+				...f.responseJSON,
+				position: e.position,
 			};
-			console.log(`Oustanding requests is ${JSON.stringify(self.outstandingRequests[data.title])}`);
-			// Process the update if the tiddler is not the subject of an outstanding request
-			if (self.outstandingRequests[data.title]) return;
-			if (data.is_deleted) {
-				self.removeTiddlerInfo(data.title);
-				delete options.syncer.tiddlerInfo[data.title];
-				options.syncer.logger.log("Deleting tiddler missing from server:", data.title);
-				options.syncer.wiki.deleteTiddler(data.title);
-				options.syncer.processTaskQueue();
-			} else {
-				var result = self.incomingUpdatesFilterFn.call(self.wiki, self.wiki.makeTiddlerIterator([data.title]));
-				if (result.length > 0) {
-					self.setTiddlerInfo(data.title, data.tiddler_id.toString(), data.bag_name);
-					options.syncer.storeTiddler(data.tiddler);
-				}
-			}
+		})));
+
+		this.bags.sort((a, b) => b.position - a.position);
 
+	}
+
+	updateBags() {
+
+		if (!this.bags) throw new Error("No bags loaded, call pollServer first");
+
+		const modified = new Set<string>(),
+			deleted = new Set<string>();
+
+		const incomingTitles = new Set<string>(this.bags.map(
+			// get the titles in each layer that aren't deleted
+			e => e.tiddlers.filter(f => !f.is_deleted).map(f => f.title)
+			// and flatten them for Set
+		).flat());
+
+		let last_revision = this.last_known_revision_id;
 
+		this.bags.forEach(bag => {
+			bag.tiddlers.forEach(tid => {
+				// if the revision is old, ignore, since deletions create a new revision
+				if (tid.revision_id <= this.last_known_revision_id) return;
+				if (tid.revision_id > last_revision) last_revision = tid.revision_id;
+				// check if this title still exists in any layer
+				if (incomingTitles.has(tid.title))
+					modified.add(tid.title);
+				else
+					deleted.add(tid.title);
+			})
 		});
+
+		return {
+			modifications: [...modified.keys()],
+			deletions: [...deleted.keys()],
+			last_revision,
+			incomingTitles,
+		}
 	}
 	/*
-	Poll the server for changes. Options include:
-  
-	callback: invoked on completion as (err,changes)
+	Queue a load for a tiddler if there has been an update for it since the specified revision
 	*/
-	async pollServer(options: {
-		callback: (err: any, changes?: { modifications: string[]; deletions: string[] }) => void;
+	private checkLastRecordedUpdate(title: string, revision: string) {
+		// var lru = this.lastRecordedUpdate[title];
+		// if (lru && lru.revision_id > revision) {
+		// 	console.log(`Checking for updates to ${title} since ${JSON.stringify(lru)} comparing to ${revision}`);
+		// 	this.syncer && this.syncer.enqueueLoadTiddler(title);
+		// }
+	}
+	private get syncer() {
+		if ($tw.syncadaptor === this) return $tw.syncer as Syncer<MWSAdaptorInfo>;
+	}
+	async loadTiddlers(options: {
+		syncer: Syncer<MWSAdaptorInfo>;
+		titles: string[];
+		onNext: (tiddlerFields: TiddlerFields) => void;
+		onDone: () => void;
+		onError: (err: Error) => void;
 	}) {
-		var self = this;
-		const [ok, err, result] = await this.httpRequest({
-			url: this.host + "recipes/" + this.recipe + "/tiddlers.json",
-			data: {
-				last_known_tiddler_id: this.last_known_tiddler_id,
-				include_deleted: "true"
-			},
-			responseType: "json",
+		const { syncer, titles, onNext, onDone, onError } = options;
+
+		const tiddlers = await this.rpcRequest({
+			key: "rpcLoadRecipeTiddlerList",
+			body: { titles: titles as PrismaField<"Tiddlers", "title">[] }
+		}).catch(err => {
+			onError(err);
+			throw new Error("Error loading tiddlers: " + err.message);
+		});
+		tiddlers.forEach(({ bag_name, tiddler: fields }) => {
+			this.setTiddlerInfo({ title: fields.title, revision_id: fields.revision_id, bag_name: bag_name });
+			onNext(fields);
 		});
+		onDone();
+	}
+	async saveTiddlers(options: {
+		syncer: Syncer<MWSAdaptorInfo>;
+		tiddlers: Tiddler[];
+		onNext: (title: string, adaptorInfo: MWSAdaptorInfo, revision: string) => void;
+		onDone: () => void;
+		onError: (err: Error) => void;
+	}) {
+		const { syncer, tiddlers, onNext, onDone, onError } = options;
 
-		if (!ok) { return options.callback(err); }
-		const { data: tiddlerInfoArray = [] } = result;
+		const fields = tiddlers.map(e => e.getFieldStrings());
 
-		var modifications: string[] = [], deletions: string[] = [];
+		const results = await this.rpcRequest({
+			key: "rpcSaveRecipeTiddlerList",
+			body: { tiddlers: fields }
+		}).catch(err => {
+			onError(err);
+			throw new Error("Error saving tiddlers: " + err.message);
+		});
 
-		$tw.utils.each(tiddlerInfoArray,
-			/**
-			 * @param {{ title: string; tiddler_id: number; is_deleted: boolean; bag_name: string; }} tiddlerInfo 
-			 */
-			function (tiddlerInfo) {
-				if (tiddlerInfo.tiddler_id > self.last_known_tiddler_id) {
-					self.last_known_tiddler_id = tiddlerInfo.tiddler_id;
-				}
-				if (tiddlerInfo.is_deleted) {
-					deletions.push(tiddlerInfo.title);
-				} else {
-					modifications.push(tiddlerInfo.title);
-				}
-			}
-		);
+		results.forEach((tiddler) => {
+			this.setTiddlerInfo({ title: tiddler.title, revision_id: tiddler.revision_id, bag_name: tiddler.bag.bag_name });
+			onNext(tiddler.title, {
+				bag: tiddler.bag.bag_name,
+				title: tiddler.title,
+				revision: tiddler.revision_id
+			}, tiddler.revision_id);
+		});
+
+		onDone();
+	}
 
-		// Invoke the callback with the results
-		options.callback(null, {
-			modifications: modifications,
-			deletions: deletions
+	async deleteTiddlers(options: { syncer: Syncer<MWSAdaptorInfo>; titles: string[]; onNext: (title: string) => void; onDone: () => void; onError: (err: Error) => void; }) {
+		const { syncer, titles, onNext, onDone, onError } = options;
+
+		const results = await this.rpcRequest({
+			key: "rpcDeleteRecipeTiddlerList",
+			body: { titles: titles as PrismaField<"Tiddlers", "title">[] }
+		}).catch(err => {
+			onError(err);
+			throw new Error("Error deleting tiddlers: " + err.message);
 		});
 
-		setTimeout(() => {
-			// If Browswer Storage tiddlers were cached on reloading the wiki, add them after sync from server completes in the above callback.
-			if ($tw.browserStorage && $tw.browserStorage.isEnabled()) {
-				$tw.browserStorage.addCachedTiddlers();
-			}
+		results.forEach(({ title, revision_id, bag_id }) => {
+			this.removeTiddlerInfo({ title, revision_id, bag_id });
+			onNext(title);
 		});
+
+		onDone();
 	}
-	/*
-	Queue a load for a tiddler if there has been an update for it since the specified revision
-	*/
-	checkLastRecordedUpdate(title: string, revision: string) {
-		var lru = this.lastRecordedUpdate[title];
-		if (lru) {
-			var numRevision = $tw.utils.getInt(revision, 0);
-			if (!numRevision) {
-				this.logger.log("Error: revision is not a number", revision);
-				return;
-			}
-			console.log(`Checking for updates to ${title} since ${JSON.stringify(revision)} comparing to ${numRevision}`);
-			if (lru.tiddler_id > numRevision) {
-				this.syncer && this.syncer.enqueueLoadTiddler(title);
-			}
-		}
-	}
-	get syncer() {
-		//@ts-expect-error
-		if ($tw.syncadaptor === this) return $tw.syncer;
-	}
+
+
+
 	/*
 	Save a tiddler and invoke the callback with (err,adaptorInfo,revision)
 	*/
 	async saveTiddler(
 		tiddler: Tiddler,
-		callback: (err: any, adaptorInfo?: { bag: string }, revision?: string) => void,
+		callback: (
+			err: any,
+			adaptorInfo?: MWSAdaptorInfo,
+			revision?: string
+		) => void,
 		options?: {}
 	) {
-		var self = this, title = tiddler.fields.title;
+		var self = this, title = tiddler.fields.title as string;
+		if (title === "$:/StoryList") {
+			return callback(null);
+		}
 		if (this.isReadOnly || title.substr(0, MWC_STATE_TIDDLER_PREFIX.length) === MWC_STATE_TIDDLER_PREFIX) {
 			return callback(null);
 		}
 		self.outstandingRequests[title] = { type: "PUT" };
-		// TODO: not using getFieldStringBlock because what happens if a field name has a colon in it?
-		let body = JSON.stringify(tiddler.getFieldStrings({ exclude: ["text"] }));
+
+		// application/x-mws-tiddler
+		// The .tid file format does not support field names with colons. 
+		// Rather than trying to catch all the unsupported variations that may appear,
+		// we'll just use JSON to send it across the wire, since that is the official fallback format anyway.
+		// However, parsing a huge string value inside a JSON object is very slow,
+		// so we split off the text field and send it after the other fields. 
+
+		const fields = tiddler.getFieldStrings({});
+		const text = fields.text;
+		delete fields.text;
+		let body = JSON.stringify(fields);
+
 		if (tiddler.hasField("text")) {
-			if (typeof tiddler.fields.text !== "string" && tiddler.fields.text)
-				return callback(new Error("Error saving tiddler " + tiddler.fields.title + ": the text field is truthy but not a string"));
-			body += `\n\n${tiddler.fields.text}`
+			if (typeof text !== "string" && text)
+				return callback(new Error("Error saving tiddler " + fields.title + ": the text field is truthy but not a string"));
+			body += `\n\n${text}`
 		}
 
-		const [ok, err, result] = await this.httpRequest({
-			url: this.host + "recipes/" + encodeURIComponent(this.recipe) + "/tiddlers/" + encodeURIComponent(title),
-			type: "PUT",
+		type t = RouteTypes["handleSaveRecipeTiddler"]
+		const [ok, err, result] = await this.recipeRequest({
+			key: "handleSaveRecipeTiddler",
+			url: "/tiddlers/" + encodeURIComponent(title),
+			method: "PUT",
+			requestBodyString: body,
 			headers: {
-				"Content-type": "application/x-mws-tiddler"
-			},
-			data: body,
-			responseType: "json",
+				"content-type": "application/x-mws-tiddler"
+			}
 		});
+
 		delete self.outstandingRequests[title];
 		if (!ok) return callback(err);
-		const { headers, data } = result;
+
+		const data = result.responseJSON;
+		if (!data) return callback(null); // a 2xx response without a body is unlikely
 
 		//If Browser-Storage plugin is present, remove tiddler from local storage after successful sync to the server
 		if ($tw.browserStorage && $tw.browserStorage.isEnabled()) {
 			$tw.browserStorage.removeTiddlerFromLocalStorage(title);
 		}
 
+
 		// Save the details of the new revision of the tiddler
-		const revision = data.tiddler_id, bag_name = data.bag_name;
+		const revision = data.revision_id, bag_name = data.bag_name;
 		console.log(`Saved ${title} with revision ${revision} and bag ${bag_name}`);
 		// If there has been a more recent update from the server then enqueue a load of this tiddler
 		self.checkLastRecordedUpdate(title, revision);
 		// Invoke the callback
-		self.setTiddlerInfo(title, revision, bag_name);
-		callback(null, { bag: bag_name }, revision);
+		self.setTiddlerInfo({ title, revision_id: revision, bag_name: bag_name });
+		callback(null, { bag: bag_name, title: title, revision: revision }, revision);
 
 	}
 	/*
@@ -480,24 +695,29 @@ class MultiWikiClientAdaptor {
 	The syncer does not pass itself into options.
 	*/
 	async loadTiddler(title: string, callback: (err: any, fields?: any) => void, options: any) {
-		var self = this;
+		const self = this;
+		// if (!self.syncer) return callback(new Error("No syncer available"));
 		self.outstandingRequests[title] = { type: "GET" };
-		const [ok, err, result] = await this.httpRequest({
-			url: this.host + "recipes/" + encodeURIComponent(this.recipe) + "/tiddlers/" + encodeURIComponent(title),
-		});
+		const [ok, err, result] = await this.recipeRequest({
+			key: "handleLoadRecipeTiddler",
+			url: "/tiddlers/" + encodeURIComponent(title),
+			method: "GET",
+		})
 		delete self.outstandingRequests[title];
-		if (err === 404) {
-			return callback(null, null);
-		} else if (!ok) {
-			return callback(err);
-		}
-		const { data, headers } = result;
-		const revision = headers["x-revision-number"], bag_name = headers["x-bag-name"];
+
+		if (!ok) return callback(err);
+
+		const { responseJSON: data, headers } = result;
+		const revision = headers.get("x-revision-number") ?? "",
+			bag_name = headers.get("x-bag-name") ?? "";
+
+		if (!revision || !bag_name || !data) return callback(null, null);
+
 		// If there has been a more recent update from the server then enqueue a load of this tiddler
 		self.checkLastRecordedUpdate(title, revision);
 		// Invoke the callback
-		self.setTiddlerInfo(title, revision, bag_name);
-		callback(null, $tw.utils.parseJSONSafe(data));
+		self.setTiddlerInfo({ title, revision_id: revision, bag_name: bag_name });
+		callback(null, data);
 	}
 	/*
 	Delete a tiddler and invoke the callback with (err)
@@ -512,23 +732,254 @@ class MultiWikiClientAdaptor {
 		// if(!bag) { return callback(null, options.tiddlerInfo.adaptorInfo); }
 		self.outstandingRequests[title] = { type: "DELETE" };
 		// Issue HTTP request to delete the tiddler
-		const [ok, err, result] = await this.httpRequest({
-			url: this.host + "recipes/" + encodeURIComponent(this.recipe) + "/tiddlers/" + encodeURIComponent(title),
-			type: "DELETE",
+		type t = RouteTypes["handleDeleteRecipeTiddler"]
+		const [ok, err, result] = await this.recipeRequest({
+			key: "handleDeleteRecipeTiddler",
+			url: "/tiddlers/" + encodeURIComponent(title),
+			method: "DELETE",
 		});
 		delete self.outstandingRequests[title];
-		if (!ok) { return callback(err); }
-		const { data } = result;
-		const revision = data.tiddler_id, bag_name = data.bag_name;
+		if (!ok) return callback(err);
+		const { responseJSON: data } = result;
+		if (!data) return callback(null);
+
+		const { revision_id, bag_name, bag_id } = data;
 		// If there has been a more recent update from the server then enqueue a load of this tiddler
-		self.checkLastRecordedUpdate(title, revision);
-		self.removeTiddlerInfo(title);
+		self.checkLastRecordedUpdate(title, revision_id);
+		self.removeTiddlerInfo({ title, revision_id, bag_id });
 		// Invoke the callback & return null adaptorInfo
 		callback(null, null);
 	}
+
+	async rpcRequest<T extends string & keyof RouteTypes>(
+		options: {
+			key: T;
+			body: zod.infer<RouteTypes[T]["T"]>;
+		}
+	): Promise<RouteTypes[T]["R"]> {
+
+		type t = RouteTypes["handleSaveRecipeTiddler"]
+		const [ok, err, result] = await this.recipeRequest({
+			key: options.key,
+			url: "/rpc/" + options.key,
+			method: "PUT",
+			requestBodyString: JSON.stringify(options.body),
+			headers: {
+				"content-type": "application/json"
+			}
+		});
+
+		if (!ok) throw err;
+
+		if (!result.responseJSON) throw new Error("No response JSON returned");
+
+		return result.responseJSON as RouteTypes[T]["R"];
+	}
+
+	/** 
+	 * This throws an error if the response status is not 2xx, but will still return the response alongside the error.
+	 * 
+	 * So if the first parameter is false, the third parameter may still contain the response.
+	 */
+	private async recipeRequest<KEY extends (string & keyof RouteTypes)>(
+		options: Omit<HttpRequestOptions<"arraybuffer">, "responseType"> & { key: KEY }
+	) {
+		if (!options.url.startsWith("/"))
+			throw new Error("The url does not start with a slash");
+
+		return await httpRequest({
+			...options,
+			responseType: "blob",
+			url: this.host + "recipe/" + encodeURIComponent(this.recipe) + options.url,
+		}).then(async e => {
+			if (!e.ok) return [
+				false, new Error(
+					`The server return a status code ${e.status} with the following reason: `
+					+ `${e.headers.get("x-reason") ?? "(no reason given)"}`
+				), {
+					...e,
+					responseString: "",
+					responseJSON: undefined,
+				}
+			] as const;
+			let responseString: string = "";
+			if (e.headers.get("x-gzip-stream") === "yes") {
+				// Browsers only decode the first stream, 
+				// so we cant use content-encoding or DecompressionStream
+
+				await new Promise<void>(async resolve => {
+
+					const gunzip = new fflate.AsyncGunzip((err, chunk, final) => {
+						if (err) return console.log(err);
+						responseString += fflate.strFromU8(chunk);
+						if (final) resolve();
+					});
+
+					if (this.isDevMode) gunzip.onmember = e => console.log("gunzip member", e);
+
+					gunzip.push(new Uint8Array(await readBlobAsArrayBuffer(e.response)));
+					// this has to be on a separate line
+					gunzip.push(new Uint8Array(), true);
+
+				});
+
+				if (this.isDevMode) console.log("gunzip result", responseString.length);
+
+			} else {
+				responseString = fflate.strFromU8(new Uint8Array(await readBlobAsArrayBuffer(e.response)));
+			}
+
+			return [true, void 0, {
+				...e,
+				responseString,
+				/** this is undefined if status is not 200 */
+				responseJSON: e.status === 200
+					? tryParseJSON(responseString) as RouteTypes[KEY]["R"]
+					: undefined,
+			}] as const;
+		}, e => [false, e, void 0] as const);
+
+		function tryParseJSON(data: string) {
+			try {
+				return JSON.parse(data);
+			} catch (e) {
+				console.error("Error parsing JSON, returning undefined", e);
+				// console.log(data);
+				return undefined;
+			}
+		}
+
+	}
+
 }
 
 
 if ($tw.browser && document.location.protocol.startsWith("http")) {
 	exports.adaptorClass = MultiWikiClientAdaptor;
 }
+
+type ParamsInput = URLSearchParams | [string, string][] | object | string | undefined;
+
+interface HttpRequestOptions<TYPE extends "arraybuffer" | "blob" | "text"> {
+	/** The request METHOD. Maybe be anything except CONNECT, TRACE, or TRACK.  */
+	method: string;
+	/** The url may also contain query params. */
+	url: string;
+	/** The response types */
+	responseType: TYPE;
+	headers?: HeadersInit;
+	/** This is parsed separately from the url and appended to it. */
+	queryParams?: ParamsInput;
+	/** 
+	 * The string to send as the request body. Not valid for GET and HEAD.
+	 * 
+	 * For `application/x-www-form-urlencoded`, use `new URLSearchParams().toString()`.
+	 * 
+	 * For `application/json`, use `JSON.stringify()`
+	 */
+	requestBodyString?: string;
+	progress?: (event: ProgressEvent<EventTarget>) => void;
+}
+
+
+function httpRequest<TYPE extends "arraybuffer" | "blob" | "text">(options: HttpRequestOptions<TYPE>) {
+
+	return new Promise<{
+		/** Shorthand to check if the response is in the 2xx range. */
+		ok: boolean;
+		status: number;
+		statusText: string;
+		headers: Headers;
+		response:
+		TYPE extends "arraybuffer" ? ArrayBuffer :
+		TYPE extends "blob" ? Blob :
+		TYPE extends "text" ? string :
+		never;
+	}>((resolve, reject) => {
+		// if this throws sync'ly, the promise will reject.
+
+		options.method = options.method.toUpperCase();
+
+		if ((options.method === "GET" || options.method === "HEAD") && options.requestBodyString)
+			throw new Error("requestBodyString must be falsy if method is GET or HEAD");
+
+
+		const url = new URL(options.url, location.href);
+		const query = paramsInput(options.queryParams);
+		query.forEach((v, k) => { url.searchParams.append(k, v); });
+
+		const headers = new Headers(options.headers || {});
+
+		const request = new XMLHttpRequest();
+		request.responseType = options.responseType || "text";
+
+		try {
+			request.open(options.method, url, true);
+		} catch (e) { console.log(e, { e }); throw e; }
+
+		if (!headers.has("content-type"))
+			headers.set("content-type", "application/x-www-form-urlencoded; charset=UTF-8");
+
+		if (!headers.has("x-requested-with"))
+			headers.set("x-requested-with", "TiddlyWiki");
+
+		headers.set("accept", "application/json");
+
+
+		headers.forEach((v, k) => {
+			request.setRequestHeader(k, v);
+		});
+
+		request.onreadystatechange = function () {
+			if (this.readyState !== 4) return;
+
+			const headers = new Headers();
+			request.getAllResponseHeaders()?.trim().split(/[\r\n]+/).forEach((line) => {
+				const parts = line.split(": ");
+				const header = parts.shift()?.toLowerCase();
+				const value = parts.join(": ");
+				if (header) headers.append(header, value);
+			});
+			resolve({
+				ok: this.status >= 200 && this.status < 300,
+				status: this.status,
+				statusText: this.statusText,
+				response: this.response,
+				headers,
+			});
+
+		};
+
+		if (options.progress)
+			request.onprogress = options.progress;
+
+		try { request.send(options.requestBodyString); } catch (e) { console.log(e, { e }); throw e; }
+
+
+	});
+
+
+	function paramsInput(input: ParamsInput) {
+		if (!input) return new URLSearchParams();
+		if (input instanceof URLSearchParams) return input;
+		if (Array.isArray(input) || typeof input === "string") return new URLSearchParams(input);
+		return new URLSearchParams(Object.entries(input));
+	}
+
+}
+
+function readBlobAsArrayBuffer(blob: Blob) {
+	const error = new Error("Error reading blob");
+	return new Promise<ArrayBuffer>((resolve, reject) => {
+		const reader = new FileReader();
+		reader.onload = () => {
+			resolve(reader.result as ArrayBuffer)
+		};
+		reader.onerror = () => {
+			reject(error);
+		};
+		reader.readAsArrayBuffer(blob);
+	});
+
+}
+