@vibesdotdev/storage 0.0.1

package/src/sync/sync-bridge.ts

@@ -0,0 +1,763 @@
+/**
+ * Sync Bridge — Default Implementation
+ *
+ * Executes bidirectional sync between local runtime descriptors and a cloud CRUD API.
+ * Each SyncBridge instance corresponds to a storage/sync-bridge descriptor.
+ *
+ * Flow:
+ * 1. Collect local descriptors via runtime.query(source.runtimeKind)
+ * 2. Fetch remote entities from target.apiPath
+ * 3. Diff local ↔ remote by idField
+ * 4. Push new/updated local items, pull new/updated remote items
+ * 5. Track state in sync-state.json alongside storage manifests
+ */
+
+import { getRuntimeEnv } from '@vibesdotdev/runtime';
+import type { SyncBridgeDescriptor } from '../schemas/sync-bridge-descriptor';
+
+import type { KindContext } from '@vibesdotdev/runtime/schemas/kind';
+import {
+	loadSyncState,
+	saveSyncState,
+	getManifestSyncState,
+	markSynced,
+	markLocalAhead,
+	markConflict,
+	markError,
+	markRemoteAhead,
+	computeHash
+} from './sync-state.consumer';
+
+export interface SyncBridgeResult {
+	bridgeId: string;
+	pushed: number;
+	pulled: number;
+	skipped: number;
+	conflicts: number;
+	errors: number;
+	details: Array<{
+		entityId: string;
+		action: 'pushed' | 'pulled' | 'skipped' | 'conflict' | 'error';
+		message?: string;
+	}>;
+}
+
+export interface SyncBridgeOptions {
+	dryRun?: boolean;
+	force?: boolean;
+	direction?: 'push' | 'pull' | 'both';
+}
+
+/**
+ * Default SyncBridge implementation.
+ * Constructed by the runtime with (descriptor, context).
+ */
+export class SyncBridge {
+	readonly descriptor: SyncBridgeDescriptor;
+
+	constructor(descriptor: SyncBridgeDescriptor, _context?: KindContext) {
+		this.descriptor = descriptor;
+	}
+
+	get id(): string {
+		return this.descriptor.id;
+	}
+
+	/** Cached h2 fetch for HTTPS local endpoints */
+	private _h2Fetch: ((input: string, init?: RequestInit) => Promise<Response>) | null = null;
+
+	/**
+	 * Get an HTTP/2-aware fetch function.
+	 * Local HTTPS endpoints (vite dev) require h2 since Bun's fetch is h1 only.
+	 */
+	private async getFetch(
+		apiEndpoint: string
+	): Promise<(input: string, init?: RequestInit) => Promise<Response>> {
+		if (this._h2Fetch) return this._h2Fetch;
+
+		const isHttps = apiEndpoint.startsWith('https://');
+		const isLocal =
+			apiEndpoint.includes('.local') ||
+			apiEndpoint.includes('localhost') ||
+			apiEndpoint.includes('127.0.0.1');
+
+		if (isHttps && isLocal) {
+			try {
+				// Set REJECT_UNAUTHORIZED=false for local self-signed certs
+				if (!getRuntimeEnv('REJECT_UNAUTHORIZED')) {
+					globalThis.process?.['env'] && (globalThis.process.env.REJECT_UNAUTHORIZED = 'false');
+				}
+				// h2 fetch lives in the app layer; local HTTP/2 dev servers fall through to regular fetch.
+			} catch {
+				// Fall through to regular fetch
+			}
+		}
+
+		this._h2Fetch = (input: string, init?: RequestInit) => fetch(input, init);
+		return this._h2Fetch;
+	}
+
+	/**
+	 * Collect local descriptors from the runtime.
+	 */
+	async collectLocal(): Promise<Record<string, unknown>[]> {
+		if (this.descriptor.collect) {
+			return this.descriptor.collect();
+		}
+
+		const { getVibesRuntime } = await import('@vibesdotdev/runtime');
+		const runtime = getVibesRuntime();
+		const descriptors = runtime
+			.query(this.descriptor.source.runtimeKind)
+			.descriptors() as unknown as Record<string, unknown>[];
+		return descriptors;
+	}
+
+	private stripSensitiveFields(entity: Record<string, unknown>): Record<string, unknown> {
+		const sanitized = { ...entity };
+		for (const field of this.descriptor.sync.sensitiveFields) {
+			delete sanitized[field];
+		}
+		return sanitized;
+	}
+
+	private buildPushShape(entity: Record<string, unknown>): Record<string, unknown> {
+		const transformed = this.descriptor.toRemote ? this.descriptor.toRemote(entity) : entity;
+		return this.stripSensitiveFields(transformed);
+	}
+
+	private buildComparableRemote(entity: Record<string, unknown>): Record<string, unknown> {
+		const localShape = this.descriptor.fromRemote ? this.descriptor.fromRemote(entity) : entity;
+		return this.buildPushShape(localShape);
+	}
+
+	private getEntityTimestamp(entity: Record<string, unknown> | undefined): number | null {
+		if (!entity) return null;
+		const value =
+			typeof entity.deletedAt === 'string'
+				? entity.deletedAt
+				: typeof entity.updatedAt === 'string'
+					? entity.updatedAt
+					: typeof entity.createdAt === 'string'
+						? entity.createdAt
+						: null;
+		if (!value) return null;
+		const timestamp = new Date(value).getTime();
+		return Number.isNaN(timestamp) ? null : timestamp;
+	}
+
+	private isDeleted(entity: Record<string, unknown> | undefined): boolean {
+		return Boolean(entity?.deletedAt);
+	}
+
+	private resolveConflictAction(
+		localItem: Record<string, unknown>,
+		remoteItem: Record<string, unknown>,
+		direction: 'push' | 'pull' | 'both'
+	): 'push' | 'pull' | 'conflict' {
+		const strategy = this.descriptor.sync.conflictStrategy;
+
+		if (strategy === 'manual') {
+			return 'conflict';
+		}
+		if (strategy === 'local-wins') {
+			return direction === 'pull' ? 'conflict' : 'push';
+		}
+		if (strategy === 'remote-wins') {
+			return direction === 'push' ? 'conflict' : 'pull';
+		}
+
+		const localTimestamp = this.getEntityTimestamp(localItem);
+		const remoteTimestamp = this.getEntityTimestamp(remoteItem);
+		if (localTimestamp === null || remoteTimestamp === null || localTimestamp === remoteTimestamp) {
+			return 'conflict';
+		}
+
+		if (localTimestamp > remoteTimestamp) {
+			return direction === 'pull' ? 'conflict' : 'push';
+		}
+
+		return direction === 'push' ? 'conflict' : 'pull';
+	}
+
+	private extractList(data: unknown): Record<string, unknown>[] {
+		if (!data || typeof data !== 'object') {
+			return [];
+		}
+
+		const record = data as Record<string, unknown>;
+		const listField = this.descriptor.target.listField;
+		if (listField && Array.isArray(record[listField])) {
+			return record[listField] as Record<string, unknown>[];
+		}
+		if (Array.isArray(data)) {
+			return data as Record<string, unknown>[];
+		}
+		for (const key of ['items', 'data', 'results', 'benchmarks', 'scenarios']) {
+			if (Array.isArray(record[key])) {
+				return record[key] as Record<string, unknown>[];
+			}
+		}
+		return [];
+	}
+
+	/**
+	 * Fetch remote entities from the cloud CRUD API.
+	 */
+	async fetchRemote(apiEndpoint: string, apiToken?: string): Promise<Record<string, unknown>[]> {
+		const { target } = this.descriptor;
+		const url = `${apiEndpoint}${target.apiPath}`;
+
+		try {
+			const headers: Record<string, string> = {
+				'Content-Type': 'application/json'
+			};
+			if (apiToken) {
+				headers['Authorization'] = `Bearer ${apiToken}`;
+			}
+
+			const doFetch = await this.getFetch(apiEndpoint);
+			const fetchPage = async (
+				page: number
+			): Promise<{
+				items: Record<string, unknown>[];
+				totalPages: number;
+			}> => {
+				const pageUrl = new URL(url);
+				pageUrl.searchParams.set('page', String(page));
+				if (!pageUrl.searchParams.has('perPage')) {
+					pageUrl.searchParams.set('perPage', '100');
+				}
+				if (target.includeDeleted && !pageUrl.searchParams.has('includeDeleted')) {
+					pageUrl.searchParams.set('includeDeleted', 'true');
+				}
+
+				const response = await doFetch(pageUrl.toString(), { headers });
+				if (!response.ok) {
+					if (response.status === 404) {
+						return { items: [], totalPages: 0 };
+					}
+					throw new Error(`API ${response.status}: ${response.statusText}`);
+				}
+
+				const data = await response.json();
+				const items = this.extractList(data);
+				const pagination =
+					data && typeof data === 'object' && 'pagination' in (data as Record<string, unknown>)
+						? ((data as Record<string, unknown>).pagination as Record<string, unknown>)
+						: null;
+				const totalPages =
+					typeof pagination?.totalPages === 'number'
+						? pagination.totalPages
+						: typeof pagination?.totalPages === 'string'
+							? Number.parseInt(pagination.totalPages, 10)
+							: 1;
+
+				return { items, totalPages: Number.isNaN(totalPages) ? 1 : totalPages };
+			};
+
+			const firstPage = await fetchPage(1);
+			if (firstPage.totalPages <= 1) {
+				return firstPage.items;
+			}
+
+			const items = [...firstPage.items];
+			for (let page = 2; page <= firstPage.totalPages; page++) {
+				const nextPage = await fetchPage(page);
+				items.push(...nextPage.items);
+			}
+
+			return items;
+		} catch (error) {
+			throw new Error(`Failed to fetch remote from ${url}: ${(error as Error).message}`);
+		}
+	}
+
+	/**
+	 * Push a single entity to the cloud.
+	 */
+	async pushEntity(
+		entity: Record<string, unknown>,
+		apiEndpoint: string,
+		apiToken?: string
+	): Promise<void> {
+		const { target, sync } = this.descriptor;
+		const entityId = entity[sync.idField] as string;
+		const entityPath = target.entityPath ?? `${target.apiPath}/:id`;
+		const url = `${apiEndpoint}${entityPath.replace(':id', encodeURIComponent(entityId))}`;
+
+		const headers: Record<string, string> = {
+			'Content-Type': 'application/json'
+		};
+		if (apiToken) {
+			headers['Authorization'] = `Bearer ${apiToken}`;
+		}
+
+		const doFetch = await this.getFetch(apiEndpoint);
+
+		// Try PUT first (upsert), fall back to POST
+		let response = await doFetch(url, {
+			method: 'PUT',
+			headers,
+			body: JSON.stringify(entity)
+		});
+
+		if (response.status === 404 || response.status === 405) {
+			// PUT not supported for this entity — try POST to collection
+			response = await doFetch(`${apiEndpoint}${target.apiPath}`, {
+				method: 'POST',
+				headers,
+				body: JSON.stringify(entity)
+			});
+		}
+
+		if (!response.ok) {
+			const body = await response.text().catch(() => '');
+			throw new Error(`Push failed (${response.status}): ${body}`);
+		}
+	}
+
+	async deleteEntity(
+		entity: Record<string, unknown>,
+		apiEndpoint: string,
+		apiToken?: string
+	): Promise<void> {
+		const { target, sync } = this.descriptor;
+		const entityId = entity[sync.idField] as string;
+		const entityPath = target.entityPath ?? `${target.apiPath}/:id`;
+		const url = `${apiEndpoint}${entityPath.replace(':id', encodeURIComponent(entityId))}`;
+
+		const headers: Record<string, string> = {};
+		if (apiToken) {
+			headers['Authorization'] = `Bearer ${apiToken}`;
+		}
+
+		const doFetch = await this.getFetch(apiEndpoint);
+		if (this.descriptor.deleteRemote) {
+			await this.descriptor.deleteRemote(entity, {
+				apiEndpoint,
+				apiToken,
+				fetch: doFetch
+			});
+			return;
+		}
+
+		const response = await doFetch(url, {
+			method: 'DELETE',
+			headers
+		});
+
+		if (response.status === 404) {
+			return;
+		}
+
+		if (!response.ok) {
+			const body = await response.text().catch(() => '');
+			throw new Error(`Delete failed (${response.status}): ${body}`);
+		}
+	}
+
+	/**
+	 * Persist pulled items locally.
+	 * Storage-backed bridges: merge into the storage manifest (localdb/json).
+	 * Descriptor-only bridges: register with runtime.
+	 */
+	private async persistPulledItems(
+		newItems: Record<string, unknown>[],
+		existingLocalItems: Record<string, unknown>[],
+		idField: string
+	): Promise<void> {
+		const { writePulled, storageManifestId } = this.descriptor;
+
+		if (writePulled) {
+			await writePulled(newItems);
+			return;
+		}
+
+		if (storageManifestId) {
+			const { StorageManifest } = await import('../manifest/base');
+			const manifest = StorageManifest.fromId<Record<string, unknown>[]>(storageManifestId);
+			const existing = (await manifest.load()) ?? [];
+			if (newItems.length > 0) {
+				const merged = new Map(existing.map((item) => [String(item[idField]), item]));
+				for (const item of newItems) {
+					merged.set(String(item[idField]), item);
+				}
+				await manifest.save([...merged.values()] as never);
+			}
+			return;
+		}
+
+		// Descriptor-only: register with runtime
+		const { getVibesRuntime } = await import('@vibesdotdev/runtime');
+		const runtime = getVibesRuntime();
+		const kindId = this.descriptor.source.runtimeKind;
+		if (!runtime.hasKind(kindId)) return;
+
+		for (const item of newItems) {
+			try {
+				runtime.registerDescriptor(
+					kindId,
+					item as unknown as Parameters<typeof runtime.registerDescriptor>[1]
+				);
+			} catch {
+				// Descriptor already registered or invalid — skip
+			}
+		}
+	}
+
+	/**
+	 * Execute a full sync cycle.
+	 */
+	async execute(
+		apiEndpoint: string,
+		options: SyncBridgeOptions = {},
+		apiToken?: string
+	): Promise<SyncBridgeResult> {
+		const { sync } = this.descriptor;
+		const { dryRun = false, force = false, direction = 'both' } = options;
+		const idField = sync.idField;
+
+		const result: SyncBridgeResult = {
+			bridgeId: this.id,
+			pushed: 0,
+			pulled: 0,
+			skipped: 0,
+			conflicts: 0,
+			errors: 0,
+			details: []
+		};
+
+		const syncState = await loadSyncState();
+
+		try {
+			let localItems = await this.collectLocal();
+
+			if (sync.ownershipFilter !== 'all' && this.descriptor.source.sourceTypeField) {
+				const field = this.descriptor.source.sourceTypeField;
+				localItems = localItems.filter((item) => {
+					const sourceType = item[field] as string | undefined;
+					if (sync.ownershipFilter === 'user-only') return sourceType !== 'system';
+					if (sync.ownershipFilter === 'system-only') return sourceType === 'system';
+					return true;
+				});
+			}
+
+			let remoteItems: Record<string, unknown>[] = [];
+			try {
+				remoteItems = await this.fetchRemote(apiEndpoint, apiToken);
+			} catch (error) {
+				if (direction === 'pull') {
+					result.errors++;
+					result.details.push({
+						entityId: '*',
+						action: 'error',
+						message: `Remote unreachable: ${(error as Error).message}`
+					});
+					return result;
+				}
+			}
+
+			const pushAllowed = direction === 'push' || direction === 'both';
+			const pullAllowed = direction === 'pull' || direction === 'both';
+			const doFetch = await this.getFetch(apiEndpoint);
+
+			const localMap = new Map(localItems.map((item) => [String(item[idField]), item]));
+			const remoteMap = new Map(
+				remoteItems.map((item) => [String(item[idField] ?? item.id), item])
+			);
+			const allEntityIds = new Set([...localMap.keys(), ...remoteMap.keys()]);
+			const pulledItems: Record<string, unknown>[] = [];
+
+			const pushLocal = async (
+				entityId: string,
+				localItem: Record<string, unknown>,
+				localHash: string
+			): Promise<void> => {
+				const stateKey = `${this.id}:${entityId}`;
+				const pushItem = this.buildPushShape(localItem);
+
+				if (dryRun) {
+					result.pushed++;
+					result.details.push({
+						entityId,
+						action: 'pushed',
+						message: '[dry-run] Would push to cloud'
+					});
+					return;
+				}
+
+				try {
+					if (this.descriptor.beforePush) {
+						await this.descriptor.beforePush(localItem, {
+							apiEndpoint,
+							apiToken,
+							fetch: doFetch
+						});
+					}
+					await this.pushEntity(pushItem, apiEndpoint, apiToken);
+					result.pushed++;
+					result.details.push({ entityId, action: 'pushed' });
+					markSynced(syncState, stateKey, localHash, 'push');
+				} catch (error) {
+					result.errors++;
+					result.details.push({
+						entityId,
+						action: 'error',
+						message: `Push failed: ${(error as Error).message}`
+					});
+					markError(syncState, stateKey, (error as Error).message);
+				}
+			};
+
+			const deleteRemote = async (
+				entityId: string,
+				localItem: Record<string, unknown>,
+				localHash: string
+			): Promise<void> => {
+				const stateKey = `${this.id}:${entityId}`;
+
+				if (dryRun) {
+					result.pushed++;
+					result.details.push({
+						entityId,
+						action: 'pushed',
+						message: '[dry-run] Would delete remote tombstone'
+					});
+					return;
+				}
+
+				try {
+					await this.deleteEntity(localItem, apiEndpoint, apiToken);
+					result.pushed++;
+					result.details.push({
+						entityId,
+						action: 'pushed',
+						message: 'Deleted remote via tombstone'
+					});
+					markSynced(syncState, stateKey, localHash, 'push');
+				} catch (error) {
+					result.errors++;
+					result.details.push({
+						entityId,
+						action: 'error',
+						message: `Delete failed: ${(error as Error).message}`
+					});
+					markError(syncState, stateKey, (error as Error).message);
+				}
+			};
+
+			const pullRemote = (
+				entityId: string,
+				remoteItem: Record<string, unknown>,
+				remoteHash: string
+			): void => {
+				const stateKey = `${this.id}:${entityId}`;
+				const localItem = this.descriptor.fromRemote
+					? this.descriptor.fromRemote(remoteItem)
+					: remoteItem;
+
+				if (dryRun) {
+					result.pulled++;
+					result.details.push({
+						entityId,
+						action: 'pulled',
+						message: '[dry-run] Would pull from cloud'
+					});
+					return;
+				}
+
+				pulledItems.push(localItem);
+				result.pulled++;
+				result.details.push({ entityId, action: 'pulled' });
+				markSynced(syncState, stateKey, remoteHash, 'pull');
+			};
+
+			for (const entityId of allEntityIds) {
+				const stateKey = `${this.id}:${entityId}`;
+				const localItem = localMap.get(entityId);
+				const remoteItem = remoteMap.get(entityId);
+				const comparableRemote = remoteItem
+					? this.descriptor.fromRemote
+						? this.descriptor.fromRemote(remoteItem)
+						: remoteItem
+					: null;
+				const localHash = localItem ? computeHash(this.buildPushShape(localItem)) : null;
+				const remoteHash = comparableRemote
+					? computeHash(this.buildPushShape(comparableRemote))
+					: null;
+				const prev = getManifestSyncState(syncState, stateKey);
+				const localDeleted = this.isDeleted(localItem);
+				const remoteDeleted = this.isDeleted(comparableRemote ?? undefined);
+
+				if (localItem && remoteItem && localHash === remoteHash) {
+					result.skipped++;
+					result.details.push({ entityId, action: 'skipped', message: 'In sync' });
+					markSynced(
+						syncState,
+						stateKey,
+						localHash ?? remoteHash ?? '',
+						direction === 'pull' ? 'pull' : 'push'
+					);
+					continue;
+				}
+
+				if (localItem && !remoteItem) {
+					if (localDeleted) {
+						result.skipped++;
+						result.details.push({
+							entityId,
+							action: 'skipped',
+							message: 'Local tombstone already applied remotely'
+						});
+						if (localHash) {
+							markSynced(syncState, stateKey, localHash, 'push');
+						}
+						continue;
+					}
+					if (pushAllowed && localHash) {
+						await pushLocal(entityId, localItem, localHash);
+					} else {
+						result.skipped++;
+						result.details.push({
+							entityId,
+							action: 'skipped',
+							message: 'Local-only item'
+						});
+						if (localHash) {
+							markLocalAhead(syncState, stateKey, localHash);
+						}
+					}
+					continue;
+				}
+
+				if (!localItem && remoteItem) {
+					if (pullAllowed && remoteHash) {
+						pullRemote(entityId, remoteItem, remoteHash);
+					} else {
+						result.skipped++;
+						result.details.push({
+							entityId,
+							action: 'skipped',
+							message: 'Remote-only item'
+						});
+						if (remoteHash) {
+							markRemoteAhead(syncState, stateKey, remoteHash);
+						}
+					}
+					continue;
+				}
+
+				if (!localItem || !remoteItem || !localHash || !remoteHash) {
+					continue;
+				}
+
+				if (localDeleted !== remoteDeleted) {
+					if (localDeleted) {
+						if (pushAllowed) {
+							await deleteRemote(entityId, localItem, localHash);
+						} else {
+							result.skipped++;
+							result.details.push({
+								entityId,
+								action: 'skipped',
+								message: 'Local tombstone pending remote delete'
+							});
+							markLocalAhead(syncState, stateKey, localHash);
+						}
+					} else if (pullAllowed) {
+						pullRemote(entityId, remoteItem, remoteHash);
+					} else {
+						result.skipped++;
+						result.details.push({
+							entityId,
+							action: 'skipped',
+							message: 'Remote tombstone pending local pull'
+						});
+						markRemoteAhead(syncState, stateKey, remoteHash);
+					}
+					continue;
+				}
+
+				const localChanged = force || localHash !== prev.localHash;
+				const remoteChanged = force || remoteHash !== prev.remoteHash;
+
+				if (pushAllowed && pullAllowed) {
+					if (localChanged && !remoteChanged) {
+						await pushLocal(entityId, localItem, localHash);
+						continue;
+					}
+					if (remoteChanged && !localChanged) {
+						pullRemote(entityId, remoteItem, remoteHash);
+						continue;
+					}
+				}
+
+				if (pushAllowed && !pullAllowed) {
+					if (localChanged) {
+						await pushLocal(entityId, localItem, localHash);
+					} else {
+						result.skipped++;
+						result.details.push({
+							entityId,
+							action: 'skipped',
+							message: 'Remote changed; pull required'
+						});
+						markRemoteAhead(syncState, stateKey, remoteHash);
+					}
+					continue;
+				}
+
+				if (pullAllowed && !pushAllowed) {
+					if (remoteChanged) {
+						pullRemote(entityId, remoteItem, remoteHash);
+					} else {
+						result.skipped++;
+						result.details.push({
+							entityId,
+							action: 'skipped',
+							message: 'Local changed; push required'
+						});
+						markLocalAhead(syncState, stateKey, localHash);
+					}
+					continue;
+				}
+
+				const conflictAction = this.resolveConflictAction(localItem, remoteItem, direction);
+				if (conflictAction === 'push') {
+					await pushLocal(entityId, localItem, localHash);
+					continue;
+				}
+				if (conflictAction === 'pull') {
+					pullRemote(entityId, remoteItem, remoteHash);
+					continue;
+				}
+
+				result.conflicts++;
+				result.details.push({
+					entityId,
+					action: 'conflict',
+					message: 'Both local and remote changed'
+				});
+				markConflict(syncState, stateKey, localHash, remoteHash);
+			}
+
+			if (!dryRun && pulledItems.length > 0) {
+				await this.persistPulledItems(pulledItems, localItems, idField);
+			}
+
+			if (!dryRun) {
+				await saveSyncState(syncState);
+			}
+		} catch (error) {
+			result.errors++;
+			result.details.push({
+				entityId: '*',
+				action: 'error',
+				message: `Bridge sync failed: ${(error as Error).message}`
+			});
+		}
+
+		return result;
+	}
+}