@vibesdotdev/storage 0.0.1

package/src/manifest/base.ts

@@ -0,0 +1,594 @@
+/**
+ * Storage Manifest Base Class
+ *
+ * Base class for working with stored data. Modules can extend this
+ * for custom methods while getting load/save for free.
+ *
+ * Uses runtime query API to resolve the appropriate adapter.
+ */
+
+import type { z } from 'zod/v4';
+import { getLogger } from '@vibesdotdev/logging';
+import { getVibesRuntime } from '@vibesdotdev/runtime/api/runtime';
+import type { KindContext } from '@vibesdotdev/runtime/schemas/kind';
+import type { StorageAdapterInterface } from '../interfaces/adapter';
+import type { StorageAdapterDescriptor } from '../schemas/adapter-descriptor';
+import type { StorageManifestDescriptor } from '../schemas/manifest-descriptor';
+import type { DocumentChangeCallback, Unsubscribe } from '../interfaces/document-store';
+import { createTracer } from '@vibesdotdev/runtime/debug';
+import { isBuildingEnvironment } from '@vibesdotdev/runtime';
+import { resolveHomePath, resolveScopePathTemplate, resolveWorkspacePath } from '../paths';
+import { toErrorContext } from '../utils/error-context';
+
+const logger = getLogger('storage:manifest');
+
+type RuntimeLike = ReturnType<typeof getVibesRuntime>;
+
+/**
+ * Base class for storage manifests.
+ *
+ * Provides document-style operations (load, save, exists, delete) using
+ * the runtime to resolve the appropriate adapter based on scope and configuration.
+ *
+ * Includes caching and subscription support for reactive updates.
+ *
+ * @template T - The type of data being stored
+ *
+ * @example
+ * ```ts
+ * // Runtime instantiation (via descriptor)
+ * // The runtime will instantiate this with the descriptor
+ *
+ * // Manual instantiation by ID
+ * const storage = StorageManifest.fromId<AIProvider[]>('ai.providers');
+ * const providers = await storage.load();
+ *
+ * // Extended class with custom methods
+ * class AIProvidersStorage extends StorageManifest<AIProvider[]> {
+ *   async getEnabled(): Promise<AIProvider[]> {
+ *     const all = await this.load() ?? [];
+ *     return all.filter(p => p.enabled);
+ *   }
+ * }
+ * ```
+ */
+export class StorageManifest<T> {
+	protected descriptor!: StorageManifestDescriptor;
+	protected context!: KindContext;
+	private adapterCache: StorageAdapterInterface | null = null;
+
+	// Lazy resolution
+	private pendingId: string | null = null;
+	private pendingContext: KindContext | undefined;
+	private resolved = false;
+
+	// Caching
+	private cachedData: T | null = null;
+	private cacheValid = false;
+
+	// Subscriptions
+	private subscribers = new Set<(data: T | null) => void>();
+	private storeUnsubscribe: Unsubscribe | null = null;
+	private subscriptionStarted = false;
+
+	private getRuntime(): RuntimeLike {
+		return (this.context.runtime as RuntimeLike | undefined) ?? getVibesRuntime();
+	}
+
+	/**
+	 * Create a storage manifest instance.
+	 *
+	 * @param descriptorOrId - Either the full descriptor (for runtime instantiation)
+	 *                         or the manifest ID string (for manual instantiation)
+	 * @param context - Runtime context (optional for manual instantiation, required for runtime)
+	 */
+	constructor(descriptorOrId: StorageManifestDescriptor | string, context?: KindContext) {
+		if (typeof descriptorOrId === 'string') {
+			// Defer descriptor resolution so module-level singletons don't crash
+			// when imported before the runtime is bootstrapped (e.g. SSR imports).
+			this.pendingId = descriptorOrId;
+			this.pendingContext = context;
+			// Set stub descriptor so property accesses before resolve don't throw
+			this.descriptor = {
+				id: descriptorOrId,
+				kind: 'storage/manifest',
+				namespace: descriptorOrId
+			} as StorageManifestDescriptor;
+			this.context = context ?? { runtime: getVibesRuntime() };
+		} else {
+			// Direct descriptor (runtime instantiation)
+			this.descriptor = descriptorOrId;
+			this.context = context ?? { runtime: getVibesRuntime() };
+			this.resolved = true;
+		}
+	}
+
+	/**
+	 * Resolve the descriptor from the runtime on first data access.
+	 * No-op if already resolved or constructed with a direct descriptor.
+	 */
+	private ensureResolved(): void {
+		if (this.resolved || !this.pendingId) return;
+
+		const runtime =
+			(this.pendingContext?.runtime as RuntimeLike | undefined) ??
+			(this.context.runtime as RuntimeLike | undefined) ??
+			getVibesRuntime();
+		const hasKind = runtime.hasKind('storage/manifest');
+
+		if (!hasKind) {
+			// Runtime not yet bootstrapped — keep the stub descriptor.
+			// It will be re-resolved on next access after bootstrap.
+			return;
+		}
+
+		const descriptor = runtime
+			.query<StorageManifestDescriptor, unknown>('storage/manifest')
+			.filter((d: StorageManifestDescriptor) => d.id === this.pendingId)
+			.firstDescriptor();
+
+		if (!descriptor) {
+			if (isBuildingEnvironment()) return;
+			throw new Error(`Storage manifest '${this.pendingId}' not found. Is it registered?`);
+		}
+
+		this.descriptor = descriptor;
+		this.context = this.pendingContext ?? { runtime };
+		this.resolved = true;
+		this.pendingId = null;
+		this.pendingContext = undefined;
+	}
+
+	/**
+	 * Create a storage manifest by looking up its ID.
+	 * Convenience static method for manual instantiation.
+	 */
+	static fromId<T>(manifestId: string): StorageManifest<T> {
+		return new StorageManifest<T>(manifestId);
+	}
+
+	/**
+	 * Get the schema for this manifest.
+	 */
+	get schema(): z.ZodType<T> {
+		this.ensureResolved();
+		return this.descriptor.schema as z.ZodType<T>;
+	}
+
+	/**
+	 * Get the namespace for this manifest.
+	 */
+	get namespace(): string {
+		this.ensureResolved();
+		return this.descriptor.namespace;
+	}
+
+	/**
+	 * Resolve the adapter for this manifest using runtime query API.
+	 *
+	 * Uses the manifest's allowed adapters list and current scope to
+	 * select the best available adapter.
+	 */
+	protected async getAdapter(): Promise<StorageAdapterInterface> {
+		this.ensureResolved();
+
+		if (!this.resolved) {
+			throw new Error(
+				`Storage manifest '${this.descriptor.id}' is not yet resolved. ` +
+					`The runtime may not be bootstrapped. Ensure 'storage/manifest' kind is registered before accessing storage.`
+			);
+		}
+
+		// Adapters are runtime singletons, so manifest-specific configuration
+		// must be re-applied on every access before reuse.
+		if (this.adapterCache) {
+			this.adapterCache.configure(this.getAdapterConfig());
+			return this.adapterCache;
+		}
+
+		const adapterTracer = createTracer(`adapter:${this.descriptor.id}`);
+		adapterTracer.start();
+
+		const runtime = this.getRuntime();
+		let adapter: StorageAdapterInterface | null = null;
+		let adapterId: string = '';
+
+		// Prefer explicit default adapter when available for current hardware.
+		if (
+			this.descriptor.defaultAdapter &&
+			this.descriptor.allowedAdapters.includes(this.descriptor.defaultAdapter)
+		) {
+			adapterId = this.descriptor.defaultAdapter;
+			try {
+				adapter = await runtime
+					.query<StorageAdapterDescriptor, StorageAdapterInterface>('storage/adapter')
+					.withId(adapterId)
+					.enabled()
+					.resolve();
+			} catch {
+				// Default adapter not registered or unavailable — fall through to allowed adapters
+				adapter = null;
+				adapterId = '';
+			}
+		}
+
+		// Fallback to first available adapter
+		if (!adapter) {
+			for (const allowedId of this.descriptor.allowedAdapters) {
+				try {
+					adapter = await runtime
+						.query<StorageAdapterDescriptor, StorageAdapterInterface>('storage/adapter')
+						.withId(allowedId)
+						.enabled()
+						.resolve();
+					adapterId = allowedId;
+					break;
+				} catch {
+					continue;
+				}
+			}
+		}
+
+		if (!adapter || !adapterId) {
+			throw new Error(
+				`No available adapter for manifest '${this.descriptor.id}'. Checked: ${this.descriptor.allowedAdapters.join(', ')}`
+			);
+		}
+
+		// Adapters are shared across manifests. Always apply this manifest's
+		// configuration before use so filename/basePath overrides stay isolated.
+		const config = this.getAdapterConfig(adapterId);
+		adapter.configure(config);
+
+		// Register per-manifest cloud endpoint overrides so cloud-api uses
+		// descriptor endpoints (e.g. /api/storage/chat/rooms) instead of the
+		// namespace fallback (/api/storage/chat.rooms).
+		if (adapterId === 'storage.cloud-api') {
+			const cloudEndpoint = this.getCloudApiEndpoint();
+			if (cloudEndpoint && this.hasCloudEndpointRegistrar(adapter)) {
+				adapter.registerEndpoint(this.descriptor.namespace, cloudEndpoint);
+			}
+		}
+
+		// Cache adapter but don't initialize yet
+		// Initialization will happen on first data access
+		this.adapterCache = adapter;
+		adapterTracer.end('resolved');
+		return adapter;
+	}
+
+	/**
+	 * Get adapter configuration based on manifest scope.
+	 * Override in subclasses for custom configuration.
+	 */
+	protected getAdapterConfig(adapterId?: string): Record<string, unknown> {
+		const workspacePath = resolveWorkspacePath(this.context);
+		const config: Record<string, unknown> = {
+			basePath: this.getBasePath(workspacePath),
+			workspacePath
+		};
+
+		// Pass through descriptor-level flags that adapters need
+		if (this.descriptor.customTables) {
+			config.customTables = true;
+		}
+		if (this.descriptor.migrationKey) {
+			config.migrationKey = this.descriptor.migrationKey;
+		}
+
+		// Pass through adapter-specific config from descriptor
+		if (this.descriptor.adapterConfig) {
+			const selectedAdapterId =
+				adapterId || this.descriptor.defaultAdapter || this.descriptor.allowedAdapters[0];
+			const adapterSpecificConfig = this.descriptor.adapterConfig[selectedAdapterId];
+			if (adapterSpecificConfig && typeof adapterSpecificConfig === 'object') {
+				Object.assign(config, adapterSpecificConfig);
+			}
+
+			// Backward-compatible normalization for cloud adapter manifests.
+			// Only promote endpoint -> apiEndpoint when endpoint is a full URL.
+			if (
+				selectedAdapterId === 'storage.cloud-api' &&
+				typeof config.endpoint === 'string' &&
+				typeof config.apiEndpoint !== 'string' &&
+				config.endpoint.startsWith('http')
+			) {
+				config.apiEndpoint = config.endpoint;
+			}
+		}
+
+		return config;
+	}
+
+	private getCloudApiEndpoint(): string | null {
+		const raw = this.descriptor.adapterConfig?.['storage.cloud-api'];
+		if (!raw || typeof raw !== 'object') return null;
+		const endpoint = (raw as Record<string, unknown>).endpoint;
+		return typeof endpoint === 'string' && endpoint.length > 0 ? endpoint : null;
+	}
+
+	private hasCloudEndpointRegistrar(
+		adapter: StorageAdapterInterface
+	): adapter is StorageAdapterInterface & {
+		registerEndpoint(namespace: string, endpoint: string): void;
+	} {
+		return typeof (adapter as unknown as Record<string, unknown>).registerEndpoint === 'function';
+	}
+
+	/**
+	 * Get base path for storage based on scope.
+	 * Uses descriptor's scopePaths if provided, otherwise defaults.
+	 */
+	protected getBasePath(workspacePath = resolveWorkspacePath(this.context)): string {
+		const home = resolveHomePath();
+		const scope = this.descriptor.scope;
+
+		// Use custom scope paths if provided
+		if (this.descriptor.scopePaths) {
+			const customPath = this.descriptor.scopePaths[scope as 'global' | 'project' | 'local'];
+			if (customPath) {
+				return resolveScopePathTemplate(customPath, workspacePath);
+			}
+		}
+
+		// Default path resolution
+		switch (scope) {
+			case 'global':
+				return `${home}/.vibes/storage`;
+			case 'project':
+			case 'local':
+				return `${workspacePath}/.vibes/storage`;
+			case 'auto':
+			default:
+				return `${workspacePath}/.vibes/storage`;
+		}
+	}
+
+	// Caching
+
+	/**
+	 * Invalidate the cache, forcing the next load to hit storage.
+	 */
+	invalidateCache(): void {
+		this.cacheValid = false;
+		this.cachedData = null;
+	}
+
+	/**
+	 * Get cached data if available.
+	 * @returns Cached data or undefined if not cached
+	 */
+	getCachedData(): T | undefined {
+		if (this.cacheValid) {
+			return this.cachedData ?? undefined;
+		}
+		return undefined;
+	}
+
+	/**
+	 * Check if data is currently cached.
+	 */
+	isCached(): boolean {
+		return this.cacheValid;
+	}
+
+	// Document Operations
+
+	/**
+	 * Load data from storage.
+	 *
+	 * @returns The stored data, or null if not found
+	 */
+	async load(): Promise<T | null> {
+		// Return cached data if valid
+		if (this.cacheValid) {
+			return this.cachedData;
+		}
+
+		const adapter = await this.getAdapter();
+		const store = adapter.documentStore(this.descriptor.namespace, this.schema, this.descriptor);
+		const data = await store.load();
+
+		// Update cache
+		this.cachedData = data;
+		this.cacheValid = true;
+
+		// Start subscription if adapter supports it and not already started
+		if (!this.subscriptionStarted && store.subscribe) {
+			this.startSubscription(store.subscribe.bind(store));
+		}
+
+		return data;
+	}
+
+	/**
+	 * Save data to storage.
+	 *
+	 * @param data - The data to save
+	 */
+	async save(data: T): Promise<void> {
+		const adapter = await this.getAdapter();
+		const store = adapter.documentStore(this.descriptor.namespace, this.schema, this.descriptor);
+		await store.save(data);
+
+		// Update cache
+		this.cachedData = data;
+		this.cacheValid = true;
+
+		// Notify subscribers
+		this.notifySubscribers(data);
+
+		// Non-blocking cloud push when a consumer-local save needs to mirror to cloud.
+		this.triggerCloudSync(adapter.id, data);
+	}
+
+	/**
+	 * Trigger a non-blocking cloud push after save.
+	 * Only fires if the manifest has cloud-api in allowedAdapters.
+	 *
+	 * Consumer modules are loaded via opaque dynamic import to prevent Vite
+	 * from tracing them into the client bundle — they contain Node.js APIs
+	 * and must only run server-side.
+	 */
+	private triggerCloudSync(activeAdapterId: string, data: T): void {
+		if (!this.descriptor.allowedAdapters.includes('storage.cloud-api')) return;
+		if (activeAdapterId === 'storage.cloud-api') return;
+		if (this.getRuntime().scope.hardware === 'cloud') return;
+
+		const consumerSyncModule = './cloud-sync.consumer';
+
+		void import(/* @vite-ignore */ consumerSyncModule)
+			.then(({ pushManifestToCloudAfterLocalSave }) =>
+				pushManifestToCloudAfterLocalSave({
+					descriptor: this.descriptor,
+					data,
+					activeAdapterId
+				})
+			)
+			.catch(() => {
+				// Non-blocking: don't fail the save operation on sync errors.
+			});
+	}
+
+	/**
+	 * Check if data exists in storage.
+	 */
+	async exists(): Promise<boolean> {
+		const adapter = await this.getAdapter();
+		const store = adapter.documentStore(this.descriptor.namespace, this.schema, this.descriptor);
+		return store.exists();
+	}
+
+	/**
+	 * Delete data from storage.
+	 */
+	async delete(): Promise<void> {
+		const adapter = await this.getAdapter();
+		const store = adapter.documentStore(this.descriptor.namespace, this.schema, this.descriptor);
+		await store.delete();
+
+		// Clear cache
+		this.cachedData = null;
+		this.cacheValid = false;
+
+		// Notify subscribers
+		this.notifySubscribers(null);
+	}
+
+	/**
+	 * Load data or return default value.
+	 *
+	 * @param defaultValue - Value to return if no data exists
+	 */
+	async loadOrDefault(defaultValue: T): Promise<T> {
+		const data = await this.load();
+		return data ?? defaultValue;
+	}
+
+	/**
+	 * Update data using a transform function.
+	 *
+	 * @param transform - Function that receives current data and returns updated data
+	 */
+	async update(transform: (current: T | null) => T): Promise<void> {
+		const current = await this.load();
+		const updated = transform(current);
+		await this.save(updated);
+	}
+
+	/**
+	 * Validate data against the manifest's schema.
+	 */
+	validate(data: unknown): z.ZodSafeParseResult<T> {
+		return this.schema.safeParse(data) as z.ZodSafeParseResult<T>;
+	}
+
+	// Subscriptions
+
+	/**
+	 * Subscribe to changes for this manifest.
+	 * Returns an unsubscribe function.
+	 *
+	 * @param callback - Function called when data changes
+	 * @returns Unsubscribe function
+	 */
+	subscribe(callback: (data: T | null) => void): Unsubscribe {
+		this.subscribers.add(callback);
+
+		// Start storage-level subscription if not already started
+		if (!this.subscriptionStarted) {
+			this.initSubscription();
+		}
+
+		// Return unsubscribe function
+		return () => {
+			this.subscribers.delete(callback);
+			// Clean up storage subscription if no more subscribers
+			if (this.subscribers.size === 0) {
+				this.stopSubscription();
+			}
+		};
+	}
+
+	/**
+	 * Initialize subscription to storage adapter.
+	 */
+	private async initSubscription(): Promise<void> {
+		if (this.subscriptionStarted) return;
+
+		const adapter = await this.getAdapter();
+		const store = adapter.documentStore(this.descriptor.namespace, this.schema, this.descriptor);
+
+		if (store.subscribe) {
+			this.startSubscription(store.subscribe.bind(store));
+		}
+	}
+
+	/**
+	 * Start subscription with the given subscribe function.
+	 */
+	private startSubscription(subscribeFn: (cb: DocumentChangeCallback<T>) => Unsubscribe): void {
+		if (this.storeUnsubscribe) return;
+
+		this.storeUnsubscribe = subscribeFn((data) => {
+			// Update cache from external change
+			this.cachedData = data;
+			this.cacheValid = true;
+			// Notify our subscribers
+			this.notifySubscribers(data);
+		});
+
+		this.subscriptionStarted = true;
+	}
+
+	/**
+	 * Stop storage-level subscription.
+	 */
+	private stopSubscription(): void {
+		if (this.storeUnsubscribe) {
+			this.storeUnsubscribe();
+			this.storeUnsubscribe = null;
+		}
+		this.subscriptionStarted = false;
+	}
+
+	/**
+	 * Notify all subscribers of a change.
+	 */
+	protected notifySubscribers(data: T | null): void {
+		for (const callback of this.subscribers) {
+			try {
+				callback(data);
+			} catch (error) {
+				logger.error('Error in manifest subscriber:', toErrorContext(error));
+			}
+		}
+	}
+
+	/**
+	 * Get the number of active subscribers.
+	 */
+	getSubscriberCount(): number {
+		return this.subscribers.size;
+	}
+}

package/src/manifest/cloud-sync.consumer.ts

@@ -0,0 +1,120 @@
+import { getVibesRuntime } from '@vibesdotdev/runtime/api/runtime';
+import type { z } from 'zod/v4';
+import type { StorageAdapterInterface } from '../interfaces/adapter';
+import type { StorageAdapterDescriptor } from '../schemas/adapter-descriptor';
+import type { StorageManifestDescriptor } from '../schemas/manifest-descriptor';
+import {
+	computeHash,
+	loadSyncState,
+	markError,
+	markLocalAhead,
+	markSynced,
+	saveSyncState
+} from '../sync/sync-state.consumer';
+
+const REMOTE_ADAPTER_IDS = new Set(['storage.cloud-api', 'storage.drizzle']);
+
+function getCloudEndpoint(descriptor: StorageManifestDescriptor): string | null {
+	const raw = descriptor.adapterConfig?.['storage.cloud-api'];
+	if (!raw || typeof raw !== 'object') return null;
+	const endpoint = (raw as Record<string, unknown>).endpoint;
+	return typeof endpoint === 'string' && endpoint.length > 0 ? endpoint : null;
+}
+
+function hasEndpointRegistrar(
+	adapter: StorageAdapterInterface
+): adapter is StorageAdapterInterface & {
+	registerEndpoint(namespace: string, endpoint: string): void;
+} {
+	return 'registerEndpoint' in adapter && typeof adapter.registerEndpoint === 'function';
+}
+
+function isZodSchema<T>(schema: object | null): schema is z.ZodType<T> {
+	return schema !== null && '_zod' in schema;
+}
+
+async function isCloudSyncEnabled(): Promise<boolean> {
+	const runtime = getVibesRuntime();
+	if (!runtime.hasKind('config/manifest')) {
+		return false;
+	}
+
+	try {
+		const manifest = (await runtime
+			.query('config/manifest')
+			.withId('ai.core')
+			.enabled()
+			.resolve()) as { load(): Promise<Record<string, unknown> | null> } | null;
+		if (!manifest) {
+			return false;
+		}
+
+		const config = await manifest.load();
+		return config?.syncToCloud === true;
+	} catch {
+		return false;
+	}
+}
+
+export async function pushManifestToCloudAfterLocalSave<T>({
+	descriptor,
+	data,
+	activeAdapterId
+}: {
+	descriptor: StorageManifestDescriptor;
+	data: T;
+	activeAdapterId: string;
+}): Promise<void> {
+	if (!descriptor.allowedAdapters.includes('storage.cloud-api')) {
+		return;
+	}
+
+	if (REMOTE_ADAPTER_IDS.has(activeAdapterId)) {
+		return;
+	}
+
+	const runtime = getVibesRuntime();
+	if (runtime.scope?.surface === 'worker' || runtime.scope?.purpose === 'worker') {
+		return;
+	}
+
+	if (!(await isCloudSyncEnabled())) {
+		return;
+	}
+
+	const localHash = computeHash(data);
+	const syncState = await loadSyncState();
+
+	try {
+		const cloudAdapter = await runtime
+			.query<StorageAdapterDescriptor, StorageAdapterInterface>('storage/adapter')
+			.withId('storage.cloud-api')
+			.enabled()
+			.resolve();
+
+		const endpoint = getCloudEndpoint(descriptor);
+		if (endpoint && hasEndpointRegistrar(cloudAdapter)) {
+			cloudAdapter.registerEndpoint(descriptor.namespace, endpoint);
+		}
+
+		await cloudAdapter.ensureReady();
+		const schema =
+			typeof descriptor.schema === 'object' && isZodSchema<T>(descriptor.schema)
+				? descriptor.schema
+				: null;
+		if (!schema) {
+			throw new Error(`Storage manifest '${descriptor.id}' is missing a valid schema`);
+		}
+
+		const store = cloudAdapter.documentStore(descriptor.namespace, schema, descriptor);
+		await store.save(data);
+
+		markSynced(syncState, descriptor.id, localHash, 'push');
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		markLocalAhead(syncState, descriptor.id, localHash);
+		markError(syncState, descriptor.id, message);
+	}
+
+	await saveSyncState(syncState);
+}