@vibesdotdev/storage 0.0.1

package/src/sync/sync-manifest.ts

@@ -0,0 +1,212 @@
+/**
+ * Sync Manifest Helper (Consumer)
+ *
+ * Core bidirectional sync logic for a single manifest.
+ * Extracted from MaintenanceStorageSyncJob to keep file size under limit.
+ */
+
+import type { SyncState } from './schemas/sync-state';
+import type { SyncHelpers } from './schemas/sync-helpers';
+import { logger } from '@vibesdotdev/logging';
+
+export type { SyncHelpers } from './schemas/sync-helpers';
+
+/**
+ * Result of syncing a single manifest.
+ * Owned by storage — workers and CLI commands consume this type.
+ */
+export interface ManifestSyncResult {
+	manifestId: string;
+	action: 'pushed' | 'pulled' | 'skipped' | 'conflict' | 'error';
+	message?: string;
+}
+
+const LOG_PREFIX = '[maintenance.storage-sync]';
+
+export async function syncManifest(
+	manifestId: string,
+	syncState: SyncState,
+	helpers: SyncHelpers
+): Promise<ManifestSyncResult> {
+	const { getVibesRuntime } = await import('@vibesdotdev/runtime');
+	const runtime = getVibesRuntime();
+	const { StorageManifest } = await import('../manifest/index.js');
+
+	// Load local data via default adapter
+	const localManifest = new StorageManifest(manifestId);
+	let localData: unknown;
+	try {
+		localData = await localManifest.load();
+	} catch (schemaError) {
+		// Schema validation failed — data exists but doesn't match current schema.
+		// This is a data migration issue, not a sync issue.
+		const msg = schemaError instanceof Error ? schemaError.message : String(schemaError);
+		helpers.markError(syncState, manifestId, `Schema validation failed: ${msg}`);
+		return {
+			manifestId,
+			action: 'error',
+			message: `Local data fails schema validation (needs migration): ${msg}`
+		};
+	}
+	const localHash = localData ? helpers.computeHash(localData) : null;
+
+	// Load remote data via cloud-api adapter
+	const { remoteData, remoteHash } = await loadRemoteData(runtime, manifestId, helpers);
+
+	const prev = helpers.getManifestSyncState(syncState, manifestId);
+
+	// Decision matrix
+	if (!localData && !remoteData) {
+		return { manifestId, action: 'skipped', message: 'No data on either side' };
+	}
+
+	if (localHash === remoteHash) {
+		if (prev.syncStatus !== 'in-sync') {
+			helpers.markSynced(syncState, manifestId, localHash ?? '', 'push');
+		}
+		return { manifestId, action: 'skipped', message: 'Already in sync' };
+	}
+
+	const localChanged = localHash !== prev.localHash;
+	const remoteChanged = remoteHash !== prev.remoteHash;
+
+	if (localChanged && remoteChanged && remoteHash !== null) {
+		helpers.markConflict(syncState, manifestId, localHash ?? '', remoteHash);
+		return { manifestId, action: 'conflict', message: 'Both local and remote changed' };
+	}
+
+	if (remoteChanged && !localChanged && remoteData) {
+		if (helpers.dryRun) {
+			return { manifestId, action: 'pulled', message: '[dry-run] Would pull from cloud' };
+		}
+		await localManifest.save(remoteData as never);
+		helpers.markSynced(syncState, manifestId, remoteHash ?? '', 'pull');
+		logger.info(`${LOG_PREFIX} Pulled ${manifestId} from cloud`);
+		return { manifestId, action: 'pulled' };
+	}
+
+	if (localData) {
+		return pushToCloud(runtime, manifestId, localData, localHash, syncState, helpers);
+	}
+
+	return { manifestId, action: 'skipped', message: 'No actionable state' };
+}
+
+async function loadRemoteData(
+	runtime: ReturnType<Awaited<typeof import('@vibesdotdev/runtime')>['getVibesRuntime']>,
+	manifestId: string,
+	helpers: SyncHelpers
+): Promise<{ remoteData: unknown; remoteHash: string | null }> {
+	try {
+		const cloudAdapter = await runtime
+			.query('storage/adapter')
+			.filter((d: { id: string }) => d.id === 'storage.cloud-api')
+			.resolve();
+
+		if (!cloudAdapter) return { remoteData: null, remoteHash: null };
+
+		const descriptor = runtime
+			.query('storage/manifest')
+			.filter((d: { id: string }) => d.id === manifestId)
+			.firstDescriptor() as unknown as {
+			namespace: string;
+			schema: unknown;
+			adapterConfig?: Record<string, Record<string, unknown>>;
+		} | null;
+
+		if (!descriptor) return { remoteData: null, remoteHash: null };
+
+		const adapter = cloudAdapter as {
+			registerEndpoint(ns: string, ep: string): void;
+			ensureReady(): Promise<void>;
+			documentStore<T>(ns: string, schema: unknown): { load(): Promise<T | null> };
+		};
+
+		const cloudConfig = descriptor.adapterConfig?.['storage.cloud-api'] as
+			| { endpoint?: string }
+			| undefined;
+		if (cloudConfig?.endpoint) {
+			adapter.registerEndpoint(descriptor.namespace, cloudConfig.endpoint);
+		}
+		await adapter.ensureReady();
+
+		const store = adapter.documentStore<unknown>(descriptor.namespace, descriptor.schema);
+		const remoteData = await store.load();
+		const remoteHash = remoteData ? helpers.computeHash(remoteData) : null;
+		return { remoteData, remoteHash };
+	} catch (error) {
+		logger.debug(`${LOG_PREFIX} Cloud read failed for ${manifestId}`, {
+			error: (error as Error).message
+		});
+		return { remoteData: null, remoteHash: null };
+	}
+}
+
+async function pushToCloud(
+	runtime: ReturnType<Awaited<typeof import('@vibesdotdev/runtime')>['getVibesRuntime']>,
+	manifestId: string,
+	localData: unknown,
+	localHash: string | null,
+	syncState: SyncState,
+	helpers: SyncHelpers
+): Promise<ManifestSyncResult> {
+	if (helpers.dryRun) {
+		return { manifestId, action: 'pushed', message: '[dry-run] Would push to cloud' };
+	}
+
+	try {
+		const cloudAdapter = await runtime
+			.query('storage/adapter')
+			.filter((d: { id: string }) => d.id === 'storage.cloud-api')
+			.resolve();
+
+		if (!cloudAdapter) {
+			helpers.markLocalAhead(syncState, manifestId, localHash ?? '');
+			return { manifestId, action: 'skipped', message: 'Cloud adapter not available for push' };
+		}
+
+		const descriptor = runtime
+			.query('storage/manifest')
+			.filter((d: { id: string }) => d.id === manifestId)
+			.firstDescriptor() as unknown as {
+			namespace: string;
+			schema: unknown;
+			adapterConfig?: Record<string, Record<string, unknown>>;
+		} | null;
+
+		const adapter = cloudAdapter as {
+			registerEndpoint(ns: string, ep: string): void;
+			ensureReady(): Promise<void>;
+			documentStore<T>(ns: string, schema: unknown): { save(data: T): Promise<void> };
+		};
+
+		if (!descriptor) {
+			helpers.markLocalAhead(syncState, manifestId, localHash ?? '');
+			return {
+				manifestId,
+				action: 'skipped',
+				message: 'Manifest descriptor not registered for push'
+			};
+		}
+
+		// Register per-manifest endpoint override
+		const cloudConfig = descriptor.adapterConfig?.['storage.cloud-api'] as
+			| { endpoint?: string }
+			| undefined;
+		if (cloudConfig?.endpoint) {
+			adapter.registerEndpoint(descriptor.namespace, cloudConfig.endpoint);
+		}
+		await adapter.ensureReady();
+
+		const store = adapter.documentStore<unknown>(descriptor.namespace, descriptor.schema);
+		await store.save(localData);
+
+		helpers.markSynced(syncState, manifestId, localHash ?? '', 'push');
+		logger.info(`${LOG_PREFIX} Pushed ${manifestId} to cloud`);
+		return { manifestId, action: 'pushed' };
+	} catch (error) {
+		const msg = (error as Error).message;
+		helpers.markError(syncState, manifestId, msg);
+		return { manifestId, action: 'error', message: `Push failed: ${msg}` };
+	}
+}

package/src/sync/sync-state.consumer.ts

@@ -0,0 +1,144 @@
+/**
+ * Storage Sync State Tracking (Consumer)
+ *
+ * Tracks per-manifest sync state: hashes, timestamps, and conflict status.
+ * Stored in ~/.vibes/sync-state.json via direct JSON I/O (intentionally
+ * bypasses StorageManifest to avoid circular dependency).
+ *
+ * Consumer-scoped: uses host fs for persistence.
+ */
+
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+import { createHash } from 'node:crypto';
+import { SyncStateSchema, type SyncState, type ManifestSyncState } from './schemas/sync-state';
+
+// Re-export types for consumers
+export type { SyncState, SyncStatus, ManifestSyncState } from './schemas/sync-state';
+
+const SYNC_STATE_DIR = join(homedir(), '.vibes');
+const SYNC_STATE_PATH = join(SYNC_STATE_DIR, 'sync-state.json');
+
+function createDefaultState(): SyncState {
+	return {
+		version: 1,
+		updatedAt: new Date().toISOString(),
+		manifests: {}
+	};
+}
+
+function createDefaultManifestState(manifestId: string): ManifestSyncState {
+	return {
+		manifestId,
+		localHash: null,
+		remoteHash: null,
+		lastLocalChange: null,
+		lastRemoteChange: null,
+		lastSyncAt: null,
+		lastSyncDirection: null,
+		syncStatus: 'never-synced',
+		errorMessage: null
+	};
+}
+
+export async function loadSyncState(): Promise<SyncState> {
+	if (!existsSync(SYNC_STATE_PATH)) {
+		return createDefaultState();
+	}
+
+	try {
+		const { readFile } = await import('node:fs/promises');
+		const raw = await readFile(SYNC_STATE_PATH, 'utf-8');
+		const parsed = JSON.parse(raw);
+		return SyncStateSchema.parse(parsed);
+	} catch {
+		return createDefaultState();
+	}
+}
+
+export async function saveSyncState(state: SyncState): Promise<void> {
+	state.updatedAt = new Date().toISOString();
+
+	if (!existsSync(SYNC_STATE_DIR)) {
+		const { mkdir } = await import('node:fs/promises');
+		await mkdir(SYNC_STATE_DIR, { recursive: true });
+	}
+
+	const { writeFile } = await import('node:fs/promises');
+	await writeFile(SYNC_STATE_PATH, JSON.stringify(state, null, 2));
+}
+
+export function getManifestSyncState(state: SyncState, manifestId: string): ManifestSyncState {
+	return state.manifests[manifestId] ?? createDefaultManifestState(manifestId);
+}
+
+export function updateManifestSyncState(
+	state: SyncState,
+	manifestId: string,
+	update: Partial<ManifestSyncState>
+): void {
+	const current = getManifestSyncState(state, manifestId);
+	state.manifests[manifestId] = { ...current, ...update, manifestId };
+}
+
+export function markSynced(
+	state: SyncState,
+	manifestId: string,
+	hash: string,
+	direction: 'push' | 'pull'
+): void {
+	const now = new Date().toISOString();
+	updateManifestSyncState(state, manifestId, {
+		localHash: hash,
+		remoteHash: hash,
+		lastSyncAt: now,
+		lastSyncDirection: direction,
+		syncStatus: 'in-sync',
+		errorMessage: null,
+		...(direction === 'push' ? { lastLocalChange: now } : { lastRemoteChange: now })
+	});
+}
+
+export function markLocalAhead(state: SyncState, manifestId: string, localHash: string): void {
+	updateManifestSyncState(state, manifestId, {
+		localHash,
+		lastLocalChange: new Date().toISOString(),
+		syncStatus: 'local-ahead'
+	});
+}
+
+export function markRemoteAhead(state: SyncState, manifestId: string, remoteHash: string): void {
+	updateManifestSyncState(state, manifestId, {
+		remoteHash,
+		lastRemoteChange: new Date().toISOString(),
+		syncStatus: 'remote-ahead'
+	});
+}
+
+export function markConflict(
+	state: SyncState,
+	manifestId: string,
+	localHash: string,
+	remoteHash: string
+): void {
+	updateManifestSyncState(state, manifestId, {
+		localHash,
+		remoteHash,
+		syncStatus: 'conflict'
+	});
+}
+
+export function markError(state: SyncState, manifestId: string, errorMessage: string): void {
+	updateManifestSyncState(state, manifestId, { errorMessage });
+}
+
+export function markAllPaused(state: SyncState): void {
+	for (const manifestId of Object.keys(state.manifests)) {
+		updateManifestSyncState(state, manifestId, { syncStatus: 'paused' });
+	}
+}
+
+export function computeHash(data: unknown): string {
+	return createHash('sha256').update(JSON.stringify(data)).digest('hex');
+}

package/src/transform/index.ts

@@ -0,0 +1,130 @@
+/**
+ * Storage transform registration + resolution helpers.
+ *
+ * Plugins register transforms via `createStorageTransform()` (returns
+ * descriptor + implementation) and `runtime.registerDescriptor` +
+ * `runtime.registerLoader`. Consumers (typically `vibesCrudCollection`)
+ * call `resolveStorageTransform(runtime, id)` to pull both the descriptor
+ * metadata and the live `fromRow` / `toRow` functions.
+ *
+ * The runtime kind itself (`storage/transform`) is declared in
+ * `storage.plugin.ts` so it's available before any app plugin tries to
+ * register against it.
+ */
+
+import { createRuntimeImplementation } from '@vibesdotdev/runtime';
+import type { RuntimeImplementationWrapper } from '@vibesdotdev/runtime';
+import {
+	StorageTransformDescriptorSchema,
+	type StorageTransformDescriptor,
+	type StorageTransformImplementation
+} from './transform-descriptor.ts';
+
+export {
+	StorageTransformDescriptorSchema,
+	type StorageTransformDescriptor,
+	type StorageTransformImplementation
+} from './transform-descriptor.ts';
+
+// NOTE: createDefaultDbTransforms and commonFieldMappings are available from
+// @vibesdotdev/storage-adapter-drizzle/transforms directly.
+
+/**
+ * Minimal shape required from the runtime for transform resolution.
+ * Avoids a hard import of `VibesRuntime` so this module stays
+ * dependency-light.
+ */
+interface RuntimeForTransform {
+	query(kind: string): {
+		withId(id: string): {
+			firstDescriptor?: () => StorageTransformDescriptor | null;
+			resolve?: () => Promise<unknown> | unknown;
+		};
+	};
+}
+
+export interface CreateStorageTransformOptions<TRow, TDescriptor> {
+	/** Stable id, typically `<manifest-id>` (e.g. `'ai.providers'`). */
+	id: string;
+	/** Storage manifest this transform is paired with. */
+	manifestId?: string;
+	/** Human description for diagnostics. */
+	description?: string;
+	/** Row → descriptor mapping (read path). */
+	fromRow: (row: TRow) => TDescriptor;
+	/** Descriptor → row mapping (write path). Omit for read-only manifests. */
+	toRow?: (descriptor: Partial<TDescriptor>) => Partial<TRow>;
+}
+
+/**
+ * Build the descriptor + runtime implementation pair for a transform.
+ *
+ * Usage from an app-local plugin / hook:
+ *
+ * ```ts
+ * const aiProvidersTransform = createStorageTransform<ProviderRow, ProviderDescriptor>({
+ *   id: 'ai.providers',
+ *   manifestId: 'ai.providers',
+ *   fromRow: (row) => ({ id: row.id, kind: 'ai/provider', ... }),
+ *   toRow: (d) => ({ name: d.name, ... })
+ * });
+ *
+ * runtime.registerDescriptor('storage/transform', aiProvidersTransform.descriptor);
+ * runtime.registerLoader(
+ *   'storage/transform',
+ *   aiProvidersTransform.descriptor.id,
+ *   async () => aiProvidersTransform.implementation
+ * );
+ * ```
+ */
+export function createStorageTransform<TRow, TDescriptor>(
+	options: CreateStorageTransformOptions<TRow, TDescriptor>
+): {
+	descriptor: StorageTransformDescriptor;
+	implementation: RuntimeImplementationWrapper<StorageTransformImplementation<TRow, TDescriptor>>;
+} {
+	const direction = options.toRow ? 'bidirectional' : 'read';
+	const descriptor = StorageTransformDescriptorSchema.parse({
+		kind: 'storage/transform',
+		id: options.id,
+		manifestId: options.manifestId,
+		description: options.description,
+		direction
+	});
+
+	const impl: StorageTransformImplementation<TRow, TDescriptor> = options.toRow
+		? { fromRow: options.fromRow, toRow: options.toRow }
+		: { fromRow: options.fromRow };
+
+	const implementation = createRuntimeImplementation({
+		id: descriptor.id,
+		kind: 'storage/transform',
+		implementation: impl
+	}) as RuntimeImplementationWrapper<StorageTransformImplementation<TRow, TDescriptor>>;
+
+	return { descriptor, implementation };
+}
+
+/**
+ * Resolve a transform's live implementation from the runtime by id.
+ *
+ * Returns `null` when no transform is registered for the id — callers
+ * should treat that as "no row↔descriptor mapping configured" and fall
+ * back to identity (the row IS the descriptor wire shape).
+ */
+export async function resolveStorageTransform<TRow = unknown, TDescriptor = unknown>(
+	runtime: RuntimeForTransform,
+	id: string
+): Promise<StorageTransformImplementation<TRow, TDescriptor> | null> {
+	const scoped = runtime.query('storage/transform').withId(id);
+	if (typeof scoped.resolve !== 'function') return null;
+	try {
+		const resolved = await scoped.resolve();
+		if (!resolved || typeof resolved !== 'object') return null;
+		const impl = resolved as Partial<StorageTransformImplementation<TRow, TDescriptor>>;
+		if (typeof impl.fromRow !== 'function') return null;
+		return impl as StorageTransformImplementation<TRow, TDescriptor>;
+	} catch {
+		return null;
+	}
+}

package/src/transform/transform-descriptor.ts

@@ -0,0 +1,53 @@
+/**
+ * Storage Transform Descriptor Schema
+ *
+ * Descriptors for `storage/transform` runtime kind. A transform converts
+ * between a storage-adapter row shape (e.g. a Drizzle row from `model_providers`)
+ * and the manifest's wire-shape descriptor (e.g. `AIProviderDescriptor`).
+ *
+ * The descriptor is data (id + metadata); the actual `fromRow` / `toRow`
+ * functions live in the runtime implementation registered via
+ * `runtime.registerLoader('storage/transform', id, ...)`. This split keeps
+ * descriptors serializable while implementations can hold closures.
+ */
+
+import * as z from 'zod/v4';
+import { RuntimeDescriptorSchema } from '@vibesdotdev/runtime/schemas/descriptor';
+
+export const StorageTransformDescriptorSchema = RuntimeDescriptorSchema.extend({
+	kind: z.literal('storage/transform'),
+
+	/**
+	 * Optional binding to a `storage/manifest` descriptor — most transforms
+	 * exist to serve one specific manifest. Helpers can verify that the
+	 * caller's descriptor matches when resolving by id.
+	 */
+	manifestId: z.string().optional(),
+
+	/**
+	 * Direction this transform supports. `'bidirectional'` is the common
+	 * case (DB row ↔ wire descriptor); `'read'` means the manifest is
+	 * read-only over HTTP (no write-path mapping needed).
+	 */
+	direction: z.enum(['read', 'write', 'bidirectional']).default('bidirectional')
+});
+
+export type StorageTransformDescriptor = z.infer<typeof StorageTransformDescriptorSchema>;
+
+/**
+ * Runtime implementation interface for `storage/transform`.
+ *
+ * `TRow` is the storage adapter's native row shape (a Drizzle `$inferSelect`
+ * type for `storage.drizzle.*`, a JSON-blob shape for `storage.json`, etc.).
+ * `TDescriptor` is the wire-shape descriptor consumed by manifests and
+ * exposed to cross-app callers via `vibesCrudCollection`.
+ */
+export interface StorageTransformImplementation<TRow = unknown, TDescriptor = unknown> {
+	/** Map a single row from the storage adapter into a wire-shape descriptor. */
+	fromRow(row: TRow): TDescriptor;
+	/**
+	 * Map a wire-shape descriptor (or partial patch) into an adapter row
+	 * shape suitable for insert/update. Omit for read-only transforms.
+	 */
+	toRow?(descriptor: Partial<TDescriptor>): Partial<TRow>;
+}

package/src/tui/storage-explorer.descriptor.ts

@@ -0,0 +1,32 @@
+/**
+ * Storage Explorer — tui/app descriptor
+ *
+ * Interactive storage browser for inspecting adapter health,
+ * browsing namespaces, and viewing stored entries.
+ *
+ * NOTE: The actual TUI types (TuiAppDescriptor) live in @vibesdotdev/tui, but
+ * storage must not depend on tui in production (that would re-create the cycle).
+ * We declare the shape inline here using the base RuntimeDescriptor-compatible fields.
+ */
+
+import type { RuntimeDescriptor } from '@vibesdotdev/runtime';
+
+const descriptor: RuntimeDescriptor & Record<string, unknown> = {
+	kind: 'tui/app',
+	id: 'storage.explorer',
+	name: 'Storage Explorer',
+	description: 'Inspect storage adapters, namespaces, and persisted runtime data.',
+	tags: ['system', 'ops'],
+	// TUI-specific fields passed through as opaque data
+	title: 'Storage Explorer',
+	icon: '🗄',
+	category: 'System',
+	audience: 'ops',
+	order: 5,
+	route: '/storage',
+	sidebar: true,
+	embeddable: true,
+	enabled: true
+} as RuntimeDescriptor & Record<string, unknown>;
+
+export default descriptor;

package/src/tui/storage-explorer.impl.consumer.ts

@@ -0,0 +1,14 @@
+/**
+ * Storage Explorer TUI implementation (consumer stub).
+ *
+ * The actual implementation requires @vibesdotdev/tui components.
+ * This stub is loaded at runtime only when the TUI stack is present.
+ * When TUI is not available, the onRegister hook in storage.plugin.ts
+ * skips registration entirely via runtime.hasKind('tui/app').
+ */
+
+const implementation: Record<string, unknown> = {
+	render: () => ({ type: 'placeholder', message: 'Storage Explorer requires TUI runtime' })
+};
+
+export default implementation;

package/src/tui/storage-open.descriptor.ts

@@ -0,0 +1,31 @@
+/**
+ * Storage Open — tui/command descriptor
+ *
+ * Navigates to the Storage Explorer.
+ * Triggered via /storage slash command or keybinding.
+ *
+ * NOTE: The actual TUI types (TuiCommandDescriptor) live in @vibesdotdev/tui, but
+ * storage must not depend on tui in production (that would re-create the cycle).
+ * We declare the shape inline here using the base RuntimeDescriptor-compatible fields.
+ */
+
+import type { RuntimeDescriptor } from '@vibesdotdev/runtime';
+
+const descriptor: RuntimeDescriptor & Record<string, unknown> = {
+	kind: 'tui/command',
+	id: 'storage.open',
+	title: 'Open Storage Explorer',
+	name: 'Open Storage Explorer',
+	description: 'Navigate to the Storage Explorer',
+	tags: ['navigation'],
+	// TUI-specific fields passed through as opaque data
+	trigger: '/storage',
+	aliases: ['/store', '/db'],
+	category: 'Navigation',
+	keybinding: 'g then d',
+	acceptsArgs: false,
+	order: 5,
+	enabled: true
+} as RuntimeDescriptor & Record<string, unknown>;
+
+export default descriptor;

package/src/tui/storage-open.impl.consumer.ts

@@ -0,0 +1,14 @@
+/**
+ * Storage Open TUI command implementation (consumer stub).
+ *
+ * The actual implementation requires @vibesdotdev/tui components.
+ * This stub is loaded at runtime only when the TUI stack is present.
+ * When TUI is not available, the onRegister hook in storage.plugin.ts
+ * skips registration entirely via runtime.hasKind('tui/command').
+ */
+
+const implementation: Record<string, unknown> = {
+	execute: () => ({ type: 'navigate', route: '/storage' })
+};
+
+export default implementation;

package/src/utils/error-context.ts

@@ -0,0 +1,19 @@
+export function toErrorContext(error: unknown): Record<string, unknown> | undefined {
+	if (error instanceof Error) {
+		return {
+			error: error.message,
+			name: error.name,
+			stack: error.stack
+		};
+	}
+
+	if (typeof error === 'object' && error !== null && !Array.isArray(error)) {
+		return error as Record<string, unknown>;
+	}
+
+	if (error === undefined) {
+		return undefined;
+	}
+
+	return { error: String(error) };
+}