@vibesdotdev/storage 0.0.1

package/src/sync/cloud-pull.ts

@@ -0,0 +1,202 @@
+/**
+ * Cloud Pull Service
+ *
+ * Pulls data from cloud storage for manifests that have cloud-api configured.
+ * Used at daemon startup to hydrate local state from cloud, and by the
+ * sync job for bidirectional sync.
+ */
+
+import { getLogger } from '@vibesdotdev/logging';
+import type { SyncState } from './schemas/sync-state';
+
+const logger = getLogger('storage:cloud-pull');
+import type { CloudPullOptions, CloudPullResult } from './schemas/cloud-pull';
+import {
+	loadSyncState,
+	saveSyncState,
+	getManifestSyncState,
+	computeHash,
+	markSynced,
+	markConflict,
+	markError
+} from './sync-state.consumer';
+
+export type { CloudPullOptions, CloudPullResult } from './schemas/cloud-pull';
+
+const LOG_PREFIX = '[cloud-pull]';
+
+/**
+ * Pull data from cloud for all eligible manifests.
+ * Called at daemon startup and periodically by the sync job.
+ */
+export async function pullFromCloud(options: CloudPullOptions = {}): Promise<CloudPullResult> {
+	const { manifestIds, force = false, dryRun = false } = options;
+
+	const result: CloudPullResult = { pulled: 0, skipped: 0, conflicts: 0, errors: 0, details: [] };
+
+	try {
+		const { getVibesRuntime } = await import('@vibesdotdev/runtime');
+		const { StorageManifest } = await import('../manifest/base');
+		const runtime = getVibesRuntime();
+
+		// Find manifests with cloud-api support
+		const descriptors = runtime.query('storage/manifest').descriptors() as unknown as Array<{
+			id: string;
+			allowedAdapters?: string[];
+			namespace: string;
+			schema: unknown;
+			adapterConfig?: Record<string, Record<string, unknown>>;
+		}>;
+
+		const eligible = descriptors.filter((d) => {
+			if (!d.allowedAdapters?.includes('storage.cloud-api')) return false;
+			if (manifestIds?.length && !manifestIds.includes(d.id)) return false;
+			return true;
+		});
+
+		if (eligible.length === 0) {
+			logger.debug(`${LOG_PREFIX} No eligible manifests for cloud pull`);
+			return result;
+		}
+
+		const syncState = await loadSyncState();
+
+		// Resolve cloud adapter once
+		const cloudAdapter = await runtime
+			.query('storage/adapter')
+			.filter((d: { id: string }) => d.id === 'storage.cloud-api')
+			.resolve();
+
+		if (!cloudAdapter) {
+			logger.debug(`${LOG_PREFIX} Cloud adapter not available`);
+			return result;
+		}
+
+		const adapter = cloudAdapter as {
+			configure(config: Record<string, unknown>): void;
+			documentStore<T>(
+				ns: string,
+				schema: unknown,
+				manifest?: unknown
+			): { load(): Promise<T | null> };
+		};
+
+		for (const descriptor of eligible) {
+			try {
+				const pullResult = await pullManifest(
+					descriptor,
+					adapter,
+					syncState,
+					new StorageManifest(descriptor.id),
+					{ force, dryRun }
+				);
+				result.details.push(pullResult);
+
+				switch (pullResult.action) {
+					case 'pulled':
+						result.pulled++;
+						break;
+					case 'conflict':
+						result.conflicts++;
+						break;
+					case 'error':
+						result.errors++;
+						break;
+					default:
+						result.skipped++;
+				}
+			} catch (error) {
+				const msg = (error as Error).message;
+				logger.warn(`${LOG_PREFIX} Pull failed for ${descriptor.id}`, { error: msg });
+				markError(syncState, descriptor.id, msg);
+				result.details.push({ manifestId: descriptor.id, action: 'error', message: msg });
+				result.errors++;
+			}
+		}
+
+		if (!dryRun) {
+			await saveSyncState(syncState);
+		}
+
+		logger.info(`${LOG_PREFIX} Pull complete`, {
+			pulled: result.pulled,
+			skipped: result.skipped,
+			conflicts: result.conflicts,
+			errors: result.errors
+		});
+	} catch (error) {
+		logger.error(`${LOG_PREFIX} Pull failed`, error as Error);
+	}
+
+	return result;
+}
+
+async function pullManifest(
+	descriptor: {
+		id: string;
+		namespace: string;
+		schema: unknown;
+		adapterConfig?: Record<string, Record<string, unknown>>;
+	},
+	adapter: {
+		configure(config: Record<string, unknown>): void;
+		documentStore<T>(ns: string, schema: unknown): { load(): Promise<T | null> };
+	},
+	syncState: SyncState,
+	localManifest: { load(): Promise<unknown>; save(data: never): Promise<void> },
+	options: { force: boolean; dryRun: boolean }
+): Promise<{
+	manifestId: string;
+	action: 'pulled' | 'skipped' | 'conflict' | 'error';
+	message?: string;
+}> {
+	const { id: manifestId, namespace, schema } = descriptor;
+	const cloudConfig = descriptor.adapterConfig?.['storage.cloud-api'] ?? {};
+
+	adapter.configure(cloudConfig);
+	const store = adapter.documentStore<unknown>(namespace, schema);
+
+	// Load remote
+	let remoteData: unknown;
+	try {
+		remoteData = await store.load();
+	} catch {
+		return { manifestId, action: 'skipped', message: 'Cloud not reachable' };
+	}
+
+	if (!remoteData) {
+		return { manifestId, action: 'skipped', message: 'No remote data' };
+	}
+
+	const remoteHash = computeHash(remoteData);
+	const prev = getManifestSyncState(syncState, manifestId);
+
+	// Load local for comparison
+	const localData = await localManifest.load();
+	const localHash = localData ? computeHash(localData) : null;
+
+	// Already in sync
+	if (localHash === remoteHash) {
+		markSynced(syncState, manifestId, remoteHash, 'pull');
+		return { manifestId, action: 'skipped', message: 'Already in sync' };
+	}
+
+	// Check for conflicts
+	const localChanged = localHash !== prev.localHash && localHash !== null;
+	const remoteChanged = remoteHash !== prev.remoteHash;
+
+	if (localChanged && remoteChanged && !options.force) {
+		markConflict(syncState, manifestId, localHash ?? '', remoteHash);
+		return { manifestId, action: 'conflict', message: 'Both sides changed' };
+	}
+
+	// Pull remote → local
+	if (options.dryRun) {
+		return { manifestId, action: 'pulled', message: '[dry-run] Would pull from cloud' };
+	}
+
+	await localManifest.save(remoteData as never);
+	markSynced(syncState, manifestId, remoteHash, 'pull');
+	logger.info(`${LOG_PREFIX} Pulled ${manifestId} from cloud`);
+	return { manifestId, action: 'pulled' };
+}

package/src/sync/cloud-record-crud-config.ts

@@ -0,0 +1,41 @@
+/**
+ * Resolve a record's cloud-API CRUD configuration from its descriptor.
+ *
+ * Reads `StorageRecordDescriptor` metadata only; consumed by storage core's
+ * `record-sync-bridge`. Storage core owns it because adapters depend on core,
+ * not the other way around.
+ */
+
+import type { StorageRecordDescriptor } from '../schemas/record';
+
+export type CloudRecordCrudConfig = {
+	apiPath: string;
+	entityPath?: string;
+	listField?: string;
+	totalField?: string;
+	createMethod?: 'POST' | 'PUT';
+	updateMethod?: 'PUT' | 'PATCH';
+	deleteMethod?: 'DELETE';
+};
+
+export function resolveCloudRecordCrudConfig(
+	descriptor: StorageRecordDescriptor
+): CloudRecordCrudConfig {
+	const remoteCrud =
+		descriptor.remote?.adapter === 'storage.cloud-api' ? descriptor.remote.crud : null;
+	if (remoteCrud) {
+		return remoteCrud as CloudRecordCrudConfig;
+	}
+
+	const raw = descriptor.adapterConfig?.['storage.cloud-api'];
+	const value =
+		raw && typeof raw === 'object' && 'record' in raw ? (raw as { record: unknown }).record : raw;
+	if (
+		!value ||
+		typeof value !== 'object' ||
+		typeof (value as { apiPath?: unknown }).apiPath !== 'string'
+	) {
+		throw new Error(`Record '${descriptor.id}' is missing storage.cloud-api remote CRUD metadata.`);
+	}
+	return value as CloudRecordCrudConfig;
+}

package/src/sync/record-sync-bridge.ts

@@ -0,0 +1,183 @@
+import { getVibesRuntime } from '@vibesdotdev/runtime';
+import { resolveWorkspacePath } from '../paths';
+import { StorageRecord } from '../record';
+import { resolveCloudRecordCrudConfig } from './cloud-record-crud-config';
+import type { StorageRecordInterface } from '../interfaces/record';
+import type {
+	StorageRecordDescriptor,
+	StorageRecordLocalSyncContext,
+	StorageRecordRemoteSyncContext
+} from '../schemas/record';
+import type { SyncBridgeDescriptor } from '../schemas/sync-bridge-descriptor';
+
+const PAGE_SIZE = 200;
+
+/** Subset of `descriptor.sync` (StorageSyncConfig) used by the bridge. */
+type RecordSyncConfig = NonNullable<StorageRecordDescriptor['sync']>;
+
+/** Hook signatures matching `StorageRecordSyncHooks`, derived locally for callable narrowing. */
+type CollectHook = (
+	context: StorageRecordLocalSyncContext
+) => Promise<Record<string, unknown>[]>;
+type ToRemoteHook = (
+	item: Record<string, unknown>,
+	context?: StorageRecordLocalSyncContext
+) => Record<string, unknown>;
+type FromRemoteHook = (item: Record<string, unknown>) => Record<string, unknown>;
+type BeforePushHook = (
+	item: Record<string, unknown>,
+	context: StorageRecordRemoteSyncContext
+) => Promise<void>;
+type DeleteRemoteHook = BeforePushHook;
+type WritePulledHook = (
+	items: Record<string, unknown>[],
+	context: StorageRecordLocalSyncContext
+) => Promise<void>;
+
+export function isSyncableStorageRecord(descriptor: StorageRecordDescriptor): boolean {
+	if (descriptor.enabled === false) return false;
+	if (!descriptor.allowedAdapters.includes('storage.cloud-api')) return false;
+	try {
+		resolveCloudRecordCrudConfig(descriptor);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+export function createRecordSyncBridgeDescriptor(
+	descriptor: StorageRecordDescriptor
+): SyncBridgeDescriptor {
+	const sync = descriptor.sync as RecordSyncConfig | undefined;
+	const remote = resolveCloudRecordCrudConfig(descriptor);
+	const beforePush = sync?.beforePush as BeforePushHook | undefined;
+	const deleteRemote = sync?.deleteRemote as DeleteRemoteHook | undefined;
+	return {
+		kind: 'storage/sync-bridge',
+		id: descriptor.id,
+		name: `${descriptor.name} Sync`,
+		description: `Syncs ${descriptor.name} through storage/record metadata`,
+		tags: [...(descriptor.tags ?? []), 'storage', 'record', 'sync'],
+		source: { runtimeKind: descriptor.id },
+		target: {
+			apiPath: remote.apiPath,
+			entityPath: remote.entityPath,
+			listField: remote.listField,
+			totalField: remote.totalField,
+			includeDeleted: Boolean(descriptor.softDelete)
+		},
+		sync: {
+			direction: 'bidirectional',
+			ownershipFilter: sync?.ownershipFilter ?? 'all',
+			conflictStrategy: sync?.conflictStrategy ?? 'local-wins',
+			sensitiveFields: sync?.sensitiveFields ?? [],
+			idField: sync?.idField ?? descriptor.primaryKey ?? 'id'
+		},
+		collect: () => collectRecordItems(descriptor, sync),
+		toRemote: sync?.toRemote as ToRemoteHook | undefined,
+		fromRemote: sync?.fromRemote as FromRemoteHook | undefined,
+		beforePush: beforePush
+			? (
+					record: Record<string, unknown>,
+					context: {
+						apiEndpoint: string;
+						apiToken?: string;
+						fetch: (input: string, init?: RequestInit) => Promise<Response>;
+					}
+				) =>
+					beforePush(record, {
+						...context,
+						runtime: getVibesRuntime()
+					} as StorageRecordRemoteSyncContext)
+			: undefined,
+		deleteRemote: deleteRemote
+			? (
+					record: Record<string, unknown>,
+					context: {
+						apiEndpoint: string;
+						apiToken?: string;
+						fetch: (input: string, init?: RequestInit) => Promise<Response>;
+					}
+				) =>
+					deleteRemote(record, {
+						...context,
+						runtime: getVibesRuntime()
+					} as StorageRecordRemoteSyncContext)
+			: undefined,
+		writePulled: (items: Record<string, unknown>[]) =>
+			writePulledRecordItems(descriptor, sync, items)
+	};
+}
+
+async function collectRecordItems(
+	descriptor: StorageRecordDescriptor,
+	sync: RecordSyncConfig | undefined
+): Promise<Record<string, unknown>[]> {
+	const runtime = getVibesRuntime();
+	const record = await resolveLocalRecord(descriptor);
+	const collect = sync?.collect as CollectHook | undefined;
+	if (collect) {
+		return collect({
+			runtime,
+			record: record as StorageRecordInterface<Record<string, unknown>>
+		});
+	}
+	return listAllRecords(record as StorageRecordInterface<Record<string, unknown>>);
+}
+
+async function writePulledRecordItems(
+	descriptor: StorageRecordDescriptor,
+	sync: RecordSyncConfig | undefined,
+	items: Record<string, unknown>[]
+): Promise<void> {
+	const runtime = getVibesRuntime();
+	const record = (await resolveLocalRecord(descriptor)) as StorageRecordInterface<
+		Record<string, unknown>
+	>;
+	const writePulled = sync?.writePulled as WritePulledHook | undefined;
+	if (writePulled) {
+		await writePulled(items, { runtime, record });
+		return;
+	}
+	const idField = sync?.idField ?? descriptor.primaryKey ?? 'id';
+	for (const item of items) {
+		const id = item[idField];
+		if (typeof id !== 'string') continue;
+		await record.delete(id, { force: true }).catch(() => undefined);
+		await record.create(item);
+	}
+}
+
+async function resolveLocalRecord(
+	descriptor: StorageRecordDescriptor
+): Promise<StorageRecord<Record<string, unknown>>> {
+	const localAdapters = descriptor.persistence?.localAdapters ?? ['storage.localdb'];
+	const localDescriptor: StorageRecordDescriptor = {
+		...descriptor,
+		defaultAdapter: localAdapters[0] ?? descriptor.defaultAdapter,
+		persistence: {
+			...(descriptor.persistence ?? { mode: 'local-first' }),
+			mode: 'local-only',
+			localAdapters,
+			remoteAdapters: descriptor.persistence?.remoteAdapters ?? ['storage.cloud-api']
+		}
+	};
+	const record = new StorageRecord<Record<string, unknown>>(localDescriptor, {
+		'runtime/workspace-path': resolveWorkspacePath()
+	});
+	await record.initialize();
+	return record;
+}
+
+async function listAllRecords(
+	record: StorageRecordInterface<Record<string, unknown>>
+): Promise<Record<string, unknown>[]> {
+	const items: Record<string, unknown>[] = [];
+	let offset = 0;
+	for (;;) {
+		const page = await record.list({ limit: PAGE_SIZE, offset, includeDeleted: true });
+		items.push(...page.items);
+		if (page.items.length < PAGE_SIZE) return items;
+		offset += PAGE_SIZE;
+	}
+}

package/src/sync/resolve-sync-target.ts

@@ -0,0 +1,57 @@
+/**
+ * Resolve Sync Target
+ *
+ * Resolves the current sync target URL, environment name, and auth token from the runtime.
+ * Used by CLI commands and sync-bridge to determine where to push/pull data.
+ *
+ * Resolution chain for sync target:
+ * 1. config/environment → apiUrl
+ * 2. VIBES_API_URL environment variable
+ * 3. 'unknown' (no target configured)
+ *
+ * Resolution chain for auth token:
+ * 1. config/environment → apiToken
+ * 2. config/manifest#auth → credentials.token (CLI credential store)
+ * 3. VIBES_AUTH_TOKEN environment variable
+ */
+
+import { getRuntimeEnv, getVibesRuntime } from '@vibesdotdev/runtime';
+
+export interface SyncTarget {
+	syncTarget: string;
+	envName: string;
+	apiToken?: string;
+}
+
+export async function resolveSyncTarget(): Promise<SyncTarget> {
+	let syncTarget = getRuntimeEnv('VIBES_API_URL') ?? 'unknown';
+	let envName = 'env';
+	let apiToken: string | undefined;
+
+	try {
+		const runtime = getVibesRuntime();
+		const env = (await runtime.query('config/environment').resolve()) as {
+			apiUrl?: string;
+			id?: string;
+			name?: string;
+			apiToken?: string;
+		} | null;
+
+		if (env?.apiUrl) syncTarget = env.apiUrl;
+		if (env?.name) envName = env.name;
+		else if (env?.id) envName = env.id;
+		if (env?.apiToken) apiToken = env.apiToken;
+	} catch {
+		// Environment resolution may fail; use fallback
+	}
+
+	// Credential store lives in the app layer; consumers must supply auth via
+	// environment config or VIBES_AUTH_TOKEN.
+
+	// Fallback to environment variable
+	if (!apiToken && getRuntimeEnv('VIBES_AUTH_TOKEN')) {
+		apiToken = getRuntimeEnv('VIBES_AUTH_TOKEN');
+	}
+
+	return { syncTarget, envName, apiToken };
+}

package/src/sync/schemas/cloud-pull.ts

@@ -0,0 +1,26 @@
+/**
+ * Cloud Pull Schemas
+ *
+ * Types for cloud pull operations — options and result structures.
+ */
+
+export interface CloudPullOptions {
+	/** Only pull specific manifests */
+	manifestIds?: string[];
+	/** Force overwrite local even if local has changes */
+	force?: boolean;
+	/** Preview without writing */
+	dryRun?: boolean;
+}
+
+export interface CloudPullResult {
+	pulled: number;
+	skipped: number;
+	conflicts: number;
+	errors: number;
+	details: Array<{
+		manifestId: string;
+		action: 'pulled' | 'skipped' | 'conflict' | 'error';
+		message?: string;
+	}>;
+}

package/src/sync/schemas/index.ts

@@ -0,0 +1,18 @@
+/**
+ * Storage Sync Schemas
+ *
+ * Re-exports all sync-related schemas and types.
+ */
+
+export {
+	SyncStatusSchema,
+	ManifestSyncStateSchema,
+	SyncStateSchema,
+	type SyncStatus,
+	type ManifestSyncState,
+	type SyncState
+} from './sync-state';
+
+export { type CloudPullOptions, type CloudPullResult } from './cloud-pull';
+
+export { type SyncHelpers } from './sync-helpers';

package/src/sync/schemas/sync-helpers.ts

@@ -0,0 +1,18 @@
+/**
+ * Sync Helpers Schema
+ *
+ * Interface for sync operation helper functions passed between sync components.
+ */
+
+import type { SyncState, ManifestSyncState } from './sync-state';
+
+export interface SyncHelpers {
+	dryRun: boolean;
+	computeHash: (data: unknown) => string;
+	getManifestSyncState: (state: SyncState, id: string) => ManifestSyncState;
+	markSynced: (state: SyncState, id: string, hash: string, dir: 'push' | 'pull') => void;
+	markLocalAhead: (state: SyncState, id: string, hash: string) => void;
+	markConflict: (state: SyncState, id: string, local: string, remote: string) => void;
+	markError: (state: SyncState, id: string, msg: string) => void;
+	markRemoteAhead: (state: SyncState, id: string, hash: string) => void;
+}

package/src/sync/schemas/sync-state.ts

@@ -0,0 +1,41 @@
+/**
+ * Sync State Schemas
+ *
+ * Zod schemas and inferred types for per-manifest sync state tracking.
+ * Pure schemas — no host/fs imports.
+ */
+
+import * as z from 'zod/v4';
+
+export const SyncStatusSchema = z.enum([
+	'in-sync',
+	'local-ahead',
+	'remote-ahead',
+	'conflict',
+	'paused',
+	'never-synced'
+]);
+
+export type SyncStatus = z.infer<typeof SyncStatusSchema>;
+
+export const ManifestSyncStateSchema = z.object({
+	manifestId: z.string(),
+	localHash: z.string().nullable(),
+	remoteHash: z.string().nullable(),
+	lastLocalChange: z.string().nullable(),
+	lastRemoteChange: z.string().nullable(),
+	lastSyncAt: z.string().nullable(),
+	lastSyncDirection: z.enum(['push', 'pull']).nullable(),
+	syncStatus: SyncStatusSchema,
+	errorMessage: z.string().nullable()
+});
+
+export type ManifestSyncState = z.infer<typeof ManifestSyncStateSchema>;
+
+export const SyncStateSchema = z.object({
+	version: z.literal(1),
+	updatedAt: z.string(),
+	manifests: z.record(z.string(), ManifestSyncStateSchema)
+});
+
+export type SyncState = z.infer<typeof SyncStateSchema>;