@vibesdotdev/storage 0.0.1 → 1.0.0-rc.3

package/src/sync/cloud-pull.ts

@@ -1,202 +0,0 @@
-/**
- * Cloud Pull Service
- *
- * Pulls data from cloud storage for manifests that have cloud-api configured.
- * Used at daemon startup to hydrate local state from cloud, and by the
- * sync job for bidirectional sync.
- */
-
-import { getLogger } from '@vibesdotdev/logging';
-import type { SyncState } from './schemas/sync-state';
-
-const logger = getLogger('storage:cloud-pull');
-import type { CloudPullOptions, CloudPullResult } from './schemas/cloud-pull';
-import {
-	loadSyncState,
-	saveSyncState,
-	getManifestSyncState,
-	computeHash,
-	markSynced,
-	markConflict,
-	markError
-} from './sync-state.consumer';
-
-export type { CloudPullOptions, CloudPullResult } from './schemas/cloud-pull';
-
-const LOG_PREFIX = '[cloud-pull]';
-
-/**
- * Pull data from cloud for all eligible manifests.
- * Called at daemon startup and periodically by the sync job.
- */
-export async function pullFromCloud(options: CloudPullOptions = {}): Promise<CloudPullResult> {
-	const { manifestIds, force = false, dryRun = false } = options;
-
-	const result: CloudPullResult = { pulled: 0, skipped: 0, conflicts: 0, errors: 0, details: [] };
-
-	try {
-		const { getVibesRuntime } = await import('@vibesdotdev/runtime');
-		const { StorageManifest } = await import('../manifest/base');
-		const runtime = getVibesRuntime();
-
-		// Find manifests with cloud-api support
-		const descriptors = runtime.query('storage/manifest').descriptors() as unknown as Array<{
-			id: string;
-			allowedAdapters?: string[];
-			namespace: string;
-			schema: unknown;
-			adapterConfig?: Record<string, Record<string, unknown>>;
-		}>;
-
-		const eligible = descriptors.filter((d) => {
-			if (!d.allowedAdapters?.includes('storage.cloud-api')) return false;
-			if (manifestIds?.length && !manifestIds.includes(d.id)) return false;
-			return true;
-		});
-
-		if (eligible.length === 0) {
-			logger.debug(`${LOG_PREFIX} No eligible manifests for cloud pull`);
-			return result;
-		}
-
-		const syncState = await loadSyncState();
-
-		// Resolve cloud adapter once
-		const cloudAdapter = await runtime
-			.query('storage/adapter')
-			.filter((d: { id: string }) => d.id === 'storage.cloud-api')
-			.resolve();
-
-		if (!cloudAdapter) {
-			logger.debug(`${LOG_PREFIX} Cloud adapter not available`);
-			return result;
-		}
-
-		const adapter = cloudAdapter as {
-			configure(config: Record<string, unknown>): void;
-			documentStore<T>(
-				ns: string,
-				schema: unknown,
-				manifest?: unknown
-			): { load(): Promise<T | null> };
-		};
-
-		for (const descriptor of eligible) {
-			try {
-				const pullResult = await pullManifest(
-					descriptor,
-					adapter,
-					syncState,
-					new StorageManifest(descriptor.id),
-					{ force, dryRun }
-				);
-				result.details.push(pullResult);
-
-				switch (pullResult.action) {
-					case 'pulled':
-						result.pulled++;
-						break;
-					case 'conflict':
-						result.conflicts++;
-						break;
-					case 'error':
-						result.errors++;
-						break;
-					default:
-						result.skipped++;
-				}
-			} catch (error) {
-				const msg = (error as Error).message;
-				logger.warn(`${LOG_PREFIX} Pull failed for ${descriptor.id}`, { error: msg });
-				markError(syncState, descriptor.id, msg);
-				result.details.push({ manifestId: descriptor.id, action: 'error', message: msg });
-				result.errors++;
-			}
-		}
-
-		if (!dryRun) {
-			await saveSyncState(syncState);
-		}
-
-		logger.info(`${LOG_PREFIX} Pull complete`, {
-			pulled: result.pulled,
-			skipped: result.skipped,
-			conflicts: result.conflicts,
-			errors: result.errors
-		});
-	} catch (error) {
-		logger.error(`${LOG_PREFIX} Pull failed`, error as Error);
-	}
-
-	return result;
-}
-
-async function pullManifest(
-	descriptor: {
-		id: string;
-		namespace: string;
-		schema: unknown;
-		adapterConfig?: Record<string, Record<string, unknown>>;
-	},
-	adapter: {
-		configure(config: Record<string, unknown>): void;
-		documentStore<T>(ns: string, schema: unknown): { load(): Promise<T | null> };
-	},
-	syncState: SyncState,
-	localManifest: { load(): Promise<unknown>; save(data: never): Promise<void> },
-	options: { force: boolean; dryRun: boolean }
-): Promise<{
-	manifestId: string;
-	action: 'pulled' | 'skipped' | 'conflict' | 'error';
-	message?: string;
-}> {
-	const { id: manifestId, namespace, schema } = descriptor;
-	const cloudConfig = descriptor.adapterConfig?.['storage.cloud-api'] ?? {};
-
-	adapter.configure(cloudConfig);
-	const store = adapter.documentStore<unknown>(namespace, schema);
-
-	// Load remote
-	let remoteData: unknown;
-	try {
-		remoteData = await store.load();
-	} catch {
-		return { manifestId, action: 'skipped', message: 'Cloud not reachable' };
-	}
-
-	if (!remoteData) {
-		return { manifestId, action: 'skipped', message: 'No remote data' };
-	}
-
-	const remoteHash = computeHash(remoteData);
-	const prev = getManifestSyncState(syncState, manifestId);
-
-	// Load local for comparison
-	const localData = await localManifest.load();
-	const localHash = localData ? computeHash(localData) : null;
-
-	// Already in sync
-	if (localHash === remoteHash) {
-		markSynced(syncState, manifestId, remoteHash, 'pull');
-		return { manifestId, action: 'skipped', message: 'Already in sync' };
-	}
-
-	// Check for conflicts
-	const localChanged = localHash !== prev.localHash && localHash !== null;
-	const remoteChanged = remoteHash !== prev.remoteHash;
-
-	if (localChanged && remoteChanged && !options.force) {
-		markConflict(syncState, manifestId, localHash ?? '', remoteHash);
-		return { manifestId, action: 'conflict', message: 'Both sides changed' };
-	}
-
-	// Pull remote → local
-	if (options.dryRun) {
-		return { manifestId, action: 'pulled', message: '[dry-run] Would pull from cloud' };
-	}
-
-	await localManifest.save(remoteData as never);
-	markSynced(syncState, manifestId, remoteHash, 'pull');
-	logger.info(`${LOG_PREFIX} Pulled ${manifestId} from cloud`);
-	return { manifestId, action: 'pulled' };
-}

package/src/sync/cloud-record-crud-config.ts

@@ -1,41 +0,0 @@
-/**
- * Resolve a record's cloud-API CRUD configuration from its descriptor.
- *
- * Reads `StorageRecordDescriptor` metadata only; consumed by storage core's
- * `record-sync-bridge`. Storage core owns it because adapters depend on core,
- * not the other way around.
- */
-
-import type { StorageRecordDescriptor } from '../schemas/record';
-
-export type CloudRecordCrudConfig = {
-	apiPath: string;
-	entityPath?: string;
-	listField?: string;
-	totalField?: string;
-	createMethod?: 'POST' | 'PUT';
-	updateMethod?: 'PUT' | 'PATCH';
-	deleteMethod?: 'DELETE';
-};
-
-export function resolveCloudRecordCrudConfig(
-	descriptor: StorageRecordDescriptor
-): CloudRecordCrudConfig {
-	const remoteCrud =
-		descriptor.remote?.adapter === 'storage.cloud-api' ? descriptor.remote.crud : null;
-	if (remoteCrud) {
-		return remoteCrud as CloudRecordCrudConfig;
-	}
-
-	const raw = descriptor.adapterConfig?.['storage.cloud-api'];
-	const value =
-		raw && typeof raw === 'object' && 'record' in raw ? (raw as { record: unknown }).record : raw;
-	if (
-		!value ||
-		typeof value !== 'object' ||
-		typeof (value as { apiPath?: unknown }).apiPath !== 'string'
-	) {
-		throw new Error(`Record '${descriptor.id}' is missing storage.cloud-api remote CRUD metadata.`);
-	}
-	return value as CloudRecordCrudConfig;
-}

package/src/sync/record-sync-bridge.ts

@@ -1,183 +0,0 @@
-import { getVibesRuntime } from '@vibesdotdev/runtime';
-import { resolveWorkspacePath } from '../paths';
-import { StorageRecord } from '../record';
-import { resolveCloudRecordCrudConfig } from './cloud-record-crud-config';
-import type { StorageRecordInterface } from '../interfaces/record';
-import type {
-	StorageRecordDescriptor,
-	StorageRecordLocalSyncContext,
-	StorageRecordRemoteSyncContext
-} from '../schemas/record';
-import type { SyncBridgeDescriptor } from '../schemas/sync-bridge-descriptor';
-
-const PAGE_SIZE = 200;
-
-/** Subset of `descriptor.sync` (StorageSyncConfig) used by the bridge. */
-type RecordSyncConfig = NonNullable<StorageRecordDescriptor['sync']>;
-
-/** Hook signatures matching `StorageRecordSyncHooks`, derived locally for callable narrowing. */
-type CollectHook = (
-	context: StorageRecordLocalSyncContext
-) => Promise<Record<string, unknown>[]>;
-type ToRemoteHook = (
-	item: Record<string, unknown>,
-	context?: StorageRecordLocalSyncContext
-) => Record<string, unknown>;
-type FromRemoteHook = (item: Record<string, unknown>) => Record<string, unknown>;
-type BeforePushHook = (
-	item: Record<string, unknown>,
-	context: StorageRecordRemoteSyncContext
-) => Promise<void>;
-type DeleteRemoteHook = BeforePushHook;
-type WritePulledHook = (
-	items: Record<string, unknown>[],
-	context: StorageRecordLocalSyncContext
-) => Promise<void>;
-
-export function isSyncableStorageRecord(descriptor: StorageRecordDescriptor): boolean {
-	if (descriptor.enabled === false) return false;
-	if (!descriptor.allowedAdapters.includes('storage.cloud-api')) return false;
-	try {
-		resolveCloudRecordCrudConfig(descriptor);
-		return true;
-	} catch {
-		return false;
-	}
-}
-
-export function createRecordSyncBridgeDescriptor(
-	descriptor: StorageRecordDescriptor
-): SyncBridgeDescriptor {
-	const sync = descriptor.sync as RecordSyncConfig | undefined;
-	const remote = resolveCloudRecordCrudConfig(descriptor);
-	const beforePush = sync?.beforePush as BeforePushHook | undefined;
-	const deleteRemote = sync?.deleteRemote as DeleteRemoteHook | undefined;
-	return {
-		kind: 'storage/sync-bridge',
-		id: descriptor.id,
-		name: `${descriptor.name} Sync`,
-		description: `Syncs ${descriptor.name} through storage/record metadata`,
-		tags: [...(descriptor.tags ?? []), 'storage', 'record', 'sync'],
-		source: { runtimeKind: descriptor.id },
-		target: {
-			apiPath: remote.apiPath,
-			entityPath: remote.entityPath,
-			listField: remote.listField,
-			totalField: remote.totalField,
-			includeDeleted: Boolean(descriptor.softDelete)
-		},
-		sync: {
-			direction: 'bidirectional',
-			ownershipFilter: sync?.ownershipFilter ?? 'all',
-			conflictStrategy: sync?.conflictStrategy ?? 'local-wins',
-			sensitiveFields: sync?.sensitiveFields ?? [],
-			idField: sync?.idField ?? descriptor.primaryKey ?? 'id'
-		},
-		collect: () => collectRecordItems(descriptor, sync),
-		toRemote: sync?.toRemote as ToRemoteHook | undefined,
-		fromRemote: sync?.fromRemote as FromRemoteHook | undefined,
-		beforePush: beforePush
-			? (
-					record: Record<string, unknown>,
-					context: {
-						apiEndpoint: string;
-						apiToken?: string;
-						fetch: (input: string, init?: RequestInit) => Promise<Response>;
-					}
-				) =>
-					beforePush(record, {
-						...context,
-						runtime: getVibesRuntime()
-					} as StorageRecordRemoteSyncContext)
-			: undefined,
-		deleteRemote: deleteRemote
-			? (
-					record: Record<string, unknown>,
-					context: {
-						apiEndpoint: string;
-						apiToken?: string;
-						fetch: (input: string, init?: RequestInit) => Promise<Response>;
-					}
-				) =>
-					deleteRemote(record, {
-						...context,
-						runtime: getVibesRuntime()
-					} as StorageRecordRemoteSyncContext)
-			: undefined,
-		writePulled: (items: Record<string, unknown>[]) =>
-			writePulledRecordItems(descriptor, sync, items)
-	};
-}
-
-async function collectRecordItems(
-	descriptor: StorageRecordDescriptor,
-	sync: RecordSyncConfig | undefined
-): Promise<Record<string, unknown>[]> {
-	const runtime = getVibesRuntime();
-	const record = await resolveLocalRecord(descriptor);
-	const collect = sync?.collect as CollectHook | undefined;
-	if (collect) {
-		return collect({
-			runtime,
-			record: record as StorageRecordInterface<Record<string, unknown>>
-		});
-	}
-	return listAllRecords(record as StorageRecordInterface<Record<string, unknown>>);
-}
-
-async function writePulledRecordItems(
-	descriptor: StorageRecordDescriptor,
-	sync: RecordSyncConfig | undefined,
-	items: Record<string, unknown>[]
-): Promise<void> {
-	const runtime = getVibesRuntime();
-	const record = (await resolveLocalRecord(descriptor)) as StorageRecordInterface<
-		Record<string, unknown>
-	>;
-	const writePulled = sync?.writePulled as WritePulledHook | undefined;
-	if (writePulled) {
-		await writePulled(items, { runtime, record });
-		return;
-	}
-	const idField = sync?.idField ?? descriptor.primaryKey ?? 'id';
-	for (const item of items) {
-		const id = item[idField];
-		if (typeof id !== 'string') continue;
-		await record.delete(id, { force: true }).catch(() => undefined);
-		await record.create(item);
-	}
-}
-
-async function resolveLocalRecord(
-	descriptor: StorageRecordDescriptor
-): Promise<StorageRecord<Record<string, unknown>>> {
-	const localAdapters = descriptor.persistence?.localAdapters ?? ['storage.localdb'];
-	const localDescriptor: StorageRecordDescriptor = {
-		...descriptor,
-		defaultAdapter: localAdapters[0] ?? descriptor.defaultAdapter,
-		persistence: {
-			...(descriptor.persistence ?? { mode: 'local-first' }),
-			mode: 'local-only',
-			localAdapters,
-			remoteAdapters: descriptor.persistence?.remoteAdapters ?? ['storage.cloud-api']
-		}
-	};
-	const record = new StorageRecord<Record<string, unknown>>(localDescriptor, {
-		'runtime/workspace-path': resolveWorkspacePath()
-	});
-	await record.initialize();
-	return record;
-}
-
-async function listAllRecords(
-	record: StorageRecordInterface<Record<string, unknown>>
-): Promise<Record<string, unknown>[]> {
-	const items: Record<string, unknown>[] = [];
-	let offset = 0;
-	for (;;) {
-		const page = await record.list({ limit: PAGE_SIZE, offset, includeDeleted: true });
-		items.push(...page.items);
-		if (page.items.length < PAGE_SIZE) return items;
-		offset += PAGE_SIZE;
-	}
-}

package/src/sync/resolve-sync-target.ts

@@ -1,57 +0,0 @@
-/**
- * Resolve Sync Target
- *
- * Resolves the current sync target URL, environment name, and auth token from the runtime.
- * Used by CLI commands and sync-bridge to determine where to push/pull data.
- *
- * Resolution chain for sync target:
- * 1. config/environment → apiUrl
- * 2. VIBES_API_URL environment variable
- * 3. 'unknown' (no target configured)
- *
- * Resolution chain for auth token:
- * 1. config/environment → apiToken
- * 2. config/manifest#auth → credentials.token (CLI credential store)
- * 3. VIBES_AUTH_TOKEN environment variable
- */
-
-import { getRuntimeEnv, getVibesRuntime } from '@vibesdotdev/runtime';
-
-export interface SyncTarget {
-	syncTarget: string;
-	envName: string;
-	apiToken?: string;
-}
-
-export async function resolveSyncTarget(): Promise<SyncTarget> {
-	let syncTarget = getRuntimeEnv('VIBES_API_URL') ?? 'unknown';
-	let envName = 'env';
-	let apiToken: string | undefined;
-
-	try {
-		const runtime = getVibesRuntime();
-		const env = (await runtime.query('config/environment').resolve()) as {
-			apiUrl?: string;
-			id?: string;
-			name?: string;
-			apiToken?: string;
-		} | null;
-
-		if (env?.apiUrl) syncTarget = env.apiUrl;
-		if (env?.name) envName = env.name;
-		else if (env?.id) envName = env.id;
-		if (env?.apiToken) apiToken = env.apiToken;
-	} catch {
-		// Environment resolution may fail; use fallback
-	}
-
-	// Credential store lives in the app layer; consumers must supply auth via
-	// environment config or VIBES_AUTH_TOKEN.
-
-	// Fallback to environment variable
-	if (!apiToken && getRuntimeEnv('VIBES_AUTH_TOKEN')) {
-		apiToken = getRuntimeEnv('VIBES_AUTH_TOKEN');
-	}
-
-	return { syncTarget, envName, apiToken };
-}

package/src/sync/schemas/cloud-pull.ts

@@ -1,26 +0,0 @@
-/**
- * Cloud Pull Schemas
- *
- * Types for cloud pull operations — options and result structures.
- */
-
-export interface CloudPullOptions {
-	/** Only pull specific manifests */
-	manifestIds?: string[];
-	/** Force overwrite local even if local has changes */
-	force?: boolean;
-	/** Preview without writing */
-	dryRun?: boolean;
-}
-
-export interface CloudPullResult {
-	pulled: number;
-	skipped: number;
-	conflicts: number;
-	errors: number;
-	details: Array<{
-		manifestId: string;
-		action: 'pulled' | 'skipped' | 'conflict' | 'error';
-		message?: string;
-	}>;
-}

package/src/sync/schemas/index.ts

@@ -1,18 +0,0 @@
-/**
- * Storage Sync Schemas
- *
- * Re-exports all sync-related schemas and types.
- */
-
-export {
-	SyncStatusSchema,
-	ManifestSyncStateSchema,
-	SyncStateSchema,
-	type SyncStatus,
-	type ManifestSyncState,
-	type SyncState
-} from './sync-state';
-
-export { type CloudPullOptions, type CloudPullResult } from './cloud-pull';
-
-export { type SyncHelpers } from './sync-helpers';

package/src/sync/schemas/sync-helpers.ts

@@ -1,18 +0,0 @@
-/**
- * Sync Helpers Schema
- *
- * Interface for sync operation helper functions passed between sync components.
- */
-
-import type { SyncState, ManifestSyncState } from './sync-state';
-
-export interface SyncHelpers {
-	dryRun: boolean;
-	computeHash: (data: unknown) => string;
-	getManifestSyncState: (state: SyncState, id: string) => ManifestSyncState;
-	markSynced: (state: SyncState, id: string, hash: string, dir: 'push' | 'pull') => void;
-	markLocalAhead: (state: SyncState, id: string, hash: string) => void;
-	markConflict: (state: SyncState, id: string, local: string, remote: string) => void;
-	markError: (state: SyncState, id: string, msg: string) => void;
-	markRemoteAhead: (state: SyncState, id: string, hash: string) => void;
-}

package/src/sync/schemas/sync-state.ts

@@ -1,41 +0,0 @@
-/**
- * Sync State Schemas
- *
- * Zod schemas and inferred types for per-manifest sync state tracking.
- * Pure schemas — no host/fs imports.
- */
-
-import * as z from 'zod/v4';
-
-export const SyncStatusSchema = z.enum([
-	'in-sync',
-	'local-ahead',
-	'remote-ahead',
-	'conflict',
-	'paused',
-	'never-synced'
-]);
-
-export type SyncStatus = z.infer<typeof SyncStatusSchema>;
-
-export const ManifestSyncStateSchema = z.object({
-	manifestId: z.string(),
-	localHash: z.string().nullable(),
-	remoteHash: z.string().nullable(),
-	lastLocalChange: z.string().nullable(),
-	lastRemoteChange: z.string().nullable(),
-	lastSyncAt: z.string().nullable(),
-	lastSyncDirection: z.enum(['push', 'pull']).nullable(),
-	syncStatus: SyncStatusSchema,
-	errorMessage: z.string().nullable()
-});
-
-export type ManifestSyncState = z.infer<typeof ManifestSyncStateSchema>;
-
-export const SyncStateSchema = z.object({
-	version: z.literal(1),
-	updatedAt: z.string(),
-	manifests: z.record(z.string(), ManifestSyncStateSchema)
-});
-
-export type SyncState = z.infer<typeof SyncStateSchema>;