@flowdular/sandbox 0.3.2 → 0.4.1

package/src/server/preview-modules.ts

@@ -2,7 +2,14 @@ import { sandboxDirectory } from './config.ts';
 import { readFile, realpath } from 'node:fs/promises';
 import { dirname, join, resolve, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { parse as parseYaml } from 'yaml';
+import {
+	RESEARCH_MODULE_ID,
+	assertRecordedAdapters,
+	readSessionAdapters,
+} from './recorded-adapters.ts';
 import type { SandboxSession } from './sessions.ts';
+import { readSpecText } from './spec.ts';
 
 export interface PreviewModuleSource {
 	readonly id: string;
@@ -16,6 +23,29 @@ export const PREVIEW_SUPPORT_ROOT = resolve(
 	'../../../../modules',
 );
 const IDENTIFIER = /^[a-z][a-z0-9-]*\.core$/;
+export const DOCUMENTS_MODULE_ID = 'documents.core';
+
+/* A draft whose spec declares templates previews them through documents.core
+   and the same renderer a deployment runs; nothing extra is seeded for it. The
+   spec-schema gate stays the authority on the section's shape. */
+async function declaresTemplates(
+	modules: Iterable<{ readonly directory: string }>,
+	draftRoot: string,
+): Promise<boolean> {
+	for (const module of modules) {
+		const text = await readSpecText(join(draftRoot, module.directory));
+		if (text === null) continue;
+		try {
+			const spec = parseYaml(text) as { templates?: unknown } | null;
+			if (Array.isArray(spec?.templates) && spec.templates.length > 0) {
+				return true;
+			}
+		} catch {
+			continue;
+		}
+	}
+	return false;
+}
 
 async function manifest(
 	path: string,
@@ -78,6 +108,19 @@ export async function resolvePreviewModules(
 		visited.add(id);
 		ordered.push({ id, directory, path, support: !draft });
 	};
+	/* A session whose spec declares research previews it through research.core on
+	   recorded fixtures, and never composes one that names a live adapter. */
+	const adapters = await readSessionAdapters(
+		session.modules.map((module) => ({
+			directory: module.directory,
+			path: join(draftRoot, module.directory),
+		})),
+	);
+	assertRecordedAdapters(adapters);
+	if (adapters.research) await visit(RESEARCH_MODULE_ID);
+	if (await declaresTemplates(drafts.values(), draftRoot)) {
+		await visit(DOCUMENTS_MODULE_ID);
+	}
 	for (const module of session.modules) await visit(module.id);
 	return ordered;
 }

package/src/server/preview-revision.ts

@@ -39,6 +39,7 @@ export async function previewRevision(
 		await file(join(source.path, 'package.json'), 'package.json');
 		await directory(join(source.path, 'src'), 'src');
 		await directory(join(source.path, 'migrations'), 'migrations');
+		await directory(join(source.path, 'preview'), 'preview');
 	}
 	return hash.digest('hex');
 }

package/src/server/preview-runtime.ts

@@ -11,6 +11,7 @@ import {
 	createPlatformAgentRegistry,
 	createPlatformCapabilityRegistry,
 	createPlatformToolRegistry,
+	ModuleSettingsError,
 	type ModuleSettingRecord,
 	type ModuleSettingValue,
 	type ModuleSettingsRuntime,
@@ -47,6 +48,12 @@ import {
 	type PreviewModuleSource,
 } from './preview-modules.ts';
 import { previewRevision } from './preview-revision.ts';
+import { seedPreviewDrafts } from './preview-seed.ts';
+import {
+	LIVE_ADAPTER_REFUSED,
+	readSessionAdapters,
+	recordedAdapterSettings,
+} from './recorded-adapters.ts';
 
 export const PREVIEW_COOKIE = 'coreloom_preview';
 const PREVIEW_TENANT = 'Preview workspace';
@@ -99,8 +106,12 @@ interface DraftModule {
 
 /* Preview settings live in memory for the life of the composition: a draft
    reads and writes them like the platform would, and nothing outlives the
-   session. */
-function memorySettings(): ModuleSettingsRuntime {
+   session. Pinned values hold for every workspace and cannot be changed. */
+export function memorySettings(
+	pinned: Readonly<
+		Record<string, Readonly<Record<string, ModuleSettingValue>>>
+	> = {},
+): ModuleSettingsRuntime {
 	const records = new Map<string, ModuleSettingRecord>();
 	const keyOf = (tenantId: string, moduleId: string, key: string) =>
 		`${tenantId}\0${moduleId}\0${key}`;
@@ -112,9 +123,17 @@ function memorySettings(): ModuleSettingsRuntime {
 					values[record.key] = record.value;
 				}
 			}
-			return values;
+			return { ...values, ...pinned[moduleId] };
 		},
 		save: async (record) => {
+			const fixed = pinned[record.moduleId]?.[record.key];
+			if (fixed !== undefined && fixed !== record.value) {
+				throw new ModuleSettingsError(
+					LIVE_ADAPTER_REFUSED,
+					`The sandbox preview holds ${record.moduleId}.${record.key} at its recorded value.`,
+					409,
+				);
+			}
 			records.set(keyOf(record.tenantId, record.moduleId, record.key), record);
 		},
 		clear: async (tenantId, moduleId, key) => {
@@ -346,6 +365,9 @@ export function createInProcessPreviewRuntime(
 			const revision = await previewRevision(sources);
 			const current = compositions.get(session.id);
 			if (current && current.revision === revision) return current;
+			const adapters = await readSessionAdapters(
+				sources.filter((source) => !source.support),
+			);
 
 			const { auth, credentials } = current
 				? { auth: current.auth, credentials: current.credentials }
@@ -370,7 +392,7 @@ export function createInProcessPreviewRuntime(
 			const context: Omit<PlatformServerContext, 'workspaceRoot'> = {
 				environment: process.env,
 				auth,
-				settings: memorySettings(),
+				settings: memorySettings(recordedAdapterSettings(adapters)),
 				agentTools: createPlatformToolRegistry() as PlatformToolRegistry,
 				agentDefinitions,
 				capabilities: createPlatformCapabilityRegistry(),
@@ -442,6 +464,21 @@ export function createInProcessPreviewRuntime(
 				if (!current) auth.dispose();
 				throw error;
 			}
+			const tenant = account?.tenants[0];
+			if (account && tenant) {
+				errors.push(
+					...(await seedPreviewDrafts({
+						modules: sources,
+						revision,
+						dataPath: paths.data,
+						context: {
+							tenantId: tenant.tenantId,
+							accountId: account.accountId,
+							databases,
+						},
+					})),
+				);
+			}
 			const all = [...routes, ...createAuthRoutes(auth)];
 			const composition: PreviewComposition = {
 				sessionId: session.id,

package/src/server/preview-seed.ts

@@ -0,0 +1,115 @@
+import { createHash } from 'node:crypto';
+import { lstat, mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import type { DatabaseProvider } from '@flowdular/sdk/database';
+import type { PreviewModuleSource } from './preview-modules.ts';
+
+export const PREVIEW_SEED_FILE = 'preview/seed.json';
+export const PREVIEW_SEED_ENTRY = 'src/preview.ts';
+export const MAX_PREVIEW_SEED_BYTES = 1024 * 1024;
+const SEED_MARKER = 'preview-seeds.json';
+
+/* What `seed()` in a draft's src/preview.ts receives: the preview workspace and
+   account, the parsed preview/seed.json, and the preview's own database
+   provider, the one the module's composition already writes through. */
+export interface PreviewSeedContext {
+	readonly tenantId: string;
+	readonly accountId: string;
+	readonly data: unknown;
+	readonly databases: DatabaseProvider;
+}
+
+interface PreviewEntry {
+	readonly seed?: unknown;
+}
+
+async function readRegular(
+	modulePath: string,
+	file: string,
+): Promise<Buffer | null> {
+	const path = join(modulePath, file);
+	const info = await lstat(path).catch((error: NodeJS.ErrnoException) => {
+		if (error.code === 'ENOENT') return null;
+		throw error;
+	});
+	if (!info) return null;
+	if (!info.isFile() || info.size > MAX_PREVIEW_SEED_BYTES) {
+		throw new Error(
+			`${file} must be a regular file of at most ${MAX_PREVIEW_SEED_BYTES} bytes.`,
+		);
+	}
+	return readFile(path);
+}
+
+async function readMarker(path: string): Promise<Record<string, string>> {
+	try {
+		const value = JSON.parse(await readFile(path, 'utf8')) as unknown;
+		return value && typeof value === 'object' && !Array.isArray(value)
+			? (value as Record<string, string>)
+			: {};
+	} catch {
+		return {};
+	}
+}
+
+/* Seeds the ephemeral database once per content of preview/seed.json and
+   src/preview.ts, after the generation started. A changed file seeds again, so
+   seed() must be idempotent. Failures are reported per module and never stop
+   the preview. */
+export async function seedPreviewDrafts(input: {
+	readonly modules: readonly PreviewModuleSource[];
+	readonly revision: string;
+	readonly dataPath: string;
+	readonly context: Omit<PreviewSeedContext, 'data'>;
+}): Promise<readonly string[]> {
+	const markerPath = join(input.dataPath, SEED_MARKER);
+	const marker = await readMarker(markerPath);
+	const errors: string[] = [];
+	for (const module of input.modules) {
+		if (module.support) continue;
+		try {
+			const seedFile = await readRegular(module.path, PREVIEW_SEED_FILE);
+			const entryFile = await readRegular(module.path, PREVIEW_SEED_ENTRY);
+			if (!seedFile || !entryFile) continue;
+			const digest = createHash('sha256')
+				.update(seedFile)
+				.update('\0')
+				.update(entryFile)
+				.digest('hex');
+			if (marker[module.id] === digest) continue;
+			let data: unknown;
+			try {
+				data = JSON.parse(seedFile.toString('utf8'));
+			} catch (error) {
+				throw new Error(
+					`${PREVIEW_SEED_FILE} is not valid JSON: ${(error as Error).message}`,
+				);
+			}
+			const entry = (await import(
+				`${pathToFileURL(join(module.path, PREVIEW_SEED_ENTRY)).href}?revision=${input.revision}`
+			)) as PreviewEntry;
+			if (typeof entry.seed !== 'function') {
+				throw new Error(
+					`${PREVIEW_SEED_ENTRY} does not export a seed function, so ${PREVIEW_SEED_FILE} was not loaded.`,
+				);
+			}
+			await entry.seed({ ...input.context, data });
+			marker[module.id] = digest;
+			await mkdir(input.dataPath, { recursive: true, mode: 0o700 });
+			await writeFile(markerPath, JSON.stringify(marker), {
+				encoding: 'utf8',
+				mode: 0o600,
+			});
+		} catch (error) {
+			errors.push(
+				`${module.id}: the preview seed failed: ${
+					error instanceof Error
+						? error.message.slice(0, 400)
+						: 'seed() threw a non-error value.'
+				}`,
+			);
+		}
+	}
+	return errors;
+}

package/src/server/recorded-adapters.ts

@@ -0,0 +1,132 @@
+import { join } from 'node:path';
+import type { ModuleSettingValue } from '@flowdular/sdk/kernel';
+import { parse as parseYaml } from 'yaml';
+import { readSpecText } from './spec.ts';
+import { SandboxSetupError } from './workspace-root.ts';
+
+export const RESEARCH_MODULE_ID = 'research.core';
+export const RESEARCH_FIXTURES_FILE = 'research-fixtures.json';
+export const LIVE_ADAPTER_REFUSED = 'SANDBOX_LIVE_ADAPTER_REFUSED';
+const LIVE_RESEARCH_ADAPTERS = new Set([
+	'model-native',
+	'searxng',
+	'firecrawl',
+	'connector',
+]);
+
+export interface DraftModuleLocation {
+	readonly directory: string;
+	readonly path: string;
+}
+
+export interface LiveAdapterDeclaration {
+	readonly module: string;
+	readonly field: string;
+	readonly value: string;
+}
+
+export interface SessionAdapters {
+	/* The first draft whose spec has a research section; the preview answers
+	   research from that module's recorded fixtures. */
+	readonly research: {
+		readonly module: string;
+		readonly fixturesPath: string;
+	} | null;
+	readonly live: readonly LiveAdapterDeclaration[];
+}
+
+function objectOf(value: unknown): Record<string, unknown> | null {
+	return value && typeof value === 'object' && !Array.isArray(value)
+		? (value as Record<string, unknown>)
+		: null;
+}
+
+/* Reads the research and adapters sections as plain YAML. The spec-schema gate
+   stays the authority on their shape, so a spec this cannot read declares
+   nothing here. */
+export async function readSessionAdapters(
+	modules: readonly DraftModuleLocation[],
+): Promise<SessionAdapters> {
+	let research: SessionAdapters['research'] = null;
+	const live: LiveAdapterDeclaration[] = [];
+	for (const module of modules) {
+		const text = await readSpecText(module.path);
+		if (text === null) continue;
+		let spec: Record<string, unknown> | null;
+		try {
+			spec = objectOf(parseYaml(text));
+		} catch {
+			continue;
+		}
+		const section = objectOf(spec?.research);
+		if (section) {
+			research ??= {
+				module: module.directory,
+				fixturesPath: join(module.path, RESEARCH_FIXTURES_FILE),
+			};
+			if (
+				typeof section.adapter === 'string' &&
+				LIVE_RESEARCH_ADAPTERS.has(section.adapter)
+			) {
+				live.push({
+					module: module.directory,
+					field: 'research.adapter',
+					value: section.adapter,
+				});
+			}
+		}
+		for (const entry of Array.isArray(spec?.adapters) ? spec.adapters : []) {
+			const adapter = objectOf(entry);
+			if (
+				adapter &&
+				(typeof adapter.recorded !== 'string' || !adapter.recorded.trim())
+			) {
+				live.push({
+					module: module.directory,
+					field: `adapters[${String(adapter.id ?? '?').slice(0, 96)}].recorded`,
+					value: 'missing, so the adapter could only call its connector',
+				});
+			}
+		}
+	}
+	return { research, live };
+}
+
+export function liveAdapterRefusal(
+	live: readonly LiveAdapterDeclaration[],
+): string | null {
+	if (live.length === 0) return null;
+	return [
+		`${LIVE_ADAPTER_REFUSED}: a sandbox session may declare only recorded adapters. An owner connects a live search or connector instance after delivery.`,
+		...live.map(
+			(entry) =>
+				`- modules/${entry.module}/spec/module.yaml ${entry.field}: ${entry.value}`,
+		),
+		`Use research.adapter recorded with ${RESEARCH_FIXTURES_FILE} in the module, and a recorded fixture adapters/<id>.recorded.json for every adapter.`,
+	].join('\n');
+}
+
+export function assertRecordedAdapters(adapters: SessionAdapters): void {
+	const refusal = liveAdapterRefusal(adapters.live);
+	if (refusal) throw new SandboxSetupError(LIVE_ADAPTER_REFUSED, refusal);
+}
+
+/* Setting values the preview holds fixed for every workspace, keyed by module.
+   The adapter chain is held at the recorded adapter alone and switched on, and
+   a chain holding recorded reads pages from the same fixtures before it looks
+   at the fetch order, which stays at the default so Firecrawl is never added. */
+export function recordedAdapterSettings(
+	adapters: SessionAdapters,
+): Readonly<Record<string, Readonly<Record<string, ModuleSettingValue>>>> {
+	return adapters.research
+		? {
+				[RESEARCH_MODULE_ID]: {
+					adapter: 'recorded',
+					searchOrder: 'recorded',
+					recordedEnabled: true,
+					fetchOrder: 'direct',
+					recordedFixturesPath: adapters.research.fixturesPath,
+				},
+			}
+		: {};
+}