@flowdular/sdk 0.1.0

package/modules/agents/src/services/worker.ts

@@ -0,0 +1,442 @@
+import type { AgentHarness, AgentProvider } from '@flowdular/sdk/harness';
+import type { AgentRunExecution, AgentWorkerStatus } from '../domain/types.ts';
+import type { AgentRepository, RecoverableRun } from './repository.ts';
+import type { AgentProviderBroker } from './provider-broker.ts';
+import type { AgentRunGrantAuthority } from './run-grant.ts';
+
+export interface AgentWorkerOptions {
+	readonly workerId: string;
+	/* A function is read at every drain so an admin change applies without a
+	   restart. Out-of-range values fall back to the last valid one. */
+	readonly concurrency: number | (() => number);
+	readonly leaseMs: number;
+	readonly runGrantAuthority?: AgentRunGrantAuthority;
+	readonly providerBroker?: AgentProviderBroker;
+	readonly now?: () => number;
+}
+
+export interface AgentProviderResolver {
+	resolve(
+		tenantId: string,
+		providerId: string,
+		modelId: string,
+	): Promise<AgentProvider | null>;
+	/* A run that answered is fresher evidence than any probe. */
+	recordRunSuccess?(
+		tenantId: string,
+		providerId: string,
+		modelId: string,
+		completedAt: number,
+		durationMs: number,
+	): void | Promise<void>;
+}
+
+const CANCELLED = 'cancelled';
+const SHUTDOWN = 'worker-shutdown';
+const LEASE_LOST = 'lease-lost';
+
+function failure(error: unknown): { code: string; message: string } {
+	if (error instanceof Error) {
+		const code =
+			'code' in error && typeof error.code === 'string'
+				? error.code
+				: 'AGENT_EXECUTION_FAILED';
+		return { code, message: error.message.slice(0, 1_000) };
+	}
+	return {
+		code: 'AGENT_EXECUTION_FAILED',
+		message: 'The agent execution failed.',
+	};
+}
+
+function validConcurrency(value: number): boolean {
+	return Number.isSafeInteger(value) && value >= 1 && value <= 16;
+}
+
+export class AgentWorker {
+	readonly #inFlight = new Map<string, AbortController>();
+	readonly #idleWaiters = new Set<() => void>();
+	readonly #now: () => number;
+	#concurrency: number;
+	#scheduled = false;
+	#draining: Promise<void> | undefined;
+	#stopped = true;
+	#lastDrainAt: number | null = null;
+	#poll: ReturnType<typeof setInterval> | undefined;
+	#kickTimer: ReturnType<typeof setTimeout> | undefined;
+
+	constructor(
+		private readonly repository: AgentRepository,
+		private readonly harness: AgentHarness,
+		private readonly options: AgentWorkerOptions,
+		private readonly providerResolver?: AgentProviderResolver,
+	) {
+		const concurrency =
+			typeof options.concurrency === 'function'
+				? options.concurrency()
+				: options.concurrency;
+		if (!validConcurrency(concurrency)) {
+			throw new Error('Agent worker concurrency must be between 1 and 16.');
+		}
+		this.#concurrency = concurrency;
+		if (
+			Boolean(options.runGrantAuthority) !== Boolean(options.providerBroker)
+		) {
+			throw new Error(
+				'Agent worker run grant authority and provider broker must be configured together.',
+			);
+		}
+		if (
+			!Number.isSafeInteger(options.leaseMs) ||
+			options.leaseMs < 1_000 ||
+			options.leaseMs > 300_000
+		) {
+			throw new Error('Agent worker lease must be between 1000 and 300000 ms.');
+		}
+		this.#now = options.now ?? Date.now;
+	}
+
+	async start(): Promise<void> {
+		this.#stopped = false;
+		await this.kick();
+		if (this.#stopped || this.#poll !== undefined) return;
+		/* Interrupted runs become claimable only once their lease expires, so a
+		   periodic drain is what makes recovery happen without a new request. */
+		this.#poll = setInterval(
+			() => void this.kick(),
+			Math.max(1_000, Math.floor(this.options.leaseMs / 2)),
+		);
+		this.#poll.unref?.();
+	}
+
+	stop(): void {
+		this.#stopped = true;
+		if (this.#poll !== undefined) clearInterval(this.#poll);
+		this.#poll = undefined;
+		if (this.#kickTimer !== undefined) clearTimeout(this.#kickTimer);
+		this.#kickTimer = undefined;
+		this.#scheduled = false;
+	}
+
+	/* Terminal worker teardown. In-flight provider calls are aborted but their
+	   persisted rows stay leased for recovery by the next worker generation. */
+	async dispose(): Promise<void> {
+		this.stop();
+		await this.#draining;
+		for (const controller of this.#inFlight.values()) {
+			controller.abort(SHUTDOWN);
+		}
+		if (this.#inFlight.size === 0) return;
+		await new Promise<void>((resolve) => this.#idleWaiters.add(resolve));
+	}
+
+	/* Online means the periodic drain runs, so queued and interrupted work
+	   is picked up without a request. */
+	status(): AgentWorkerStatus {
+		return {
+			workerId: this.options.workerId,
+			online: this.#poll !== undefined && !this.#stopped,
+			concurrency: this.#currentConcurrency(),
+			inFlight: this.#inFlight.size,
+			leaseMs: this.options.leaseMs,
+			lastDrainAt: this.#lastDrainAt,
+		};
+	}
+
+	/* Aborts the in-flight execution of a run this process owns. The caller
+	   has already moved the row to cancelled; the worker only stops working. */
+	cancel(runId: string): boolean {
+		const controller = this.#inFlight.get(runId);
+		if (!controller) return false;
+		controller.abort(CANCELLED);
+		return true;
+	}
+
+	async kick(): Promise<void> {
+		if (this.#scheduled || this.#draining || this.#stopped) return;
+		this.#scheduled = true;
+		this.#kickTimer = setTimeout(async () => {
+			this.#kickTimer = undefined;
+			this.#scheduled = false;
+			try {
+				this.#draining = this.#drain();
+				await this.#draining;
+			} catch (error) {
+				/* A failed transactional claim leaves the run recoverable. The
+				   periodic poll retries it without crashing the host process. */
+				console.error(
+					'[agents] run worker drain failed:',
+					error instanceof Error ? error.message : error,
+				);
+			} finally {
+				this.#draining = undefined;
+			}
+		}, 0);
+	}
+
+	#currentConcurrency(): number {
+		if (typeof this.options.concurrency === 'function') {
+			const value = this.options.concurrency();
+			if (validConcurrency(value)) this.#concurrency = value;
+		}
+		return this.#concurrency;
+	}
+
+	async #drain(): Promise<void> {
+		if (this.#stopped) return;
+		this.#lastDrainAt = this.#now();
+		const slots = this.#currentConcurrency() - this.#inFlight.size;
+		if (slots <= 0) return;
+		const candidates = await this.repository.listRecoverableRuns(
+			this.#now(),
+			slots * 2,
+		);
+		let claimed = 0;
+		for (const candidate of candidates) {
+			if (this.#stopped) return;
+			if (claimed >= slots || this.#inFlight.has(candidate.runId)) continue;
+			const now = this.#now();
+			const execution = await this.repository.claimRun(
+				candidate.tenantId,
+				candidate.runId,
+				this.options.workerId,
+				now,
+				now + this.options.leaseMs,
+				{
+					tenantId: candidate.tenantId,
+					actorId: this.options.workerId,
+					action: 'agent-run.claimed',
+					subjectType: 'agent-run',
+					subjectId: candidate.runId,
+					metadata: {},
+					occurredAt: now,
+				},
+			);
+			if (this.#stopped) return;
+			if (!execution) continue;
+			claimed += 1;
+			const controller = new AbortController();
+			this.#inFlight.set(candidate.runId, controller);
+			void this.#execute(candidate, execution, controller)
+				.catch((error: unknown) => {
+					console.error(
+						`[agents] worker failed to settle run ${candidate.runId}:`,
+						error instanceof Error ? error.message : error,
+					);
+				})
+				.finally(() => {
+					this.#inFlight.delete(candidate.runId);
+					if (this.#inFlight.size === 0) {
+						for (const resolve of this.#idleWaiters) resolve();
+						this.#idleWaiters.clear();
+					}
+					void this.kick();
+				});
+		}
+	}
+
+	async #execute(
+		candidate: RecoverableRun,
+		execution: AgentRunExecution,
+		controller: AbortController,
+	): Promise<void> {
+		let eventWrites = Promise.resolve();
+		let eventFailure: unknown;
+		const renewal = setInterval(
+			async () => {
+				try {
+					const renewed = await this.repository.renewLease(
+						candidate.tenantId,
+						candidate.runId,
+						this.options.workerId,
+						this.#now() + this.options.leaseMs,
+					);
+					if (!renewed) controller.abort(LEASE_LOST);
+				} catch {
+					/* A database error means ownership cannot be proven. Stop local work
+					   and leave the durable row for lease-based recovery. */
+					controller.abort(LEASE_LOST);
+				}
+			},
+			Math.max(500, Math.floor(this.options.leaseMs / 2)),
+		);
+		try {
+			let provider: AgentProvider | null | undefined;
+			if (this.options.runGrantAuthority && this.options.providerBroker) {
+				if (execution.run.leaseExpiresAt === null) {
+					throw new Error('Claimed agent run has no worker lease.');
+				}
+				const grant = this.options.runGrantAuthority.issue({
+					tenantId: candidate.tenantId,
+					runId: candidate.runId,
+					providerId: execution.run.provider,
+					modelId: execution.run.model,
+					workerId: this.options.workerId,
+					attempt: execution.run.attempt,
+					permissions: execution.run.permissionSnapshot,
+					toolGrants: execution.run.toolGrants,
+					leaseExpiresAt: execution.run.leaseExpiresAt,
+				});
+				await this.repository.appendAuditEvent({
+					tenantId: candidate.tenantId,
+					actorId: this.options.workerId,
+					action: 'agent-run.grant-issued',
+					subjectType: 'agent-run',
+					subjectId: candidate.runId,
+					metadata: {
+						grantId: grant.claims.grantId,
+						providerId: grant.claims.providerId,
+						modelId: grant.claims.modelId,
+						expiresAt: grant.claims.expiresAt,
+						attempt: grant.claims.attempt,
+					},
+					occurredAt: grant.claims.issuedAt,
+				});
+				provider = (await this.options.providerBroker.exchange(grant.token))
+					.provider;
+			} else {
+				provider = await this.providerResolver?.resolve(
+					candidate.tenantId,
+					execution.run.provider,
+					execution.run.model,
+				);
+			}
+			const onEvent = (
+				event: Parameters<AgentRepository['appendRunEvent']>[2],
+			): void => {
+				// The harness callback is synchronous. Own and drain its async writes.
+				eventWrites = eventWrites.then(() =>
+					this.repository.appendRunEvent(
+						candidate.tenantId,
+						candidate.runId,
+						event,
+					),
+				);
+				void eventWrites.catch((error) => {
+					eventFailure = error;
+					controller.abort('event-persistence-failed');
+				});
+			};
+			const result = await this.harness.execute(
+				{
+					runId: execution.run.id,
+					tenantId: execution.run.tenantId,
+					requestedBy: execution.run.requestedBy,
+					requestedActor: execution.run.requestedActor,
+					...(execution.run.authorizationSubject
+						? { authorizationSubject: execution.run.authorizationSubject }
+						: {}),
+					trigger: execution.run.trigger,
+					input: execution.run.input,
+					definition: execution.definition,
+					permissionSnapshot: execution.run.permissionSnapshot,
+					toolGrants: execution.run.toolGrants,
+					outputContract: execution.run.outputContract,
+				},
+				provider
+					? { onEvent, provider, signal: controller.signal }
+					: { onEvent, signal: controller.signal },
+			);
+			await eventWrites;
+			await this.repository.completeRun(
+				candidate.tenantId,
+				candidate.runId,
+				this.options.workerId,
+				result,
+				{
+					tenantId: candidate.tenantId,
+					actorId: this.options.workerId,
+					action: 'agent-run.succeeded',
+					subjectType: 'agent-run',
+					subjectId: candidate.runId,
+					metadata: {
+						durationMs: result.completedAt - result.startedAt,
+						totalTokens: result.usage.totalTokens,
+					},
+					occurredAt: result.completedAt,
+				},
+			);
+			await this.#recordRunSuccess(
+				execution,
+				result.completedAt,
+				result.startedAt,
+			);
+		} catch (error) {
+			await eventWrites.catch(() => undefined);
+			/* The service already moved a cancelled row and wrote its audit event;
+			   the worker only had to stop. */
+			if (controller.signal.aborted && controller.signal.reason === CANCELLED) {
+				return;
+			}
+			if (controller.signal.aborted && controller.signal.reason === SHUTDOWN) {
+				return;
+			}
+			if (
+				controller.signal.aborted &&
+				controller.signal.reason === LEASE_LOST
+			) {
+				return;
+			}
+			const failed =
+				eventFailure !== undefined
+					? failure(eventFailure)
+					: controller.signal.aborted
+						? {
+								code: 'AGENT_LEASE_LOST',
+								message: 'The worker lost its lease on the run.',
+							}
+						: failure(error);
+			const completedAt = this.#now();
+			/* A lost lease means another worker owns the row now; the failure
+			   record belongs to it, so a rejected write here is not an error. */
+			try {
+				await this.repository.failRun(
+					candidate.tenantId,
+					candidate.runId,
+					this.options.workerId,
+					failed.code,
+					failed.message,
+					completedAt,
+					{
+						tenantId: candidate.tenantId,
+						actorId: this.options.workerId,
+						action: 'agent-run.failed',
+						subjectType: 'agent-run',
+						subjectId: candidate.runId,
+						metadata: { code: failed.code },
+						occurredAt: completedAt,
+					},
+				);
+			} catch (settleError) {
+				console.error(
+					`[agents] run ${candidate.runId} failed with ${failed.code} and could not be settled:`,
+					settleError instanceof Error ? settleError.message : settleError,
+				);
+			}
+		} finally {
+			clearInterval(renewal);
+			await eventWrites.catch(() => undefined);
+		}
+	}
+
+	async #recordRunSuccess(
+		execution: AgentRunExecution,
+		completedAt: number,
+		startedAt: number,
+	): Promise<void> {
+		try {
+			await this.providerResolver?.recordRunSuccess?.(
+				execution.run.tenantId,
+				execution.run.provider,
+				execution.run.model,
+				completedAt,
+				completedAt - startedAt,
+			);
+		} catch (error) {
+			console.error(
+				`[agents] readiness refresh after run ${execution.run.id} failed:`,
+				error instanceof Error ? error.message : error,
+			);
+		}
+	}
+}

package/modules/agents/src/settings.ts

@@ -0,0 +1,244 @@
+import {
+	defineModuleSettings,
+	PLATFORM_SETTINGS_TENANT,
+	type ModuleSettingsDeclaration,
+	type ModuleSettingsRuntime,
+	type ModuleSettingValue,
+} from '@flowdular/sdk/kernel';
+import { providerHostAllowlist } from './services/outbound-policy.ts';
+import { agentRuntimeOptionsFromEnvironment } from './server/runtime.ts';
+
+export const AGENTS_MODULE_ID = 'agents.core';
+
+export const AGENTS_MODULE_SETTINGS = defineModuleSettings({
+	moduleId: AGENTS_MODULE_ID,
+	settings: {
+		workerConcurrency: {
+			type: 'number',
+			defaultValue: 2,
+			min: 1,
+			max: 16,
+			visibility: 'private',
+			client: false,
+			scope: 'platform',
+			labelKey: 'agents.settings.workerConcurrency.label',
+			label: 'Worker concurrency',
+			descriptionKey: 'agents.settings.workerConcurrency.description',
+			description:
+				'Runs one process executes at the same time. Applied at the next queue drain.',
+		},
+		workerLeaseMs: {
+			type: 'number',
+			defaultValue: 30_000,
+			min: 1_000,
+			max: 300_000,
+			visibility: 'private',
+			client: false,
+			scope: 'platform',
+			labelKey: 'agents.settings.workerLeaseMs.label',
+			label: 'Worker lease (ms)',
+			descriptionKey: 'agents.settings.workerLeaseMs.description',
+			description:
+				'How long a claimed run stays owned by a worker before another worker may recover it. Applied to new claims.',
+		},
+		providerReadinessTtlMs: {
+			type: 'number',
+			defaultValue: 86_400_000,
+			min: 10_000,
+			max: 86_400_000,
+			visibility: 'private',
+			client: false,
+			scope: 'platform',
+			labelKey: 'agents.settings.providerReadinessTtlMs.label',
+			label: 'Model readiness validity (ms)',
+			descriptionKey: 'agents.settings.providerReadinessTtlMs.description',
+			description:
+				'How long a successful model test or run counts as proof that the model answers.',
+		},
+		providerHostAllowlist: {
+			type: 'string',
+			defaultValue: '',
+			max: 4_096,
+			visibility: 'private',
+			client: false,
+			scope: 'platform',
+			labelKey: 'agents.settings.providerHostAllowlist.label',
+			label: 'OpenAI-compatible host allowlist',
+			descriptionKey: 'agents.settings.providerHostAllowlist.description',
+			description:
+				'Comma-separated public HTTPS hostnames an OpenAI-compatible connection may target.',
+		},
+		defaultMaxOutputTokens: {
+			type: 'number',
+			defaultValue: 4_096,
+			min: 256,
+			max: 65_536,
+			visibility: 'private',
+			client: false,
+			scope: 'tenant',
+			labelKey: 'agents.settings.defaultMaxOutputTokens.label',
+			label: 'Default output budget (tokens)',
+			descriptionKey: 'agents.settings.defaultMaxOutputTokens.description',
+			description: 'Used when an agent definition does not set its own.',
+		},
+		defaultProvider: {
+			type: 'string',
+			defaultValue: '',
+			max: 128,
+			visibility: 'private',
+			client: false,
+			scope: 'tenant',
+			labelKey: 'agents.settings.defaultProvider.label',
+			label: 'Default provider connection',
+			descriptionKey: 'agents.settings.defaultProvider.description',
+			description:
+				'Provider connection id preselected for new agents. Empty means none.',
+		},
+		defaultModel: {
+			type: 'string',
+			defaultValue: '',
+			max: 160,
+			visibility: 'private',
+			client: false,
+			scope: 'tenant',
+			labelKey: 'agents.settings.defaultModel.label',
+			label: 'Default model',
+			descriptionKey: 'agents.settings.defaultModel.description',
+			description: 'Model id preselected for new agents. Empty means none.',
+		},
+		monthlyCostCapUsd: {
+			type: 'number',
+			defaultValue: 0,
+			min: 0,
+			max: 1_000_000,
+			visibility: 'private',
+			client: false,
+			scope: 'tenant',
+			labelKey: 'agents.settings.monthlyCostCapUsd.label',
+			label: 'Monthly agent budget (USD)',
+			descriptionKey: 'agents.settings.monthlyCostCapUsd.description',
+			description:
+				'Enqueue is refused once the calendar month costs this much. Zero means no cap. A run already executing is never stopped.',
+		},
+		agentMonthlyCostCapUsd: {
+			type: 'number',
+			defaultValue: 0,
+			min: 0,
+			max: 1_000_000,
+			visibility: 'private',
+			client: false,
+			scope: 'tenant',
+			labelKey: 'agents.settings.agentMonthlyCostCapUsd.label',
+			label: 'Monthly budget per agent (USD)',
+			descriptionKey: 'agents.settings.agentMonthlyCostCapUsd.description',
+			description:
+				'Applied to every agent on its own, in addition to the workspace budget. Zero means no cap.',
+		},
+	},
+});
+
+/* The deployment's environment is the declared default an admin sees and
+   overrides, so the same declaration is rebuilt with those values at boot. */
+export function agentsModuleSettingsFromEnvironment(
+	environment: NodeJS.ProcessEnv,
+): ModuleSettingsDeclaration {
+	const options = agentRuntimeOptionsFromEnvironment(environment);
+	const defaults: Record<string, ModuleSettingValue> = {
+		workerConcurrency: options.workerConcurrency,
+		workerLeaseMs: options.workerLeaseMs,
+		providerReadinessTtlMs: options.providerReadinessTtlMs,
+		providerHostAllowlist:
+			environment.FD_AGENT_PROVIDER_HOST_ALLOWLIST?.trim() ?? '',
+	};
+	return defineModuleSettings({
+		moduleId: AGENTS_MODULE_ID,
+		settings: Object.fromEntries(
+			Object.entries(AGENTS_MODULE_SETTINGS.settings).map(
+				([key, definition]) => [
+					key,
+					key in defaults
+						? { ...definition, defaultValue: defaults[key]! }
+						: definition,
+				],
+			),
+		),
+	});
+}
+
+export interface AgentSettingsReader {
+	workerConcurrency(): number;
+	workerLeaseMs(): number;
+	providerReadinessTtlMs(): number;
+	providerHostAllowlist(): ReadonlySet<string>;
+	defaultMaxOutputTokens(tenantId: string): number;
+	defaultProvider(tenantId: string): string;
+	defaultModel(tenantId: string): string;
+	monthlyCostCapUsd(tenantId: string): number;
+	agentMonthlyCostCapUsd(tenantId: string): number;
+}
+
+function settingsSource(value: unknown): ModuleSettingsRuntime | null {
+	return value &&
+		typeof value === 'object' &&
+		typeof (value as ModuleSettingsRuntime).get === 'function'
+		? (value as ModuleSettingsRuntime)
+		: null;
+}
+
+/* Reads are live and per call. Without a settings runtime, or when a read
+   fails, the environment decides exactly as before. */
+export function agentSettings(context: {
+	readonly environment: NodeJS.ProcessEnv;
+	readonly settings?: unknown;
+}): AgentSettingsReader {
+	const fallback = agentRuntimeOptionsFromEnvironment(context.environment);
+	const source = settingsSource(context.settings);
+	const read = <T extends ModuleSettingValue>(
+		tenantId: string,
+		key: keyof typeof AGENTS_MODULE_SETTINGS.settings,
+		fallbackValue: T,
+	): T => {
+		if (!source) return fallbackValue;
+		try {
+			const value: unknown = source.get(tenantId, AGENTS_MODULE_ID, key);
+			return typeof value === typeof fallbackValue
+				? (value as T)
+				: fallbackValue;
+		} catch {
+			return fallbackValue;
+		}
+	};
+	return {
+		workerConcurrency: () =>
+			read(
+				PLATFORM_SETTINGS_TENANT,
+				'workerConcurrency',
+				fallback.workerConcurrency,
+			),
+		workerLeaseMs: () =>
+			read(PLATFORM_SETTINGS_TENANT, 'workerLeaseMs', fallback.workerLeaseMs),
+		providerReadinessTtlMs: () =>
+			read(
+				PLATFORM_SETTINGS_TENANT,
+				'providerReadinessTtlMs',
+				fallback.providerReadinessTtlMs,
+			),
+		providerHostAllowlist: () =>
+			source
+				? providerHostAllowlist(
+						read(
+							PLATFORM_SETTINGS_TENANT,
+							'providerHostAllowlist',
+							context.environment.FD_AGENT_PROVIDER_HOST_ALLOWLIST ?? '',
+						),
+					)
+				: fallback.providerHostAllowlist,
+		defaultMaxOutputTokens: (tenantId) =>
+			read(tenantId, 'defaultMaxOutputTokens', 4_096),
+		defaultProvider: (tenantId) => read(tenantId, 'defaultProvider', ''),
+		defaultModel: (tenantId) => read(tenantId, 'defaultModel', ''),
+		monthlyCostCapUsd: (tenantId) => read(tenantId, 'monthlyCostCapUsd', 0),
+		agentMonthlyCostCapUsd: (tenantId) =>
+			read(tenantId, 'agentMonthlyCostCapUsd', 0),
+	};
+}