btca-server 1.0.20

package/src/resources/impls/git.ts

@@ -0,0 +1,156 @@
+import { promises as fs } from 'node:fs';
+
+import { Metrics } from '../../metrics/index.ts';
+import { ResourceError } from '../helpers.ts';
+import type { BtcaFsResource, BtcaGitResourceArgs } from '../types.ts';
+
+const isValidGitUrl = (url: string) => /^https?:\/\//.test(url) || /^git@/.test(url);
+const isValidBranch = (branch: string) => /^[\w\-./]+$/.test(branch);
+const isValidPath = (path: string) => !path.includes('..') && /^[\w\-./]*$/.test(path);
+
+const directoryExists = async (path: string): Promise<boolean> => {
+	try {
+		const stat = await fs.stat(path);
+		return stat.isDirectory();
+	} catch {
+		return false;
+	}
+};
+
+const runGit = async (args: string[], options: { cwd?: string; quiet: boolean }) => {
+	const stdio = options.quiet ? 'ignore' : 'inherit';
+	const proc = Bun.spawn(['git', ...args], {
+		cwd: options.cwd,
+		stdout: stdio,
+		stderr: stdio
+	});
+	const exitCode = await proc.exited;
+	if (exitCode !== 0) {
+		throw new ResourceError({
+			message: `git ${args[0]} failed`,
+			cause: new Error(String(exitCode))
+		});
+	}
+};
+
+const gitClone = async (args: {
+	repoUrl: string;
+	repoBranch: string;
+	repoSubPath: string;
+	localAbsolutePath: string;
+	quiet: boolean;
+}) => {
+	if (!isValidGitUrl(args.repoUrl)) {
+		throw new ResourceError({
+			message: 'Invalid git URL',
+			cause: new Error('URL validation failed')
+		});
+	}
+	if (!isValidBranch(args.repoBranch)) {
+		throw new ResourceError({
+			message: 'Invalid branch name',
+			cause: new Error('Branch validation failed')
+		});
+	}
+	if (args.repoSubPath && !isValidPath(args.repoSubPath)) {
+		throw new ResourceError({
+			message: 'Invalid path',
+			cause: new Error('Path validation failed')
+		});
+	}
+
+	const needsSparseCheckout = args.repoSubPath && args.repoSubPath !== '/';
+	const cloneArgs = needsSparseCheckout
+		? [
+				'clone',
+				'--filter=blob:none',
+				'--no-checkout',
+				'--sparse',
+				'-b',
+				args.repoBranch,
+				args.repoUrl,
+				args.localAbsolutePath
+			]
+		: ['clone', '--depth', '1', '-b', args.repoBranch, args.repoUrl, args.localAbsolutePath];
+
+	await runGit(cloneArgs, { quiet: args.quiet });
+
+	if (needsSparseCheckout) {
+		await runGit(['sparse-checkout', 'set', args.repoSubPath], {
+			cwd: args.localAbsolutePath,
+			quiet: args.quiet
+		});
+		await runGit(['checkout'], { cwd: args.localAbsolutePath, quiet: args.quiet });
+	}
+};
+
+const gitUpdate = async (args: { localAbsolutePath: string; branch: string; quiet: boolean }) => {
+	await runGit(['fetch', '--depth', '1', 'origin', args.branch], {
+		cwd: args.localAbsolutePath,
+		quiet: args.quiet
+	});
+	await runGit(['reset', '--hard', `origin/${args.branch}`], {
+		cwd: args.localAbsolutePath,
+		quiet: args.quiet
+	});
+};
+
+const ensureGitResource = async (config: BtcaGitResourceArgs): Promise<string> => {
+	const localPath = `${config.resourcesDirectoryPath}/${config.name}`;
+
+	return Metrics.span(
+		'resource.git.ensure',
+		async () => {
+			const exists = await directoryExists(localPath);
+
+			if (exists) {
+				Metrics.info('resource.git.update', {
+					name: config.name,
+					branch: config.branch,
+					repoSubPath: config.repoSubPath
+				});
+				await gitUpdate({
+					localAbsolutePath: localPath,
+					branch: config.branch,
+					quiet: config.quiet
+				});
+				return localPath;
+			}
+
+			Metrics.info('resource.git.clone', {
+				name: config.name,
+				branch: config.branch,
+				repoSubPath: config.repoSubPath
+			});
+
+			try {
+				await fs.mkdir(config.resourcesDirectoryPath, { recursive: true });
+			} catch (cause) {
+				throw new ResourceError({ message: 'Failed to create resources directory', cause });
+			}
+
+			await gitClone({
+				repoUrl: config.url,
+				repoBranch: config.branch,
+				repoSubPath: config.repoSubPath,
+				localAbsolutePath: localPath,
+				quiet: config.quiet
+			});
+
+			return localPath;
+		},
+		{ resource: config.name }
+	);
+};
+
+export const loadGitResource = async (config: BtcaGitResourceArgs): Promise<BtcaFsResource> => {
+	const localPath = await ensureGitResource(config);
+	return {
+		_tag: 'fs-based',
+		name: config.name,
+		type: 'git',
+		repoSubPath: config.repoSubPath,
+		specialAgentInstructions: config.specialAgentInstructions,
+		getAbsoluteDirectoryPath: async () => localPath
+	};
+};

package/src/resources/index.ts

@@ -0,0 +1,10 @@
+export { ResourceError } from './helpers.ts';
+export { Resources } from './service.ts';
+export {
+	GitResourceSchema,
+	ResourceDefinitionSchema,
+	isGitResource,
+	type GitResource,
+	type ResourceDefinition
+} from './schema.ts';
+export { FS_RESOURCE_SYSTEM_NOTE, type BtcaFsResource, type BtcaGitResourceArgs } from './types.ts';

package/src/resources/schema.ts

@@ -0,0 +1,178 @@
+import { z } from 'zod';
+
+import { LIMITS } from '../validation/index.ts';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Validation Patterns
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Resource name: must start with a letter, followed by alphanumeric and hyphens only.
+ * Prevents path traversal, git option injection, and shell metacharacters.
+ */
+const RESOURCE_NAME_REGEX = /^[a-zA-Z][a-zA-Z0-9-]*$/;
+
+/**
+ * Branch name: alphanumeric, forward slashes, dots, underscores, and hyphens.
+ * Must not start with hyphen to prevent git option injection.
+ */
+const BRANCH_NAME_REGEX = /^[a-zA-Z0-9/_.-]+$/;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Field Schemas
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Resource name field with security validation.
+ */
+const ResourceNameSchema = z
+	.string()
+	.min(1, 'Resource name cannot be empty')
+	.max(LIMITS.RESOURCE_NAME_MAX, `Resource name too long (max ${LIMITS.RESOURCE_NAME_MAX} chars)`)
+	.regex(
+		RESOURCE_NAME_REGEX,
+		'Resource name must start with a letter and contain only alphanumeric characters and hyphens'
+	);
+
+/**
+ * Git URL field with security validation.
+ * Only allows HTTPS URLs, no credentials, no private IPs.
+ */
+const GitUrlSchema = z
+	.string()
+	.min(1, 'Git URL cannot be empty')
+	.refine(
+		(url) => {
+			try {
+				const parsed = new URL(url);
+				return parsed.protocol === 'https:';
+			} catch {
+				return false;
+			}
+		},
+		{ message: 'Git URL must be a valid HTTPS URL' }
+	)
+	.refine(
+		(url) => {
+			try {
+				const parsed = new URL(url);
+				return !parsed.username && !parsed.password;
+			} catch {
+				return true; // Let the URL check above handle invalid URLs
+			}
+		},
+		{ message: 'Git URL must not contain embedded credentials' }
+	)
+	.refine(
+		(url) => {
+			try {
+				const parsed = new URL(url);
+				const hostname = parsed.hostname.toLowerCase();
+				return !(
+					hostname === 'localhost' ||
+					hostname.startsWith('127.') ||
+					hostname.startsWith('192.168.') ||
+					hostname.startsWith('10.') ||
+					hostname.match(/^172\.(1[6-9]|2[0-9]|3[0-1])\./) ||
+					hostname === '::1' ||
+					hostname === '0.0.0.0'
+				);
+			} catch {
+				return true;
+			}
+		},
+		{ message: 'Git URL must not point to localhost or private IP addresses' }
+	);
+
+/**
+ * Branch name field with security validation.
+ */
+const BranchNameSchema = z
+	.string()
+	.min(1, 'Branch name cannot be empty')
+	.max(LIMITS.BRANCH_NAME_MAX, `Branch name too long (max ${LIMITS.BRANCH_NAME_MAX} chars)`)
+	.regex(
+		BRANCH_NAME_REGEX,
+		'Branch name must contain only alphanumeric characters, forward slashes, dots, underscores, and hyphens'
+	)
+	.refine((branch) => !branch.startsWith('-'), {
+		message: "Branch name must not start with '-' to prevent git option injection"
+	});
+
+/**
+ * Search path field with security validation.
+ */
+const SearchPathSchema = z
+	.string()
+	.max(LIMITS.SEARCH_PATH_MAX, `Search path too long (max ${LIMITS.SEARCH_PATH_MAX} chars)`)
+	.refine((path) => !path.includes('\n') && !path.includes('\r'), {
+		message: 'Search path must not contain newline characters'
+	})
+	.refine((path) => !path.includes('..'), {
+		message: 'Search path must not contain path traversal sequences (..)'
+	})
+	.refine((path) => !path.startsWith('/') && !path.match(/^[a-zA-Z]:\\/), {
+		message: 'Search path must not be an absolute path'
+	})
+	.optional();
+
+/**
+ * Local path field with basic validation.
+ */
+const LocalPathSchema = z
+	.string()
+	.min(1, 'Local path cannot be empty')
+	.refine((path) => !path.includes('\0'), {
+		message: 'Path must not contain null bytes'
+	})
+	.refine((path) => path.startsWith('/') || path.match(/^[a-zA-Z]:\\/), {
+		message: 'Local path must be an absolute path'
+	});
+
+/**
+ * Special notes field with length and content validation.
+ */
+const SpecialNotesSchema = z
+	.string()
+	.max(LIMITS.NOTES_MAX, `Notes too long (max ${LIMITS.NOTES_MAX} chars)`)
+	.refine(
+		// eslint-disable-next-line no-control-regex
+		(notes) => !/[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]/.test(notes),
+		{ message: 'Notes contain invalid control characters' }
+	)
+	.optional();
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Resource Schemas
+// ─────────────────────────────────────────────────────────────────────────────
+
+export const GitResourceSchema = z.object({
+	type: z.literal('git'),
+	name: ResourceNameSchema,
+	url: GitUrlSchema,
+	branch: BranchNameSchema,
+	searchPath: SearchPathSchema,
+	specialNotes: SpecialNotesSchema
+});
+
+export const LocalResourceSchema = z.object({
+	type: z.literal('local'),
+	name: ResourceNameSchema,
+	path: LocalPathSchema,
+	specialNotes: SpecialNotesSchema
+});
+
+export const ResourceDefinitionSchema = z.discriminatedUnion('type', [
+	GitResourceSchema,
+	LocalResourceSchema
+]);
+
+export type GitResource = z.infer<typeof GitResourceSchema>;
+export type LocalResource = z.infer<typeof LocalResourceSchema>;
+export type ResourceDefinition = z.infer<typeof ResourceDefinitionSchema>;
+
+export const isGitResource = (value: ResourceDefinition): value is GitResource =>
+	value.type === 'git';
+
+export const isLocalResource = (value: ResourceDefinition): value is LocalResource =>
+	value.type === 'local';

package/src/resources/service.ts

@@ -0,0 +1,75 @@
+import { Config } from '../config/index.ts';
+
+import { ResourceError } from './helpers.ts';
+import { loadGitResource } from './impls/git.ts';
+import {
+	isGitResource,
+	type ResourceDefinition,
+	type GitResource,
+	type LocalResource
+} from './schema.ts';
+import type { BtcaFsResource, BtcaGitResourceArgs, BtcaLocalResourceArgs } from './types.ts';
+
+export namespace Resources {
+	export type Service = {
+		load: (
+			name: string,
+			options?: {
+				quiet?: boolean;
+			}
+		) => Promise<BtcaFsResource>;
+	};
+
+	const definitionToGitArgs = (
+		definition: GitResource,
+		resourcesDirectory: string,
+		quiet: boolean
+	): BtcaGitResourceArgs => ({
+		type: 'git',
+		name: definition.name,
+		url: definition.url,
+		branch: definition.branch,
+		repoSubPath: definition.searchPath ?? '',
+		resourcesDirectoryPath: resourcesDirectory,
+		specialAgentInstructions: definition.specialNotes ?? '',
+		quiet
+	});
+
+	const definitionToLocalArgs = (definition: LocalResource): BtcaLocalResourceArgs => ({
+		type: 'local',
+		name: definition.name,
+		path: definition.path,
+		specialAgentInstructions: definition.specialNotes ?? ''
+	});
+
+	const loadLocalResource = (args: BtcaLocalResourceArgs): BtcaFsResource => ({
+		_tag: 'fs-based',
+		name: args.name,
+		type: 'local',
+		repoSubPath: '',
+		specialAgentInstructions: args.specialAgentInstructions,
+		getAbsoluteDirectoryPath: async () => args.path
+	});
+
+	export const create = (config: Config.Service): Service => {
+		const getDefinition = (name: string): ResourceDefinition => {
+			const definition = config.getResource(name);
+			if (!definition)
+				throw new ResourceError({ message: `Resource \"${name}\" not found in config` });
+			return definition;
+		};
+
+		return {
+			load: async (name, options) => {
+				const quiet = options?.quiet ?? false;
+				const definition = getDefinition(name);
+
+				if (isGitResource(definition)) {
+					return loadGitResource(definitionToGitArgs(definition, config.resourcesDirectory, quiet));
+				} else {
+					return loadLocalResource(definitionToLocalArgs(definition));
+				}
+			}
+		};
+	};
+}

package/src/resources/types.ts

@@ -0,0 +1,29 @@
+export const FS_RESOURCE_SYSTEM_NOTE =
+	'This is a btca resource - a searchable knowledge source the agent can reference.';
+
+export type BtcaFsResource = {
+	readonly _tag: 'fs-based';
+	readonly name: string;
+	readonly type: 'git' | 'local';
+	readonly repoSubPath: string;
+	readonly specialAgentInstructions: string;
+	readonly getAbsoluteDirectoryPath: () => Promise<string>;
+};
+
+export type BtcaGitResourceArgs = {
+	readonly type: 'git';
+	readonly name: string;
+	readonly url: string;
+	readonly branch: string;
+	readonly repoSubPath: string;
+	readonly resourcesDirectoryPath: string;
+	readonly specialAgentInstructions: string;
+	readonly quiet: boolean;
+};
+
+export type BtcaLocalResourceArgs = {
+	readonly type: 'local';
+	readonly name: string;
+	readonly path: string;
+	readonly specialAgentInstructions: string;
+};

package/src/stream/index.ts

@@ -0,0 +1,19 @@
+export { StreamService } from './service.ts';
+export {
+	BtcaStreamEventSchema,
+	BtcaStreamMetaEventSchema,
+	BtcaStreamTextDeltaEventSchema,
+	BtcaStreamReasoningDeltaEventSchema,
+	BtcaStreamToolUpdatedEventSchema,
+	BtcaStreamDoneEventSchema,
+	BtcaStreamErrorEventSchema
+} from './types.ts';
+export type {
+	BtcaStreamEvent,
+	BtcaStreamMetaEvent,
+	BtcaStreamTextDeltaEvent,
+	BtcaStreamReasoningDeltaEvent,
+	BtcaStreamToolUpdatedEvent,
+	BtcaStreamDoneEvent,
+	BtcaStreamErrorEvent
+} from './types.ts';

package/src/stream/service.ts

@@ -0,0 +1,161 @@
+import type { OcEvent } from '../agent/types.ts';
+import { getErrorMessage, getErrorTag } from '../errors.ts';
+import { Metrics } from '../metrics/index.ts';
+
+import type {
+	BtcaStreamDoneEvent,
+	BtcaStreamErrorEvent,
+	BtcaStreamEvent,
+	BtcaStreamMetaEvent,
+	BtcaStreamReasoningDeltaEvent,
+	BtcaStreamTextDeltaEvent,
+	BtcaStreamToolUpdatedEvent
+} from './types.ts';
+
+type Accumulator = {
+	partIds: string[];
+	partText: Map<string, string>;
+	combined: string;
+};
+
+const makeAccumulator = (): Accumulator => ({ partIds: [], partText: new Map(), combined: '' });
+
+const updateAccumulator = (acc: Accumulator, partId: string, nextText: string): string => {
+	if (!acc.partIds.includes(partId)) acc.partIds.push(partId);
+	acc.partText.set(partId, nextText);
+
+	const nextCombined = acc.partIds.map((id) => acc.partText.get(id) ?? '').join('');
+	const delta = nextCombined.startsWith(acc.combined)
+		? nextCombined.slice(acc.combined.length)
+		: nextCombined;
+	acc.combined = nextCombined;
+	return delta;
+};
+
+const toSse = (event: BtcaStreamEvent): string => {
+	// Standard SSE: an event name + JSON payload.
+	return `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`;
+};
+
+export namespace StreamService {
+	export const createSseStream = (args: {
+		meta: BtcaStreamMetaEvent;
+		eventStream: AsyncIterable<OcEvent>;
+	}): ReadableStream<Uint8Array> => {
+		const encoder = new TextEncoder();
+
+		const text = makeAccumulator();
+		const reasoning = makeAccumulator();
+		const toolsByCallId = new Map<string, Omit<BtcaStreamToolUpdatedEvent, 'type'>>();
+
+		let toolUpdates = 0;
+		let textEvents = 0;
+		let reasoningEvents = 0;
+
+		const emit = (
+			controller: ReadableStreamDefaultController<Uint8Array>,
+			event: BtcaStreamEvent
+		) => {
+			controller.enqueue(encoder.encode(toSse(event)));
+		};
+
+		return new ReadableStream<Uint8Array>({
+			start(controller) {
+				Metrics.info('stream.start', {
+					collectionKey: args.meta.collection.key,
+					resources: args.meta.resources,
+					model: args.meta.model
+				});
+
+				emit(controller, args.meta);
+
+				(async () => {
+					try {
+						for await (const event of args.eventStream) {
+							if (event.type === 'message.part.updated') {
+								const part: any = (event.properties as any).part;
+								if (!part || typeof part !== 'object') continue;
+
+								if (part.type === 'text') {
+									const partId = String(part.id);
+									const nextText = String(part.text ?? '');
+									const delta = updateAccumulator(text, partId, nextText);
+									if (delta.length > 0) {
+										textEvents += 1;
+										const msg: BtcaStreamTextDeltaEvent = { type: 'text.delta', delta };
+										emit(controller, msg);
+									}
+									continue;
+								}
+
+								if (part.type === 'reasoning') {
+									const partId = String(part.id);
+									const nextText = String(part.text ?? '');
+									const delta = updateAccumulator(reasoning, partId, nextText);
+									if (delta.length > 0) {
+										reasoningEvents += 1;
+										const msg: BtcaStreamReasoningDeltaEvent = { type: 'reasoning.delta', delta };
+										emit(controller, msg);
+									}
+									continue;
+								}
+
+								if (part.type === 'tool') {
+									const callID = String(part.callID);
+									const tool = String(part.tool);
+									const state = part.state as any;
+
+									const update: BtcaStreamToolUpdatedEvent = {
+										type: 'tool.updated',
+										callID,
+										tool,
+										state
+									};
+									toolUpdates += 1;
+									toolsByCallId.set(callID, { callID, tool, state });
+									emit(controller, update);
+									continue;
+								}
+							}
+
+							if (event.type === 'session.idle') {
+								const tools = Array.from(toolsByCallId.values());
+								Metrics.info('stream.done', {
+									collectionKey: args.meta.collection.key,
+									textLength: text.combined.length,
+									reasoningLength: reasoning.combined.length,
+									toolCount: tools.length,
+									toolUpdates,
+									textEvents,
+									reasoningEvents
+								});
+								const done: BtcaStreamDoneEvent = {
+									type: 'done',
+									text: text.combined,
+									reasoning: reasoning.combined,
+									tools
+								};
+								emit(controller, done);
+								continue;
+							}
+						}
+					} catch (cause) {
+						Metrics.error('stream.error', {
+							collectionKey: args.meta.collection.key,
+							error: Metrics.errorInfo(cause)
+						});
+						const err: BtcaStreamErrorEvent = {
+							type: 'error',
+							tag: getErrorTag(cause),
+							message: getErrorMessage(cause)
+						};
+						emit(controller, err);
+					} finally {
+						Metrics.info('stream.closed', { collectionKey: args.meta.collection.key });
+						controller.close();
+					}
+				})();
+			}
+		});
+	};
+}