@roj-ai/sdk 0.0.2

package/src/plugins/uploads/plugin.ts

@@ -0,0 +1,406 @@
+import z from 'zod/v4'
+import { ValidationErrors } from '~/core/errors.js'
+import type { FileStore } from '~/core/file-store/types.js'
+import { definePlugin } from '~/core/plugins/plugin-builder.js'
+import { Err, Ok } from '~/lib/utils/result.js'
+import type { PreprocessorRegistry } from './preprocessor.js'
+import { generateUploadId, type MessageAttachment, UploadId, type UploadMetadata } from './schema.js'
+import { type PendingUpload, uploadEvents, type UploadsState } from './state.js'
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+const MAX_FILE_SIZE = 10 * 1024 * 1024 // 10MB
+
+/** Allowed MIME type patterns */
+const ALLOWED_MIME_TYPES = [
+	'image/',
+	'text/',
+	'application/pdf',
+	'application/json',
+	'application/xml',
+	'application/rtf',
+	'application/epub+zip',
+	'application/xhtml+xml',
+	'application/zip',
+	'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+	'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+	'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+	'application/vnd.oasis.opendocument.text',
+]
+
+const PROCESSING_TIMEOUT_MS = 120_000 // 120 seconds
+
+// ============================================================================
+// Config
+// ============================================================================
+
+export interface UploadsPluginConfig {
+	dataFileStore: FileStore
+	preprocessorRegistry?: PreprocessorRegistry
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+function isAllowedMimeType(mimeType: string): boolean {
+	return ALLOWED_MIME_TYPES.some((allowed) =>
+		allowed.endsWith('/')
+			? mimeType.startsWith(allowed)
+			: mimeType === allowed
+	)
+}
+
+function sleep(ms: number): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+function formatUploadsForLLM(uploads: PendingUpload[], sessionRoot: string): string {
+	const blocks = uploads.map((u) => {
+		const basePath = `${sessionRoot}/uploads/${u.uploadId}`
+		const content = u.extractedContent ?? `[File uploaded: ${u.filename}]`
+		return `<attachment uploadId="${u.uploadId}" filename="${u.filename}" type="${u.mimeType}" basePath="${basePath}">\n${content}\n</attachment>`
+	})
+	return blocks.join('\n')
+}
+
+// ============================================================================
+// Plugin
+// ============================================================================
+
+export const uploadsPlugin = definePlugin('uploads')
+	.pluginConfig<UploadsPluginConfig>()
+	.events([uploadEvents])
+	.state<UploadsState>({
+		key: 'uploads',
+		initial: (): UploadsState => ({ pending: [] }),
+		reduce: (state, event) => {
+			switch (event.type) {
+				case 'attachment_uploaded': {
+					if (event.status !== 'ready') return state
+					const upload: PendingUpload = {
+						uploadId: event.uploadId,
+						filename: event.filename,
+						mimeType: event.mimeType,
+						size: event.size,
+						status: event.status,
+						extractedContent: event.extractedContent,
+						derivedPaths: event.derivedPaths,
+					}
+					return { ...state, pending: [...state.pending, upload] }
+				}
+				case 'attachments_consumed': {
+					const consumedIds = new Set(event.uploadIds.map(String))
+					return {
+						...state,
+						pending: state.pending.filter((u) => !consumedIds.has(String(u.uploadId))),
+					}
+				}
+				default:
+					return state
+			}
+		},
+	})
+	.dequeue({
+		hasPendingMessages: (ctx) => {
+			const uploads = ctx.pluginState
+			return uploads.pending.length > 0
+		},
+		getPendingMessages: (ctx) => {
+			const uploads = ctx.pluginState
+			if (uploads.pending.length === 0) return null
+			const sessionRoot = ctx.files.getRoots().session
+			return {
+				messages: [{
+					role: 'user',
+					content: formatUploadsForLLM(uploads.pending, sessionRoot),
+				}],
+				token: uploads.pending.map((u) => u.uploadId),
+			}
+		},
+		markConsumed: async (ctx, token) => {
+			await ctx.emitEvent(uploadEvents.create('attachments_consumed', {
+				agentId: ctx.agentId,
+				uploadIds: token.map(UploadId),
+			}))
+
+			// Also mark as used on disk
+			const { dataFileStore } = ctx.pluginConfig
+			for (const uploadIdStr of token) {
+				const uploadStore = dataFileStore.scoped(`sessions/${ctx.sessionId}/uploads/${uploadIdStr}`)
+				const metaResult = await uploadStore.read('meta.json')
+				if (metaResult.ok) {
+					const meta: UploadMetadata = JSON.parse(metaResult.value)
+					meta.usedInMessageId = 'auto-dequeued'
+					await uploadStore.write('meta.json', JSON.stringify(meta, null, 2))
+				}
+			}
+		},
+	})
+	.method('listPending', {
+		input: z.object({
+			sessionId: z.string(),
+		}),
+		output: z.object({
+			uploads: z.array(z.object({
+				uploadId: z.string(),
+				filename: z.string(),
+				mimeType: z.string(),
+				size: z.number(),
+				status: z.enum(['processing', 'ready', 'failed']),
+				createdAt: z.number(),
+			})),
+		}),
+		handler: async (ctx, input) => {
+			const { dataFileStore } = ctx.pluginConfig
+			const sessionId = input.sessionId
+
+			const uploadsPath = `sessions/${sessionId}/uploads`
+			const listResult = await dataFileStore.list(uploadsPath)
+
+			if (!listResult.ok) {
+				return Ok({ uploads: [] })
+			}
+
+			const uploads: Array<{
+				uploadId: string
+				filename: string
+				mimeType: string
+				size: number
+				status: 'processing' | 'ready' | 'failed'
+				createdAt: number
+			}> = []
+
+			for (const entry of listResult.value) {
+				if (entry.type !== 'directory') continue
+				const metaResult = await dataFileStore.read(`${uploadsPath}/${entry.name}/meta.json`)
+				if (!metaResult.ok) continue
+
+				const meta: UploadMetadata = JSON.parse(metaResult.value)
+
+				// Skip used, deleted, or non-matching session uploads
+				if (meta.usedInMessageId) continue
+				if (meta.sessionId !== sessionId) continue
+				if (meta.status === 'deleted') continue
+
+				const { status } = meta
+				if (status === 'processing' || status === 'ready' || status === 'failed') {
+					uploads.push({
+						uploadId: meta.uploadId,
+						filename: meta.filename,
+						mimeType: meta.mimeType,
+						size: meta.size,
+						status,
+						createdAt: meta.createdAt,
+					})
+				}
+			}
+
+			return Ok({ uploads })
+		},
+	})
+	.method('delete', {
+		input: z.object({
+			sessionId: z.string(),
+			uploadId: z.string(),
+		}),
+		output: z.object({}),
+		handler: async (ctx, input) => {
+			const { dataFileStore } = ctx.pluginConfig
+			const uploadStore = dataFileStore.scoped(`sessions/${input.sessionId}/uploads/${input.uploadId}`)
+			const metaResult = await uploadStore.read('meta.json')
+
+			if (!metaResult.ok) {
+				return Err(ValidationErrors.invalid(`Upload not found: ${input.uploadId}`))
+			}
+
+			const meta: UploadMetadata = JSON.parse(metaResult.value)
+
+			if (meta.sessionId !== input.sessionId) {
+				return Err(ValidationErrors.invalid('Upload does not belong to this session'))
+			}
+
+			if (meta.usedInMessageId) {
+				return Err(ValidationErrors.invalid('Cannot delete an upload that has been sent in a message'))
+			}
+
+			// Mark as deleted
+			meta.status = 'deleted'
+			await uploadStore.write('meta.json', JSON.stringify(meta, null, 2))
+
+			return Ok({})
+		},
+	})
+	.method('loadAttachments', {
+		input: z.object({
+			sessionId: z.string(),
+			uploadIds: z.array(z.string()),
+		}),
+		output: z.object({
+			attachments: z.array(z.unknown()),
+		}),
+		handler: async (ctx, input) => {
+			const { dataFileStore } = ctx.pluginConfig
+			const sessionRoot = ctx.files.getRoots().session
+			const attachments: MessageAttachment[] = []
+
+			for (const uploadIdStr of input.uploadIds) {
+				const metaResult = await dataFileStore.read(`sessions/${input.sessionId}/uploads/${uploadIdStr}/meta.json`)
+
+				if (!metaResult.ok) {
+					return Err(ValidationErrors.invalid(`Upload not found: ${uploadIdStr}`))
+				}
+
+				const meta: UploadMetadata = JSON.parse(metaResult.value)
+
+				if (meta.sessionId !== input.sessionId) {
+					return Err(ValidationErrors.invalid('Upload does not belong to this session'))
+				}
+				if (meta.status !== 'ready') {
+					return Err(ValidationErrors.invalid(`Upload not ready: ${meta.status}`))
+				}
+
+				attachments.push({
+					uploadId: UploadId(uploadIdStr),
+					filename: meta.filename,
+					mimeType: meta.mimeType,
+					size: meta.size,
+					path: `${sessionRoot}/uploads/${uploadIdStr}/${meta.filename}`,
+					extractedContent: meta.extractedContent,
+					derivedPaths: meta.derivedPaths,
+				})
+			}
+
+			return Ok({ attachments })
+		},
+	})
+	.method('markUsed', {
+		input: z.object({
+			sessionId: z.string(),
+			uploadIds: z.array(z.string()),
+			messageId: z.string(),
+		}),
+		output: z.object({}),
+		handler: async (ctx, input) => {
+			const { dataFileStore } = ctx.pluginConfig
+
+			for (const uploadIdStr of input.uploadIds) {
+				const uploadStore = dataFileStore.scoped(`sessions/${input.sessionId}/uploads/${uploadIdStr}`)
+				const metaResult = await uploadStore.read('meta.json')
+				if (metaResult.ok) {
+					const meta: UploadMetadata = JSON.parse(metaResult.value)
+					meta.usedInMessageId = input.messageId
+					await uploadStore.write('meta.json', JSON.stringify(meta, null, 2))
+				}
+			}
+
+			return Ok({})
+		},
+	})
+	.method('upload', {
+		input: z.object({
+			sessionId: z.string(),
+			filename: z.string(),
+			mimeType: z.string(),
+			size: z.number(),
+			fileBuffer: z.custom<Buffer>(),
+		}),
+		output: z.object({
+			uploadId: z.string(),
+			status: z.enum(['ready', 'failed']),
+			extractedContent: z.string().optional(),
+		}),
+		handler: async (ctx, input) => {
+			const { dataFileStore, preprocessorRegistry } = ctx.pluginConfig
+
+			// Validate
+			if (input.size > MAX_FILE_SIZE) {
+				return Err(ValidationErrors.invalid(`File too large: max ${MAX_FILE_SIZE / (1024 * 1024)}MB`))
+			}
+
+			if (!isAllowedMimeType(input.mimeType)) {
+				return Err(ValidationErrors.invalid(`Unsupported file type: ${input.mimeType}`))
+			}
+
+			// Generate upload ID and scoped store
+			const uploadId = generateUploadId()
+			const uploadStore = dataFileStore.scoped(`sessions/${input.sessionId}/uploads/${uploadId}`)
+
+			// Write file to disk
+			const writeResult = await uploadStore.write(input.filename, input.fileBuffer)
+
+			if (!writeResult.ok) {
+				return Err(ValidationErrors.invalid('Failed to write file'))
+			}
+
+			const filePath = writeResult.value.path
+
+			// Run preprocessor (with timeout)
+			let processingResult: 'success' | 'failed' | 'skipped' = 'skipped'
+			let extractedContent: string | undefined
+			let derivedPaths: string[] | undefined
+
+			const preprocessor = preprocessorRegistry?.getForMimeType(input.mimeType)
+
+			if (preprocessor) {
+				const processPromise = preprocessor.process(filePath, input.mimeType, {
+					files: uploadStore,
+				})
+
+				const timeoutPromise = sleep(PROCESSING_TIMEOUT_MS).then(() => ({
+					ok: false as const,
+					error: new Error('Processing timeout'),
+				}))
+
+				const result = await Promise.race([processPromise, timeoutPromise])
+
+				if (result.ok) {
+					processingResult = 'success'
+					extractedContent = result.value.extractedContent
+					derivedPaths = result.value.derivedPaths
+				} else {
+					processingResult = 'failed'
+				}
+			}
+
+			// Create upload metadata
+			const now = Date.now()
+			const uploadStatus = processingResult === 'failed' ? 'failed' as const : 'ready' as const
+			const metadata: UploadMetadata = {
+				uploadId,
+				sessionId: ctx.sessionId,
+				filename: input.filename,
+				mimeType: input.mimeType,
+				size: input.size,
+				path: filePath,
+				status: uploadStatus,
+				extractedContent,
+				derivedPaths,
+				createdAt: now,
+				completedAt: now,
+			}
+
+			// Save metadata
+			await uploadStore.write('meta.json', JSON.stringify(metadata, null, 2))
+
+			// Emit event
+			await ctx.emitEvent(uploadEvents.create('attachment_uploaded', {
+				uploadId,
+				filename: input.filename,
+				mimeType: input.mimeType,
+				size: input.size,
+				status: uploadStatus,
+				extractedContent,
+				derivedPaths,
+			}))
+
+			return Ok({
+				uploadId: String(uploadId),
+				status: uploadStatus,
+				extractedContent,
+			})
+		},
+	})
+	.build()

package/src/plugins/uploads/preprocessor.ts

@@ -0,0 +1,111 @@
+/**
+ * Preprocessor - Interface for file upload preprocessing
+ *
+ * Preprocessors extract content from uploaded files to make them
+ * accessible to LLMs. Examples include:
+ * - Image description via vision LLM
+ * - PDF text extraction
+ * - Document conversion
+ */
+
+import type { FileStore } from '~/core/file-store/types.js'
+import type { Result } from '~/lib/utils/result.js'
+
+// ============================================================================
+// Preprocessor Context
+// ============================================================================
+
+/**
+ * Context provided to preprocessors during file processing.
+ */
+export interface PreprocessorContext {
+	/** FileStore scoped to upload directory for writing derived files */
+	files: FileStore
+}
+
+// ============================================================================
+// Preprocessor Result
+// ============================================================================
+
+/**
+ * Result of preprocessing a file.
+ */
+export interface PreprocessorResult {
+	/** Extracted text content for LLM consumption */
+	extractedContent?: string
+	/** Paths to derived files (e.g., extracted images from PDF) */
+	derivedPaths?: string[]
+}
+
+// ============================================================================
+// Preprocessor Interface
+// ============================================================================
+
+/**
+ * Interface for file preprocessors.
+ * Each preprocessor handles specific MIME types.
+ */
+export interface Preprocessor {
+	/** Name of the preprocessor for logging */
+	name: string
+	/** MIME types this preprocessor can handle (supports wildcards like "image/*") */
+	supportedMimeTypes: string[]
+	/**
+	 * Process a file and extract content.
+	 *
+	 * @param filePath - Absolute path to the uploaded file
+	 * @param mimeType - MIME type of the file
+	 * @param ctx - Context with artifact store for writing derived files
+	 * @returns Result with extracted content or error
+	 */
+	process(
+		filePath: string,
+		mimeType: string,
+		ctx: PreprocessorContext,
+	): Promise<Result<PreprocessorResult, Error>>
+}
+
+// ============================================================================
+// Preprocessor Registry
+// ============================================================================
+
+/**
+ * Registry for file preprocessors.
+ * Manages preprocessor registration and lookup by MIME type.
+ */
+export class PreprocessorRegistry {
+	private readonly processors: Preprocessor[] = []
+
+	/**
+	 * Register a preprocessor.
+	 */
+	register(processor: Preprocessor): void {
+		this.processors.push(processor)
+	}
+
+	/**
+	 * Get a preprocessor for the given MIME type.
+	 * Returns null if no preprocessor supports this type.
+	 *
+	 * @param mimeType - MIME type to find a preprocessor for
+	 * @returns The first matching preprocessor or null
+	 */
+	getForMimeType(mimeType: string): Preprocessor | null {
+		return (
+			this.processors.find((p) =>
+				p.supportedMimeTypes.some((t) =>
+					t.endsWith('/*')
+						? mimeType.startsWith(t.slice(0, -1))
+						: t === mimeType
+				)
+			) ?? null
+		)
+	}
+
+	/**
+	 * Get all registered preprocessors.
+	 */
+	getAll(): readonly Preprocessor[] {
+		return this.processors
+	}
+}

package/src/plugins/uploads/preprocessors/image-classifier.ts

@@ -0,0 +1,176 @@
+/**
+ * Image Classifier Preprocessor
+ *
+ * Sends images to a vision-capable LLM for description.
+ * Falls back to basic metadata if vision is not available.
+ */
+
+import type { LLMProvider } from '~/core/llm/provider.js'
+import { ModelId } from '~/core/llm/schema.js'
+import type { Result } from '~/lib/utils/result.js'
+import { Err, Ok } from '~/lib/utils/result.js'
+import type { FileSystem } from '~/platform/fs.js'
+import type { Logger } from '../../../lib/logger/logger.js'
+import type { Preprocessor, PreprocessorContext, PreprocessorResult } from '../preprocessor.js'
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+export interface ImageClassifierConfig {
+	/** LLM provider for vision inference */
+	llmProvider: LLMProvider
+	/** Model to use for vision (should be vision-capable) */
+	visionModel?: string
+	/** Logger for debug output */
+	logger: Logger
+	/** FileSystem adapter (for checking + reading image files) */
+	fs: FileSystem
+	/** Whether to skip vision and just return metadata */
+	skipVision?: boolean
+}
+
+// ============================================================================
+// Image Classifier
+// ============================================================================
+
+/**
+ * Image classifier preprocessor.
+ * Describes images using a vision-capable LLM.
+ */
+export class ImageClassifierPreprocessor implements Preprocessor {
+	readonly name = 'image-classifier'
+	readonly supportedMimeTypes = ['image/*']
+
+	private readonly llmProvider: LLMProvider
+	private readonly visionModel: ModelId
+	private readonly logger: Logger
+	private readonly fs: FileSystem
+	private readonly skipVision: boolean
+
+	constructor(config: ImageClassifierConfig) {
+		this.llmProvider = config.llmProvider
+		this.visionModel = config.visionModel ? ModelId(config.visionModel) : ModelId('anthropic/claude-haiku-4.5')
+		this.logger = config.logger
+		this.fs = config.fs
+		this.skipVision = config.skipVision ?? false
+	}
+
+	async process(
+		filePath: string,
+		mimeType: string,
+		ctx: PreprocessorContext,
+	): Promise<Result<PreprocessorResult, Error>> {
+		try {
+			// Check + stat image file
+			if (!(await this.fs.exists(filePath))) {
+				return Err(new Error(`Image file not found: ${filePath}`))
+			}
+
+			const size = (await this.fs.stat(filePath)).size
+			const filename = filePath.split('/').pop() ?? 'image'
+
+			// Skip vision if configured
+			if (this.skipVision) {
+				return Ok({
+					extractedContent: `[Image: ${filename}, ${this.formatSize(size)}, ${mimeType}]`,
+				})
+			}
+
+			// Try vision inference
+			const description = await this.describeImage(filePath, mimeType)
+
+			if (description) {
+				// Save description to file
+				const writeResult = await ctx.files.write('description.txt', description)
+
+				this.logger.debug('Image described successfully', {
+					filename,
+					descriptionLength: description.length,
+				})
+
+				return Ok({
+					extractedContent: `[Image: ${description}]`,
+					derivedPaths: writeResult.ok ? ['description.txt'] : [],
+				})
+			}
+
+			// Fallback to basic metadata
+			return Ok({
+				extractedContent: `[Image: ${filename}, ${this.formatSize(size)}, ${mimeType}]`,
+			})
+		} catch (error) {
+			this.logger.error(
+				'Image classification failed',
+				error instanceof Error ? error : undefined,
+				{ filePath },
+			)
+
+			// Return basic info on error instead of failing
+			const filename = filePath.split('/').pop() ?? 'image'
+			return Ok({
+				extractedContent: `[Image: ${filename} (description unavailable)]`,
+			})
+		}
+	}
+
+	/**
+	 * Describe image using vision LLM.
+	 * Returns null if vision is not available or fails.
+	 */
+	private async describeImage(
+		filePath: string,
+		mimeType: string,
+	): Promise<string | null> {
+		try {
+			// Use file:// URL - resolved to base64 lazily in LLM provider
+			const result = await this.llmProvider.inference({
+				model: this.visionModel,
+				systemPrompt: 'You are an image description assistant. Describe images concisely in 1-2 sentences.',
+				messages: [
+					{
+						role: 'user',
+						content: [
+							{
+								type: 'text',
+								text: 'Please describe this image concisely in 1-2 sentences. Focus on the main subject and any text visible.',
+							},
+							{
+								type: 'image_url',
+								imageUrl: { url: `file://${filePath}` },
+							},
+						],
+					},
+				],
+				maxTokens: 200,
+				temperature: 0.3,
+			})
+
+			if (result.ok && result.value.content) {
+				return result.value.content.trim()
+			}
+
+			return null
+		} catch (error) {
+			this.logger.warn('Vision inference failed', {
+				error: error instanceof Error ? error.message : String(error),
+			})
+			return null
+		}
+	}
+
+	private formatSize(bytes: number): string {
+		if (bytes < 1024) return `${bytes}B`
+		if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`
+		return `${(bytes / (1024 * 1024)).toFixed(1)}MB`
+	}
+}
+
+/**
+ * Create an image classifier preprocessor.
+ */
+export function createImageClassifierPreprocessor(
+	config: ImageClassifierConfig,
+): ImageClassifierPreprocessor {
+	return new ImageClassifierPreprocessor(config)
+}

package/src/plugins/uploads/preprocessors/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Preprocessors module exports
+ */
+
+export { createImageClassifierPreprocessor, type ImageClassifierConfig, ImageClassifierPreprocessor } from './image-classifier.js'
+export { MarkitdownPreprocessor, type MarkitdownPreprocessorConfig } from './markitdown-preprocessor.js'
+export { ZipPreprocessor, type ZipPreprocessorConfig } from './zip-preprocessor.js'