@roj-ai/sdk 0.0.2

package/src/plugins/uploads/preprocessors/markitdown-preprocessor.ts

@@ -0,0 +1,262 @@
+/**
+ * Markitdown Preprocessor
+ *
+ * Converts documents to markdown using Microsoft's markitdown CLI.
+ * Supports PDF, DOCX, XLSX, PPTX, HTML, CSV, JSON, XML, EPUB, and more.
+ *
+ * Image extraction:
+ * - PDF: uses pdfimages (poppler-utils)
+ * - DOCX/ODT/EPUB: uses pandoc --extract-media
+ *
+ * Extracted images are classified via the image classifier preprocessor.
+ * Full content is written to disk; extractedContent contains a structured manifest.
+ */
+
+import { dirname } from 'node:path'
+import type { Result } from '~/lib/utils/result.js'
+import { Err, Ok } from '~/lib/utils/result.js'
+import type { FileSystem } from '~/platform/fs.js'
+import type { ProcessRunner } from '~/platform/process.js'
+import type { FileStore } from '../../../core/file-store/types.js'
+import type { Logger } from '../../../lib/logger/logger.js'
+import type { Preprocessor, PreprocessorContext, PreprocessorRegistry, PreprocessorResult } from '../preprocessor.js'
+
+const MAX_IMAGES = 50
+
+function makeExec(processRunner: ProcessRunner) {
+	return (cmd: string, args: string[]) => processRunner.execFile(cmd, args, { timeout: 60_000, maxBuffer: 50 * 1024 * 1024 })
+}
+
+/** MIME types where markitdown converts to markdown (non-ZIP, non-image) */
+const SUPPORTED_MIME_TYPES = [
+	'application/pdf',
+	'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+	'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+	'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+	'application/vnd.oasis.opendocument.text',
+	'application/rtf',
+	'application/epub+zip',
+	'text/html',
+	'application/xhtml+xml',
+	'text/csv',
+	'application/json',
+	'application/xml',
+	'text/xml',
+]
+
+/** MIME types where pandoc can extract embedded media */
+const PANDOC_EXTRACT_MIMES = new Set([
+	'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+	'application/vnd.oasis.opendocument.text',
+	'application/epub+zip',
+])
+
+const PANDOC_FORMAT_MAP: Record<string, string> = {
+	'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',
+	'application/vnd.oasis.opendocument.text': 'odt',
+	'application/epub+zip': 'epub',
+}
+
+/** MIME types where pdfimages can extract images */
+const PDFIMAGES_MIMES = new Set([
+	'application/pdf',
+])
+
+export interface MarkitdownPreprocessorConfig {
+	registry: PreprocessorRegistry
+	logger: Logger
+	fs: FileSystem
+	process: ProcessRunner
+}
+
+export class MarkitdownPreprocessor implements Preprocessor {
+	readonly name = 'markitdown'
+	readonly supportedMimeTypes = SUPPORTED_MIME_TYPES
+
+	private readonly registry: PreprocessorRegistry
+	private readonly logger: Logger
+	private readonly fs: FileSystem
+	private readonly exec: (cmd: string, args: string[]) => Promise<{ stdout: string; stderr: string }>
+
+	constructor(config: MarkitdownPreprocessorConfig) {
+		this.registry = config.registry
+		this.logger = config.logger
+		this.fs = config.fs
+		this.exec = makeExec(config.process)
+	}
+
+	async process(
+		filePath: string,
+		mimeType: string,
+		ctx: PreprocessorContext,
+	): Promise<Result<PreprocessorResult, Error>> {
+		const derivedPaths: string[] = []
+		const imageEntries: string[] = []
+
+		// 1. Convert to markdown via markitdown
+		const contentPathResult = ctx.files.realPath('content.md')
+		if (!contentPathResult.ok) {
+			return Err(new Error('Failed to resolve output path'))
+		}
+
+		try {
+			await this.fs.mkdir(dirname(contentPathResult.value), { recursive: true })
+			await this.exec('markitdown', [filePath, '-o', contentPathResult.value])
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error)
+			if (message.includes('ENOENT')) {
+				return Err(new Error('markitdown not found. Install with: pip install "markitdown[all]"'))
+			}
+			return Err(new Error(`markitdown failed: ${message}`))
+		}
+
+		const contentResult = await ctx.files.read('content.md')
+		const markdown = contentResult.ok ? contentResult.value : ''
+
+		derivedPaths.push('content.md')
+
+		// 2. Extract images based on file type
+		if (PANDOC_EXTRACT_MIMES.has(mimeType)) {
+			const images = await this.extractImagesWithPandoc(filePath, mimeType, ctx)
+			for (const img of images) {
+				derivedPaths.push(img.relativePath)
+				imageEntries.push(`- ${img.relativePath} — ${img.description}`)
+			}
+		} else if (PDFIMAGES_MIMES.has(mimeType)) {
+			const images = await this.extractImagesWithPdfimages(filePath, ctx)
+			for (const img of images) {
+				derivedPaths.push(img.relativePath)
+				imageEntries.push(`- ${img.relativePath} — ${img.description}`)
+			}
+		}
+
+		// 3. Build manifest
+		const manifestLines: string[] = ['Extracted files:']
+		manifestLines.push(`- content.md (markdown, ${markdown.length} chars)`)
+		manifestLines.push(...imageEntries)
+
+		this.logger.debug('Markitdown processed', {
+			filePath,
+			mimeType,
+			contentLength: markdown.length,
+			imagesExtracted: imageEntries.length,
+		})
+
+		return Ok({
+			extractedContent: manifestLines.join('\n'),
+			derivedPaths,
+		})
+	}
+
+	private async extractImagesWithPandoc(
+		filePath: string,
+		mimeType: string,
+		ctx: PreprocessorContext,
+	): Promise<Array<{ relativePath: string; description: string }>> {
+		const mediaStore = ctx.files.scoped('media')
+		const mediaDirResult = mediaStore.realPath('')
+		if (!mediaDirResult.ok) return []
+
+		const format = PANDOC_FORMAT_MAP[mimeType]
+		if (!format) return []
+
+		try {
+			await this.exec('pandoc', [
+				'-f',
+				format,
+				'-t',
+				'gfm',
+				filePath,
+				'-o',
+				'/dev/null',
+				`--extract-media=${mediaDirResult.value}`,
+			])
+		} catch {
+			this.logger.warn('pandoc --extract-media failed', { filePath })
+			return []
+		}
+
+		return classifyExtractedImages(mediaStore, 'media', ctx, this.registry, this.logger)
+	}
+
+	private async extractImagesWithPdfimages(
+		filePath: string,
+		ctx: PreprocessorContext,
+	): Promise<Array<{ relativePath: string; description: string }>> {
+		const imageStore = ctx.files.scoped('images')
+		const imagesDirResult = imageStore.realPath('')
+		if (!imagesDirResult.ok) return []
+
+		try {
+			await this.fs.mkdir(imagesDirResult.value, { recursive: true })
+			await this.exec('pdfimages', ['-png', filePath, `${imagesDirResult.value}/img`])
+		} catch {
+			return []
+		}
+
+		return classifyExtractedImages(imageStore, 'images', ctx, this.registry, this.logger)
+	}
+}
+
+// ============================================================================
+// Shared image helpers
+// ============================================================================
+
+const IMAGE_EXT_RE = /\.(png|jpe?g|gif|webp|tiff?|bmp|svg)$/i
+
+const IMAGE_MIME_MAP: Record<string, string> = {
+	png: 'image/png',
+	jpg: 'image/jpeg',
+	jpeg: 'image/jpeg',
+	gif: 'image/gif',
+	webp: 'image/webp',
+	svg: 'image/svg+xml',
+	tif: 'image/tiff',
+	tiff: 'image/tiff',
+	bmp: 'image/bmp',
+}
+
+export function guessImageMime(filename: string): string {
+	const ext = filename.split('.').pop()?.toLowerCase()
+	return IMAGE_MIME_MAP[ext ?? ''] ?? 'image/png'
+}
+
+export async function classifyExtractedImages(
+	imageStore: FileStore,
+	relativePrefix: string,
+	ctx: PreprocessorContext,
+	registry: PreprocessorRegistry,
+	logger: Logger,
+): Promise<Array<{ relativePath: string; description: string }>> {
+	const results: Array<{ relativePath: string; description: string }> = []
+
+	const listResult = await imageStore.list('', { maxDepth: 3 })
+	if (!listResult.ok) return results
+
+	const imageFiles = listResult.value
+		.filter(e => e.type === 'file' && IMAGE_EXT_RE.test(e.name))
+		.sort((a, b) => a.name.localeCompare(b.name))
+		.slice(0, MAX_IMAGES)
+
+	for (const imgFile of imageFiles) {
+		const imgPathResult = imageStore.realPath(imgFile.name)
+		if (!imgPathResult.ok) continue
+
+		const imgMime = guessImageMime(imgFile.name)
+		let description = imgMime
+
+		const classifier = registry.getForMimeType(imgMime)
+		if (classifier) {
+			const classifyResult = await classifier.process(imgPathResult.value, imgMime, {
+				files: ctx.files.scoped(`${relativePrefix}/${imgFile.name}-meta`),
+			})
+			if (classifyResult.ok && classifyResult.value.extractedContent) {
+				description = classifyResult.value.extractedContent
+			}
+		}
+
+		results.push({ relativePath: `${relativePrefix}/${imgFile.name}`, description })
+	}
+
+	return results
+}

package/src/plugins/uploads/preprocessors/zip-preprocessor.ts

@@ -0,0 +1,210 @@
+/**
+ * ZIP Preprocessor
+ *
+ * Extracts ZIP contents via `unzip` CLI and recursively preprocesses files.
+ * Each extracted file is processed by its matching preprocessor (markitdown for
+ * documents, image classifier for images, nested zip for archives).
+ * Full manifest is written to disk; extractedContent contains a structured summary.
+ */
+
+import { extname } from 'node:path'
+import type { Result } from '~/lib/utils/result.js'
+import { Err, Ok } from '~/lib/utils/result.js'
+import type { ProcessRunner } from '~/platform/process.js'
+import type { Logger } from '../../../lib/logger/logger.js'
+import type { Preprocessor, PreprocessorContext, PreprocessorRegistry, PreprocessorResult } from '../preprocessor.js'
+
+const MAX_DEPTH = 3
+const MAX_FILES = 500
+const MAX_TOTAL_SIZE = 100 * 1024 * 1024 // 100MB
+
+const MIME_MAP: Record<string, string> = {
+	'.pdf': 'application/pdf',
+	'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+	'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+	'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+	'.odt': 'application/vnd.oasis.opendocument.text',
+	'.rtf': 'application/rtf',
+	'.epub': 'application/epub+zip',
+	'.zip': 'application/zip',
+	'.html': 'text/html',
+	'.htm': 'text/html',
+	'.csv': 'text/csv',
+	'.json': 'application/json',
+	'.xml': 'application/xml',
+	'.png': 'image/png',
+	'.jpg': 'image/jpeg',
+	'.jpeg': 'image/jpeg',
+	'.gif': 'image/gif',
+	'.webp': 'image/webp',
+	'.txt': 'text/plain',
+	'.md': 'text/markdown',
+}
+
+function getMimeType(filename: string): string | null {
+	const ext = extname(filename).toLowerCase()
+	return MIME_MAP[ext] ?? null
+}
+
+function makeExec(processRunner: ProcessRunner) {
+	return (cmd: string, args: string[]) => processRunner.execFile(cmd, args, { timeout: 60_000, maxBuffer: 50 * 1024 * 1024 })
+}
+
+function formatSize(bytes: number): string {
+	if (bytes < 1024) return `${bytes}B`
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`
+	return `${(bytes / (1024 * 1024)).toFixed(1)}MB`
+}
+
+export interface ZipPreprocessorConfig {
+	registry: PreprocessorRegistry
+	logger: Logger
+	process: ProcessRunner
+	depth?: number
+}
+
+export class ZipPreprocessor implements Preprocessor {
+	readonly name = 'zip'
+	readonly supportedMimeTypes = ['application/zip']
+
+	private readonly registry: PreprocessorRegistry
+	private readonly logger: Logger
+	private readonly processRunner: ProcessRunner
+	private readonly exec: (cmd: string, args: string[]) => Promise<{ stdout: string; stderr: string }>
+	private readonly depth: number
+
+	constructor(config: ZipPreprocessorConfig) {
+		this.registry = config.registry
+		this.logger = config.logger
+		this.processRunner = config.process
+		this.exec = makeExec(config.process)
+		this.depth = config.depth ?? 0
+	}
+
+	async process(
+		filePath: string,
+		_mimeType: string,
+		ctx: PreprocessorContext,
+	): Promise<Result<PreprocessorResult, Error>> {
+		if (this.depth >= MAX_DEPTH) {
+			return Err(new Error(`ZIP nesting depth limit reached (max ${MAX_DEPTH})`))
+		}
+
+		// Extract to disk via unzip
+		const extractStore = ctx.files.scoped('extracted')
+		const extractDirResult = extractStore.realPath('')
+		if (!extractDirResult.ok) {
+			return Err(new Error('Failed to resolve extraction directory'))
+		}
+
+		try {
+			await this.exec('unzip', ['-o', '-q', filePath, '-d', extractDirResult.value])
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error)
+			if (message.includes('ENOENT')) {
+				return Err(new Error('unzip not found'))
+			}
+			// unzip returns exit code 1 for warnings (e.g. skipped dirs) — still usable
+			if (!message.includes('exit code 1')) {
+				return Err(new Error(`unzip failed: ${message}`))
+			}
+		}
+
+		// List extracted files
+		const listResult = await extractStore.list('', { maxDepth: 10 })
+		if (!listResult.ok) {
+			return Err(new Error('Failed to list extracted files'))
+		}
+
+		const derivedPaths: string[] = []
+		const manifest: string[] = []
+		let fileCount = 0
+		let totalSize = 0
+
+		const files = listResult.value
+			.filter(e => e.type === 'file')
+			.sort((a, b) => a.name.localeCompare(b.name))
+
+		for (const file of files) {
+			if (fileCount >= MAX_FILES) {
+				manifest.push(`... (truncated, ${files.length - fileCount} more files)`)
+				break
+			}
+
+			const fileSize = file.size ?? 0
+			totalSize += fileSize
+			if (totalSize > MAX_TOTAL_SIZE) {
+				manifest.push('... (total size limit reached)')
+				break
+			}
+
+			fileCount++
+
+			const fileRealPath = extractStore.realPath(file.name)
+			if (!fileRealPath.ok) {
+				manifest.push(`- ${file.name} (path resolution failed)`)
+				continue
+			}
+
+			const relativePath = `extracted/${file.name}`
+			derivedPaths.push(relativePath)
+
+			const mime = getMimeType(file.name)
+			let contentSummary = ''
+
+			if (mime) {
+				// For nested ZIPs, create a new preprocessor with incremented depth
+				let preprocessor = this.registry.getForMimeType(mime)
+				if (mime === 'application/zip') {
+					preprocessor = new ZipPreprocessor({
+						registry: this.registry,
+						logger: this.logger,
+						process: this.processRunner,
+						depth: this.depth + 1,
+					})
+				}
+
+				if (preprocessor) {
+					const subResult = await preprocessor.process(fileRealPath.value, mime, {
+						files: ctx.files.scoped(`extracted/${file.name}-content`),
+					})
+					if (subResult.ok) {
+						if (subResult.value.derivedPaths) {
+							for (const dp of subResult.value.derivedPaths) {
+								derivedPaths.push(`extracted/${file.name}-content/${dp}`)
+							}
+						}
+						if (subResult.value.extractedContent) {
+							// Indent sub-content as nested lines, strip "Extracted files:" prefix
+							const subContent = subResult.value.extractedContent.replace(/^Extracted files:\n/m, '')
+							const indented = subContent.split('\n').map(l => `  ${l}`).join('\n')
+							contentSummary = `\n${indented}`
+						}
+					} else {
+						this.logger.warn('Sub-preprocessor failed', { file: file.name, error: subResult.error.message })
+					}
+				}
+			}
+
+			manifest.push(`- ${file.name} (${formatSize(fileSize)})${contentSummary}`)
+		}
+
+		const fullManifest = `## ZIP Contents (${fileCount} files)\n\n${manifest.join('\n')}`
+
+		// Write full manifest to disk
+		await ctx.files.write('content.txt', fullManifest)
+		derivedPaths.push('content.txt')
+
+		this.logger.debug('ZIP processed', {
+			filePath,
+			filesExtracted: fileCount,
+			totalSize,
+			depth: this.depth,
+		})
+
+		return Ok({
+			extractedContent: fullManifest,
+			derivedPaths,
+		})
+	}
+}

package/src/plugins/uploads/schema.ts

@@ -0,0 +1,92 @@
+/**
+ * Upload domain types and schemas
+ *
+ * Contains all types related to uploads:
+ * - Branded ID type and constructor
+ * - Upload metadata types
+ * - Zod schemas for validation
+ */
+
+import { uuidv7 } from 'uuidv7'
+import z from 'zod/v4'
+import type { SessionId } from '../../core/sessions/schema.js'
+
+// ============================================================================
+// UploadId - Branded type
+// ============================================================================
+
+/** UploadId schema - validates any string and brands as UploadId. */
+export const uploadIdSchema = z.string().brand('UploadId')
+
+/** Branded UploadId type */
+export type UploadId = z.infer<typeof uploadIdSchema>
+
+/** Constructor for UploadId */
+export const UploadId = (id: string): UploadId => id as UploadId
+
+/** Generate a new UploadId (UUIDv7) */
+export const generateUploadId = (): UploadId => UploadId(uuidv7())
+
+// ============================================================================
+// UploadId - Zod schemas
+// ============================================================================
+
+// ============================================================================
+// Upload types
+// ============================================================================
+
+/**
+ * Information about an uploaded file stored on disk.
+ */
+export interface UploadedFile {
+	/** Relative path within upload directory */
+	filename: string
+	/** Absolute path to the file */
+	path: string
+	/** File size in bytes */
+	size: number
+	/** MIME type of the file */
+	mimeType: string
+}
+
+/**
+ * Attachment included with a mailbox message.
+ * Represents a user-uploaded file that has been processed.
+ */
+export interface MessageAttachment {
+	/** Unique identifier for this upload */
+	uploadId: UploadId
+	/** Original filename as uploaded by user */
+	filename: string
+	/** MIME type of the original file */
+	mimeType: string
+	/** File size in bytes */
+	size: number
+	/** Absolute path to the original file */
+	path: string
+	/** Text content extracted by pre-processor (image description, PDF text, etc.) */
+	extractedContent?: string
+	/** Paths to derived files (extracted images, converted documents, etc.) */
+	derivedPaths?: string[]
+}
+
+/**
+ * Metadata for an upload stored on disk.
+ * Used to track upload status and associate with messages later.
+ */
+export interface UploadMetadata {
+	uploadId: UploadId
+	sessionId: SessionId
+	filename: string
+	mimeType: string
+	size: number
+	path: string
+	status: 'processing' | 'ready' | 'failed' | 'deleted'
+	extractedContent?: string
+	derivedPaths?: string[]
+	createdAt: number
+	completedAt?: number
+	error?: string
+	/** Set when the upload is attached to a message via sendMessage */
+	usedInMessageId?: string
+}

package/src/plugins/uploads/state.ts

@@ -0,0 +1,42 @@
+import z4 from 'zod/v4'
+import { createEventsFactory } from '~/core/events/types'
+import { uploadIdSchema } from './schema'
+
+export const uploadEvents = createEventsFactory({
+	events: {
+		attachment_uploaded: z4.object({
+			uploadId: uploadIdSchema,
+			filename: z4.string(),
+			mimeType: z4.string(),
+			size: z4.number(),
+			status: z4.enum(['ready', 'failed']),
+			extractedContent: z4.string().optional(),
+			derivedPaths: z4.array(z4.string()).optional(),
+			error: z4.string().optional(),
+		}),
+		attachments_consumed: z4.object({
+			agentId: z4.string(),
+			uploadIds: z4.array(uploadIdSchema),
+		}),
+	},
+})
+
+export type AttachmentUploadedEvent = (typeof uploadEvents)['Events']['attachment_uploaded']
+export type AttachmentsConsumedEvent = (typeof uploadEvents)['Events']['attachments_consumed']
+
+/** A pending upload tracked in session state */
+export interface PendingUpload {
+	uploadId: string
+	filename: string
+	mimeType: string
+	size: number
+	status: 'ready' | 'failed'
+	extractedContent?: string
+	derivedPaths?: string[]
+}
+
+/** Uploads plugin state slice — tracks pending (unconsumed) uploads */
+export interface UploadsState {
+	/** Uploads that have been uploaded but not yet consumed by an agent */
+	pending: PendingUpload[]
+}