@roj-ai/sdk 0.1.14 → 0.1.16

package/src/plugins/uploads/preprocessors/markitdown-preprocessor.ts

@@ -25,8 +25,16 @@ import type { Preprocessor, PreprocessorContext, PreprocessorRegistry, Preproces
 const MAX_IMAGES = 50
 const IMAGE_CLASSIFY_CONCURRENCY = 10
 
+// markitdown converts a text-only document; even large PDFs finish in seconds.
+const MARKITDOWN_TIMEOUT_MS = 60_000
+// Image extractors (pdfimages, pandoc --extract-media) scale with image count
+// and resolution. Real-world large brand PDFs (40 pages, 5MB images) can take
+// 60–90s. Upload preprocessing is async/background, so allow generous headroom.
+const IMAGE_EXTRACT_TIMEOUT_MS = 5 * 60_000
+
 function makeExec(processRunner: ProcessRunner) {
-	return (cmd: string, args: string[]) => processRunner.execFile(cmd, args, { timeout: 60_000, maxBuffer: 50 * 1024 * 1024 })
+	return (cmd: string, args: string[], timeoutMs: number = MARKITDOWN_TIMEOUT_MS) =>
+		processRunner.execFile(cmd, args, { timeout: timeoutMs, maxBuffer: 50 * 1024 * 1024 })
 }
 
 /** MIME types where markitdown converts to markdown (non-ZIP, non-image) */
@@ -78,7 +86,7 @@ export class MarkitdownPreprocessor implements Preprocessor {
 	private readonly registry: PreprocessorRegistry
 	private readonly logger: Logger
 	private readonly fs: FileSystem
-	private readonly exec: (cmd: string, args: string[]) => Promise<{ stdout: string; stderr: string }>
+	private readonly exec: (cmd: string, args: string[], timeoutMs?: number) => Promise<{ stdout: string; stderr: string }>
 
 	constructor(config: MarkitdownPreprocessorConfig) {
 		this.registry = config.registry
@@ -92,20 +100,29 @@ export class MarkitdownPreprocessor implements Preprocessor {
 		mimeType: string,
 		ctx: PreprocessorContext,
 	): Promise<Result<PreprocessorResult, Error>> {
+		const totalStart = Date.now()
 		const derivedPaths: string[] = []
 		const imageEntries: string[] = []
 
+		this.logger.info('Markitdown processing started', { filePath, mimeType })
+
 		// 1. Convert to markdown via markitdown
 		const contentPathResult = ctx.files.realPath('content.md')
 		if (!contentPathResult.ok) {
 			return Err(new Error('Failed to resolve output path'))
 		}
 
+		const markitdownStart = Date.now()
 		try {
 			await this.fs.mkdir(dirname(contentPathResult.value), { recursive: true })
 			await this.exec('markitdown', [filePath, '-o', contentPathResult.value])
 		} catch (error) {
 			const message = error instanceof Error ? error.message : String(error)
+			this.logger.error(
+				'markitdown CLI failed',
+				error instanceof Error ? error : undefined,
+				{ filePath, mimeType, durationMs: Date.now() - markitdownStart },
+			)
 			if (message.includes('ENOENT')) {
 				return Err(new Error('markitdown not found. Install with: pip install "markitdown[all]"'))
 			}
@@ -117,7 +134,15 @@ export class MarkitdownPreprocessor implements Preprocessor {
 
 		derivedPaths.push('content.md')
 
+		this.logger.info('Markitdown conversion complete', {
+			filePath,
+			mimeType,
+			durationMs: Date.now() - markitdownStart,
+			contentLength: markdown.length,
+		})
+
 		// 2. Extract images based on file type
+		const imagePhaseStart = Date.now()
 		if (PANDOC_EXTRACT_MIMES.has(mimeType)) {
 			const images = await this.extractImagesWithPandoc(filePath, mimeType, ctx)
 			for (const img of images) {
@@ -131,17 +156,20 @@ export class MarkitdownPreprocessor implements Preprocessor {
 				imageEntries.push(`- ${img.relativePath} — ${img.description}`)
 			}
 		}
+		const imagePhaseDurationMs = Date.now() - imagePhaseStart
 
 		// 3. Build manifest
 		const manifestLines: string[] = ['Extracted files:']
 		manifestLines.push(`- content.md (markdown, ${markdown.length} chars)`)
 		manifestLines.push(...imageEntries)
 
-		this.logger.debug('Markitdown processed', {
+		this.logger.info('Markitdown processing complete', {
 			filePath,
 			mimeType,
 			contentLength: markdown.length,
 			imagesExtracted: imageEntries.length,
+			imagePhaseDurationMs,
+			totalDurationMs: Date.now() - totalStart,
 		})
 
 		return Ok({
@@ -162,23 +190,39 @@ export class MarkitdownPreprocessor implements Preprocessor {
 		const format = PANDOC_FORMAT_MAP[mimeType]
 		if (!format) return []
 
+		const pandocStart = Date.now()
+		let extractSucceeded = true
 		try {
-			await this.exec('pandoc', [
-				'-f',
-				format,
-				'-t',
-				'gfm',
+			await this.exec(
+				'pandoc',
+				['-f', format, '-t', 'gfm', filePath, '-o', '/dev/null', `--extract-media=${mediaDirResult.value}`],
+				IMAGE_EXTRACT_TIMEOUT_MS,
+			)
+		} catch (error) {
+			extractSucceeded = false
+			this.logger.warn('pandoc --extract-media failed (will classify any partial output)', {
+				filePath,
+				durationMs: Date.now() - pandocStart,
+				error: error instanceof Error ? error.message : String(error),
+			})
+		}
+		if (extractSucceeded) {
+			this.logger.info('pandoc --extract-media complete', {
 				filePath,
-				'-o',
-				'/dev/null',
-				`--extract-media=${mediaDirResult.value}`,
-			])
-		} catch {
-			this.logger.warn('pandoc --extract-media failed', { filePath })
-			return []
+				format,
+				durationMs: Date.now() - pandocStart,
+			})
 		}
 
-		return classifyExtractedImages(mediaStore, 'media', ctx, this.registry, this.logger)
+		const classifyStart = Date.now()
+		const images = await classifyExtractedImages(mediaStore, 'media', ctx, this.registry, this.logger)
+		this.logger.info('Image classification complete', {
+			source: 'pandoc',
+			count: images.length,
+			partial: !extractSucceeded,
+			durationMs: Date.now() - classifyStart,
+		})
+		return images
 	}
 
 	private async extractImagesWithPdfimages(
@@ -189,14 +233,39 @@ export class MarkitdownPreprocessor implements Preprocessor {
 		const imagesDirResult = imageStore.realPath('')
 		if (!imagesDirResult.ok) return []
 
+		const pdfimagesStart = Date.now()
+		let extractSucceeded = true
 		try {
 			await this.fs.mkdir(imagesDirResult.value, { recursive: true })
-			await this.exec('pdfimages', ['-png', filePath, `${imagesDirResult.value}/img`])
-		} catch {
-			return []
+			await this.exec(
+				'pdfimages',
+				['-png', filePath, `${imagesDirResult.value}/img`],
+				IMAGE_EXTRACT_TIMEOUT_MS,
+			)
+		} catch (error) {
+			extractSucceeded = false
+			this.logger.warn('pdfimages failed (will classify any partial output)', {
+				filePath,
+				durationMs: Date.now() - pdfimagesStart,
+				error: error instanceof Error ? error.message : String(error),
+			})
+		}
+		if (extractSucceeded) {
+			this.logger.info('pdfimages complete', {
+				filePath,
+				durationMs: Date.now() - pdfimagesStart,
+			})
 		}
 
-		return classifyExtractedImages(imageStore, 'images', ctx, this.registry, this.logger)
+		const classifyStart = Date.now()
+		const images = await classifyExtractedImages(imageStore, 'images', ctx, this.registry, this.logger)
+		this.logger.info('Image classification complete', {
+			source: 'pdfimages',
+			count: images.length,
+			partial: !extractSucceeded,
+			durationMs: Date.now() - classifyStart,
+		})
+		return images
 	}
 }
 

package/src/plugins/user-chat/plugin.ts

@@ -208,6 +208,12 @@ const askUserFlatInputSchema = z.object({
 		.optional()
 		.describe("Placeholder text for text input"),
 	multiline: z.boolean().optional().describe("Allow multiline text input"),
+	allowAttachments: z
+		.boolean()
+		.optional()
+		.describe(
+			"Render a file-attach control alongside the text field (text inputType only). Use when the answer needs to come with a file the user already has on hand — logo, brand PDF, source docs, supporting screenshots. Uploaded files reach you the same way as drag-drop attachments in plain chat.",
+		),
 	// rating options
 	min: z
 		.number()
@@ -248,6 +254,7 @@ function transformToAskUserInputType(
 				type: "text",
 				placeholder: input.placeholder,
 				multiline: input.multiline,
+				allowAttachments: input.allowAttachments,
 			};
 		case "confirm":
 			return {
@@ -303,6 +310,56 @@ function formatPendingForLLM(pending: PendingInboundMessage[]): string {
 	return parts.join("\n");
 }
 
+// Some models (notably routed through OpenRouter — Gemini/Llama/Qwen) emit
+// non-ASCII tool argument strings as double-escaped JSON: the wire form is
+// `"\\u0159"`, which JSON.parse turns into a literal 6-char `ř` instead
+// of `ř`. The user then sees raw escape sequences in their UI. Applied only
+// to user-facing display fields (question, placeholder, labels, message body)
+// — never to identifiers like `option.value` (must round-trip back to the LLM
+// unchanged) or to code-bearing inputs of other tools.
+function decodeUnicodeEscapes(value: string): string;
+function decodeUnicodeEscapes(value: string | undefined): string | undefined;
+function decodeUnicodeEscapes(value: string | undefined): string | undefined {
+	if (value === undefined) return undefined;
+	if (!value.includes("\\u")) return value;
+	return value.replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) =>
+		String.fromCharCode(parseInt(hex, 16)),
+	);
+}
+
+function decodeAskUserDisplayStrings(input: AskUserInputType): AskUserInputType {
+	switch (input.type) {
+		case "text":
+			return { ...input, placeholder: decodeUnicodeEscapes(input.placeholder) };
+		case "single_choice":
+		case "multi_choice":
+			return {
+				...input,
+				options: input.options.map((o) => ({
+					...o,
+					label: decodeUnicodeEscapes(o.label),
+					description: decodeUnicodeEscapes(o.description),
+				})),
+			};
+		case "confirm":
+			return {
+				...input,
+				confirmLabel: decodeUnicodeEscapes(input.confirmLabel),
+				cancelLabel: decodeUnicodeEscapes(input.cancelLabel),
+			};
+		case "rating":
+			return input.labels
+				? {
+					...input,
+					labels: {
+						min: decodeUnicodeEscapes(input.labels.min),
+						max: decodeUnicodeEscapes(input.labels.max),
+					},
+				}
+				: input;
+	}
+}
+
 // ============================================================================
 // Plugin
 // ============================================================================
@@ -687,7 +744,7 @@ export const userChatPlugin = definePlugin("user-chat")
 					const format = input.format ?? "text";
 					const result = await ctx.self.tellUser({
 						agentId: context.agentId,
-						message: input.message,
+						message: decodeUnicodeEscapes(input.message),
 						format,
 					});
 					if (!result.ok)
@@ -710,8 +767,8 @@ export const userChatPlugin = definePlugin("user-chat")
 					const inputType = transformToAskUserInputType(input);
 					const result = await ctx.self.askQuestion({
 						agentId: context.agentId,
-						question: input.question,
-						inputType,
+						question: decodeUnicodeEscapes(input.question),
+						inputType: decodeAskUserDisplayStrings(inputType),
 					});
 					if (!result.ok)
 						return Err({ message: result.error.message, recoverable: false });

package/src/plugins/user-chat/schema.ts

@@ -19,9 +19,17 @@ export type AskUserOption = {
 
 /**
  * Input type for ask_user tool - defines how the user should respond.
+ *
+ * `text.allowAttachments` opts the question into an inline file-attach control
+ * next to the text field — used when the agent needs the answer to come with a
+ * file (logo, brand PDF, supporting docs). Attachments piggy-back on the
+ * existing pending-attachments machinery: clients reuse `uploadFile()` /
+ * `pendingAttachments`, so the answer payload itself stays a plain string and
+ * the agent sees uploaded files via the normal `<attachment>` blocks in its
+ * inbox alongside the question answer.
  */
 export type AskUserInputType =
-	| { type: 'text'; placeholder?: string; multiline?: boolean }
+	| { type: 'text'; placeholder?: string; multiline?: boolean; allowAttachments?: boolean }
 	| { type: 'single_choice'; options: AskUserOption[] }
 	| {
 		type: 'multi_choice'
@@ -47,6 +55,7 @@ export const askUserInputTypeSchema = z.discriminatedUnion('type', [
 		type: z.literal('text'),
 		placeholder: z.string().optional(),
 		multiline: z.boolean().optional(),
+		allowAttachments: z.boolean().optional(),
 	}),
 	z.object({
 		type: z.literal('single_choice'),

package/src/plugins/user-chat/user-chat.integration.test.ts

@@ -564,6 +564,105 @@ describe('user-chat plugin', () => {
 		})
 	})
 
+	// =========================================================================
+	// Unicode escape decoding (defensive fix for models that double-escape
+	// non-ASCII in tool argument JSON — see plugin.ts decodeUnicodeEscapes).
+	// =========================================================================
+
+	describe('unicode escape decoding in user-facing fields', () => {
+		it('ask_user (text) → literal \\uXXXX in question/placeholder is decoded', async () => {
+			const harness = new TestHarness({
+				presets: [createTestPreset()],
+				llmProvider: MockLLMProvider.withSequence([
+					{
+						toolCalls: [{
+							id: ToolCallId('tc1'),
+							name: 'ask_user',
+							input: {
+								question: 'Pro\\u010d ne?',
+								inputType: 'text',
+								placeholder: 'Nap\\u0159. ano',
+							},
+						}],
+					},
+					{ content: 'Done', toolCalls: [] },
+				]),
+			})
+
+			const session = await harness.createSession('test')
+			await session.sendAndWaitForIdle('Hi')
+
+			const askNotifications = harness.notifications.getByType('user-chat', 'askUser')
+			expect(askNotifications[0].payload).toMatchObject({
+				question: 'Proč ne?',
+				inputType: { type: 'text', placeholder: 'Např. ano' },
+			})
+
+			await harness.shutdown()
+		})
+
+		it('ask_user (single_choice) → option labels decoded, values left intact', async () => {
+			const harness = new TestHarness({
+				presets: [createTestPreset()],
+				llmProvider: MockLLMProvider.withSequence([
+					{
+						toolCalls: [{
+							id: ToolCallId('tc1'),
+							name: 'ask_user',
+							input: {
+								question: 'Pick',
+								inputType: 'single_choice',
+								options: [
+									{ value: 'kun_ze_\\u0159adu', label: 'K\\u016f\\u0148 ze \\u0159adu' },
+								],
+							},
+						}],
+					},
+					{ content: 'Done', toolCalls: [] },
+				]),
+			})
+
+			const session = await harness.createSession('test')
+			await session.sendAndWaitForIdle('Hi')
+
+			const askNotifications = harness.notifications.getByType('user-chat', 'askUser')
+			expect(askNotifications[0].payload).toMatchObject({
+				inputType: {
+					type: 'single_choice',
+					// Value preserved verbatim so answer round-trip stays consistent
+					// with what the LLM emitted.
+					options: [{ value: 'kun_ze_\\u0159adu', label: 'Kůň ze řadu' }],
+				},
+			})
+
+			await harness.shutdown()
+		})
+
+		it('tell_user → literal \\uXXXX in message is decoded', async () => {
+			const harness = new TestHarness({
+				presets: [createTestPreset()],
+				llmProvider: MockLLMProvider.withSequence([
+					{
+						toolCalls: [{
+							id: ToolCallId('tc1'),
+							name: 'tell_user',
+							input: { message: 'Ahoj sv\\u011bte' },
+						}],
+					},
+					{ content: 'Done', toolCalls: [] },
+				]),
+			})
+
+			const session = await harness.createSession('test')
+			await session.sendAndWaitForIdle('Hi')
+
+			const msgs = harness.notifications.getByType('user-chat', 'agentMessage')
+			expect(msgs[0].payload).toMatchObject({ content: 'Ahoj světe' })
+
+			await harness.shutdown()
+		})
+	})
+
 	// =========================================================================
 	// XML mode
 	// =========================================================================