@ixo/oracle-runtime-workers 0.2.0

package/src/attachments/extract.ts

@@ -0,0 +1,241 @@
+/**
+ * The "model-extract" lane — turn an attachment into text with the helper
+ * (vision) model through the provider's OpenAI-compatible chat endpoint, the
+ * way the Node `FileProcessingService` does: images are described, audio is
+ * transcribed (`input_audio`), video described (`video_url`), documents
+ * extracted verbatim (`file` part — the provider parses PDFs server-side).
+ * Plain text is read locally and never costs a model call.
+ *
+ * Reduced vs Node: there is no local PDF/office parser on workerd
+ * (`loadFileFromBuffer` is Node-only), so every non-plain-text document goes
+ * straight to the helper model's `file` lane instead of "local parse, then
+ * model on failure".
+ */
+import { bytesToBase64 } from '../plugins/base64';
+import { validateUrlTarget } from './download';
+import type { FileCategory } from './magic';
+import { isPlainTextType } from './magic';
+import { sanitizeAttachmentFilename } from './sanitize';
+import type { AttachmentInput } from './types';
+
+/** Provider wiring for the helper model — the platform key, never a BYO one. */
+export interface ExtractionProvider {
+  baseURL: string;
+  apiKey: string;
+  headers: Record<string, string>;
+  /** The `vision` role's model id. */
+  model: string;
+}
+
+export interface ExtractionUsage {
+  cost?: number;
+  promptTokens?: number;
+  completionTokens?: number;
+}
+
+export interface ExtractionResult {
+  content: string;
+  usage?: ExtractionUsage;
+}
+
+export interface ExtractOptions {
+  fetchImpl?: typeof fetch;
+  signal?: AbortSignal;
+  timeoutMs?: number;
+}
+
+export const MAX_TEXT_LENGTH = 50_000;
+const AI_PROCESS_TIMEOUT_MS = 120_000;
+const MAX_ERROR_BODY_LENGTH = 1024;
+
+export const PROMPTS: Record<Exclude<FileCategory, 'unsupported'>, string> = {
+  document: 'Extract all text content from this document verbatim.',
+  image:
+    'Describe this image in detail. Include all text, numbers, labels, and visual elements.',
+  audio: 'Transcribe this audio completely. Include all spoken words.',
+  video:
+    'Describe this video in detail. Include actions, text overlays, and spoken content.',
+};
+
+export function formatContent(
+  label: string,
+  filename: string,
+  content: string,
+): string {
+  return `[${label} of ${filename}]:\n${content}`;
+}
+
+export function truncateText(text: string): string {
+  if (text.length <= MAX_TEXT_LENGTH) return text;
+  return `${text.slice(0, MAX_TEXT_LENGTH)}\n\n[...truncated]`;
+}
+
+export function getAudioFormat(
+  mimetype: string,
+): 'mp3' | 'wav' | 'ogg' | 'flac' | 'webm' | 'mp4' | 'aac' {
+  if (mimetype.includes('mp3') || mimetype.includes('mpeg')) return 'mp3';
+  if (mimetype.includes('wav')) return 'wav';
+  if (mimetype.includes('ogg')) return 'ogg';
+  if (mimetype.includes('flac')) return 'flac';
+  if (mimetype.includes('webm')) return 'webm';
+  if (mimetype.includes('mp4') || mimetype.includes('m4a')) return 'mp4';
+  if (mimetype.includes('aac')) return 'aac';
+  return 'mp3';
+}
+
+interface ChatCompletion {
+  choices?: Array<{ message?: { content?: string | null } }>;
+  usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+    total_tokens?: number;
+    cost?: number;
+  };
+  model?: string;
+}
+
+async function chatCompletion(
+  provider: ExtractionProvider,
+  contentParts: Array<Record<string, unknown>>,
+  opts: ExtractOptions,
+): Promise<ExtractionResult> {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const controller = new AbortController();
+  const onAbort = (): void => controller.abort();
+  opts.signal?.addEventListener('abort', onAbort);
+  if (opts.signal?.aborted) controller.abort();
+  const timer = setTimeout(onAbort, opts.timeoutMs ?? AI_PROCESS_TIMEOUT_MS);
+  try {
+    const response = await fetchImpl(
+      `${provider.baseURL.replace(/\/+$/, '')}/chat/completions`,
+      {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${provider.apiKey}`,
+          'Content-Type': 'application/json',
+          ...provider.headers,
+        },
+        body: JSON.stringify({
+          model: provider.model,
+          messages: [{ role: 'user', content: contentParts }],
+        }),
+        signal: controller.signal,
+      },
+    );
+    if (!response.ok) {
+      const errorText = await response.text().catch(() => '');
+      throw new Error(
+        `AI processing failed (${response.status}): ${errorText.slice(0, MAX_ERROR_BODY_LENGTH)}`,
+      );
+    }
+    const raw: unknown = await response.json();
+    const result = raw as ChatCompletion;
+    return {
+      content: result.choices?.[0]?.message?.content ?? '',
+      ...(result.usage
+        ? {
+            usage: {
+              cost: result.usage.cost,
+              promptTokens: result.usage.prompt_tokens,
+              completionTokens: result.usage.completion_tokens,
+            },
+          }
+        : {}),
+    };
+  } finally {
+    clearTimeout(timer);
+    opts.signal?.removeEventListener('abort', onAbort);
+  }
+}
+
+/**
+ * Hand a public image/video URL to the provider directly (no re-upload of
+ * the bytes). The URL is SSRF-validated first.
+ */
+export async function aiProcessFromUrl(
+  url: string,
+  category: 'image' | 'video',
+  provider: ExtractionProvider,
+  opts: ExtractOptions = {},
+): Promise<ExtractionResult> {
+  validateUrlTarget(url);
+  const parts: Array<Record<string, unknown>> = [
+    { type: 'text', text: PROMPTS[category] },
+    category === 'image'
+      ? { type: 'image_url', image_url: { url } }
+      : { type: 'video_url', video_url: { url } },
+  ];
+  return chatCompletion(provider, parts, opts);
+}
+
+/** Send the bytes inline (data URI / base64) with the category's prompt. */
+export async function aiProcess(
+  bytes: Uint8Array,
+  mimetype: string,
+  category: Exclude<FileCategory, 'unsupported'>,
+  filename: string,
+  provider: ExtractionProvider,
+  opts: ExtractOptions = {},
+): Promise<ExtractionResult> {
+  const base64 = bytesToBase64(bytes);
+  const dataUri = `data:${mimetype};base64,${base64}`;
+  const parts: Array<Record<string, unknown>> = [
+    { type: 'text', text: PROMPTS[category] },
+  ];
+  if (category === 'image') {
+    parts.push({ type: 'image_url', image_url: { url: dataUri } });
+  } else if (category === 'audio') {
+    parts.push({
+      type: 'input_audio',
+      input_audio: { data: base64, format: getAudioFormat(mimetype) },
+    });
+  } else if (category === 'document') {
+    parts.push({ type: 'file', file: { filename, file_data: dataUri } });
+  } else {
+    parts.push({ type: 'video_url', video_url: { url: dataUri } });
+  }
+  return chatCompletion(provider, parts, opts);
+}
+
+/**
+ * Text for one downloaded attachment by category — the Node
+ * `processDocument` / `processImage` / `processAudio` / `processVideo`
+ * dispatch. Plain text never touches the model.
+ */
+export async function extractText(
+  bytes: Uint8Array,
+  attachment: AttachmentInput,
+  category: Exclude<FileCategory, 'unsupported'>,
+  provider: ExtractionProvider | null,
+  opts: ExtractOptions = {},
+): Promise<{ text: string; usage?: ExtractionUsage }> {
+  const safeName = sanitizeAttachmentFilename(attachment.filename);
+  if (category === 'document' && isPlainTextType(attachment.mimetype)) {
+    const text = new TextDecoder().decode(bytes);
+    return { text: formatContent('Content', safeName, truncateText(text)) };
+  }
+  if (!provider) {
+    throw new Error(
+      'no helper model configured for attachment extraction on this host',
+    );
+  }
+  const label =
+    category === 'document'
+      ? 'Content'
+      : category === 'audio'
+        ? 'Transcription'
+        : 'Description';
+  const { content, usage } = await aiProcess(
+    bytes,
+    attachment.mimetype,
+    category,
+    attachment.filename,
+    provider,
+    opts,
+  );
+  const body = category === 'document' ? truncateText(content) : content;
+  return {
+    text: formatContent(label, safeName, body),
+    ...(usage ? { usage } : {}),
+  };
+}

package/src/attachments/index.ts

@@ -0,0 +1,84 @@
+export { classifyAttachment, type AttachmentKind } from './classify';
+export {
+  buildUserMessageContent,
+  type NativeAttachment,
+} from './content-blocks';
+export {
+  ALLOWED_URI_SCHEMES,
+  downloadFromUrl,
+  loadAttachmentBytes,
+  MAX_FILE_SIZE,
+  MAX_TOTAL_SIZE,
+  validateUrlTarget,
+  type DownloadOptions,
+  type MatrixMediaSource,
+} from './download';
+export {
+  aiProcess,
+  aiProcessFromUrl,
+  extractText,
+  formatContent,
+  getAudioFormat,
+  MAX_TEXT_LENGTH,
+  PROMPTS,
+  truncateText,
+  type ExtractionProvider,
+  type ExtractionResult,
+  type ExtractionUsage,
+} from './extract';
+export {
+  categorizeFile,
+  detectMimeFromMagicBytes,
+  isDocumentType,
+  isPlainTextType,
+  verifyMagicBytes,
+  type FileCategory,
+} from './magic';
+export {
+  prepareAttachments,
+  SANDBOX_TRUNCATE_LIMIT,
+  type AttachmentPipelineDeps,
+  type ExtractedAttachment,
+  type PrepareAttachmentsParams,
+  type PreparedAttachments,
+} from './pipeline';
+export { routeAttachment, type AttachmentStrategy } from './route';
+export {
+  buildAnalysisMarkdown,
+  uploadToSandbox,
+  type SandboxUploadConfig,
+  type SandboxUploadResult,
+} from './sandbox-archive';
+export {
+  SANDBOX_OUTPUT_PREFIX,
+  sanitizeAttachmentFilename,
+  sanitizeSandboxPath,
+} from './sanitize';
+export {
+  MAX_ATTACHMENTS,
+  parseAttachmentInputs,
+  type AttachmentInput,
+} from './types';
+export {
+  applyAttachmentRetention,
+  ATTACHMENT_PAYLOAD_TURNS,
+  ATTACHMENT_PLACEHOLDER_PREFIX,
+  ATTACHMENT_VIEW_SOURCE,
+  attachmentMetas,
+  attachmentRef,
+  buildPlaceholderText,
+  hasInlinePayload,
+  INLINE_PAYLOAD_NEEDLE,
+  isAttachmentPlaceholderText,
+  isAttachmentViewMessage,
+  isInlinePayloadBlock,
+  offloadInlinePayloads,
+  VIEW_ATTACHMENT_TOOL_NAME,
+  type AttachmentRetentionResult,
+} from './retention';
+export {
+  viewAttachment,
+  type AttachmentView,
+  type AttachmentViewSurface,
+  type ViewAttachmentDeps,
+} from './view';

package/src/attachments/magic.ts

@@ -0,0 +1,133 @@
+/**
+ * Content sniffing + the extraction lane's coarse file category — ports of
+ * the Node `FileProcessingService` helpers (`detectMimeFromMagicBytes`,
+ * `verifyMagicBytes`, `categorizeFile`, `isDocumentType`, `isPlainTextType`).
+ *
+ * Note the two taxonomies deliberately differ, as on Node: `classify.ts`
+ * drives ROUTING (plain text is its own kind), while `FileCategory` drives
+ * EXTRACTION (plain text counts as a document that is read locally).
+ */
+
+export type FileCategory =
+  | 'document'
+  | 'image'
+  | 'audio'
+  | 'video'
+  | 'unsupported';
+
+const MAGIC_BYTES: Array<{ bytes: number[]; offset?: number; mime: string }> = [
+  // Images
+  { bytes: [0x89, 0x50, 0x4e, 0x47], mime: 'image/png' },
+  { bytes: [0xff, 0xd8, 0xff], mime: 'image/jpeg' },
+  { bytes: [0x47, 0x49, 0x46, 0x38], mime: 'image/gif' },
+  { bytes: [0x52, 0x49, 0x46, 0x46], mime: 'image/webp' }, // RIFF (WebP container)
+  { bytes: [0x42, 0x4d], mime: 'image/bmp' },
+  { bytes: [0x49, 0x49, 0x2a, 0x00], mime: 'image/tiff' },
+  { bytes: [0x4d, 0x4d, 0x00, 0x2a], mime: 'image/tiff' },
+  // Documents
+  { bytes: [0x25, 0x50, 0x44, 0x46], mime: 'application/pdf' },
+  { bytes: [0x50, 0x4b, 0x03, 0x04], mime: 'application/zip' }, // docx/xlsx/…
+  { bytes: [0xd0, 0xcf, 0x11, 0xe0], mime: 'application/msword' }, // OLE2
+  // Audio
+  { bytes: [0x49, 0x44, 0x33], mime: 'audio/mpeg' }, // ID3
+  { bytes: [0xff, 0xfb], mime: 'audio/mpeg' },
+  { bytes: [0xff, 0xf3], mime: 'audio/mpeg' },
+  { bytes: [0x4f, 0x67, 0x67, 0x53], mime: 'audio/ogg' },
+  { bytes: [0x66, 0x4c, 0x61, 0x43], mime: 'audio/flac' },
+  // Video
+  { bytes: [0x1a, 0x45, 0xdf, 0xa3], mime: 'video/webm' }, // EBML (WebM/MKV)
+  { bytes: [0x66, 0x74, 0x79, 0x70], offset: 4, mime: 'video/mp4' }, // `ftyp`
+];
+
+const MAGIC_MIME_CATEGORIES: Record<string, FileCategory[]> = {
+  'image/png': ['image'],
+  'image/jpeg': ['image'],
+  'image/gif': ['image'],
+  'image/webp': ['image'],
+  'image/bmp': ['image'],
+  'image/tiff': ['image'],
+  'application/pdf': ['document'],
+  'application/zip': ['document'],
+  'application/msword': ['document'],
+  'audio/mpeg': ['audio'],
+  'audio/ogg': ['audio'],
+  'audio/flac': ['audio'],
+  'video/webm': ['video', 'audio'],
+  'video/mp4': ['video', 'audio'],
+};
+
+/** The MIME a file's magic bytes announce, or null when no signature matches. */
+export function detectMimeFromMagicBytes(bytes: Uint8Array): string | null {
+  if (bytes.length < 4) return null;
+  for (const sig of MAGIC_BYTES) {
+    const offset = sig.offset ?? 0;
+    if (bytes.length < offset + sig.bytes.length) continue;
+    let match = true;
+    for (let i = 0; i < sig.bytes.length; i += 1) {
+      if (bytes[offset + i] !== sig.bytes[i]) {
+        match = false;
+        break;
+      }
+    }
+    if (match) return sig.mime;
+  }
+  return null;
+}
+
+export function isPlainTextType(mimetype: string): boolean {
+  return (
+    mimetype.startsWith('text/') ||
+    mimetype === 'application/json' ||
+    mimetype === 'application/xml' ||
+    mimetype === 'application/rtf'
+  );
+}
+
+export function isDocumentType(mimetype: string): boolean {
+  return (
+    mimetype.startsWith('text/') ||
+    mimetype === 'application/pdf' ||
+    mimetype === 'application/msword' ||
+    mimetype === 'application/json' ||
+    mimetype === 'application/xml' ||
+    mimetype === 'application/rtf' ||
+    mimetype.startsWith('application/vnd.openxmlformats-officedocument.') ||
+    mimetype === 'application/vnd.ms-excel' ||
+    mimetype === 'application/vnd.ms-powerpoint'
+  );
+}
+
+export function categorizeFile(mimetype: string): FileCategory {
+  if (isDocumentType(mimetype)) return 'document';
+  if (mimetype.startsWith('image/')) return 'image';
+  if (mimetype.startsWith('audio/')) return 'audio';
+  if (mimetype.startsWith('video/')) return 'video';
+  return 'unsupported';
+}
+
+/**
+ * Verify that the bytes are consistent with the claimed category. Strict for
+ * binary formats; skipped for text-based mimetypes (no reliable signature).
+ * Throws on a mismatch (e.g. an executable claiming to be a PNG).
+ */
+export function verifyMagicBytes(
+  bytes: Uint8Array,
+  claimedCategory: FileCategory,
+  attachment: { mimetype: string; filename: string },
+  warn?: (message: string) => void,
+): void {
+  if (isPlainTextType(attachment.mimetype)) return;
+  const detected = detectMimeFromMagicBytes(bytes);
+  if (!detected) {
+    warn?.(
+      `No magic bytes match for ${attachment.filename} (claimed: ${attachment.mimetype})`,
+    );
+    return;
+  }
+  const allowed = MAGIC_MIME_CATEGORIES[detected];
+  if (!allowed || !allowed.includes(claimedCategory)) {
+    throw new Error(
+      `File content mismatch: claimed ${attachment.mimetype} but detected ${detected}`,
+    );
+  }
+}