@echofiles/echo-pdf 0.2.0

package/src/pdf-agent.ts

@@ -0,0 +1,224 @@
+import type { Env, FileStore, ReturnMode } from "./types"
+import type { AgentTraceEvent, EchoPdfConfig, PdfOperationRequest } from "./pdf-types"
+import { resolveModelForProvider, resolveProviderAlias } from "./agent-defaults"
+import { fromBase64, normalizeReturnMode, toDataUrl } from "./file-utils"
+import { extractPdfPageText, getPdfPageCount, renderPdfPageToPng, toBytes } from "./pdfium-engine"
+import { visionRecognize } from "./provider-client"
+
+interface RuntimeOptions {
+  readonly trace?: (event: AgentTraceEvent) => void
+  readonly fileStore: FileStore
+}
+
+const traceStep = (
+  opts: RuntimeOptions,
+  phase: AgentTraceEvent["phase"],
+  name: string,
+  payload?: unknown,
+  level?: AgentTraceEvent["level"]
+): void => {
+  if (!opts.trace) return
+  opts.trace({ kind: "step", phase, name, payload, level })
+}
+
+const ensurePages = (pages: ReadonlyArray<number>, pageCount: number, maxPages: number): number[] => {
+  if (pages.length === 0) throw new Error("At least one page is required")
+  if (pages.length > maxPages) throw new Error(`Page count exceeds maxPagesPerRequest (${maxPages})`)
+  for (const page of pages) {
+    if (!Number.isInteger(page) || page < 1 || page > pageCount) {
+      throw new Error(`Page ${page} out of range 1..${pageCount}`)
+    }
+  }
+  return [...new Set(pages)].sort((a, b) => a - b)
+}
+
+export const ingestPdfFromPayload = async (
+  config: EchoPdfConfig,
+  input: {
+    readonly fileId?: string
+    readonly url?: string
+    readonly base64?: string
+    readonly filename?: string
+  },
+  opts: RuntimeOptions
+): Promise<{ id: string; filename: string; bytes: Uint8Array }> => {
+  if (input.fileId) {
+    const existing = await opts.fileStore.get(input.fileId)
+    if (!existing) {
+      throw new Error(`File not found: ${input.fileId}`)
+    }
+    return {
+      id: existing.id,
+      filename: existing.filename,
+      bytes: existing.bytes,
+    }
+  }
+
+  let bytes: Uint8Array | null = null
+  let filename = input.filename ?? "document.pdf"
+
+  if (input.url) {
+    traceStep(opts, "start", "file.fetch.url", { url: input.url })
+    bytes = await toBytes(input.url)
+    try {
+      const u = new URL(input.url)
+      filename = decodeURIComponent(u.pathname.split("/").pop() || filename)
+    } catch {
+      // ignore URL parse failure
+    }
+    traceStep(opts, "end", "file.fetch.url", { sizeBytes: bytes.byteLength })
+  } else if (input.base64) {
+    traceStep(opts, "start", "file.decode.base64")
+    bytes = fromBase64(input.base64)
+    traceStep(opts, "end", "file.decode.base64", { sizeBytes: bytes.byteLength })
+  }
+
+  if (!bytes) {
+    throw new Error("Missing file input. Provide fileId, url or base64")
+  }
+  if (bytes.byteLength > config.service.maxPdfBytes) {
+    throw new Error(`PDF exceeds max size (${config.service.maxPdfBytes} bytes)`)
+  }
+
+  const meta = await opts.fileStore.put({
+    filename,
+    mimeType: "application/pdf",
+    bytes,
+  })
+  traceStep(opts, "end", "file.stored", { fileId: meta.id, sizeBytes: meta.sizeBytes })
+  return {
+    id: meta.id,
+    filename: meta.filename,
+    bytes,
+  }
+}
+
+const resolveReturnMode = (value: ReturnMode | undefined): ReturnMode => normalizeReturnMode(value)
+
+const stripCodeFences = (value: string): string => {
+  const text = value.trim()
+  const fenced = text.match(/^```[a-zA-Z0-9_-]*\n([\s\S]*?)\n```$/)
+  return typeof fenced?.[1] === "string" ? fenced[1].trim() : text
+}
+
+const extractTabularLatex = (value: string): string => {
+  const text = stripCodeFences(value)
+  const blocks = text.match(/\\begin\{tabular\}[\s\S]*?\\end\{tabular\}/g)
+  if (!blocks || blocks.length === 0) return ""
+  return blocks.map((b) => b.trim()).join("\n\n")
+}
+
+export const runPdfAgent = async (
+  config: EchoPdfConfig,
+  env: Env,
+  request: PdfOperationRequest,
+  opts: RuntimeOptions
+): Promise<unknown> => {
+  traceStep(opts, "start", "pdf.operation", { operation: request.operation })
+  const file = await ingestPdfFromPayload(config, request, opts)
+  const pageCount = await getPdfPageCount(config, file.bytes)
+  traceStep(opts, "log", "pdf.meta", { fileId: file.id, pageCount })
+
+  const pages = ensurePages(request.pages, pageCount, config.service.maxPagesPerRequest)
+  const scale = request.renderScale ?? config.service.defaultRenderScale
+  const returnMode = resolveReturnMode(request.returnMode)
+  if (returnMode === "url") {
+    throw new Error("returnMode=url is not implemented; use inline or file_id")
+  }
+
+  if (request.operation === "extract_pages") {
+    const images: Array<{ page: number; mimeType: string; data?: string; fileId?: string; url?: string | null }> = []
+    for (const page of pages) {
+      traceStep(opts, "start", "render.page", { page })
+      const rendered = await renderPdfPageToPng(config, file.bytes, page - 1, scale)
+      if (returnMode === "file_id") {
+        const stored = await opts.fileStore.put({
+          filename: `${file.filename}-p${page}.png`,
+          mimeType: "image/png",
+          bytes: rendered.png,
+        })
+        images.push({ page, mimeType: "image/png", fileId: stored.id })
+      } else {
+        images.push({
+          page,
+          mimeType: "image/png",
+          data: toDataUrl(rendered.png, "image/png"),
+        })
+      }
+      traceStep(opts, "end", "render.page", { page, width: rendered.width, height: rendered.height })
+    }
+    const result = { fileId: file.id, pageCount, returnMode, images }
+    traceStep(opts, "end", "pdf.operation", { operation: request.operation })
+    return result
+  }
+
+  const providerAlias = resolveProviderAlias(config, request.provider)
+  const model = resolveModelForProvider(config, providerAlias, request.model)
+  if (!model) {
+    throw new Error("model is required for OCR or table extraction; set agent.defaultModel")
+  }
+
+  if (request.operation === "ocr_pages") {
+    const results: Array<{ page: number; text: string }> = []
+    for (const page of pages) {
+      traceStep(opts, "start", "ocr.page", { page })
+      const rendered = await renderPdfPageToPng(config, file.bytes, page - 1, scale)
+      const imageDataUrl = toDataUrl(rendered.png, "image/png")
+      const fallbackText = await extractPdfPageText(config, file.bytes, page - 1)
+      const prompt = request.prompt?.trim() || config.agent.ocrPrompt
+      const llmText = await visionRecognize({
+        config,
+        env,
+        providerAlias,
+        model,
+        prompt,
+        imageDataUrl,
+        runtimeApiKeys: request.providerApiKeys,
+      })
+      const text = stripCodeFences(llmText || fallbackText || "")
+      results.push({ page, text })
+      traceStep(opts, "end", "ocr.page", { page, chars: text.length })
+    }
+    const result = {
+      fileId: file.id,
+      pageCount,
+      provider: providerAlias,
+      model,
+      pages: results,
+    }
+    traceStep(opts, "end", "pdf.operation", { operation: request.operation })
+    return result
+  }
+
+  const tables: Array<{ page: number; latex: string }> = []
+  for (const page of pages) {
+    traceStep(opts, "start", "table.page", { page })
+    const rendered = await renderPdfPageToPng(config, file.bytes, page - 1, scale)
+    const imageDataUrl = toDataUrl(rendered.png, "image/png")
+    const prompt = request.prompt?.trim() || config.agent.tablePrompt
+    const rawLatex = await visionRecognize({
+      config,
+      env,
+      providerAlias,
+      model,
+      prompt,
+      imageDataUrl,
+      runtimeApiKeys: request.providerApiKeys,
+    })
+    const latex = extractTabularLatex(rawLatex)
+    if (!latex) {
+      throw new Error(`table extraction did not return valid LaTeX tabular for page ${page}`)
+    }
+    tables.push({ page, latex })
+    traceStep(opts, "end", "table.page", { page, chars: latex.length })
+  }
+  const result = {
+    fileId: file.id,
+    pageCount,
+    provider: providerAlias,
+    model,
+    pages: tables,
+  }
+  traceStep(opts, "end", "pdf.operation", { operation: request.operation })
+  return result
+}

package/src/pdf-config.ts

@@ -0,0 +1,105 @@
+import rawConfig from "../echo-pdf.config.json"
+import type { Env, JsonObject, JsonValue } from "./types"
+import type { EchoPdfConfig } from "./pdf-types"
+
+const ENV_PATTERN = /\$\{([A-Z0-9_]+)\}/g
+
+const isObject = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value)
+
+const interpolateEnv = (input: string, env: Env): string =>
+  input.replace(ENV_PATTERN, (_, name: string) => {
+    const value = env[name]
+    return typeof value === "string" ? value : `\${${name}}`
+  })
+
+const resolveEnvRefs = (value: JsonValue, env: Env): JsonValue => {
+  if (typeof value === "string") return interpolateEnv(value, env)
+  if (Array.isArray(value)) return value.map((item) => resolveEnvRefs(item, env))
+  if (isObject(value)) {
+    const out: JsonObject = {}
+    for (const [key, nested] of Object.entries(value)) {
+      out[key] = resolveEnvRefs(nested as JsonValue, env)
+    }
+    return out
+  }
+  return value
+}
+
+const validateConfig = (config: EchoPdfConfig): EchoPdfConfig => {
+  if (!config.service?.name) throw new Error("service.name is required")
+  if (!config.pdfium?.wasmUrl) throw new Error("pdfium.wasmUrl is required")
+  if (!config.service?.storage) throw new Error("service.storage is required")
+  if (!Number.isFinite(config.service.storage.maxFileBytes) || config.service.storage.maxFileBytes <= 0) {
+    throw new Error("service.storage.maxFileBytes must be positive")
+  }
+  if (config.service.storage.maxFileBytes < config.service.maxPdfBytes) {
+    throw new Error("service.storage.maxFileBytes must be >= service.maxPdfBytes")
+  }
+  if (!Number.isFinite(config.service.storage.maxTotalBytes) || config.service.storage.maxTotalBytes <= 0) {
+    throw new Error("service.storage.maxTotalBytes must be positive")
+  }
+  if (config.service.storage.maxTotalBytes < config.service.storage.maxFileBytes) {
+    throw new Error("service.storage.maxTotalBytes must be >= maxFileBytes")
+  }
+  if (!Number.isFinite(config.service.storage.ttlHours) || config.service.storage.ttlHours <= 0) {
+    throw new Error("service.storage.ttlHours must be positive")
+  }
+  if (!Number.isFinite(config.service.storage.cleanupBatchSize) || config.service.storage.cleanupBatchSize <= 0) {
+    throw new Error("service.storage.cleanupBatchSize must be positive")
+  }
+  if (!config.agent?.defaultProvider) throw new Error("agent.defaultProvider is required")
+  if (!config.providers?.[config.agent.defaultProvider]) {
+    throw new Error(`default provider "${config.agent.defaultProvider}" missing`)
+  }
+  if (typeof config.agent.defaultModel !== "string") {
+    throw new Error("agent.defaultModel must be a string")
+  }
+  return config
+}
+
+export const loadEchoPdfConfig = (env: Env): EchoPdfConfig => {
+  const fromEnv = env.ECHO_PDF_CONFIG_JSON?.trim()
+  const configJson = fromEnv ? JSON.parse(fromEnv) : rawConfig
+  const resolved = resolveEnvRefs(configJson as unknown as JsonValue, env) as unknown as EchoPdfConfig
+
+  const providerOverride = env.ECHO_PDF_DEFAULT_PROVIDER
+  const modelOverride = env.ECHO_PDF_DEFAULT_MODEL
+  const withOverrides: EchoPdfConfig = {
+    ...resolved,
+    agent: {
+      ...resolved.agent,
+      defaultProvider:
+        typeof providerOverride === "string" && providerOverride.trim().length > 0
+          ? providerOverride.trim()
+          : resolved.agent.defaultProvider,
+      defaultModel:
+        typeof modelOverride === "string" && modelOverride.trim().length > 0
+          ? modelOverride.trim()
+          : resolved.agent.defaultModel,
+    },
+  }
+
+  return validateConfig(withOverrides)
+}
+
+export const readRequiredEnv = (env: Env, key: string): string => {
+  const read = (name: string): string | null => {
+    const value = env[name]
+    return typeof value === "string" && value.trim().length > 0 ? value.trim() : null
+  }
+  const direct = read(key)
+  if (direct) return direct
+
+  // Backward compatibility: allow *_KEY and *_API_KEY aliases.
+  if (key.endsWith("_API_KEY")) {
+    const alt = read(key.replace(/_API_KEY$/, "_KEY"))
+    if (alt) return alt
+  }
+  if (key.endsWith("_KEY")) {
+    const alt = read(key.replace(/_KEY$/, "_API_KEY"))
+    if (alt) return alt
+  }
+
+  throw new Error(`Missing required env var "${key}"`)
+}

package/src/pdf-storage.ts

@@ -0,0 +1,94 @@
+import { DurableObjectFileStore } from "./file-store-do"
+import type { EchoPdfConfig } from "./pdf-types"
+import type { Env, FileStore, StoredFileMeta, StoredFileRecord } from "./types"
+
+class InMemoryFileStore implements FileStore {
+  private readonly store = new Map<string, StoredFileRecord>()
+
+  async put(input: {
+    readonly filename: string
+    readonly mimeType: string
+    readonly bytes: Uint8Array
+  }): Promise<StoredFileMeta> {
+    const id = crypto.randomUUID()
+    const record: StoredFileRecord = {
+      id,
+      filename: input.filename,
+      mimeType: input.mimeType,
+      sizeBytes: input.bytes.byteLength,
+      createdAt: new Date().toISOString(),
+      bytes: input.bytes,
+    }
+    this.store.set(id, record)
+    return this.toMeta(record)
+  }
+
+  async get(fileId: string): Promise<StoredFileRecord | null> {
+    return this.store.get(fileId) ?? null
+  }
+
+  async list(): Promise<ReadonlyArray<StoredFileMeta>> {
+    return [...this.store.values()].map((record) => this.toMeta(record))
+  }
+
+  async delete(fileId: string): Promise<boolean> {
+    return this.store.delete(fileId)
+  }
+
+  private toMeta(record: StoredFileRecord): StoredFileMeta {
+    return {
+      id: record.id,
+      filename: record.filename,
+      mimeType: record.mimeType,
+      sizeBytes: record.sizeBytes,
+      createdAt: record.createdAt,
+    }
+  }
+}
+
+const fallbackStore = new InMemoryFileStore()
+
+export interface RuntimeFileStoreBundle {
+  readonly store: FileStore
+  stats: () => Promise<unknown>
+  cleanup: () => Promise<unknown>
+}
+
+export const getRuntimeFileStore = (env: Env, config: EchoPdfConfig): RuntimeFileStoreBundle => {
+  if (env.FILE_STORE_DO) {
+    const store = new DurableObjectFileStore(env.FILE_STORE_DO, config.service.storage)
+    return {
+      store,
+      stats: async () => store.stats(),
+      cleanup: async () => store.cleanup(),
+    }
+  }
+
+  return {
+    store: fallbackStore,
+    stats: async () => {
+      const files = await fallbackStore.list()
+      const totalBytes = files.reduce((sum, file) => sum + file.sizeBytes, 0)
+      return {
+        backend: "memory",
+        policy: config.service.storage,
+        stats: {
+          fileCount: files.length,
+          totalBytes,
+        },
+      }
+    },
+    cleanup: async () => ({
+      backend: "memory",
+      deletedExpired: 0,
+      deletedEvicted: 0,
+      stats: await (async () => {
+        const files = await fallbackStore.list()
+        return {
+          fileCount: files.length,
+          totalBytes: files.reduce((sum, file) => sum + file.sizeBytes, 0),
+        }
+      })(),
+    }),
+  }
+}

package/src/pdf-types.ts

@@ -0,0 +1,79 @@
+import type { ProviderType, ReturnMode } from "./types"
+
+export interface EchoPdfProviderConfig {
+  readonly type: ProviderType
+  readonly apiKeyEnv: string
+  readonly baseUrl?: string
+  readonly headers?: Record<string, string>
+  readonly timeoutMs?: number
+  readonly endpoints?: {
+    readonly chatCompletionsPath?: string
+    readonly modelsPath?: string
+  }
+}
+
+export interface StoragePolicy {
+  readonly maxFileBytes: number
+  readonly maxTotalBytes: number
+  readonly ttlHours: number
+  readonly cleanupBatchSize: number
+}
+
+export interface EchoPdfConfig {
+  readonly service: {
+    readonly name: string
+    readonly maxPdfBytes: number
+    readonly maxPagesPerRequest: number
+    readonly defaultRenderScale: number
+    readonly storage: StoragePolicy
+  }
+  readonly pdfium: {
+    readonly wasmUrl: string
+  }
+  readonly agent: {
+    readonly defaultProvider: string
+    readonly defaultModel: string
+    readonly ocrPrompt: string
+    readonly tablePrompt: string
+  }
+  readonly providers: Record<string, EchoPdfProviderConfig>
+  readonly mcp: {
+    readonly serverName: string
+    readonly version: string
+    readonly authHeader?: string
+    readonly authEnv?: string
+  }
+}
+
+export interface AgentTraceEvent {
+  readonly kind: "step"
+  readonly phase: "start" | "end" | "log"
+  readonly name: string
+  readonly level?: "info" | "error"
+  readonly payload?: unknown
+}
+
+export interface PdfOperationRequest {
+  readonly operation: "extract_pages" | "ocr_pages" | "tables_to_latex"
+  readonly fileId?: string
+  readonly url?: string
+  readonly base64?: string
+  readonly filename?: string
+  readonly pages: ReadonlyArray<number>
+  readonly renderScale?: number
+  readonly provider?: string
+  readonly model: string
+  readonly providerApiKeys?: Record<string, string>
+  readonly returnMode?: ReturnMode
+  readonly prompt?: string
+}
+
+export interface ToolSchema {
+  readonly name: string
+  readonly description: string
+  readonly inputSchema: Record<string, unknown>
+  readonly source: {
+    readonly kind: "local"
+    readonly toolName: string
+  }
+}