@mcptoolshop/toolshopstudio 1.1.0-toolshop

package/src/pandoc/full.test.ts

@@ -0,0 +1,211 @@
+import { describe, it, expect, vi } from "vitest";
+import {
+  convertDocument,
+  type ConvertDocumentContext,
+} from "./convert.js";
+import { createPandocCRUD } from "./crud.js";
+import { ensureCorrectExtension } from "./output-polish.js";
+import { PANDOC_PRESET_SPECS } from "./preset-spec.js";
+import type { PandocDocumentAsset } from "./schemas.js";
+import type { PandocNotification } from "./types.js";
+import type { PandocInputCheck, PandocAssertionResult } from "./preflight.js";
+import type { PandocPresetSpec } from "./preset-spec.js";
+
+// ── Fixtures ─────────────────────────────────────────────────────
+
+const GOOD_INPUT_CHECK: PandocInputCheck = {
+  ok: true,
+  warnings: [],
+  sizeBytes: 5_000,
+  detectedFormat: "markdown",
+};
+
+const GOOD_ASSERTION: PandocAssertionResult = {
+  ok: true,
+  warnings: [],
+};
+
+const BAD_ASSERTION: PandocAssertionResult = {
+  ok: false,
+  warnings: ["Output file is empty — pandoc produced no content."],
+};
+
+function makeCtx(
+  crud: ReturnType<typeof createPandocCRUD>,
+  overrides?: Partial<ConvertDocumentContext>,
+): {
+  ctx: ConvertDocumentContext;
+  notifications: PandocNotification[];
+} {
+  const notifications: PandocNotification[] = [];
+  const ac = new AbortController();
+
+  const ctx: ConvertDocumentContext = {
+    signal: ac.signal,
+    userId: "user1",
+    notify: (n) => notifications.push(n),
+    createAsset: async (a) => { await crud.create(a); },
+    runPandoc: vi.fn(async (_args, _signal, onProgress) => {
+      onProgress(50);
+      onProgress(99);
+    }),
+    checkInput: vi.fn(async () => GOOD_INPUT_CHECK),
+    assertOutput: vi.fn(async () => GOOD_ASSERTION),
+    statFile: vi.fn(async () => ({ size: 9_200 })),
+    ...overrides,
+  };
+
+  return { ctx, notifications };
+}
+
+// ── Tests ────────────────────────────────────────────────────────
+
+describe("pandoc full integration", () => {
+  it("blog-post happy path: polish + CRUD create + correct metadata", async () => {
+    const crud = createPandocCRUD();
+    const { ctx, notifications } = makeCtx(crud);
+
+    const asset = await convertDocument(
+      {
+        inputPath: "/data/sandbox/user1/readme.md",
+        outputPath: "/data/sandbox/user1/readme.html",
+        preset: "blog-post",
+      },
+      ctx,
+    );
+
+    // Asset shape
+    expect(asset.id).toBeTruthy();
+    expect(asset.preset).toBe("blog-post");
+    expect(asset.outputPath).toBe("/data/sandbox/user1/readme.html");
+    expect(asset.inputMetadata.format).toBe("markdown");
+    expect(asset.inputMetadata.sizeBytes).toBe(5_000);
+    expect(asset.outputMetadata.format).toBe("html5");
+    expect(asset.outputMetadata.sizeBytes).toBe(9_200);
+    expect(asset.expiresAt).toBeTruthy();
+    expect(new Date(asset.expiresAt).getTime()).toBeGreaterThan(Date.now());
+
+    // CRUD persisted
+    const stored = await crud.read(asset.id);
+    expect(stored).not.toBeNull();
+    expect(stored?.id).toBe(asset.id);
+
+    const all = await crud.list();
+    expect(all).toHaveLength(1);
+
+    // Filter by preset
+    const filtered = await crud.list({ preset: "academic-pdf" });
+    expect(filtered).toHaveLength(0);
+
+    // Notifications: progress → ready
+    const readyNotifs = notifications.filter((n) => n.type === "pandoc:ready");
+    expect(readyNotifs).toHaveLength(1);
+    expect(
+      readyNotifs[0].type === "pandoc:ready" && readyNotifs[0].sizeBytes,
+    ).toBe(9_200);
+  });
+
+  it("newsletter fallback → correct preset in asset + warning emitted", async () => {
+    let assertCallCount = 0;
+    const crud = createPandocCRUD();
+    const { ctx, notifications } = makeCtx(crud, {
+      assertOutput: vi.fn(async () => {
+        assertCallCount++;
+        return assertCallCount === 1 ? BAD_ASSERTION : GOOD_ASSERTION;
+      }),
+    });
+
+    const asset = await convertDocument(
+      {
+        inputPath: "/data/sandbox/user1/news.md",
+        outputPath: "/data/sandbox/user1/news.html",
+        preset: "newsletter",
+      },
+      ctx,
+    );
+
+    // Should have fallen back to blog-post
+    expect(asset.preset).toBe("blog-post");
+    expect(asset.outputPath).toBe("/data/sandbox/user1/news.html");
+    expect(asset.outputMetadata.format).toBe("html5");
+
+    // CRUD has the fallback asset
+    const stored = await crud.read(asset.id);
+    expect(stored?.preset).toBe("blog-post");
+
+    // Warning about fallback was emitted
+    const warningNotifs = notifications.filter(
+      (n) => n.type === "pandoc:warning",
+    );
+    expect(warningNotifs.length).toBeGreaterThanOrEqual(1);
+    expect(
+      warningNotifs.some(
+        (n) =>
+          n.type === "pandoc:warning" &&
+          n.warnings.some((w) => w.includes("falling back")),
+      ),
+    ).toBe(true);
+  });
+
+  it("dual preset smoke: blog-post + academic-pdf in same CRUD store", async () => {
+    const crud = createPandocCRUD();
+
+    // Blog-post
+    const { ctx: ctx1 } = makeCtx(crud);
+    const blogAsset = await convertDocument(
+      {
+        inputPath: "/data/sandbox/user1/post.md",
+        outputPath: "/data/sandbox/user1/post.html",
+        preset: "blog-post",
+      },
+      ctx1,
+    );
+
+    // Academic PDF
+    const { ctx: ctx2 } = makeCtx(crud);
+    const pdfAsset = await convertDocument(
+      {
+        inputPath: "/data/sandbox/user1/thesis.md",
+        outputPath: "/data/sandbox/user1/thesis.pdf",
+        preset: "academic-pdf",
+      },
+      ctx2,
+    );
+
+    // Both stored
+    const all = await crud.list();
+    expect(all).toHaveLength(2);
+
+    // Filter works
+    const blogs = await crud.list({ preset: "blog-post" });
+    expect(blogs).toHaveLength(1);
+    expect(blogs[0].id).toBe(blogAsset.id);
+
+    const pdfs = await crud.list({ preset: "academic-pdf" });
+    expect(pdfs).toHaveLength(1);
+    expect(pdfs[0].id).toBe(pdfAsset.id);
+
+    // Different output formats
+    expect(blogAsset.outputMetadata.format).toBe("html5");
+    expect(pdfAsset.outputMetadata.format).toBe("pdf");
+  });
+});
+
+// ── Output polish unit tests ─────────────────────────────────────
+
+describe("ensureCorrectExtension", () => {
+  it("returns unchanged if extension matches", () => {
+    const spec = PANDOC_PRESET_SPECS["blog-post"];
+    expect(ensureCorrectExtension("out.html", spec)).toBe("out.html");
+  });
+
+  it("appends extension if missing", () => {
+    const spec = PANDOC_PRESET_SPECS["academic-pdf"];
+    expect(ensureCorrectExtension("thesis", spec)).toBe("thesis.pdf");
+  });
+
+  it("appends correct extension if different", () => {
+    const spec = PANDOC_PRESET_SPECS["ebook"];
+    expect(ensureCorrectExtension("book.txt", spec)).toBe("book.txt.epub");
+  });
+});

package/src/pandoc/index.ts

@@ -0,0 +1,10 @@
+export * from "./schemas.js";
+export * from "./preset-spec.js";
+export * from "./preflight.js";
+export * from "./types.js";
+export * from "./build-args.js";
+export * from "./progress.js";
+export * from "./exec.js";
+export * from "./convert.js";
+export * from "./crud.js";
+export * from "./output-polish.js";

package/src/pandoc/output-polish.ts

@@ -0,0 +1,60 @@
+import { extname } from "node:path";
+import type { PandocPresetSpec } from "./preset-spec.js";
+import type { PandocOutputMetadata } from "./schemas.js";
+
+// ── Output polish — pure post-processing helpers ─────────────────
+
+/**
+ * Ensure the output path has the correct extension for the preset.
+ * If the path already ends with the right extension, returns it unchanged.
+ * Otherwise appends the correct extension.
+ *
+ * Pure function — no I/O.
+ */
+export function ensureCorrectExtension(
+  outputPath: string,
+  spec: PandocPresetSpec,
+): string {
+  const expectedExt = `.${spec.outputExt}`;
+  const currentExt = extname(outputPath).toLowerCase();
+
+  if (currentExt === expectedExt) {
+    return outputPath;
+  }
+
+  // If there's no extension, append it
+  if (!currentExt) {
+    return `${outputPath}${expectedExt}`;
+  }
+
+  // Different extension — append (don't replace, user may have reasons)
+  return `${outputPath}${expectedExt}`;
+}
+
+/**
+ * Build PandocOutputMetadata from known values.
+ *
+ * Pure function — caller provides sizeBytes from stat.
+ */
+export function buildOutputMetadata(
+  spec: PandocPresetSpec,
+  sizeBytes: number,
+): PandocOutputMetadata {
+  return {
+    format: spec.to,
+    sizeBytes,
+    // pages is only meaningful for PDF — we'd need pdfinfo or
+    // a real PDF parser for that. Left undefined until Phase 5+.
+    ...(spec.to === "pdf" ? { pages: undefined } : {}),
+  };
+}
+
+/**
+ * Compute the asset expiration timestamp.
+ * Default: 24 hours from now.
+ */
+export function computeExpiresAt(
+  ttlMs: number = 86_400_000,
+): string {
+  return new Date(Date.now() + ttlMs).toISOString();
+}

package/src/pandoc/pipeline.test.ts

@@ -0,0 +1,170 @@
+import { describe, it, expect, vi } from "vitest";
+import {
+  convertDocument,
+  type ConvertDocumentContext,
+} from "./convert.js";
+import type { PandocDocumentAsset } from "./schemas.js";
+import type { PandocNotification } from "./types.js";
+import type { PandocInputCheck, PandocAssertionResult } from "./preflight.js";
+import type { PandocPresetSpec } from "./preset-spec.js";
+
+// ── Fixtures ─────────────────────────────────────────────────────
+
+const GOOD_INPUT_CHECK: PandocInputCheck = {
+  ok: true,
+  warnings: [],
+  sizeBytes: 5_000,
+  detectedFormat: "markdown",
+};
+
+const GOOD_ASSERTION: PandocAssertionResult = {
+  ok: true,
+  warnings: [],
+};
+
+const BAD_ASSERTION: PandocAssertionResult = {
+  ok: false,
+  warnings: ["Output file is empty — pandoc produced no content."],
+};
+
+function makeCtx(
+  overrides?: Partial<ConvertDocumentContext>,
+): {
+  ctx: ConvertDocumentContext;
+  notifications: PandocNotification[];
+  assets: PandocDocumentAsset[];
+} {
+  const notifications: PandocNotification[] = [];
+  const assets: PandocDocumentAsset[] = [];
+  const ac = new AbortController();
+
+  const ctx: ConvertDocumentContext = {
+    signal: ac.signal,
+    userId: "user1",
+    notify: (n) => notifications.push(n),
+    createAsset: vi.fn(async (a) => {
+      assets.push(a);
+    }),
+    runPandoc: vi.fn(async (_args, _signal, onProgress) => {
+      onProgress(50);
+      onProgress(99);
+    }),
+    checkInput: vi.fn(async () => GOOD_INPUT_CHECK),
+    assertOutput: vi.fn(async () => GOOD_ASSERTION),
+    statFile: vi.fn(async () => ({ size: 12_000 })),
+    ...overrides,
+  };
+
+  return { ctx, notifications, assets };
+}
+
+const VALID_REQ = {
+  inputPath: "/data/sandbox/user1/doc.md",
+  outputPath: "/data/sandbox/user1/doc.html",
+  preset: "blog-post" as const,
+  timeoutSeconds: 60,
+  maxOutputBytes: 0,
+};
+
+// ── Tests ────────────────────────────────────────────────────────
+
+describe("convertDocument pipeline", () => {
+  it("blog-post happy path: produces asset with correct shape", async () => {
+    const { ctx, notifications, assets } = makeCtx();
+
+    const asset = await convertDocument(VALID_REQ, ctx);
+
+    expect(asset.id).toBeTruthy();
+    expect(asset.inputPath).toBe(VALID_REQ.inputPath);
+    expect(asset.outputPath).toBe(VALID_REQ.outputPath);
+    expect(asset.preset).toBe("blog-post");
+    expect(asset.inputMetadata.format).toBe("markdown");
+    expect(asset.inputMetadata.sizeBytes).toBe(5_000);
+    expect(asset.outputMetadata.format).toBe("html5");
+    expect(asset.outputMetadata.sizeBytes).toBe(12_000);
+    expect(assets).toHaveLength(1);
+
+    // Should have progress + ready notifications
+    const progressNotifs = notifications.filter(
+      (n) => n.type === "pandoc:progress",
+    );
+    const readyNotifs = notifications.filter(
+      (n) => n.type === "pandoc:ready",
+    );
+    expect(progressNotifs.length).toBeGreaterThan(0);
+    expect(readyNotifs).toHaveLength(1);
+
+    // Final progress must be 100%
+    const last100 = progressNotifs.filter(
+      (n) => n.type === "pandoc:progress" && n.percent === 100,
+    );
+    expect(last100.length).toBeGreaterThanOrEqual(1);
+
+    // Ready notification has correct shape
+    const ready = readyNotifs[0];
+    expect(ready.type === "pandoc:ready" && ready.preset).toBe("blog-post");
+    expect(ready.type === "pandoc:ready" && ready.sizeBytes).toBe(12_000);
+  });
+
+  it("newsletter → fallback to blog-post on assertion fail + warning emitted", async () => {
+    let assertCallCount = 0;
+    const { ctx, notifications } = makeCtx({
+      assertOutput: vi.fn(async () => {
+        assertCallCount++;
+        // First call (newsletter) → fails assertion
+        // Second call (blog-post fallback) → passes
+        return assertCallCount === 1 ? BAD_ASSERTION : GOOD_ASSERTION;
+      }),
+    });
+
+    const asset = await convertDocument(
+      {
+        ...VALID_REQ,
+        preset: "newsletter",
+      },
+      ctx,
+    );
+
+    expect(asset).toBeDefined();
+    // Should have fallen back to blog-post
+    expect(asset.preset).toBe("blog-post");
+
+    // Should have a warning notification about fallback
+    const warningNotifs = notifications.filter(
+      (n) => n.type === "pandoc:warning",
+    );
+    expect(warningNotifs.length).toBeGreaterThanOrEqual(1);
+    expect(
+      warningNotifs.some(
+        (n) =>
+          n.type === "pandoc:warning" &&
+          n.warnings.some((w) => w.includes("falling back")),
+      ),
+    ).toBe(true);
+  });
+
+  it("AbortSignal cancels immediately", async () => {
+    const ac = new AbortController();
+    ac.abort(); // pre-abort
+
+    const { ctx } = makeCtx({ signal: ac.signal });
+
+    await expect(convertDocument(VALID_REQ, ctx)).rejects.toThrow("Aborted");
+  });
+
+  it("maxOutputBytes preflight rejects when estimate exceeds limit", async () => {
+    const { ctx } = makeCtx();
+
+    // blog-post → html5 → factor 1.8 → 5000 * 1.8 = 9000
+    // Set maxOutputBytes to 100 (way below 9000)
+    await expect(
+      convertDocument(
+        {
+          ...VALID_REQ,
+          maxOutputBytes: 100,
+        },
+        ctx,
+      ),
+    ).rejects.toThrow("exceeds maxOutputBytes");
+  });
+});

package/src/pandoc/preflight.ts

@@ -0,0 +1,209 @@
+import { stat } from "node:fs/promises";
+import { extname } from "node:path";
+import type { PandocPreset } from "./schemas.js";
+import type { PandocPresetSpec } from "./preset-spec.js";
+
+// Re-export validateSandboxPath from root for convenience
+export { validateSandboxPath } from "../preflight.js";
+
+// ── Input validation ──────────────────────────────────────────────
+
+/** Max input file size: 50 MB (documents shouldn't be bigger) */
+const MAX_INPUT_BYTES = 50 * 1024 * 1024;
+
+/** Allowed input extensions per format */
+const ALLOWED_EXTENSIONS: Record<string, string[]> = {
+  markdown: [".md", ".markdown", ".mkd", ".txt"],
+  docx: [".docx"],
+  latex: [".tex", ".latex"],
+  html: [".html", ".htm"],
+  rst: [".rst"],
+  org: [".org"],
+};
+
+export interface PandocInputCheck {
+  ok: boolean;
+  warnings: string[];
+  sizeBytes: number;
+  detectedFormat: string;
+}
+
+/**
+ * Validate a Pandoc input file: exists, not too large, recognized extension.
+ */
+export async function checkPandocInput(
+  filePath: string,
+): Promise<PandocInputCheck> {
+  const warnings: string[] = [];
+
+  // Check file exists and get size
+  let sizeBytes: number;
+  try {
+    const s = await stat(filePath);
+    sizeBytes = s.size;
+  } catch {
+    return {
+      ok: false,
+      warnings: [`Input file not found: "${filePath}".`],
+      sizeBytes: 0,
+      detectedFormat: "unknown",
+    };
+  }
+
+  if (sizeBytes > MAX_INPUT_BYTES) {
+    return {
+      ok: false,
+      warnings: [
+        `Input file is ${(sizeBytes / 1024 / 1024).toFixed(1)} MB, ` +
+          `exceeds maximum ${MAX_INPUT_BYTES / 1024 / 1024} MB.`,
+      ],
+      sizeBytes,
+      detectedFormat: "unknown",
+    };
+  }
+
+  if (sizeBytes === 0) {
+    warnings.push("Input file is empty.");
+  }
+
+  // Detect format from extension
+  const ext = filePath.slice(filePath.lastIndexOf(".")).toLowerCase();
+  let detectedFormat = "markdown"; // default
+
+  for (const [format, exts] of Object.entries(ALLOWED_EXTENSIONS)) {
+    if (exts.includes(ext)) {
+      detectedFormat = format;
+      break;
+    }
+  }
+
+  if (!Object.values(ALLOWED_EXTENSIONS).flat().includes(ext)) {
+    warnings.push(
+      `Unrecognized extension "${ext}"; assuming markdown. ` +
+        `Supported: ${Object.values(ALLOWED_EXTENSIONS).flat().join(", ")}.`,
+    );
+  }
+
+  return { ok: true, warnings, sizeBytes, detectedFormat };
+}
+
+// ── Output size estimation ────────────────────────────────────────
+
+/** Rough expansion multipliers per output format */
+const EXPANSION_FACTORS: Record<string, number> = {
+  html5: 1.8,    // HTML + embedded resources
+  pdf: 0.8,      // PDFs are often smaller than source
+  epub: 1.2,     // EPUB with compression
+  revealjs: 2.5, // Slides embed reveal.js
+  html: 1.5,     // Plain HTML
+};
+
+/**
+ * Estimate output file size in bytes based on input size and preset.
+ */
+export function estimatePandocOutputBytes(
+  inputSizeBytes: number,
+  preset: PandocPreset,
+): number {
+  // Map preset to output format for factor lookup
+  const formatMap: Record<PandocPreset, string> = {
+    "blog-post": "html5",
+    "academic-pdf": "pdf",
+    "ebook": "epub",
+    "slides": "revealjs",
+    "newsletter": "html",
+  };
+
+  const factor = EXPANSION_FACTORS[formatMap[preset]] ?? 1.5;
+  return Math.ceil(inputSizeBytes * factor);
+}
+
+// ── Output assertions ────────────────────────────────────────────
+
+export interface PandocAssertionResult {
+  ok: boolean;
+  warnings: string[];
+}
+
+/**
+ * Assert that the pandoc output file matches the preset spec.
+ *
+ * Checks:
+ * 1. Output file exists
+ * 2. Output file is non-empty
+ * 3. Output extension matches spec
+ * 4. Output size is within maxOutputBytes (if set)
+ */
+export async function assertPandocOutput(
+  spec: PandocPresetSpec,
+  outputPath: string,
+  maxOutputBytes: number,
+): Promise<PandocAssertionResult> {
+  const warnings: string[] = [];
+
+  // Check file exists and get size
+  let sizeBytes: number;
+  try {
+    const s = await stat(outputPath);
+    sizeBytes = s.size;
+  } catch {
+    return {
+      ok: false,
+      warnings: [`Output file not found: "${outputPath}".`],
+    };
+  }
+
+  // Check non-empty
+  if (sizeBytes === 0) {
+    return {
+      ok: false,
+      warnings: ["Output file is empty — pandoc produced no content."],
+    };
+  }
+
+  // Check extension matches spec
+  const actualExt = extname(outputPath).slice(1).toLowerCase();
+  if (actualExt && actualExt !== spec.outputExt) {
+    warnings.push(
+      `Output extension ".${actualExt}" does not match expected ".${spec.outputExt}".`,
+    );
+  }
+
+  // Check maxOutputBytes (0 = unlimited)
+  if (maxOutputBytes > 0 && sizeBytes > maxOutputBytes) {
+    return {
+      ok: false,
+      warnings: [
+        `Output is ${(sizeBytes / 1024 / 1024).toFixed(1)} MB, ` +
+          `exceeds maximum ${(maxOutputBytes / 1024 / 1024).toFixed(1)} MB.`,
+      ],
+    };
+  }
+
+  return { ok: true, warnings };
+}
+
+// ── Input format validation (pure) ───────────────────────────────
+
+/**
+ * Validate that detected input format is compatible with the preset's
+ * expected --from format. Returns a warning if there's a mismatch.
+ */
+export function checkFormatCompatibility(
+  detectedFormat: string,
+  presetFrom: string,
+): { ok: boolean; warning?: string } {
+  // Pandoc is flexible — markdown presets accept most text formats
+  if (presetFrom === "markdown") return { ok: true };
+
+  if (detectedFormat !== presetFrom) {
+    return {
+      ok: true,
+      warning:
+        `Detected format "${detectedFormat}" differs from preset's ` +
+        `expected "${presetFrom}"; pandoc will attempt conversion anyway.`,
+    };
+  }
+
+  return { ok: true };
+}