@knpkv/confluence-to-markdown 0.2.0

package/src/ConfluenceError.ts

@@ -0,0 +1,193 @@
+/**
+ * Error types for Confluence operations.
+ *
+ * @module
+ */
+import * as Data from "effect/Data"
+
+/**
+ * Error thrown when .confluence.json is not found.
+ *
+ * @example
+ * ```typescript
+ * import { Effect } from "effect"
+ * import { ConfigNotFoundError } from "@knpkv/confluence-to-markdown/ConfluenceError"
+ *
+ * Effect.gen(function* () {
+ *   // ... operation that needs config
+ * }).pipe(
+ *   Effect.catchTag("ConfigNotFoundError", (error) =>
+ *     Effect.sync(() => console.error(`Config not found at: ${error.path}`))
+ *   )
+ * )
+ * ```
+ *
+ * @category Errors
+ */
+export class ConfigNotFoundError extends Data.TaggedError("ConfigNotFoundError")<{
+  readonly path: string
+}> {}
+
+/**
+ * Error thrown when .confluence.json parsing fails.
+ *
+ * @category Errors
+ */
+export class ConfigParseError extends Data.TaggedError("ConfigParseError")<{
+  readonly path: string
+  readonly cause: unknown
+}> {}
+
+/**
+ * Error thrown when configuration validation fails.
+ *
+ * @category Errors
+ */
+export class ConfigError extends Data.TaggedError("ConfigError")<{
+  readonly message: string
+}> {}
+
+/**
+ * Error thrown when authentication is missing.
+ *
+ * @example
+ * ```typescript
+ * import { Effect } from "effect"
+ * import { AuthMissingError } from "@knpkv/confluence-to-markdown/ConfluenceError"
+ *
+ * Effect.gen(function* () {
+ *   // ... operation that needs auth
+ * }).pipe(
+ *   Effect.catchTag("AuthMissingError", () =>
+ *     Effect.sync(() => console.error("Set CONFLUENCE_API_KEY or run: confluence auth login"))
+ *   )
+ * )
+ * ```
+ *
+ * @category Errors
+ */
+export class AuthMissingError extends Data.TaggedError("AuthMissingError")<{
+  readonly message: string
+}> {
+  constructor() {
+    super({ message: "CONFLUENCE_API_KEY env var or OAuth2 credentials required" })
+  }
+}
+
+/**
+ * Error thrown when Confluence API request fails.
+ *
+ * @category Errors
+ */
+export class ApiError extends Data.TaggedError("ApiError")<{
+  readonly status: number
+  readonly message: string
+  readonly endpoint: string
+  readonly pageId?: string
+}> {}
+
+/**
+ * Error thrown when rate limit is exceeded.
+ *
+ * @category Errors
+ */
+export class RateLimitError extends Data.TaggedError("RateLimitError")<{
+  readonly retryAfter?: number
+}> {}
+
+/**
+ * Error thrown when HTML/Markdown conversion fails.
+ *
+ * @category Errors
+ */
+export class ConversionError extends Data.TaggedError("ConversionError")<{
+  readonly direction: "htmlToMarkdown" | "markdownToHtml"
+  readonly cause: unknown
+}> {}
+
+/**
+ * Error thrown when sync conflict is detected.
+ *
+ * @category Errors
+ */
+export class ConflictError extends Data.TaggedError("ConflictError")<{
+  readonly pageId: string
+  readonly localVersion: number
+  readonly remoteVersion: number
+  readonly path: string
+}> {}
+
+/**
+ * Error thrown when file system operation fails.
+ *
+ * @category Errors
+ */
+export class FileSystemError extends Data.TaggedError("FileSystemError")<{
+  readonly operation: "read" | "write" | "delete" | "mkdir" | "rename"
+  readonly path: string
+  readonly cause: unknown
+}> {}
+
+/**
+ * Error thrown when OAuth2 flow fails.
+ *
+ * @category Errors
+ */
+export class OAuthError extends Data.TaggedError("OAuthError")<{
+  readonly step: "authorize" | "token" | "refresh"
+  readonly cause: unknown
+}> {}
+
+/**
+ * Error thrown when front-matter parsing fails.
+ *
+ * @category Errors
+ */
+export class FrontMatterError extends Data.TaggedError("FrontMatterError")<{
+  readonly path: string
+  readonly cause: unknown
+}> {}
+
+/**
+ * Union of all Confluence errors.
+ *
+ * @category Errors
+ */
+export type ConfluenceError =
+  | ConfigNotFoundError
+  | ConfigParseError
+  | ConfigError
+  | AuthMissingError
+  | ApiError
+  | RateLimitError
+  | ConversionError
+  | ConflictError
+  | FileSystemError
+  | OAuthError
+  | FrontMatterError
+
+/**
+ * Type guard to check if error is a ConfluenceError.
+ *
+ * @param error - The error to check
+ * @returns True if error is a ConfluenceError
+ *
+ * @category Utilities
+ */
+export const isConfluenceError = (error: unknown): error is ConfluenceError =>
+  typeof error === "object" &&
+  error !== null &&
+  "_tag" in error &&
+  [
+    "ConfigNotFoundError",
+    "ConfigParseError",
+    "ConfigError",
+    "AuthMissingError",
+    "ApiError",
+    "RateLimitError",
+    "ConversionError",
+    "ConflictError",
+    "FileSystemError",
+    "OAuthError",
+    "FrontMatterError"
+  ].includes((error as { _tag: string })._tag)

package/src/LocalFileSystem.ts

@@ -0,0 +1,225 @@
+/**
+ * Local file system operations for markdown files.
+ *
+ * @module
+ */
+import * as FileSystem from "@effect/platform/FileSystem"
+import * as Path from "@effect/platform/Path"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import type { ContentHash } from "./Brand.js"
+import type { FrontMatterError } from "./ConfluenceError.js"
+import { FileSystemError } from "./ConfluenceError.js"
+import type { ParsedMarkdown } from "./internal/frontmatter.js"
+import { parseMarkdown, serializeMarkdown } from "./internal/frontmatter.js"
+import { computeHash } from "./internal/hashUtils.js"
+import { pageToDir, pageToPath } from "./internal/pathUtils.js"
+import type { PageFrontMatter } from "./Schemas.js"
+
+/**
+ * Local markdown file representation.
+ */
+export interface LocalFile {
+  readonly path: string
+  readonly frontMatter: PageFrontMatter | null
+  readonly content: string
+  readonly contentHash: ContentHash
+  readonly isNew: boolean
+}
+
+/**
+ * Local file system service for markdown operations.
+ *
+ * @example
+ * ```typescript
+ * import { LocalFileSystem } from "@knpkv/confluence-to-markdown/LocalFileSystem"
+ * import { Effect } from "effect"
+ *
+ * const program = Effect.gen(function* () {
+ *   const fs = yield* LocalFileSystem
+ *   const files = yield* fs.listMarkdownFiles(".docs/confluence")
+ *   console.log(files)
+ * })
+ * ```
+ *
+ * @category FileSystem
+ */
+export class LocalFileSystem extends Context.Tag(
+  "@knpkv/confluence-to-markdown/LocalFileSystem"
+)<
+  LocalFileSystem,
+  {
+    /**
+     * Read a markdown file with front-matter.
+     */
+    readonly readMarkdownFile: (filePath: string) => Effect.Effect<LocalFile, FileSystemError | FrontMatterError>
+
+    /**
+     * Write a markdown file with front-matter.
+     */
+    readonly writeMarkdownFile: (
+      filePath: string,
+      frontMatter: PageFrontMatter,
+      content: string
+    ) => Effect.Effect<void, FileSystemError>
+
+    /**
+     * List all markdown files in a directory recursively.
+     */
+    readonly listMarkdownFiles: (dirPath: string) => Effect.Effect<ReadonlyArray<string>, FileSystemError>
+
+    /**
+     * Ensure a directory exists.
+     */
+    readonly ensureDir: (dirPath: string) => Effect.Effect<void, FileSystemError>
+
+    /**
+     * Delete a file.
+     */
+    readonly deleteFile: (filePath: string) => Effect.Effect<void, FileSystemError>
+
+    /**
+     * Check if a file exists.
+     */
+    readonly exists: (filePath: string) => Effect.Effect<boolean, FileSystemError>
+
+    /**
+     * Get the file path for a page.
+     */
+    readonly getPagePath: (
+      title: string,
+      hasChildren: boolean,
+      parentPath: string
+    ) => string
+
+    /**
+     * Get the directory path for a page's children.
+     */
+    readonly getPageDir: (title: string, parentPath: string) => string
+  }
+>() {}
+
+/**
+ * Layer that provides LocalFileSystem.
+ *
+ * @category Layers
+ */
+export const layer: Layer.Layer<LocalFileSystem, never, FileSystem.FileSystem | Path.Path> = Layer.effect(
+  LocalFileSystem,
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+    const pathService = yield* Path.Path
+
+    const readMarkdownFile = (
+      filePath: string
+    ): Effect.Effect<LocalFile, FileSystemError | FrontMatterError> =>
+      Effect.gen(function*() {
+        const content = yield* fs.readFileString(filePath).pipe(
+          Effect.mapError((cause) => new FileSystemError({ operation: "read", path: filePath, cause }))
+        )
+
+        const parsed: ParsedMarkdown = yield* parseMarkdown(filePath, content)
+        const contentHash = computeHash(parsed.content)
+
+        return {
+          path: filePath,
+          frontMatter: parsed.frontMatter && "pageId" in parsed.frontMatter
+            ? parsed.frontMatter as PageFrontMatter
+            : null,
+          content: parsed.content,
+          contentHash,
+          isNew: parsed.isNew
+        }
+      })
+
+    const writeMarkdownFile = (
+      filePath: string,
+      frontMatter: PageFrontMatter,
+      content: string
+    ): Effect.Effect<void, FileSystemError> =>
+      Effect.gen(function*() {
+        const dir = pathService.dirname(filePath)
+        yield* fs.makeDirectory(dir, { recursive: true }).pipe(
+          Effect.catchAll(() => Effect.void)
+        )
+
+        const serialized = serializeMarkdown(frontMatter, content)
+
+        // Atomic write: write to temp file, then rename
+        const tempPath = `${filePath}.tmp.${Date.now()}`
+        yield* fs.writeFileString(tempPath, serialized).pipe(
+          Effect.mapError((cause) => new FileSystemError({ operation: "write", path: filePath, cause }))
+        )
+        yield* fs.rename(tempPath, filePath).pipe(
+          Effect.mapError((cause) => new FileSystemError({ operation: "rename", path: filePath, cause }))
+        )
+      })
+
+    const listMarkdownFiles = (
+      dirPath: string
+    ): Effect.Effect<ReadonlyArray<string>, FileSystemError> =>
+      Effect.gen(function*() {
+        const exists = yield* fs.exists(dirPath).pipe(
+          Effect.mapError((cause) => new FileSystemError({ operation: "read", path: dirPath, cause }))
+        )
+
+        if (!exists) {
+          return []
+        }
+
+        const files: Array<string> = []
+
+        const walkDir = (dir: string): Effect.Effect<void, FileSystemError> =>
+          Effect.gen(function*() {
+            const entries = yield* fs.readDirectory(dir).pipe(
+              Effect.mapError((cause) => new FileSystemError({ operation: "read", path: dir, cause }))
+            )
+
+            for (const entryName of entries) {
+              const fullPath = pathService.join(dir, entryName)
+
+              const stat = yield* fs.stat(fullPath).pipe(
+                Effect.mapError((cause) => new FileSystemError({ operation: "read", path: fullPath, cause }))
+              )
+
+              if (stat.type === "Directory") {
+                yield* walkDir(fullPath)
+              } else if (stat.type === "File" && entryName.endsWith(".md")) {
+                files.push(fullPath)
+              }
+            }
+          })
+
+        yield* walkDir(dirPath)
+        return files
+      })
+
+    const ensureDir = (dirPath: string): Effect.Effect<void, FileSystemError> =>
+      fs.makeDirectory(dirPath, { recursive: true }).pipe(
+        Effect.mapError((cause) => new FileSystemError({ operation: "mkdir", path: dirPath, cause })),
+        Effect.asVoid
+      )
+
+    const deleteFile = (filePath: string): Effect.Effect<void, FileSystemError> =>
+      fs.remove(filePath).pipe(
+        Effect.mapError((cause) => new FileSystemError({ operation: "delete", path: filePath, cause }))
+      )
+
+    const exists = (filePath: string): Effect.Effect<boolean, FileSystemError> =>
+      fs.exists(filePath).pipe(
+        Effect.mapError((cause) => new FileSystemError({ operation: "read", path: filePath, cause }))
+      )
+
+    return LocalFileSystem.of({
+      readMarkdownFile,
+      writeMarkdownFile,
+      listMarkdownFiles,
+      ensureDir,
+      deleteFile,
+      exists,
+      getPagePath: pageToPath,
+      getPageDir: pageToDir
+    })
+  })
+)

package/src/MarkdownConverter.ts

@@ -0,0 +1,187 @@
+/**
+ * HTML to Markdown conversion service using unified/remark/rehype.
+ *
+ * @module
+ */
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import rehypeParse from "rehype-parse"
+import rehypeRemark from "rehype-remark"
+import rehypeStringify from "rehype-stringify"
+import remarkGfm from "remark-gfm"
+import remarkParse from "remark-parse"
+import remarkRehype from "remark-rehype"
+import remarkStringify from "remark-stringify"
+import { unified } from "unified"
+import { ConversionError } from "./ConfluenceError.js"
+
+/**
+ * Markdown conversion service for HTML <-> GFM conversion.
+ *
+ * @example
+ * ```typescript
+ * import { MarkdownConverter } from "@knpkv/confluence-to-markdown/MarkdownConverter"
+ * import { Effect } from "effect"
+ *
+ * const program = Effect.gen(function* () {
+ *   const converter = yield* MarkdownConverter
+ *   const md = yield* converter.htmlToMarkdown("<h1>Hello</h1><p>World</p>")
+ *   console.log(md) // # Hello\n\nWorld
+ * })
+ *
+ * Effect.runPromise(
+ *   program.pipe(Effect.provide(MarkdownConverter.layer))
+ * )
+ * ```
+ *
+ * @category Conversion
+ */
+export class MarkdownConverter extends Context.Tag(
+  "@knpkv/confluence-to-markdown/MarkdownConverter"
+)<
+  MarkdownConverter,
+  {
+    /**
+     * Convert Confluence storage format (HTML) to GitHub Flavored Markdown.
+     */
+    readonly htmlToMarkdown: (html: string) => Effect.Effect<string, ConversionError>
+
+    /**
+     * Convert GitHub Flavored Markdown to HTML (Confluence storage format).
+     */
+    readonly markdownToHtml: (markdown: string) => Effect.Effect<string, ConversionError>
+  }
+>() {}
+
+/** Maximum HTML input size (1MB) to prevent ReDoS attacks */
+const MAX_HTML_SIZE = 1024 * 1024
+
+/**
+ * Strip Confluence-specific macros while preserving text content.
+ * Uses iterative approach to avoid ReDoS with nested content.
+ */
+const stripConfluenceMacros = (html: string): Effect.Effect<string, ConversionError> =>
+  Effect.gen(function*() {
+    // Limit input size to prevent ReDoS
+    if (html.length > MAX_HTML_SIZE) {
+      return yield* Effect.fail(
+        new ConversionError({
+          direction: "htmlToMarkdown",
+          cause: `HTML input too large: ${html.length} bytes (max ${MAX_HTML_SIZE})`
+        })
+      )
+    }
+
+    let result = html
+
+    // Process structured macros iteratively to handle nesting safely
+    let iterations = 0
+    const maxIterations = 100 // Prevent infinite loops
+
+    while (iterations < maxIterations) {
+      const macroStart = result.indexOf("<ac:structured-macro")
+      if (macroStart === -1) break
+
+      // Find matching closing tag by counting nesting
+      let depth = 1
+      let pos = macroStart + 20 // Skip past opening tag start
+      let endPos = -1
+
+      while (pos < result.length && depth > 0) {
+        if (result.slice(pos, pos + 20) === "<ac:structured-macro") {
+          depth++
+          pos += 20
+        } else if (result.slice(pos, pos + 21) === "</ac:structured-macro") {
+          depth--
+          if (depth === 0) {
+            endPos = result.indexOf(">", pos) + 1
+          }
+          pos += 21
+        } else {
+          pos++
+        }
+      }
+
+      if (endPos === -1) break // Malformed HTML, stop processing
+
+      const macroContent = result.slice(macroStart, endPos)
+
+      // Extract content from macro
+      let replacement = ""
+      const plainBodyStart = macroContent.indexOf("<ac:plain-text-body><![CDATA[")
+      const plainBodyEnd = macroContent.indexOf("]]></ac:plain-text-body>")
+      if (plainBodyStart !== -1 && plainBodyEnd !== -1) {
+        const content = macroContent.slice(plainBodyStart + 29, plainBodyEnd)
+        replacement = `<pre><code>${content}</code></pre>`
+      } else {
+        const richBodyStart = macroContent.indexOf("<ac:rich-text-body>")
+        const richBodyEnd = macroContent.indexOf("</ac:rich-text-body>")
+        if (richBodyStart !== -1 && richBodyEnd !== -1) {
+          replacement = macroContent.slice(richBodyStart + 19, richBodyEnd)
+        }
+      }
+
+      result = result.slice(0, macroStart) + replacement + result.slice(endPos)
+      iterations++
+    }
+
+    // Remove remaining simple tags with non-greedy bounded patterns
+    result = result
+      .replace(/<ac:parameter[^>]{0,1000}>[^<]{0,10000}<\/ac:parameter>/gi, "")
+      .replace(/<\/?ac:[a-z-]{1,50}[^>]{0,1000}>/gi, "")
+      .replace(/<\/?ri:[a-z-]{1,50}[^>]{0,1000}\/?>/gi, "")
+
+    return result
+  })
+
+/**
+ * Create the markdown converter processor for HTML -> Markdown.
+ */
+const createHtmlToMdProcessor = () =>
+  unified()
+    .use(rehypeParse, { fragment: true })
+    .use(rehypeRemark)
+    .use(remarkGfm)
+    .use(remarkStringify)
+
+/**
+ * Create the markdown converter processor for Markdown -> HTML.
+ */
+const createMdToHtmlProcessor = () =>
+  unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkRehype)
+    .use(rehypeStringify)
+
+/**
+ * Layer that provides the MarkdownConverter service.
+ *
+ * @category Layers
+ */
+export const layer: Layer.Layer<MarkdownConverter> = Layer.succeed(
+  MarkdownConverter,
+  MarkdownConverter.of({
+    htmlToMarkdown: (html) =>
+      Effect.gen(function*() {
+        const cleaned = yield* stripConfluenceMacros(html)
+        return yield* Effect.try({
+          try: () => {
+            const result = createHtmlToMdProcessor().processSync(cleaned)
+            return String(result).trim()
+          },
+          catch: (cause) => new ConversionError({ direction: "htmlToMarkdown", cause })
+        })
+      }),
+
+    markdownToHtml: (markdown) =>
+      Effect.try({
+        try: () => {
+          const result = createMdToHtmlProcessor().processSync(markdown)
+          return String(result).trim()
+        },
+        catch: (cause) => new ConversionError({ direction: "markdownToHtml", cause })
+      })
+  })
+)