@knpkv/confluence-to-markdown 0.2.0 → 0.4.2

package/src/LocalFileSystem.ts

@@ -12,10 +12,10 @@ import type { ContentHash } from "./Brand.js"
 import type { FrontMatterError } from "./ConfluenceError.js"
 import { FileSystemError } from "./ConfluenceError.js"
 import type { ParsedMarkdown } from "./internal/frontmatter.js"
-import { parseMarkdown, serializeMarkdown } from "./internal/frontmatter.js"
-import { computeHash } from "./internal/hashUtils.js"
+import { parseMarkdown, serializeMarkdown, serializeNewPageMarkdown } from "./internal/frontmatter.js"
+import { computeHash, HashServiceLive } from "./internal/hashUtils.js"
 import { pageToDir, pageToPath } from "./internal/pathUtils.js"
-import type { PageFrontMatter } from "./Schemas.js"
+import type { NewPageFrontMatter, PageFrontMatter } from "./Schemas.js"
 
 /**
  * Local markdown file representation.
@@ -28,6 +28,20 @@ export interface LocalFile {
   readonly isNew: boolean
 }
 
+/**
+ * Page tree node for directory structure.
+ */
+export interface PageTreeNode {
+  /** File path relative to docsPath */
+  readonly path: string
+  /** Page title */
+  readonly title: string
+  /** Page ID (null if new page) */
+  readonly pageId: string | null
+  /** Child pages */
+  readonly children: ReadonlyArray<PageTreeNode>
+}
+
 /**
  * Local file system service for markdown operations.
  *
@@ -38,7 +52,7 @@ export interface LocalFile {
  *
  * const program = Effect.gen(function* () {
  *   const fs = yield* LocalFileSystem
- *   const files = yield* fs.listMarkdownFiles(".docs/confluence")
+ *   const files = yield* fs.listMarkdownFiles(".confluence/docs")
  *   console.log(files)
  * })
  * ```
@@ -91,12 +105,35 @@ export class LocalFileSystem extends Context.Tag(
       title: string,
       hasChildren: boolean,
       parentPath: string
-    ) => string
+    ) => Effect.Effect<string>
 
     /**
      * Get the directory path for a page's children.
      */
-    readonly getPageDir: (title: string, parentPath: string) => string
+    readonly getPageDir: (title: string, parentPath: string) => Effect.Effect<string>
+
+    /**
+     * Write a raw file (e.g., source HTML).
+     */
+    readonly writeFile: (filePath: string, content: string) => Effect.Effect<void, FileSystemError>
+
+    /**
+     * Build a tree of pages from directory structure.
+     */
+    readonly buildPageTree: (
+      docsPath: string,
+      rootPageId: string,
+      rootTitle: string
+    ) => Effect.Effect<PageTreeNode, FileSystemError | FrontMatterError>
+
+    /**
+     * Write a new page file with minimal front-matter.
+     */
+    readonly writeNewPageFile: (
+      filePath: string,
+      frontMatter: NewPageFrontMatter,
+      content: string
+    ) => Effect.Effect<void, FileSystemError>
   }
 >() {}
 
@@ -120,7 +157,9 @@ export const layer: Layer.Layer<LocalFileSystem, never, FileSystem.FileSystem |
         )
 
         const parsed: ParsedMarkdown = yield* parseMarkdown(filePath, content)
-        const contentHash = computeHash(parsed.content)
+        const contentHash = yield* computeHash(parsed.content).pipe(
+          Effect.provide(HashServiceLive)
+        )
 
         return {
           path: filePath,
@@ -211,6 +250,134 @@ export const layer: Layer.Layer<LocalFileSystem, never, FileSystem.FileSystem |
         Effect.mapError((cause) => new FileSystemError({ operation: "read", path: filePath, cause }))
       )
 
+    const writeFile = (
+      filePath: string,
+      content: string
+    ): Effect.Effect<void, FileSystemError> =>
+      Effect.gen(function*() {
+        const dir = pathService.dirname(filePath)
+        yield* fs.makeDirectory(dir, { recursive: true }).pipe(
+          Effect.catchAll(() => Effect.void)
+        )
+
+        // Atomic write: write to temp file, then rename
+        const tempPath = `${filePath}.tmp.${Date.now()}`
+        yield* fs.writeFileString(tempPath, content).pipe(
+          Effect.mapError((cause) => new FileSystemError({ operation: "write", path: filePath, cause }))
+        )
+        yield* fs.rename(tempPath, filePath).pipe(
+          Effect.mapError((cause) => new FileSystemError({ operation: "rename", path: filePath, cause }))
+        )
+      })
+
+    // Wrap path functions to provide Path service
+    const getPagePath = (title: string, hasChildren: boolean, parentPath: string) =>
+      pageToPath(title, hasChildren, parentPath).pipe(
+        Effect.provide(Layer.succeed(Path.Path, pathService))
+      )
+
+    const getPageDir = (title: string, parentPath: string) =>
+      pageToDir(title, parentPath).pipe(
+        Effect.provide(Layer.succeed(Path.Path, pathService))
+      )
+
+    const writeNewPageFile = (
+      filePath: string,
+      frontMatter: NewPageFrontMatter,
+      content: string
+    ): Effect.Effect<void, FileSystemError> =>
+      Effect.gen(function*() {
+        const dir = pathService.dirname(filePath)
+        yield* fs.makeDirectory(dir, { recursive: true }).pipe(
+          Effect.catchAll(() => Effect.void)
+        )
+
+        const serialized = serializeNewPageMarkdown(frontMatter, content)
+
+        // Atomic write: write to temp file, then rename
+        const tempPath = `${filePath}.tmp.${Date.now()}`
+        yield* fs.writeFileString(tempPath, serialized).pipe(
+          Effect.mapError((cause) => new FileSystemError({ operation: "write", path: filePath, cause }))
+        )
+        yield* fs.rename(tempPath, filePath).pipe(
+          Effect.mapError((cause) => new FileSystemError({ operation: "rename", path: filePath, cause }))
+        )
+      })
+
+    const buildPageTree = (
+      docsPath: string,
+      rootPageId: string,
+      rootTitle: string
+    ): Effect.Effect<PageTreeNode, FileSystemError | FrontMatterError> =>
+      Effect.gen(function*() {
+        // Build a map of directory -> page info
+        const pagesByDir = new Map<string, { path: string; title: string; pageId: string | null }>()
+
+        // Get all markdown files
+        const files = yield* listMarkdownFiles(docsPath)
+
+        // Parse each file and map to its directory
+        for (const filePath of files) {
+          const localFile = yield* readMarkdownFile(filePath)
+          const relativePath = pathService.relative(docsPath, filePath)
+          const dirPath = pathService.dirname(relativePath)
+          const baseName = pathService.basename(filePath, ".md")
+
+          // Determine the page's "key" - the directory it owns children in
+          // e.g., "foo.md" owns "foo/" directory
+          const pageKey = dirPath === "." ? baseName : pathService.join(dirPath, baseName)
+
+          pagesByDir.set(pageKey, {
+            path: relativePath,
+            title: localFile.frontMatter?.title ?? baseName,
+            pageId: localFile.frontMatter?.pageId ?? null
+          })
+        }
+
+        // Build tree recursively
+        const buildNode = (
+          key: string,
+          path: string,
+          title: string,
+          pageId: string | null
+        ): PageTreeNode => {
+          // Find children: pages whose parent directory matches this key
+          const children: Array<PageTreeNode> = []
+
+          for (const [childKey, childInfo] of pagesByDir.entries()) {
+            const childDir = pathService.dirname(childKey)
+            if (childDir === key) {
+              children.push(buildNode(childKey, childInfo.path, childInfo.title, childInfo.pageId))
+            }
+          }
+
+          // Sort children by title
+          children.sort((a, b) => a.title.localeCompare(b.title))
+
+          return { path, title, pageId, children }
+        }
+
+        // Find root-level pages (those in docsPath root, or the root page itself)
+        const rootChildren: Array<PageTreeNode> = []
+
+        for (const [key, info] of pagesByDir.entries()) {
+          const dir = pathService.dirname(key)
+          if (dir === ".") {
+            rootChildren.push(buildNode(key, info.path, info.title, info.pageId))
+          }
+        }
+
+        // Sort children by title
+        rootChildren.sort((a, b) => a.title.localeCompare(b.title))
+
+        return {
+          path: "",
+          title: rootTitle,
+          pageId: rootPageId,
+          children: rootChildren
+        }
+      })
+
     return LocalFileSystem.of({
       readMarkdownFile,
       writeMarkdownFile,
@@ -218,8 +385,11 @@ export const layer: Layer.Layer<LocalFileSystem, never, FileSystem.FileSystem |
       ensureDir,
       deleteFile,
       exists,
-      getPagePath: pageToPath,
-      getPageDir: pageToDir
+      getPagePath,
+      getPageDir,
+      writeFile,
+      buildPageTree,
+      writeNewPageFile
     })
   })
 )

package/src/MarkdownConverter.ts

@@ -1,20 +1,23 @@
 /**
- * HTML to Markdown conversion service using unified/remark/rehype.
+ * HTML to Markdown conversion service using AST-based approach.
  *
  * @module
  */
 import * as Context from "effect/Context"
 import * as Effect from "effect/Effect"
 import * as Layer from "effect/Layer"
-import rehypeParse from "rehype-parse"
-import rehypeRemark from "rehype-remark"
-import rehypeStringify from "rehype-stringify"
-import remarkGfm from "remark-gfm"
-import remarkParse from "remark-parse"
-import remarkRehype from "remark-rehype"
-import remarkStringify from "remark-stringify"
-import { unified } from "unified"
+import * as Schema from "effect/Schema"
+import type { Document } from "./ast/Document.js"
 import { ConversionError } from "./ConfluenceError.js"
+import { parseConfluenceHtml } from "./parsers/ConfluenceParser.js"
+import { parseMarkdown } from "./parsers/MarkdownParser.js"
+import { ParseError, type SerializeError } from "./SchemaConverterError.js"
+import { ConfluenceToMarkdown, DocumentFromHast, DocumentFromMdast } from "./schemas/ConversionSchema.js"
+import { HastFromHtml } from "./schemas/hast/index.js"
+import { MdastFromMarkdown } from "./schemas/mdast/index.js"
+import { PreprocessedHtmlFromConfluence } from "./schemas/preprocessing/index.js"
+import { serializeToConfluence } from "./serializers/ConfluenceSerializer.js"
+import { type SerializeOptions, serializeToMarkdown } from "./serializers/MarkdownSerializer.js"
 
 /**
  * Markdown conversion service for HTML <-> GFM conversion.
@@ -45,143 +48,144 @@ export class MarkdownConverter extends Context.Tag(
     /**
      * Convert Confluence storage format (HTML) to GitHub Flavored Markdown.
      */
-    readonly htmlToMarkdown: (html: string) => Effect.Effect<string, ConversionError>
+    readonly htmlToMarkdown: (
+      html: string,
+      options?: SerializeOptions
+    ) => Effect.Effect<string, ConversionError>
 
     /**
      * Convert GitHub Flavored Markdown to HTML (Confluence storage format).
      */
     readonly markdownToHtml: (markdown: string) => Effect.Effect<string, ConversionError>
+
+    /**
+     * Parse Confluence HTML to Document AST.
+     */
+    readonly htmlToAst: (html: string) => Effect.Effect<Document, ParseError>
+
+    /**
+     * Parse Markdown to Document AST.
+     */
+    readonly markdownToAst: (markdown: string) => Effect.Effect<Document, ParseError>
+
+    /**
+     * Serialize Document AST to Confluence HTML.
+     */
+    readonly astToHtml: (doc: Document) => Effect.Effect<string, SerializeError>
+
+    /**
+     * Serialize Document AST to Markdown.
+     */
+    readonly astToMarkdown: (doc: Document) => Effect.Effect<string, SerializeError>
   }
 >() {}
 
-/** Maximum HTML input size (1MB) to prevent ReDoS attacks */
-const MAX_HTML_SIZE = 1024 * 1024
-
 /**
- * Strip Confluence-specific macros while preserving text content.
- * Uses iterative approach to avoid ReDoS with nested content.
+ * Layer that provides the MarkdownConverter service.
+ *
+ * @category Layers
  */
-const stripConfluenceMacros = (html: string): Effect.Effect<string, ConversionError> =>
-  Effect.gen(function*() {
-    // Limit input size to prevent ReDoS
-    if (html.length > MAX_HTML_SIZE) {
-      return yield* Effect.fail(
-        new ConversionError({
-          direction: "htmlToMarkdown",
-          cause: `HTML input too large: ${html.length} bytes (max ${MAX_HTML_SIZE})`
-        })
-      )
-    }
-
-    let result = html
-
-    // Process structured macros iteratively to handle nesting safely
-    let iterations = 0
-    const maxIterations = 100 // Prevent infinite loops
-
-    while (iterations < maxIterations) {
-      const macroStart = result.indexOf("<ac:structured-macro")
-      if (macroStart === -1) break
-
-      // Find matching closing tag by counting nesting
-      let depth = 1
-      let pos = macroStart + 20 // Skip past opening tag start
-      let endPos = -1
-
-      while (pos < result.length && depth > 0) {
-        if (result.slice(pos, pos + 20) === "<ac:structured-macro") {
-          depth++
-          pos += 20
-        } else if (result.slice(pos, pos + 21) === "</ac:structured-macro") {
-          depth--
-          if (depth === 0) {
-            endPos = result.indexOf(">", pos) + 1
-          }
-          pos += 21
-        } else {
-          pos++
-        }
-      }
-
-      if (endPos === -1) break // Malformed HTML, stop processing
-
-      const macroContent = result.slice(macroStart, endPos)
-
-      // Extract content from macro
-      let replacement = ""
-      const plainBodyStart = macroContent.indexOf("<ac:plain-text-body><![CDATA[")
-      const plainBodyEnd = macroContent.indexOf("]]></ac:plain-text-body>")
-      if (plainBodyStart !== -1 && plainBodyEnd !== -1) {
-        const content = macroContent.slice(plainBodyStart + 29, plainBodyEnd)
-        replacement = `<pre><code>${content}</code></pre>`
-      } else {
-        const richBodyStart = macroContent.indexOf("<ac:rich-text-body>")
-        const richBodyEnd = macroContent.indexOf("</ac:rich-text-body>")
-        if (richBodyStart !== -1 && richBodyEnd !== -1) {
-          replacement = macroContent.slice(richBodyStart + 19, richBodyEnd)
-        }
-      }
+export const layer: Layer.Layer<MarkdownConverter> = Layer.succeed(
+  MarkdownConverter,
+  MarkdownConverter.of({
+    htmlToMarkdown: (html, options) =>
+      Effect.gen(function*() {
+        // Use AST-based approach to preserve colors, underlines, etc.
+        const doc = yield* parseConfluenceHtml(html).pipe(
+          Effect.mapError((e) => new ConversionError({ direction: "htmlToMarkdown", cause: e.message }))
+        )
+        return yield* serializeToMarkdown(doc, options).pipe(
+          Effect.mapError((e) => new ConversionError({ direction: "htmlToMarkdown", cause: e.message }))
+        )
+      }),
 
-      result = result.slice(0, macroStart) + replacement + result.slice(endPos)
-      iterations++
-    }
+    markdownToHtml: (markdown) =>
+      Effect.gen(function*() {
+        // Use AST-based approach for consistency
+        const doc = yield* parseMarkdown(markdown).pipe(
+          Effect.mapError((e) => new ConversionError({ direction: "markdownToHtml", cause: e.message }))
+        )
+        return yield* serializeToConfluence(doc).pipe(
+          Effect.mapError((e) => new ConversionError({ direction: "markdownToHtml", cause: e.message }))
+        )
+      }),
 
-    // Remove remaining simple tags with non-greedy bounded patterns
-    result = result
-      .replace(/<ac:parameter[^>]{0,1000}>[^<]{0,10000}<\/ac:parameter>/gi, "")
-      .replace(/<\/?ac:[a-z-]{1,50}[^>]{0,1000}>/gi, "")
-      .replace(/<\/?ri:[a-z-]{1,50}[^>]{0,1000}\/?>/gi, "")
+    htmlToAst: (html) => parseConfluenceHtml(html),
 
-    return result
-  })
+    markdownToAst: (markdown) => parseMarkdown(markdown),
 
-/**
- * Create the markdown converter processor for HTML -> Markdown.
- */
-const createHtmlToMdProcessor = () =>
-  unified()
-    .use(rehypeParse, { fragment: true })
-    .use(rehypeRemark)
-    .use(remarkGfm)
-    .use(remarkStringify)
+    astToHtml: (doc) => serializeToConfluence(doc),
 
-/**
- * Create the markdown converter processor for Markdown -> HTML.
- */
-const createMdToHtmlProcessor = () =>
-  unified()
-    .use(remarkParse)
-    .use(remarkGfm)
-    .use(remarkRehype)
-    .use(rehypeStringify)
+    astToMarkdown: (doc) => serializeToMarkdown(doc)
+  })
+)
 
 /**
- * Layer that provides the MarkdownConverter service.
+ * Schema-based layer for MarkdownConverter using Effect Schema transforms.
+ *
+ * This is an alternative implementation that uses the new Schema-based
+ * conversion pipeline. It provides the same API as the default layer.
+ *
+ * Note: For full fidelity, continue to use the default layer. This schema-based
+ * layer is useful for simpler use cases or when you want to leverage Schema
+ * composition.
+ *
+ * @example
+ * ```typescript
+ * import { MarkdownConverter, schemaBasedLayer } from "@knpkv/confluence-to-markdown/MarkdownConverter"
+ * import { Effect } from "effect"
+ *
+ * const program = Effect.gen(function* () {
+ *   const converter = yield* MarkdownConverter
+ *   const md = yield* converter.htmlToMarkdown("<h1>Hello</h1>")
+ * })
+ *
+ * Effect.runPromise(
+ *   program.pipe(Effect.provide(schemaBasedLayer))
+ * )
+ * ```
  *
  * @category Layers
  */
-export const layer: Layer.Layer<MarkdownConverter> = Layer.succeed(
+export const schemaBasedLayer: Layer.Layer<MarkdownConverter> = Layer.succeed(
   MarkdownConverter,
   MarkdownConverter.of({
-    htmlToMarkdown: (html) =>
+    // Note: Schema-based layer doesn't support includeRawSource option yet
+    htmlToMarkdown: (html, options) =>
       Effect.gen(function*() {
-        const cleaned = yield* stripConfluenceMacros(html)
-        return yield* Effect.try({
-          try: () => {
-            const result = createHtmlToMdProcessor().processSync(cleaned)
-            return String(result).trim()
-          },
-          catch: (cause) => new ConversionError({ direction: "htmlToMarkdown", cause })
-        })
-      }),
+        if (options?.includeRawSource !== undefined) {
+          yield* Effect.logWarning("schemaBasedLayer: includeRawSource option is not supported, use default layer")
+        }
+        return yield* Schema.decode(ConfluenceToMarkdown)(html)
+      }).pipe(
+        Effect.mapError((e) => new ConversionError({ direction: "htmlToMarkdown", cause: e.message }))
+      ),
 
     markdownToHtml: (markdown) =>
-      Effect.try({
-        try: () => {
-          const result = createMdToHtmlProcessor().processSync(markdown)
-          return String(result).trim()
-        },
-        catch: (cause) => new ConversionError({ direction: "markdownToHtml", cause })
-      })
+      Schema.encode(ConfluenceToMarkdown)(markdown).pipe(
+        Effect.mapError((e) => new ConversionError({ direction: "markdownToHtml", cause: e.message }))
+      ),
+
+    htmlToAst: (html) =>
+      Effect.gen(function*() {
+        const preprocessed = yield* Schema.decode(PreprocessedHtmlFromConfluence)(html)
+        const hast = yield* Schema.decode(HastFromHtml)(preprocessed)
+        return yield* Schema.decode(DocumentFromHast)(hast)
+      }).pipe(
+        Effect.mapError((e) => new ParseError({ source: "confluence", message: e.message }))
+      ),
+
+    markdownToAst: (markdown) =>
+      Effect.gen(function*() {
+        const mdast = yield* Schema.decode(MdastFromMarkdown)(markdown)
+        return yield* Schema.decode(DocumentFromMdast)(mdast)
+      }).pipe(
+        Effect.mapError((e) => new ParseError({ source: "markdown", message: e.message }))
+      ),
+
+    // For serialization, continue using the existing serializers for full fidelity
+    astToHtml: (doc) => serializeToConfluence(doc),
+
+    astToMarkdown: (doc) => serializeToMarkdown(doc)
   })
 )

package/src/SchemaConverterError.ts

@@ -0,0 +1,108 @@
+/**
+ * Error types for schema-based conversion.
+ *
+ * @module
+ */
+import * as Data from "effect/Data"
+
+/**
+ * Error thrown when parsing HTML or Markdown fails.
+ *
+ * @example
+ * ```typescript
+ * import { Effect } from "effect"
+ * import { ParseError } from "@knpkv/confluence-to-markdown/SchemaConverterError"
+ *
+ * Effect.gen(function* () {
+ *   // ... parsing operation
+ * }).pipe(
+ *   Effect.catchTag("ParseError", (error) =>
+ *     Effect.sync(() => console.error(`Parse error: ${error.message}`))
+ *   )
+ * )
+ * ```
+ *
+ * @category Errors
+ */
+export class ParseError extends Data.TaggedError("ParseError")<{
+  readonly source: "confluence" | "markdown"
+  readonly message: string
+  readonly position?: { readonly line: number; readonly column: number }
+  readonly rawContent?: string
+}> {}
+
+/**
+ * Error thrown when serializing AST to HTML or Markdown fails.
+ *
+ * @example
+ * ```typescript
+ * import { Effect } from "effect"
+ * import { SerializeError } from "@knpkv/confluence-to-markdown/SchemaConverterError"
+ *
+ * Effect.gen(function* () {
+ *   // ... serialization operation
+ * }).pipe(
+ *   Effect.catchTag("SerializeError", (error) =>
+ *     Effect.sync(() => console.error(`Serialize error: ${error.message}`))
+ *   )
+ * )
+ * ```
+ *
+ * @category Errors
+ */
+export class SerializeError extends Data.TaggedError("SerializeError")<{
+  readonly target: "confluence" | "markdown"
+  readonly nodeType: string
+  readonly message: string
+}> {}
+
+/**
+ * Error thrown when migrating between schema versions fails.
+ *
+ * @example
+ * ```typescript
+ * import { Effect } from "effect"
+ * import { MigrationError } from "@knpkv/confluence-to-markdown/SchemaConverterError"
+ *
+ * Effect.gen(function* () {
+ *   // ... migration operation
+ * }).pipe(
+ *   Effect.catchTag("MigrationError", (error) =>
+ *     Effect.sync(() =>
+ *       console.error(`Migration error: ${error.nodeType} v${error.fromVersion} -> v${error.toVersion}`)
+ *     )
+ *   )
+ * )
+ * ```
+ *
+ * @category Errors
+ */
+export class MigrationError extends Data.TaggedError("MigrationError")<{
+  readonly nodeType: string
+  readonly fromVersion: number
+  readonly toVersion: number
+  readonly message: string
+}> {}
+
+/**
+ * Union of all schema converter errors.
+ *
+ * @category Errors
+ */
+export type SchemaConverterError = ParseError | SerializeError | MigrationError
+
+/**
+ * Type guard to check if error is a SchemaConverterError.
+ *
+ * @param error - The error to check
+ * @returns True if error is a SchemaConverterError
+ *
+ * @category Utilities
+ */
+export const isSchemaConverterError = (error: unknown): error is SchemaConverterError =>
+  typeof error === "object" &&
+  error !== null &&
+  "_tag" in error &&
+  ["ParseError", "SerializeError", "MigrationError"].includes(
+    (error as { _tag: string })._tag
+  )