@knpkv/confluence-to-markdown 0.5.0 → 0.6.0

package/src/SyncEngine.ts

@@ -101,6 +101,26 @@ type SyncError =
   | GitServiceError
   | StructureError
 
+/**
+ * Pretty-print an ADF JSON wire string for the `.source.json` companion file.
+ * Falls back to the raw string if it can't be parsed as JSON.
+ */
+const prettyAdf = (raw: string): string => {
+  try {
+    return JSON.stringify(JSON.parse(raw), null, 2)
+  } catch {
+    return raw
+  }
+}
+
+/**
+ * Hash a markdown body in the form `readMarkdownFile` will later return it.
+ * `parseMarkdown` trims the body when reading, so the hash we store on
+ * frontmatter must hash the trimmed body too — otherwise every subsequent
+ * read sees a "changed" file and `push` keeps pushing a no-op update.
+ */
+const hashBody = (markdown: string) => computeHash(markdown.trim()).pipe(Effect.provide(HashServiceLive))
+
 /**
  * Sync engine service for Confluence <-> Markdown operations.
  *
@@ -334,11 +354,9 @@ export const layer: Layer.Layer<
       position?: number
     ): Effect.Effect<{ markdown: string; frontMatter: PageFrontMatter }, SyncError> =>
       Effect.gen(function*() {
-        const htmlContent = version.body?.storage?.value ?? ""
-        const markdown = yield* converter.htmlToMarkdown(htmlContent, {
-          includeRawSource: config.saveSource
-        })
-        const contentHash = yield* computeHash(markdown).pipe(Effect.provide(HashServiceLive))
+        const adfJson = version.body?.atlas_doc_format?.value ?? ""
+        const markdown = yield* converter.adfToMarkdown(adfJson)
+        const contentHash = yield* hashBody(markdown)
 
         // Get author info
         const author = version.authorId ? yield* getUser(version.authorId) : undefined
@@ -434,7 +452,7 @@ export const layer: Layer.Layer<
             }
 
             // Check if body content is available from the versions list
-            const bodyContent = versionInfo.page?.body?.storage?.value
+            const bodyContent = versionInfo.page?.body?.atlas_doc_format?.value
             if (!bodyContent) {
               // No body content available - history replay not supported
               historyReplayFailed = true
@@ -448,9 +466,9 @@ export const layer: Layer.Layer<
               createdAt: versionInfo.createdAt,
               message: versionInfo.message,
               body: {
-                storage: {
+                atlas_doc_format: {
                   value: bodyContent,
-                  representation: "storage" as const
+                  representation: "atlas_doc_format" as const
                 }
               }
             }
@@ -465,10 +483,10 @@ export const layer: Layer.Layer<
             // Write file
             yield* localFs.writeMarkdownFile(filePath, frontMatter, markdown)
 
-            // Save source HTML if configured
-            if (config.saveSource && versionContent.body?.storage?.value) {
-              const sourceFilePath = filePath.replace(/\.md$/, ".html")
-              yield* localFs.writeFile(sourceFilePath, versionContent.body.storage.value)
+            // Save source ADF JSON if configured
+            if (config.saveSource && versionContent.body?.atlas_doc_format?.value) {
+              const sourceFilePath = filePath.replace(/\.md$/, ".source.json")
+              yield* localFs.writeFile(sourceFilePath, prettyAdf(versionContent.body.atlas_doc_format.value))
             }
 
             // Commit this version
@@ -501,10 +519,8 @@ export const layer: Layer.Layer<
           }
 
           // Simple pull without history replay
-          const htmlContent = fullPage.body?.storage?.value ?? ""
-          let markdown = yield* converter.htmlToMarkdown(htmlContent, {
-            includeRawSource: config.saveSource
-          })
+          const adfJson = fullPage.body?.atlas_doc_format?.value ?? ""
+          let markdown = yield* converter.adfToMarkdown(adfJson)
 
           // Add child page links for index pages
           if (hasChildren && config.spaceKey) {
@@ -517,7 +533,7 @@ export const layer: Layer.Layer<
             markdown = markdown.trim() + "\n\n## Child Pages\n\n" + childLinks + "\n"
           }
 
-          const contentHash = yield* computeHash(markdown).pipe(Effect.provide(HashServiceLive))
+          const contentHash = yield* hashBody(markdown)
 
           // Get author info
           const author = fullPage.version.authorId ? yield* getUser(fullPage.version.authorId) : undefined
@@ -538,10 +554,10 @@ export const layer: Layer.Layer<
 
           yield* localFs.writeMarkdownFile(filePath, frontMatter, markdown)
 
-          // Save source HTML if configured
-          if (config.saveSource && htmlContent) {
-            const sourceFilePath = filePath.replace(/\.md$/, ".html")
-            yield* localFs.writeFile(sourceFilePath, htmlContent)
+          // Save source ADF JSON if configured
+          if (config.saveSource && adfJson) {
+            const sourceFilePath = filePath.replace(/\.md$/, ".source.json")
+            yield* localFs.writeFile(sourceFilePath, prettyAdf(adfJson))
           }
         }
 
@@ -576,31 +592,55 @@ export const layer: Layer.Layer<
           yield* git.checkout("origin/confluence")
         }
 
-        const rootPage = yield* client.getPage(config.rootPageId)
-        const result = yield* pullPage(rootPage, docsPath, options, gitInitialized)
+        const pullAndMerge = Effect.gen(function*() {
+          const rootPage = yield* client.getPage(config.rootPageId)
+          const result = yield* pullPage(rootPage, docsPath, options, gitInitialized)
 
-        // If git is initialized and we have changes but didn't replay history, auto-commit
-        if (gitInitialized && !options.replayHistory && result.pulled > 0) {
-          yield* git.addAll()
-          yield* git.commit({
-            message: `Pull from Confluence (${result.pulled} page${result.pulled !== 1 ? "s" : ""})`
-          }).pipe(Effect.catchTag("GitNoChangesError", () => Effect.void))
-        }
+          // If git is initialized and we have changes but didn't replay history, auto-commit
+          if (gitInitialized && !options.replayHistory && result.pulled > 0) {
+            yield* git.addAll()
+            yield* git.commit({
+              message: `Pull from Confluence (${result.pulled} page${result.pulled !== 1 ? "s" : ""})`
+            }).pipe(Effect.catchTag("GitNoChangesError", () => Effect.void))
+          }
 
-        // Two-branch model: merge origin/confluence into current branch
-        if (hasRemoteBranch && originalBranch && originalBranch !== "origin/confluence") {
-          yield* git.checkout(originalBranch)
-          yield* git.merge("origin/confluence", {
-            message: `Merge remote changes from Confluence`
-          }).pipe(Effect.catchAll(() => Effect.void)) // May fail if no changes
-        }
+          // Two-branch model: merge origin/confluence into current branch
+          if (hasRemoteBranch && originalBranch && originalBranch !== "origin/confluence") {
+            yield* git.checkout(originalBranch)
+            yield* git.merge("origin/confluence", {
+              message: `Merge remote changes from Confluence`
+            }).pipe(Effect.catchAll(() => Effect.void)) // May fail if no changes
+          }
 
-        return {
-          pulled: result.pulled,
-          skipped: 0,
-          commits: result.commits,
-          errors: [] as ReadonlyArray<string>
-        }
+          return {
+            pulled: result.pulled,
+            skipped: 0,
+            commits: result.commits,
+            errors: [] as ReadonlyArray<string>
+          }
+        })
+
+        // Any failure after the origin/confluence checkout (API, conversion,
+        // git) must not strand the repo on a detached HEAD — same protection
+        // as findUnpushedCommits. On success this re-checkout is a no-op.
+        // The restore itself can fail (a half-written pull leaves dirty files
+        // that differ between branches); that must be loud, not swallowed.
+        return yield* pullAndMerge.pipe(
+          Effect.ensuring(
+            hasRemoteBranch && originalBranch
+              ? git.checkout(originalBranch).pipe(
+                Effect.catchAll((error) =>
+                  Effect.logWarning(
+                    `pull: could not restore branch '${originalBranch}' — the repo may still be on origin/confluence. ` +
+                      `Restore manually with \`git checkout ${originalBranch}\` (stash local changes first if needed). Cause: ${
+                        String(error)
+                      }`
+                  )
+                )
+              )
+              : Effect.void
+          )
+        )
       })
 
     /**
@@ -641,8 +681,8 @@ export const layer: Layer.Layer<
           // Resolve parent from directory structure
           const parentId = yield* resolveParent(filePath, pageIdMap)
 
-          // Convert markdown to HTML
-          const html = yield* converter.markdownToHtml(localFile.content)
+          // Convert markdown to ADF
+          const adfValue = yield* converter.markdownToAdf(localFile.content)
 
           // Create the page
           const createdPage = yield* client.createPage({
@@ -650,8 +690,8 @@ export const layer: Layer.Layer<
             parentId,
             title,
             body: {
-              representation: "storage",
-              value: html
+              representation: "atlas_doc_format",
+              value: adfValue
             }
           })
 
@@ -665,11 +705,9 @@ export const layer: Layer.Layer<
 
           // Fetch canonical content back from Confluence
           const canonicalPage = yield* client.getPage(createdPage.id as PageId)
-          const canonicalHtml = canonicalPage.body?.storage?.value ?? ""
-          const canonicalMarkdown = yield* converter.htmlToMarkdown(canonicalHtml, {
-            includeRawSource: config.saveSource
-          })
-          const canonicalHash = yield* computeHash(canonicalMarkdown).pipe(Effect.provide(HashServiceLive))
+          const canonicalAdf = canonicalPage.body?.atlas_doc_format?.value ?? ""
+          const canonicalMarkdown = yield* converter.adfToMarkdown(canonicalAdf)
+          const canonicalHash = yield* hashBody(canonicalMarkdown)
 
           // Write canonical content with full front-matter
           const newFrontMatter: PageFrontMatter = {
@@ -698,7 +736,7 @@ export const layer: Layer.Layer<
 
         // Fetch current version to avoid conflicts
         const remotePage = yield* client.getPage(fm.pageId)
-        const html = yield* converter.markdownToHtml(localFile.content)
+        const adfValue = yield* converter.markdownToAdf(localFile.content)
         const updatedPage = yield* client.updatePage({
           id: fm.pageId,
           title: fm.title,
@@ -708,18 +746,16 @@ export const layer: Layer.Layer<
             message: revisionMessage
           },
           body: {
-            representation: "storage",
-            value: html
+            representation: "atlas_doc_format",
+            value: adfValue
           }
         })
 
         // Fetch canonical content back from Confluence
         const canonicalPage = yield* client.getPage(fm.pageId)
-        const canonicalHtml = canonicalPage.body?.storage?.value ?? ""
-        const canonicalMarkdown = yield* converter.htmlToMarkdown(canonicalHtml, {
-          includeRawSource: config.saveSource
-        })
-        const canonicalHash = yield* computeHash(canonicalMarkdown).pipe(Effect.provide(HashServiceLive))
+        const canonicalAdf = canonicalPage.body?.atlas_doc_format?.value ?? ""
+        const canonicalMarkdown = yield* converter.adfToMarkdown(canonicalAdf)
+        const canonicalHash = yield* hashBody(canonicalMarkdown)
 
         // Write canonical content with updated front-matter
         const newFrontMatter: PageFrontMatter = {
@@ -759,34 +795,45 @@ export const layer: Layer.Layer<
         const allCommits = yield* git.log({ n: 100 })
         if (allCommits.length === 0) return []
 
-        const unpushed: Array<{ hash: string; message: string }> = []
-
-        for (const commit of allCommits) {
-          yield* git.checkout(commit.hash)
-
-          let hasChanges = false
-          for (const filePath of files) {
-            const exists = yield* localFs.exists(filePath)
-            if (!exists) continue
+        // We checkout each commit to compare content; restore HEAD before returning
+        // so subsequent pushFile() calls see the working-tree content the user staged,
+        // not whatever commit we last inspected.
+        const originalBranch = yield* git.getCurrentBranch()
+
+        const collect = Effect.gen(function*() {
+          const unpushed: Array<{ hash: string; message: string }> = []
+          for (const commit of allCommits) {
+            yield* git.checkout(commit.hash)
+
+            let hasChanges = false
+            for (const filePath of files) {
+              const exists = yield* localFs.exists(filePath)
+              if (!exists) continue
+
+              const localFile = yield* localFs.readMarkdownFile(filePath)
+              if (!localFile.frontMatter) {
+                hasChanges = true
+                break
+              }
 
-            const localFile = yield* localFs.readMarkdownFile(filePath)
-            if (!localFile.frontMatter) {
-              hasChanges = true
-              break
+              const currentHash = yield* computeHash(localFile.content).pipe(Effect.provide(HashServiceLive))
+              if (currentHash !== localFile.frontMatter.contentHash) {
+                hasChanges = true
+                break
+              }
             }
 
-            const currentHash = yield* computeHash(localFile.content).pipe(Effect.provide(HashServiceLive))
-            if (currentHash !== localFile.frontMatter.contentHash) {
-              hasChanges = true
-              break
-            }
+            if (!hasChanges) break
+            unpushed.push({ hash: commit.hash, message: commit.message })
           }
+          return unpushed.reverse()
+        })
 
-          if (!hasChanges) break
-          unpushed.push({ hash: commit.hash, message: commit.message })
-        }
-
-        return unpushed.reverse()
+        return yield* collect.pipe(
+          Effect.ensuring(
+            originalBranch ? git.checkout(originalBranch).pipe(Effect.ignore) : Effect.void
+          )
+        )
       })
 
     const push = (options: { dryRun: boolean; message?: string }): Effect.Effect<PushResult, SyncError> =>

package/src/atlaskit-adf-schema.d.ts

@@ -0,0 +1,3 @@
+declare module "@atlaskit/adf-schema/dist/cjs/schema/default-schema.js" {
+  export { defaultSchema, defaultSchemaConfig } from "@atlaskit/adf-schema/schema-default"
+}

package/src/commands/clone.ts

@@ -8,6 +8,8 @@ import * as Console from "effect/Console"
 import * as Effect from "effect/Effect"
 import * as Layer from "effect/Layer"
 import * as Option from "effect/Option"
+import { layer as AdfSchemaValidatorLayer } from "../AdfSchemaValidator.js"
+import { layer as AtlaskitTransformersLayer } from "../AtlaskitTransformers.js"
 import type { PageId } from "../Brand.js"
 import { type ConfluenceClientConfig, layer as ConfluenceClientLayer } from "../ConfluenceClient.js"
 import { createConfigFile, layerFromValues as ConfluenceConfigLayerFromValues } from "../ConfluenceConfig.js"
@@ -19,6 +21,11 @@ import { layer as MarkdownConverterLayer } from "../MarkdownConverter.js"
 import { layer as SyncEngineLayer, SyncEngine } from "../SyncEngine.js"
 import { getAuth } from "./shared.js"
 
+const ConverterPipeline = MarkdownConverterLayer.pipe(
+  Layer.provide(AtlaskitTransformersLayer),
+  Layer.provide(AdfSchemaValidatorLayer)
+)
+
 const rootPageIdOption = Options.text("root-page-id").pipe(
   Options.withDescription("Confluence root page ID to sync from"),
   Options.optional
@@ -115,7 +122,7 @@ export const cloneCommand = Command.make(
         Layer.provideMerge(UserCacheLayer),
         Layer.provideMerge(GitServiceLayer),
         Layer.provideMerge(clientLayer),
-        Layer.provideMerge(MarkdownConverterLayer),
+        Layer.provideMerge(ConverterPipeline),
         Layer.provideMerge(LocalFileSystemLayer),
         Layer.provideMerge(configLayer),
         Layer.provideMerge(NodeContext.layer)

package/src/commands/layers.ts

@@ -6,6 +6,8 @@ import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient"
 import * as NodeTerminal from "@effect/platform-node/NodeTerminal"
 import * as Effect from "effect/Effect"
 import * as Layer from "effect/Layer"
+import { layer as AdfSchemaValidatorLayer } from "../AdfSchemaValidator.js"
+import { layer as AtlaskitTransformersLayer } from "../AtlaskitTransformers.js"
 import type { PageId } from "../Brand.js"
 import { ConfluenceAuth, layer as ConfluenceAuthLayer } from "../ConfluenceAuth.js"
 import { ConfluenceClient, type ConfluenceClientConfig, layer as ConfluenceClientLayer } from "../ConfluenceClient.js"
@@ -21,6 +23,11 @@ import { layer as MarkdownConverterLayer } from "../MarkdownConverter.js"
 import { layer as SyncEngineLayer, SyncEngine } from "../SyncEngine.js"
 import { getAuth } from "./shared.js"
 
+const ConverterPipeline = MarkdownConverterLayer.pipe(
+  Layer.provide(AtlaskitTransformersLayer),
+  Layer.provide(AdfSchemaValidatorLayer)
+)
+
 // Dummy config layer for help/init
 const DummyConfigLayer = ConfluenceConfigLayerFromValues({
   rootPageId: "dummy" as PageId,
@@ -136,7 +143,7 @@ export const AppLayer = SyncEngineLayer.pipe(
   Layer.provideMerge(UserCacheLayer),
   Layer.provideMerge(GitServiceLayer),
   Layer.provideMerge(ConfluenceClientLive),
-  Layer.provideMerge(MarkdownConverterLayer),
+  Layer.provideMerge(ConverterPipeline),
   Layer.provideMerge(LocalFileSystemLayer),
   Layer.provideMerge(ConfluenceConfigLayer()),
   Layer.provideMerge(AuthLive),
@@ -152,7 +159,7 @@ export const AuthOnlyLayer = DummySyncEngineLayer.pipe(
   Layer.provideMerge(DummyGitServiceLayer),
   Layer.provideMerge(DummyConfluenceClientLayer),
   Layer.provideMerge(AuthLive),
-  Layer.provideMerge(MarkdownConverterLayer),
+  Layer.provideMerge(ConverterPipeline),
   Layer.provideMerge(LocalFileSystemLayer),
   Layer.provideMerge(DummyConfigLayer),
   Layer.provideMerge(NodeHttpClient.layer),
@@ -167,7 +174,7 @@ export const MinimalLayer = DummySyncEngineLayer.pipe(
   Layer.provideMerge(GitServiceLayer),
   Layer.provideMerge(DummyConfluenceClientLayer),
   Layer.provideMerge(DummyConfluenceAuthLayer),
-  Layer.provideMerge(MarkdownConverterLayer),
+  Layer.provideMerge(ConverterPipeline),
   Layer.provideMerge(LocalFileSystemLayer),
   Layer.provideMerge(DummyConfigLayer),
   Layer.provideMerge(NodeTerminal.layer),
@@ -182,7 +189,7 @@ export const CloneLayer = DummySyncEngineLayer.pipe(
   Layer.provideMerge(GitServiceLayer),
   Layer.provideMerge(DummyConfluenceClientLayer),
   Layer.provideMerge(AuthLive),
-  Layer.provideMerge(MarkdownConverterLayer),
+  Layer.provideMerge(ConverterPipeline),
   Layer.provideMerge(LocalFileSystemLayer),
   Layer.provideMerge(DummyConfigLayer),
   Layer.provideMerge(NodeHttpClient.layer),

package/src/index.ts

@@ -6,6 +6,9 @@
  * @packageDocumentation
  */
 
+export { AdfSchemaValidator, layer as AdfSchemaValidatorLayer } from "./AdfSchemaValidator.js"
+export { type WalkerWarning, type WalkResult } from "./AdfWalker.js"
+export { AtlaskitTransformers, layer as AtlaskitTransformersLayer, type Transformers } from "./AtlaskitTransformers.js"
 export * from "./Brand.js"
 export { ConfluenceAuth, type ConfluenceAuthService, layer as ConfluenceAuthLayer } from "./ConfluenceAuth.js"
 export {
@@ -35,21 +38,3 @@ export {
   SyncEngine,
   type SyncStatus
 } from "./SyncEngine.js"
-
-// AST types
-export * from "./ast/index.js"
-
-// Parsers
-export { parseConfluenceHtml } from "./parsers/ConfluenceParser.js"
-export { parseMarkdown } from "./parsers/MarkdownParser.js"
-
-// Serializers
-export { serializeToConfluence } from "./serializers/ConfluenceSerializer.js"
-export { serializeToMarkdown } from "./serializers/MarkdownSerializer.js"
-
-// Bi-directional schemas
-export { DocumentFromConfluence } from "./schemas/ConfluenceSchema.js"
-export { DocumentFromMarkdown } from "./schemas/MarkdownSchema.js"
-
-// Schema converter errors
-export { MigrationError, ParseError, SerializeError } from "./SchemaConverterError.js"