@knpkv/confluence-to-markdown 0.2.0

package/src/Schemas.ts

@@ -0,0 +1,235 @@
+/**
+ * Schema definitions for configuration and data structures.
+ *
+ * @module
+ */
+import * as Schema from "effect/Schema"
+import { ContentHashSchema, PageIdSchema, SpaceKeySchema } from "./Brand.js"
+
+/**
+ * Schema for .confluence.json configuration file.
+ *
+ * @example
+ * ```typescript
+ * import { ConfluenceConfigFileSchema } from "@knpkv/confluence-to-markdown/Schemas"
+ * import * as Schema from "effect/Schema"
+ *
+ * const config = Schema.decodeUnknownSync(ConfluenceConfigFileSchema)({
+ *   rootPageId: "12345",
+ *   baseUrl: "https://mysite.atlassian.net"
+ * })
+ * ```
+ *
+ * @category Schema
+ */
+export const ConfluenceConfigFileSchema = Schema.Struct({
+  /** Root page ID to sync from */
+  rootPageId: PageIdSchema,
+  /** Confluence Cloud base URL */
+  baseUrl: Schema.String.pipe(
+    Schema.pattern(/^https:\/\/[a-z0-9-]+\.atlassian\.net$/)
+  ),
+  /** Optional space key */
+  spaceKey: Schema.optional(SpaceKeySchema),
+  /** Local docs path (default: .docs/confluence) */
+  docsPath: Schema.optionalWith(Schema.String, { default: () => ".docs/confluence" }),
+  /** Glob patterns to exclude from sync */
+  excludePatterns: Schema.optionalWith(Schema.Array(Schema.String), { default: () => [] })
+})
+
+/**
+ * Type for .confluence.json configuration file.
+ *
+ * @category Types
+ */
+export type ConfluenceConfigFile = Schema.Schema.Type<typeof ConfluenceConfigFileSchema>
+
+/**
+ * Schema for page front-matter in markdown files.
+ *
+ * @example
+ * ```typescript
+ * import { PageFrontMatterSchema } from "@knpkv/confluence-to-markdown/Schemas"
+ * import * as Schema from "effect/Schema"
+ *
+ * const frontMatter = Schema.decodeUnknownSync(PageFrontMatterSchema)({
+ *   pageId: "12345",
+ *   version: 3,
+ *   title: "My Page",
+ *   updated: "2025-01-15T10:30:00Z",
+ *   contentHash: "a".repeat(64)
+ * })
+ * ```
+ *
+ * @category Schema
+ */
+export const PageFrontMatterSchema = Schema.Struct({
+  /** Confluence page ID */
+  pageId: PageIdSchema,
+  /** Page version number */
+  version: Schema.Number.pipe(Schema.int(), Schema.positive()),
+  /** Page title */
+  title: Schema.String.pipe(Schema.nonEmptyString()),
+  /** Last updated timestamp (ISO8601) */
+  updated: Schema.DateFromString,
+  /** Parent page ID (optional) */
+  parentId: Schema.optional(PageIdSchema),
+  /** Position among siblings (optional) */
+  position: Schema.optional(Schema.Number),
+  /** SHA256 hash of content for change detection */
+  contentHash: ContentHashSchema
+})
+
+/**
+ * Type for page front-matter.
+ *
+ * @category Types
+ */
+export type PageFrontMatter = Schema.Schema.Type<typeof PageFrontMatterSchema>
+
+/**
+ * Schema for new page front-matter (no pageId yet).
+ *
+ * @category Schema
+ */
+export const NewPageFrontMatterSchema = Schema.Struct({
+  /** Page title */
+  title: Schema.String.pipe(Schema.nonEmptyString()),
+  /** Parent page ID (optional, determined by directory structure) */
+  parentId: Schema.optional(PageIdSchema)
+})
+
+/**
+ * Type for new page front-matter.
+ *
+ * @category Types
+ */
+export type NewPageFrontMatter = Schema.Schema.Type<typeof NewPageFrontMatterSchema>
+
+/**
+ * Schema for sync state file (.sync-state.json).
+ *
+ * @category Schema
+ */
+export const SyncStateSchema = Schema.Struct({
+  /** Last sync timestamp */
+  lastSync: Schema.DateFromString,
+  /** Map of page ID to sync info */
+  pages: Schema.Record({
+    key: Schema.String,
+    value: Schema.Struct({
+      /** Local file path */
+      localPath: Schema.String,
+      /** Last synced version */
+      version: Schema.Number,
+      /** Content hash at last sync */
+      contentHash: ContentHashSchema
+    })
+  })
+})
+
+/**
+ * Type for sync state.
+ *
+ * @category Types
+ */
+export type SyncState = Schema.Schema.Type<typeof SyncStateSchema>
+
+/**
+ * Schema for Confluence page API response (full, from getPage).
+ *
+ * @category Schema
+ */
+export const PageResponseSchema = Schema.Struct({
+  id: Schema.String,
+  title: Schema.String,
+  status: Schema.optional(Schema.String),
+  version: Schema.Struct({
+    number: Schema.Number,
+    createdAt: Schema.optional(Schema.String)
+  }),
+  body: Schema.optional(
+    Schema.Struct({
+      storage: Schema.optional(
+        Schema.Struct({
+          value: Schema.String,
+          representation: Schema.optional(Schema.String)
+        })
+      )
+    })
+  ),
+  parentId: Schema.optional(Schema.String),
+  position: Schema.optional(Schema.Number),
+  _links: Schema.optional(
+    Schema.Struct({
+      webui: Schema.optional(Schema.String)
+    })
+  )
+})
+
+/**
+ * Type for page API response.
+ *
+ * @category Types
+ */
+export type PageResponse = Schema.Schema.Type<typeof PageResponseSchema>
+
+/**
+ * Schema for page list item (children list, version optional).
+ *
+ * @category Schema
+ */
+export const PageListItemSchema = Schema.Struct({
+  id: Schema.String,
+  title: Schema.String,
+  status: Schema.optional(Schema.String),
+  version: Schema.optional(Schema.Struct({
+    number: Schema.Number,
+    createdAt: Schema.optional(Schema.String)
+  })),
+  body: Schema.optional(
+    Schema.Struct({
+      storage: Schema.optional(
+        Schema.Struct({
+          value: Schema.String,
+          representation: Schema.optional(Schema.String)
+        })
+      )
+    })
+  ),
+  parentId: Schema.optional(Schema.String),
+  position: Schema.optional(Schema.Number),
+  _links: Schema.optional(
+    Schema.Struct({
+      webui: Schema.optional(Schema.String)
+    })
+  )
+})
+
+/**
+ * Type for page list item.
+ *
+ * @category Types
+ */
+export type PageListItem = Schema.Schema.Type<typeof PageListItemSchema>
+
+/**
+ * Schema for page children API response.
+ *
+ * @category Schema
+ */
+export const PageChildrenResponseSchema = Schema.Struct({
+  results: Schema.Array(PageListItemSchema),
+  _links: Schema.optional(
+    Schema.Struct({
+      next: Schema.optional(Schema.String)
+    })
+  )
+})
+
+/**
+ * Type for page children response.
+ *
+ * @category Types
+ */
+export type PageChildrenResponse = Schema.Schema.Type<typeof PageChildrenResponseSchema>

package/src/SyncEngine.ts

@@ -0,0 +1,429 @@
+/**
+ * Sync engine for bidirectional Confluence <-> Markdown synchronization.
+ *
+ * @module
+ */
+import * as Path from "@effect/platform/Path"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import type { PageId } from "./Brand.js"
+import { ConfluenceClient } from "./ConfluenceClient.js"
+import { ConfluenceConfig } from "./ConfluenceConfig.js"
+import type {
+  ApiError,
+  ConflictError,
+  ConversionError,
+  FileSystemError,
+  FrontMatterError,
+  RateLimitError
+} from "./ConfluenceError.js"
+import { computeHash } from "./internal/hashUtils.js"
+import { LocalFileSystem } from "./LocalFileSystem.js"
+import { MarkdownConverter } from "./MarkdownConverter.js"
+import type { PageFrontMatter, PageListItem, PageResponse } from "./Schemas.js"
+
+/**
+ * Sync status for a single page.
+ */
+export type SyncStatus =
+  | { readonly _tag: "Synced"; readonly path: string }
+  | { readonly _tag: "LocalOnly"; readonly path: string; readonly title: string }
+  | { readonly _tag: "RemoteOnly"; readonly page: PageResponse }
+  | { readonly _tag: "LocalModified"; readonly path: string; readonly page: PageResponse }
+  | { readonly _tag: "RemoteModified"; readonly path: string; readonly page: PageResponse }
+  | {
+    readonly _tag: "Conflict"
+    readonly path: string
+    readonly page: PageResponse
+    readonly localVersion: number
+    readonly remoteVersion: number
+  }
+
+/**
+ * Result of a pull operation.
+ */
+export interface PullResult {
+  readonly pulled: number
+  readonly skipped: number
+  readonly errors: ReadonlyArray<string>
+}
+
+/**
+ * Result of a push operation.
+ */
+export interface PushResult {
+  readonly pushed: number
+  readonly created: number
+  readonly skipped: number
+  readonly errors: ReadonlyArray<string>
+}
+
+/**
+ * Result of a sync operation.
+ */
+export interface SyncResult {
+  readonly pulled: number
+  readonly pushed: number
+  readonly created: number
+  readonly conflicts: number
+  readonly errors: ReadonlyArray<string>
+}
+
+/**
+ * Result of a status operation.
+ */
+export interface StatusResult {
+  readonly synced: number
+  readonly localModified: number
+  readonly remoteModified: number
+  readonly conflicts: number
+  readonly localOnly: number
+  readonly remoteOnly: number
+  readonly files: ReadonlyArray<SyncStatus>
+}
+
+type SyncError = ApiError | RateLimitError | ConversionError | FileSystemError | FrontMatterError
+
+/**
+ * Sync engine service for Confluence <-> Markdown operations.
+ *
+ * @example
+ * ```typescript
+ * import { SyncEngine } from "@knpkv/confluence-to-markdown/SyncEngine"
+ * import { Effect } from "effect"
+ *
+ * const program = Effect.gen(function* () {
+ *   const engine = yield* SyncEngine
+ *   const result = yield* engine.pull({ force: false })
+ *   console.log(`Pulled ${result.pulled} pages`)
+ * })
+ * ```
+ *
+ * @category Sync
+ */
+export class SyncEngine extends Context.Tag(
+  "@knpkv/confluence-to-markdown/SyncEngine"
+)<
+  SyncEngine,
+  {
+    /**
+     * Pull pages from Confluence to local markdown.
+     */
+    readonly pull: (options: { force: boolean }) => Effect.Effect<PullResult, SyncError>
+
+    /**
+     * Push local markdown changes to Confluence.
+     */
+    readonly push: (options: { dryRun: boolean }) => Effect.Effect<PushResult, SyncError>
+
+    /**
+     * Bidirectional sync with conflict detection.
+     */
+    readonly sync: () => Effect.Effect<SyncResult, SyncError | ConflictError>
+
+    /**
+     * Get sync status for all files.
+     */
+    readonly status: () => Effect.Effect<StatusResult, SyncError>
+  }
+>() {}
+
+/**
+ * Layer that provides SyncEngine.
+ *
+ * @category Layers
+ */
+export const layer: Layer.Layer<
+  SyncEngine,
+  never,
+  ConfluenceClient | ConfluenceConfig | MarkdownConverter | LocalFileSystem | Path.Path
+> = Layer.effect(
+  SyncEngine,
+  Effect.gen(function*() {
+    const client = yield* ConfluenceClient
+    const config = yield* ConfluenceConfig
+    const converter = yield* MarkdownConverter
+    const localFs = yield* LocalFileSystem
+    const pathService = yield* Path.Path
+
+    const docsPath = pathService.join(process.cwd(), config.docsPath)
+
+    /**
+     * Pull a single page and its children recursively.
+     */
+    const pullPage = (
+      page: PageListItem | PageResponse,
+      parentPath: string,
+      force: boolean
+    ): Effect.Effect<number, SyncError> =>
+      Effect.gen(function*() {
+        // Get children to determine if this is a folder
+        const children = yield* client.getAllChildren(page.id as PageId)
+        const hasChildren = children.length > 0
+
+        const filePath = localFs.getPagePath(page.title, hasChildren, parentPath)
+        const dirPath = hasChildren ? localFs.getPageDir(page.title, parentPath) : parentPath
+
+        // Get page content
+        const fullPage = yield* client.getPage(page.id as PageId)
+        const htmlContent = fullPage.body?.storage?.value ?? ""
+        let markdown = yield* converter.htmlToMarkdown(htmlContent)
+
+        // Add child page links for index pages
+        if (hasChildren && config.spaceKey) {
+          const childLinks = children
+            .map((child) => {
+              const pageUrl = `${config.baseUrl}/wiki/spaces/${config.spaceKey}/pages/${child.id}`
+              return `- [${child.title}](${pageUrl})`
+            })
+            .join("\n")
+          markdown = markdown.trim() + "\n\n## Child Pages\n\n" + childLinks + "\n"
+        }
+
+        const contentHash = computeHash(markdown)
+
+        // Check if we need to update
+        if (!force) {
+          const exists = yield* localFs.exists(filePath)
+          if (exists) {
+            const localFile = yield* localFs.readMarkdownFile(filePath)
+            if (
+              localFile.frontMatter &&
+              localFile.frontMatter.version === fullPage.version.number &&
+              localFile.frontMatter.contentHash === contentHash
+            ) {
+              // Skip - already in sync
+              let count = 0
+              for (const child of children) {
+                count += yield* pullPage(child, dirPath, force)
+              }
+              return count
+            }
+          }
+        }
+
+        // Ensure directory exists
+        if (hasChildren) {
+          yield* localFs.ensureDir(dirPath)
+        }
+
+        // Write file
+        const frontMatter: PageFrontMatter = {
+          pageId: page.id as PageId,
+          version: fullPage.version.number,
+          title: fullPage.title,
+          updated: fullPage.version.createdAt ? new Date(fullPage.version.createdAt) : new Date(),
+          ...(page.parentId ? { parentId: page.parentId as PageId } : {}),
+          ...(page.position !== undefined ? { position: page.position } : {}),
+          contentHash
+        }
+
+        yield* localFs.writeMarkdownFile(filePath, frontMatter, markdown)
+
+        // Pull children
+        let count = 1
+        for (const child of children) {
+          count += yield* pullPage(child, dirPath, force)
+        }
+
+        return count
+      })
+
+    const pull = (options: { force: boolean }): Effect.Effect<PullResult, SyncError> =>
+      Effect.gen(function*() {
+        yield* localFs.ensureDir(docsPath)
+
+        const rootPage = yield* client.getPage(config.rootPageId)
+        const pulled = yield* pullPage(rootPage, docsPath, options.force)
+
+        return {
+          pulled,
+          skipped: 0,
+          errors: [] as ReadonlyArray<string>
+        }
+      })
+
+    const push = (options: { dryRun: boolean }): Effect.Effect<PushResult, SyncError> =>
+      Effect.gen(function*() {
+        const files = yield* localFs.listMarkdownFiles(docsPath)
+        let pushed = 0
+        let created = 0
+        let skipped = 0
+        const errors: Array<string> = []
+
+        for (const filePath of files) {
+          yield* Effect.gen(function*() {
+            const localFile = yield* localFs.readMarkdownFile(filePath)
+
+            if (localFile.isNew || !localFile.frontMatter) {
+              // New file - create page
+              if (!options.dryRun) {
+                // For now, skip page creation - need space ID in config
+                errors.push(
+                  `Page creation requires space ID in config (not yet supported): ${filePath}`
+                )
+              }
+              created++
+              return
+            }
+
+            const fm = localFile.frontMatter
+            const currentHash = computeHash(localFile.content)
+
+            if (currentHash === fm.contentHash) {
+              skipped++
+              return
+            }
+
+            if (!options.dryRun) {
+              // Fetch current version to avoid conflicts
+              const remotePage = yield* client.getPage(fm.pageId)
+              const html = yield* converter.markdownToHtml(localFile.content)
+              const updatedPage = yield* client.updatePage({
+                id: fm.pageId,
+                title: fm.title,
+                status: "current",
+                version: {
+                  number: remotePage.version.number + 1,
+                  message: "Updated via confluence-to-markdown"
+                },
+                body: {
+                  representation: "storage",
+                  value: html
+                }
+              })
+
+              // Update front-matter with new version
+              const newFrontMatter: PageFrontMatter = {
+                ...fm,
+                version: updatedPage.version.number,
+                updated: new Date(),
+                contentHash: currentHash
+              }
+              yield* localFs.writeMarkdownFile(filePath, newFrontMatter, localFile.content)
+            }
+
+            pushed++
+          }).pipe(
+            Effect.catchAll((error) =>
+              Effect.sync(() => {
+                errors.push(
+                  `Failed to push ${filePath}: ${error._tag === "ApiError" ? error.message : error._tag}`
+                )
+              })
+            )
+          )
+        }
+
+        return { pushed, created, skipped, errors: errors as ReadonlyArray<string> }
+      })
+
+    const sync = (): Effect.Effect<SyncResult, SyncError | ConflictError> =>
+      Effect.gen(function*() {
+        // First, check for conflicts
+        const statusResult = yield* status()
+        const conflictErrors: Array<string> = []
+
+        if (statusResult.conflicts > 0) {
+          for (const file of statusResult.files) {
+            if (file._tag === "Conflict") {
+              conflictErrors.push(
+                `Conflict in ${file.path}: local v${file.localVersion} vs remote v${file.remoteVersion}`
+              )
+            }
+          }
+        }
+
+        // Pull remote changes
+        const pullResult = yield* pull({ force: false })
+
+        // Push local changes
+        const pushResult = yield* push({ dryRun: false })
+
+        return {
+          pulled: pullResult.pulled,
+          pushed: pushResult.pushed,
+          created: pushResult.created,
+          conflicts: statusResult.conflicts,
+          errors: [...conflictErrors, ...pullResult.errors, ...pushResult.errors] as ReadonlyArray<string>
+        }
+      })
+
+    const status = (): Effect.Effect<StatusResult, SyncError> =>
+      Effect.gen(function*() {
+        const files = yield* localFs.listMarkdownFiles(docsPath)
+        const statuses: Array<SyncStatus> = []
+
+        let synced = 0
+        let localModified = 0
+        let remoteModified = 0
+        let conflicts = 0
+        let localOnly = 0
+        const remoteOnly = 0
+
+        for (const filePath of files) {
+          const localFile = yield* localFs.readMarkdownFile(filePath)
+
+          if (localFile.isNew || !localFile.frontMatter) {
+            statuses.push({ _tag: "LocalOnly", path: filePath, title: pathService.basename(filePath, ".md") })
+            localOnly++
+            continue
+          }
+
+          const fm = localFile.frontMatter
+          const currentHash = computeHash(localFile.content)
+
+          // Fetch remote page
+          const remotePage = yield* Effect.either(client.getPage(fm.pageId))
+
+          if (remotePage._tag === "Left") {
+            statuses.push({ _tag: "LocalOnly", path: filePath, title: fm.title })
+            localOnly++
+            continue
+          }
+
+          const page = remotePage.right
+          const localChanged = currentHash !== fm.contentHash
+          const remoteChanged = page.version.number > fm.version
+
+          if (localChanged && remoteChanged) {
+            statuses.push({
+              _tag: "Conflict",
+              path: filePath,
+              page,
+              localVersion: fm.version,
+              remoteVersion: page.version.number
+            })
+            conflicts++
+          } else if (localChanged) {
+            statuses.push({ _tag: "LocalModified", path: filePath, page })
+            localModified++
+          } else if (remoteChanged) {
+            statuses.push({ _tag: "RemoteModified", path: filePath, page })
+            remoteModified++
+          } else {
+            statuses.push({ _tag: "Synced", path: filePath })
+            synced++
+          }
+        }
+
+        return {
+          synced,
+          localModified,
+          remoteModified,
+          conflicts,
+          localOnly,
+          remoteOnly,
+          files: statuses as ReadonlyArray<SyncStatus>
+        }
+      })
+
+    return SyncEngine.of({
+      pull,
+      push,
+      sync,
+      status
+    })
+  })
+)