@baiyibai-antora/confluence-retriever 1.0.0-beta.5

package/bin/confluence-retrieve.js

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+'use strict'
+
+// confluence-retrieve: pull a Confluence space (or a page subtree) down as
+// AsciiDoc files plus attachments. Credentials from the environment only.
+
+const fsp = require('node:fs/promises')
+const ospath = require('node:path')
+const { AtlassianClient, credentialsFromEnv } = require('@baiyibai-antora/atlassian-client')
+const { retrieveSpace } = require('../lib')
+
+const USAGE = `Usage: confluence-retrieve [options] <SPACE-KEY>
+
+Writes every page of the space as an .adoc file (attachments alongside in
+_attachments/), mirroring the page hierarchy. Pages published by
+confluence-publisher are restored to their original component paths.
+
+Options:
+  --out <dir>        output directory (default: ./<SPACE-KEY>)
+  --page <id>        only this page and its descendants
+  --keep-colors      keep text/background colors as role spans
+  --no-keys          ignore publisher page keys; use slugified titles only
+  -h, --help         show this help
+
+Environment: ATLASSIAN_BASE_URL (or CONFLUENCE_BASE_URL), ATLASSIAN_EMAIL,
+ATLASSIAN_API_TOKEN`
+
+function parseArgs (argv) {
+  const opts = { out: null, page: null, styles: 'drop', restoreKeys: true, space: null }
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]
+    if (arg === '-h' || arg === '--help') return { help: true }
+    else if (arg === '--out') opts.out = argv[++i]
+    else if (arg === '--page') opts.page = argv[++i]
+    else if (arg === '--keep-colors') opts.styles = 'roles'
+    else if (arg === '--no-keys') opts.restoreKeys = false
+    else if (arg.startsWith('-')) throw new Error(`unknown option ${arg}`)
+    else if (opts.space) throw new Error('only one space key may be given')
+    else opts.space = arg
+  }
+  if (!opts.space) throw new Error('a space key is required')
+  return opts
+}
+
+async function main () {
+  let opts
+  try {
+    opts = parseArgs(process.argv.slice(2))
+  } catch (err) {
+    process.stderr.write(`confluence-retrieve: ${err.message}\n${USAGE}\n`)
+    return 1
+  }
+  if (opts.help) {
+    process.stdout.write(USAGE + '\n')
+    return 0
+  }
+
+  let client
+  try {
+    client = new AtlassianClient({ ...credentialsFromEnv(), serviceName: 'Confluence' })
+  } catch (err) {
+    process.stderr.write(`confluence-retrieve: ${err.message}\n`)
+    return 1
+  }
+
+  const logger = { warn: (msg) => process.stderr.write(`confluence-retrieve: ${msg}\n`) }
+  const { files, pages } = await retrieveSpace({
+    client,
+    spaceKey: opts.space,
+    styles: opts.styles,
+    restoreKeys: opts.restoreKeys,
+    rootPageId: opts.page,
+    logger,
+  })
+
+  const outDir = opts.out || opts.space
+  for (const file of files) {
+    const dest = ospath.join(outDir, ...file.path.split('/'))
+    await fsp.mkdir(ospath.dirname(dest), { recursive: true })
+    await fsp.writeFile(dest, file.contents)
+  }
+  process.stdout.write(`retrieved ${pages.length} page(s) and ${files.length - pages.length} attachment(s) to ${outDir}\n`)
+  return 0
+}
+
+main().then(
+  (code) => {
+    process.exitCode = code
+  },
+  (err) => {
+    process.stderr.write(`confluence-retrieve: ${err.message}\n`)
+    process.exitCode = 1
+  }
+)

package/lib/index.js

@@ -0,0 +1,5 @@
+'use strict'
+
+const { retrieveSpace, slugify, keyToPath } = require('./retrieve')
+
+module.exports = { retrieveSpace, slugify, keyToPath }

package/lib/retrieve.js

@@ -0,0 +1,143 @@
+'use strict'
+
+const { posix: path } = require('node:path')
+const { toAdoc } = require('@baiyibai-antora/asciidoc-adf-converter')
+
+// Retrieve a Confluence space as an in-memory AsciiDoc file tree. The core
+// is Antora-agnostic: it returns { files, pages, warnings } where files are
+// { path, contents } entries (pages as strings, attachments as Buffers) and
+// callers decide where they land (the CLI writes them to a directory).
+//
+// Page paths mirror the Confluence page hierarchy with slugified titles.
+// Pages that confluence-publisher published carry their original source
+// identity in the `antora-adf-publisher` content property
+// (`component:version:module:file`); with restoreKeys (default) those pages
+// come back under their original component/module paths instead.
+
+const PAGE_PROPERTY_KEY = 'antora-adf-publisher'
+
+function slugify (title) {
+  const slug = String(title)
+    .toLowerCase()
+    .normalize('NFKD')
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+  return slug || 'untitled'
+}
+
+function keyToPath (pageKey) {
+  // docs:1.2.0:ROOT:guides/intro.adoc -> docs/1.2.0/ROOT/guides/intro.adoc
+  const parts = String(pageKey).split(':')
+  if (parts.length < 4) return null
+  const [component, version, module_, ...file] = parts
+  return path.join(component, version, module_, file.join(':'))
+}
+
+async function retrieveSpace ({ client, spaceKey, styles, restoreKeys = true, rootPageId, logger }) {
+  const warnings = []
+  const warn = (msg) => {
+    warnings.push(msg)
+    if (logger) logger.warn(msg)
+  }
+
+  const spaces = await client.request('GET', `/wiki/api/v2/spaces?keys=${encodeURIComponent(spaceKey)}`)
+  const space = spaces && spaces.results && spaces.results[0]
+  if (!space) throw new Error(`space "${spaceKey}" not found`)
+
+  const raw = await client.getAll(`/wiki/api/v2/spaces/${space.id}/pages?body-format=atlas_doc_format&limit=100`)
+  const byId = new Map(raw.map((p) => [String(p.id), p]))
+
+  // Optional subtree filter: keep pages that descend from rootPageId.
+  const inScope = (page) => {
+    if (!rootPageId) return true
+    for (let id = String(page.id); id; ) {
+      if (id === String(rootPageId)) return true
+      const parent = byId.get(id)
+      id = parent && parent.parentId != null ? String(parent.parentId) : null
+    }
+    return false
+  }
+  const pagesRaw = raw.filter(inScope)
+
+  // Ancestor slug chains (the space homepage is the tree root, not a folder).
+  const homeId = space.homepageId == null ? null : String(space.homepageId)
+  const slugById = new Map(pagesRaw.map((p) => [String(p.id), slugify(p.title)]))
+  const dirFor = (page) => {
+    const chain = []
+    for (let id = page.parentId == null ? null : String(page.parentId); id && id !== homeId; ) {
+      const parent = byId.get(id)
+      if (!parent) break
+      chain.unshift(slugById.get(id) || slugify(parent.title))
+      id = parent.parentId == null ? null : String(parent.parentId)
+    }
+    return chain.join('/')
+  }
+
+  const files = []
+  const pages = []
+  const usedPaths = new Set()
+  const uniquePath = (candidate) => {
+    let result = candidate
+    for (let n = 2; usedPaths.has(result); n++) {
+      result = candidate.replace(/\.adoc$/, `-${n}.adoc`)
+    }
+    usedPaths.add(result)
+    return result
+  }
+
+  for (const page of pagesRaw) {
+    const id = String(page.id)
+    const body = page.body && page.body.atlas_doc_format && page.body.atlas_doc_format.value
+    if (!body) {
+      warn(`${page.title} (${id}): no atlas_doc_format body; skipped`)
+      continue
+    }
+
+    let key = null
+    if (restoreKeys) {
+      const props = await client.request(
+        'GET',
+        `/wiki/api/v2/pages/${id}/properties?key=${PAGE_PROPERTY_KEY}`
+      )
+      const prop = props && props.results && props.results[0]
+      key = (prop && prop.value && prop.value.pageKey) || null
+    }
+    const pagePath = uniquePath(
+      (key && keyToPath(key)) || path.join(dirFor(page), `${slugById.get(id)}.adoc`)
+    )
+
+    // Attachments: ADF media ids reference the attachment *file id*; map
+    // them onto downloaded files next to the page.
+    const attachments = await client.getAll(`/wiki/api/v2/pages/${id}/attachments?limit=100`)
+    const byFileId = new Map(attachments.map((att) => [String(att.fileId), att]))
+    const wanted = new Map()
+    const attachmentDir = path.join(path.dirname(pagePath), '_attachments')
+    const resolveMedia = ({ id: mediaId }) => {
+      const att = byFileId.get(String(mediaId))
+      if (!att) return null
+      const target = path.join(attachmentDir, att.title)
+      wanted.set(target, att)
+      return path.relative(path.dirname(pagePath), target)
+    }
+
+    const result = toAdoc(JSON.parse(body), { styles, resolveMedia })
+    for (const warning of result.warnings) warn(`${page.title} (${id}): ${warning}`)
+
+    for (const [target, att] of wanted) {
+      const downloadPath = att._links && att._links.download
+      const bytes = downloadPath ? await client.download(`/wiki${downloadPath}`) : null
+      if (bytes) {
+        files.push({ path: target, contents: bytes })
+      } else {
+        warn(`${page.title} (${id}): attachment ${att.title} could not be downloaded`)
+      }
+    }
+
+    files.push({ path: pagePath, contents: `= ${page.title}\n\n${result.adoc}` })
+    pages.push({ id, title: page.title, path: pagePath, key })
+  }
+
+  return { files, pages, warnings }
+}
+
+module.exports = { retrieveSpace, slugify, keyToPath }

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@baiyibai-antora/confluence-retriever",
+  "version": "1.0.0-beta.5",
+  "description": "Retrieve a Confluence Cloud space as AsciiDoc: pages via the v2 REST API in ADF, converted with @baiyibai-antora/asciidoc-adf-converter, attachments downloaded alongside",
+  "license": "MIT",
+  "author": "baiyibai",
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/baiyibai-antora/confluora.git",
+    "directory": "packages/confluence-retriever"
+  },
+  "type": "commonjs",
+  "main": "lib/index.js",
+  "bin": {
+    "confluence-retrieve": "bin/confluence-retrieve.js"
+  },
+  "files": [
+    "lib",
+    "bin"
+  ],
+  "scripts": {
+    "test": "node --test test/*-test.js"
+  },
+  "dependencies": {
+    "@baiyibai-antora/asciidoc-adf-converter": "^1.0.0-beta.5",
+    "@baiyibai-antora/atlassian-client": "^1.0.0-beta.5"
+  },
+  "keywords": [
+    "confluora",
+    "confluence",
+    "asciidoc",
+    "adf",
+    "retriever",
+    "export"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}