@stravica-ai/digital-operator-pa 0.1.39 → 0.1.41

package/dist/engineering-pa-core-extensions-atlassian/src/confluence/commands/mirror-folder.mjs

@@ -0,0 +1,487 @@
+// mirror-folder.mjs - Mirror a Confluence folder subtree to local markdown.
+//
+// Usage:
+//   confluence mirror-folder <FOLDER_ID> [--target DIR] [--dry-run] [--no-reindex]
+//
+// Behaviour:
+//   1. Fetch the folder's descendants (folders + pages, recursively).
+//   2. For each page, fetch the storage-format body and convert to markdown.
+//   3. Write every page into a staging directory that mirrors the Confluence
+//      hierarchy.
+//   4. Atomically swap the staging directory into the target: preserve
+//      README.md and any dotfiles, remove everything else, move staged
+//      content in.
+//   5. Run `npm run reindex` against the repo root so domains.json refreshes.
+//
+// Design notes:
+//   - A partial/failed run must never leave the target directory broken.
+//     All page fetches happen into staging first; the swap is last.
+//   - Zero-deps: everything uses node: built-ins and the existing client.
+//   - Progress goes to stderr (console.error) so --dry-run output on stdout
+//     can be piped cleanly.
+
+import { mkdir, writeFile, rm, readdir, rename, stat, readFile } from 'node:fs/promises';
+import { basename, join, resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { spawnSync } from 'node:child_process';
+
+import { createClient, ConfluenceError } from '../client.mjs';
+import { storageToMarkdown } from '../storage.mjs';
+import {
+  sanitiseTitle,
+  disambiguate,
+  composeMarkdown,
+  planTree,
+  splitForSwap,
+  pageUrl,
+  buildManifest,
+  serialiseManifest,
+} from '../mirror.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+// From operator/extensions/tools/confluence/src/commands/ → five levels up.
+const DEFAULT_REPO_ROOT = resolve(__dirname, '..', '..', '..', '..', '..', '..');
+
+const HELP = `confluence mirror-folder - Mirror a Confluence folder to local markdown
+
+Usage:
+  confluence mirror-folder <FOLDER_ID> [options]
+
+Arguments:
+  FOLDER_ID          Numeric Confluence folder ID.
+
+Options:
+  --target DIR       Target directory (default: operator/knowledge/mirror/).
+  --dry-run          Walk the tree and print the plan; write nothing.
+  --no-reindex       Skip the npm run reindex step after a successful mirror.
+  --help, -h         Show this help.
+
+Behaviour:
+  Walks the folder's descendants (folders + pages, recursively), fetches each
+  page as markdown, and mirrors the hierarchy into the target directory.
+  README.md in the target is preserved across runs; everything else is
+  wiped and replaced.
+
+Examples:
+  confluence mirror-folder 3390046580
+  confluence mirror-folder 3390046580 --target /tmp/scratch --dry-run
+`;
+
+/**
+ * Run the mirror-folder command.
+ *
+ * @param {string[]} argv  Remaining args after 'mirror-folder'
+ * @param {object} [deps]  Dependency overrides for testing
+ * @returns {Promise<number>} Exit code
+ */
+export async function run(argv = [], deps = {}) {
+  const log = deps.log || console.log;
+  const logProgress = deps.logProgress || ((msg) => console.error(msg));
+  const logError = deps.logError || console.error;
+  const { parseArgs } = await import('../cli.mjs');
+  const args = parseArgs(argv);
+
+  if (args.help || args.h) {
+    log(HELP);
+    return 0;
+  }
+
+  const folderId = args._[0];
+  if (!folderId) {
+    logError('Usage: confluence mirror-folder <FOLDER_ID> [--target DIR] [--dry-run]');
+    return 1;
+  }
+
+  const repoRoot = deps.repoRoot || process.env.DAVE_ROOT || DEFAULT_REPO_ROOT;
+  const target = args.target
+    ? resolve(args.target)
+    : join(repoRoot, 'operator', 'knowledge', 'mirror');
+  const dryRun = !!args['dry-run'];
+  const skipReindex = !!args['no-reindex'];
+
+  const client = deps.client || createClient();
+
+  // Resolve credentials for URL construction - this also verifies auth
+  // is available before we do any heavy lifting.
+  let domain;
+  try {
+    domain = client.getCredentials().domain;
+  } catch (err) {
+    if (err instanceof ConfluenceError) {
+      logError(`Failed to load credentials: ${err.message}`);
+      return err.exitCode;
+    }
+    throw err;
+  }
+
+  try {
+    // ── 1. Build the tree ─────────────────────────────────────────
+    logProgress(`Walking Confluence folder ${folderId}…`);
+    const rootMeta = await fetchFolderMeta(client, folderId);
+    logProgress(`  Folder: ${rootMeta.title} (space ${rootMeta.spaceId})`);
+
+    const rootChildren = await collectFolderChildren(client, folderId, logProgress);
+    const pageCount = countPages(rootChildren);
+    logProgress(`  Discovered ${pageCount} page(s) in the subtree.`);
+
+    if (pageCount === 0) {
+      logError('No pages found under folder. Refusing to wipe target.');
+      return 2;
+    }
+
+    // ── 2. Plan the write ─────────────────────────────────────────
+    const plan = planTree(rootChildren);
+    if (dryRun) {
+      log(`Target: ${target}`);
+      log(`Folder: ${rootMeta.title} (${folderId})`);
+      log(`Plan (${plan.length} entries):`);
+      for (const entry of plan) {
+        log(`  [${entry.kind}] ${entry.relPath} - ${entry.node.title}`);
+      }
+      return 0;
+    }
+
+    // ── 3. Stage the mirror tree ──────────────────────────────────
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const stagingDir = join(target, `.mirror-tmp-${stamp}`);
+    logProgress(`Staging directory: ${stagingDir}`);
+
+    await mkdir(target, { recursive: true });
+    await mkdir(stagingDir, { recursive: true });
+
+    try {
+      const mirrorRunAt = new Date().toISOString();
+      const mirroredPageIds = [];
+      let writtenFiles = 0;
+      for (const entry of plan) {
+        const absPath = join(stagingDir, ...entry.relPath.split('/'));
+        if (entry.kind === 'dir') {
+          await mkdir(absPath, { recursive: true });
+          continue;
+        }
+        // File - fetch page body and write markdown.
+        logProgress(`  Fetching page ${entry.node.id} - ${entry.node.title}`);
+        const pageBody = await fetchPageMarkdown(client, entry.node.id);
+        const md = composeMarkdown(
+          {
+            title: entry.node.title,
+            pageId: entry.node.id,
+            url: pageUrl(domain, entry.node.id, pageBody.webuiPath),
+            space: rootMeta.spaceId ? String(rootMeta.spaceId) : undefined,
+            version: pageBody.version,
+            updatedAt: pageBody.updatedAt,
+            breadcrumb: entry.breadcrumb,
+          },
+          pageBody.markdown,
+        );
+        await mkdir(dirname(absPath), { recursive: true });
+        await writeFile(absPath, md, 'utf-8');
+        mirroredPageIds.push(String(entry.node.id));
+        writtenFiles++;
+      }
+      logProgress(`  Wrote ${writtenFiles} file(s) to staging.`);
+
+      // ── 3b. Write the run manifest into staging ────────────────
+      // Lives at the domain root as a sibling of README.md. Written
+      // BEFORE the swap so that if any preceding page write fails
+      // we never publish a half-truthful manifest. The swap is
+      // intentionally NOT preserving manifest.json across runs -
+      // a stale manifest after a partial run would be misleading.
+      const mirrorToolVersion = await readToolVersion();
+      const manifest = buildManifest({
+        domainId: basename(target),
+        sourceFolderId: folderId,
+        sourceSpaceKey: rootMeta.spaceId ? String(rootMeta.spaceId) : undefined,
+        mirrorRunAt,
+        pageIds: mirroredPageIds,
+        mirrorToolVersion,
+      });
+      const manifestPath = join(stagingDir, 'manifest.json');
+      await writeFile(manifestPath, serialiseManifest(manifest), 'utf-8');
+      logProgress(`  Wrote manifest.json (${manifest.pageCount} page IDs).`);
+
+      // ── 4. Atomic swap ──────────────────────────────────────────
+      logProgress('Swapping staged content into target…');
+      await swapIntoTarget(target, stagingDir);
+      logProgress('Swap complete.');
+    } finally {
+      // Best-effort cleanup of staging (only still exists if we failed
+      // mid-way or the swap logic chose to leave residue).
+      await rm(stagingDir, { recursive: true, force: true }).catch(() => {});
+    }
+
+    // ── 5. Reindex ────────────────────────────────────────────────
+    if (!skipReindex) {
+      logProgress('Running npm run reindex…');
+      const rc = runReindex(repoRoot, logProgress);
+      if (rc !== 0) {
+        // Soft-fail (acceptance ruling C2, 2026-07-14): on the composed v2
+        // workspace there is no local `reindex` script - domains.json is
+        // estate-composed and refreshed by the librarian, not by this tool.
+        // A completed mirror is a success; a stale local index is a warning.
+        logError(`Reindex exited with code ${rc}. Mirror completed; domains.json refresh is estate-side on v2 (warning only).`);
+      }
+    }
+
+    logProgress(`Mirror complete: ${pageCount} page(s) written to ${target}.`);
+    return 0;
+  } catch (err) {
+    if (err instanceof ConfluenceError) {
+      logError(`Mirror failed: ${err.message}`);
+      return err.exitCode;
+    }
+    logError(`Mirror failed: ${err.message || err}`);
+    return 2;
+  }
+}
+
+// ── API walkers ──────────────────────────────────────────────────────
+
+/**
+ * Fetch folder metadata (title, spaceId).
+ * @param {object} client
+ * @param {string} folderId
+ */
+async function fetchFolderMeta(client, folderId) {
+  const { data } = await client.get(`/wiki/api/v2/folders/${encodeURIComponent(folderId)}`);
+  return data;
+}
+
+/**
+ * Fetch the direct children of a folder, paginating through all pages.
+ * @param {object} client
+ * @param {string} folderId
+ */
+async function fetchFolderDirectChildren(client, folderId) {
+  const results = [];
+  let cursor = null;
+  // The v2 direct-children endpoint returns _links.next as a relative path.
+  // Guard against unbounded loops with a generous cap.
+  for (let i = 0; i < 200; i++) {
+    const query = new URLSearchParams({ limit: '100' });
+    if (cursor) query.set('cursor', cursor);
+    const path = `/wiki/api/v2/folders/${encodeURIComponent(folderId)}/direct-children?${query.toString()}`;
+    const { data } = await client.get(path);
+    for (const item of data.results || []) results.push(item);
+    const nextPath = data._links?.next;
+    if (!nextPath) break;
+    const nextCursor = extractCursor(nextPath);
+    if (!nextCursor || nextCursor === cursor) break;
+    cursor = nextCursor;
+  }
+  return results;
+}
+
+/**
+ * Fetch a page's direct child pages. v2 exposes /pages/{id}/children which
+ * returns pages only (folders cannot be children of a page).
+ * @param {object} client
+ * @param {string} pageId
+ */
+async function fetchPageChildren(client, pageId) {
+  const results = [];
+  let cursor = null;
+  for (let i = 0; i < 200; i++) {
+    const query = new URLSearchParams({ limit: '100' });
+    if (cursor) query.set('cursor', cursor);
+    const path = `/wiki/api/v2/pages/${encodeURIComponent(pageId)}/children?${query.toString()}`;
+    const { data } = await client.get(path);
+    for (const item of data.results || []) results.push(item);
+    const nextPath = data._links?.next;
+    if (!nextPath) break;
+    const nextCursor = extractCursor(nextPath);
+    if (!nextCursor || nextCursor === cursor) break;
+    cursor = nextCursor;
+  }
+  return results;
+}
+
+/**
+ * Extract the `cursor` query parameter from a v2 _links.next path.
+ * @param {string} nextPath
+ * @returns {string|null}
+ */
+function extractCursor(nextPath) {
+  try {
+    const match = nextPath.match(/[?&]cursor=([^&]+)/);
+    if (match) return decodeURIComponent(match[1]);
+  } catch {
+    // fall through
+  }
+  return null;
+}
+
+/**
+ * Recursively collect the full subtree under a folder as a list of
+ * tree nodes suitable for planTree().
+ * @param {object} client
+ * @param {string} folderId
+ * @param {Function} logProgress
+ */
+async function collectFolderChildren(client, folderId, logProgress) {
+  const raw = await fetchFolderDirectChildren(client, folderId);
+  const nodes = [];
+  for (const child of raw) {
+    if (child.type === 'folder') {
+      logProgress(`  ↳ folder: ${child.title}`);
+      const subChildren = await collectFolderChildren(client, child.id, logProgress);
+      nodes.push({
+        type: 'folder',
+        id: String(child.id),
+        title: child.title || 'untitled',
+        children: subChildren,
+      });
+    } else if (child.type === 'page') {
+      const subChildren = await collectPageDescendants(client, child.id, logProgress);
+      nodes.push({
+        type: 'page',
+        id: String(child.id),
+        title: child.title || 'untitled',
+        children: subChildren,
+      });
+    }
+    // Other types (attachment, whiteboard, etc) are skipped deliberately.
+  }
+  return nodes;
+}
+
+/**
+ * Recursively collect a page's descendants (child pages and their children).
+ */
+async function collectPageDescendants(client, pageId, logProgress) {
+  const children = await fetchPageChildren(client, pageId);
+  const nodes = [];
+  for (const child of children) {
+    const sub = await collectPageDescendants(client, child.id, logProgress);
+    nodes.push({
+      type: 'page',
+      id: String(child.id),
+      title: child.title || 'untitled',
+      children: sub,
+    });
+  }
+  return nodes;
+}
+
+/**
+ * Count the page nodes in a tree (ignores folders).
+ */
+function countPages(nodes) {
+  let n = 0;
+  for (const node of nodes) {
+    if (node.type === 'page') n++;
+    if (Array.isArray(node.children)) n += countPages(node.children);
+  }
+  return n;
+}
+
+// ── Page fetching ────────────────────────────────────────────────────
+
+/**
+ * Fetch a page's storage body and convert to markdown.
+ * Also returns version and updated-at metadata.
+ * @param {object} client
+ * @param {string} pageId
+ */
+async function fetchPageMarkdown(client, pageId) {
+  const { data } = await client.get(
+    `/wiki/api/v2/pages/${encodeURIComponent(pageId)}?body-format=storage`,
+  );
+  const storageBody = data.body?.storage?.value || '';
+  const markdown = storageToMarkdown(storageBody);
+  return {
+    markdown,
+    version: data.version?.number,
+    updatedAt: data.version?.createdAt,
+    webuiPath: data._links?.webui,
+  };
+}
+
+// ── Atomic swap ──────────────────────────────────────────────────────
+
+/**
+ * Move staged content into the target directory, preserving README.md and
+ * dotfiles. Removes everything else currently in the target.
+ *
+ * @param {string} target
+ * @param {string} stagingDir
+ */
+async function swapIntoTarget(target, stagingDir) {
+  const existing = await readdir(target);
+  const { toRemove } = splitForSwap(existing.filter((e) => !isStagingDir(e)));
+  for (const name of toRemove) {
+    await rm(join(target, name), { recursive: true, force: true });
+  }
+
+  // Move staged entries into the target.
+  const staged = await readdir(stagingDir);
+  for (const name of staged) {
+    const from = join(stagingDir, name);
+    const to = join(target, name);
+    // If a preserved entry (e.g. README.md) collided with staged output, leave
+    // the preserved one alone and drop the staged collision.
+    try {
+      await stat(to);
+      // Collision - skip.
+      continue;
+    } catch {
+      // Doesn't exist, safe to move.
+    }
+    await rename(from, to);
+  }
+}
+
+function isStagingDir(name) {
+  return name.startsWith('.mirror-tmp-');
+}
+
+// ── Tool version ─────────────────────────────────────────────────────
+
+/**
+ * Read the confluence tool's package.json version. Best-effort - if the
+ * file is missing or unparseable we return undefined so the manifest
+ * just omits the field.
+ *
+ * @returns {Promise<string|undefined>}
+ */
+async function readToolVersion() {
+  try {
+    // From src/commands/ → up two levels to the tool root.
+    const pkgPath = resolve(__dirname, '..', '..', 'package.json');
+    const raw = await readFile(pkgPath, 'utf-8');
+    const pkg = JSON.parse(raw);
+    if (typeof pkg.version === 'string' && pkg.version.length > 0) {
+      return pkg.version;
+    }
+    return undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+// ── Reindex ──────────────────────────────────────────────────────────
+
+/**
+ * Run `npm run reindex` from the repo root. Returns the exit code.
+ * Uses spawnSync so output is streamed to the parent stdio.
+ */
+function runReindex(repoRoot, logProgress) {
+  const result = spawnSync('npm', ['run', 'reindex'], {
+    cwd: repoRoot,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    encoding: 'utf-8',
+  });
+  if (result.stdout) logProgress(result.stdout.trimEnd());
+  if (result.stderr) logProgress(result.stderr.trimEnd());
+  return result.status ?? 1;
+}
+
+// ── Re-exports for testing ───────────────────────────────────────────
+
+export const _test = {
+  extractCursor,
+  countPages,
+  swapIntoTarget,
+  isStagingDir,
+  readToolVersion,
+};