@serve.zone/gitops 2.13.1

package/ts/classes/jobrunners/autobookstackdocs.runner.ts

@@ -0,0 +1,435 @@
+import * as plugins from '../../plugins.js';
+import { logger } from '../../logging.js';
+import type * as interfaces from '../../../ts_interfaces/index.js';
+import type { BaseProvider } from '../../providers/classes.baseprovider.js';
+import type { StorageManager } from '../../storage/index.js';
+import { BaseJobRunner, type IJobRunContext } from './base.jobrunner.js';
+
+const KEYCHAIN_PREFIX = 'keychain:';
+const BATCH_SIZE = 5;
+const BATCH_DELAY_MS = 200;
+const HASH_STORAGE_PREFIX = '/job-hashes/';
+
+/** Simple SHA-256 hex hash */
+async function sha256(content: string): Promise<string> {
+  const data = new TextEncoder().encode(content);
+  const hashBuffer = await crypto.subtle.digest('SHA-256', data);
+  return Array.from(new Uint8Array(hashBuffer)).map((b) => b.toString(16).padStart(2, '0')).join('');
+}
+
+/**
+ * If the first line of markdown is a heading matching the book name (e.g. "# @org/repo"),
+ * strip it — the book title already provides that context.
+ */
+const REDUNDANT_HEADINGS = ['changelog'];
+
+function stripRedundantHeading(markdown: string, bookName: string): string {
+  const firstNewline = markdown.indexOf('\n');
+  const firstLine = (firstNewline === -1 ? markdown : markdown.slice(0, firstNewline)).trim();
+  if (firstLine.startsWith('# ')) {
+    const heading = firstLine.slice(2).trim().toLowerCase();
+    const fullName = bookName.toLowerCase();
+    const repoName = bookName.split('/').pop()?.toLowerCase() || '';
+    // Strip headings matching the book/repo name (with optional trailing decorations)
+    if (heading === fullName || heading === repoName
+      || heading.startsWith(fullName + ' ') || heading.startsWith(repoName + ' ')) {
+      return firstNewline === -1 ? '' : markdown.slice(firstNewline + 1);
+    }
+    // Strip generic redundant headings (e.g. "# Changelog")
+    if (REDUNDANT_HEADINGS.includes(heading)) {
+      return firstNewline === -1 ? '' : markdown.slice(firstNewline + 1);
+    }
+  }
+  return markdown;
+}
+
+/** Persisted content hashes for a job — avoids unnecessary BookStack updates */
+type ContentHashMap = Record<string, string>; // page key → sha256 hash
+
+/**
+ * Mapping:
+ *   Git org/group  →  BookStack Shelf
+ *   Git repository →  BookStack Book (named @group/repo, assigned to shelf)
+ *   readme.md      →  BookStack Page (named "readme.md for @group/repo")
+ *   docs/*.md      →  BookStack Pages (named "docs/file.md for @group/repo")
+ *
+ * Content hashes are persisted in storage to skip updates when content hasn't changed.
+ */
+export class AutoBookstackDocsRunner extends BaseJobRunner {
+  readonly jobType: interfaces.data.TJobType = 'autobookstackdocs';
+
+  async execute(context: IJobRunContext): Promise<void> {
+    const config = context.jobConfig.autoBookstackDocsConfig;
+    if (!config) {
+      throw new Error('Missing autoBookstackDocsConfig');
+    }
+
+    // 1. Resolve BookStack token from keychain
+    let tokenSecret = config.bookstackTarget.tokenSecret;
+    if (tokenSecret.startsWith(KEYCHAIN_PREFIX)) {
+      const keychainId = tokenSecret.slice(KEYCHAIN_PREFIX.length);
+      const resolved = await context.smartSecret.getSecret(keychainId);
+      if (!resolved) {
+        throw new Error('Could not retrieve BookStack token from keychain');
+      }
+      tokenSecret = resolved;
+    }
+
+    // 2. Connect to BookStack
+    const bookstack = new plugins.bookstackClient.BookStackAccount(
+      config.bookstackTarget.baseUrl,
+      config.bookstackTarget.tokenId,
+      tokenSecret,
+    );
+    await bookstack.testConnection();
+    logger.jobLog('info', `Connected to BookStack at ${config.bookstackTarget.baseUrl}`, 'bookstack');
+
+    // 3. Load persisted content hashes (for diff-based updates)
+    const hashKey = `${HASH_STORAGE_PREFIX}${context.jobConfig.id}.json`;
+    const oldHashes: ContentHashMap = (await context.storageManager.getJSON<ContentHashMap>(hashKey)) || {};
+    const newHashes: ContentHashMap = {};
+
+    // 4. Pre-load existing shelves for find-or-create
+    const existingShelves = await bookstack.getShelves();
+    const shelfMap = new Map<string, plugins.bookstackClient.BookStackShelf>();
+    for (const shelf of existingShelves) {
+      shelfMap.set(shelf.name.toLowerCase(), shelf);
+    }
+
+    // Track desired state for deletion reconciliation
+    const syncedShelfNames = new Set<string>();
+    const syncedBookNames = new Map<string, Set<string>>();
+    const shelfBookIds = new Map<string, number[]>();
+
+    let pagesCreated = 0;
+    let pagesUpdated = 0;
+    let pagesSkipped = 0;
+
+    // 5. Process each source connection
+    for (const connectionId of config.sourceConnectionIds) {
+      const conn = context.connectionManager.getConnection(connectionId);
+      if (!conn || conn.status === 'paused') {
+        logger.jobLog('warn', `Skipping connection ${connectionId} (not found or paused)`, 'bookstack');
+        continue;
+      }
+
+      const provider = context.connectionManager.getProvider(connectionId);
+      const groups = await provider.getGroups();
+      logger.jobLog('info', `Connection "${conn.name}": ${groups.length} groups`, 'bookstack');
+
+      for (const group of groups) {
+        // Apply group filters
+        if (config.includeGroups && config.includeGroups.length > 0) {
+          if (!config.includeGroups.some((g) => g.toLowerCase() === group.name.toLowerCase() || g.toLowerCase() === group.fullPath.toLowerCase())) {
+            continue;
+          }
+        }
+        if (config.excludeGroups && config.excludeGroups.length > 0) {
+          if (config.excludeGroups.some((g) => g.toLowerCase() === group.name.toLowerCase() || g.toLowerCase() === group.fullPath.toLowerCase())) {
+            continue;
+          }
+        }
+
+        // Find or create shelf for this org/group
+        const shelfName = group.fullPath || group.name;
+        const shelfKey = shelfName.toLowerCase();
+        syncedShelfNames.add(shelfKey);
+        if (!syncedBookNames.has(shelfKey)) {
+          syncedBookNames.set(shelfKey, new Set());
+        }
+        if (!shelfBookIds.has(shelfKey)) {
+          shelfBookIds.set(shelfKey, []);
+        }
+
+        let shelf = shelfMap.get(shelfKey);
+        if (!shelf) {
+          shelf = await bookstack.createShelf({
+            name: shelfName,
+            description: group.description || `Documentation for ${shelfName}`,
+          });
+          shelfMap.set(shelfKey, shelf);
+          logger.jobLog('info', `Created shelf: ${shelfName}`, 'bookstack');
+        }
+
+        // Get projects in this group
+        const projects = await provider.getGroupProjects(group.id);
+
+        // Process repos in batches
+        for (let i = 0; i < projects.length; i += BATCH_SIZE) {
+          if (i > 0) await new Promise((r) => setTimeout(r, BATCH_DELAY_MS));
+          const batch = projects.slice(i, i + BATCH_SIZE);
+          const results = await Promise.all(
+            batch.map((project) => {
+              const bookName = `@${group.fullPath}/${project.name}`;
+              syncedBookNames.get(shelfKey)!.add(bookName.toLowerCase());
+              return this.syncRepoAsBook(
+                provider, bookstack, context.storageManager, project, bookName, config,
+                group.visibility, oldHashes, newHashes,
+              );
+            }),
+          );
+          for (const result of results) {
+            if (result) {
+              shelfBookIds.get(shelfKey)!.push(result.bookId);
+              pagesCreated += result.created;
+              pagesUpdated += result.updated;
+              pagesSkipped += result.skipped;
+            }
+          }
+        }
+
+        // Assign all synced books to this shelf (only if book list changed)
+        const bookIds = shelfBookIds.get(shelfKey)!;
+        if (bookIds.length > 0) {
+          const shelfHashKey = `__shelf__${shelfKey}`;
+          const desiredShelfState = bookIds.slice().sort((a, b) => a - b).join(',');
+          if (oldHashes[shelfHashKey] !== desiredShelfState) {
+            await shelf.update({ books: bookIds });
+            logger.jobLog('info', `Updated shelf "${shelfName}" book assignments`, 'bookstack');
+          }
+          newHashes[shelfHashKey] = desiredShelfState;
+        }
+      }
+    }
+
+    // 6. Deletion reconciliation
+    if (config.propagateDeletes) {
+      await this.reconcileDeletes(bookstack, shelfMap, syncedShelfNames, syncedBookNames);
+    }
+
+    // 7. Persist updated content hashes
+    await context.storageManager.setJSON(hashKey, newHashes);
+
+    logger.jobLog('success', `Sync complete — ${pagesCreated} created, ${pagesUpdated} updated, ${pagesSkipped} unchanged`, 'bookstack');
+  }
+
+  /**
+   * Syncs a single git repository as a BookStack Book.
+   * Returns book ID + page stats, or null if skipped (no docs).
+   */
+  private async syncRepoAsBook(
+    provider: BaseProvider,
+    bookstack: plugins.bookstackClient.BookStackAccount,
+    storageManager: StorageManager,
+    project: interfaces.data.IProject,
+    bookName: string,
+    config: interfaces.data.IAutoBookstackDocsConfig,
+    groupVisibility: string,
+    oldHashes: ContentHashMap,
+    newHashes: ContentHashMap,
+  ): Promise<{ bookId: number; created: number; updated: number; skipped: number } | null> {
+    try {
+      // 0. Collect tags from package.json keywords + git repo topics
+      const tags = await this.collectTags(provider, project);
+
+      // Collect all pages to sync for this repo
+      const pagesToSync: { name: string; content: string; hash: string }[] = [];
+
+      // 1. Fetch explicit root files (e.g. readme.md)
+      for (const filePath of config.filePaths) {
+        let content = await provider.getFileContent(project.fullPath, filePath);
+        if (content) {
+          content = stripRedundantHeading(content, bookName);
+          const tagsStr = tags.map((t) => t.name).join(',');
+          const hash = await sha256(content + '\0' + tagsStr);
+          const pageName = `${filePath} for ${bookName}`;
+          pagesToSync.push({ name: pageName, content, hash });
+          newHashes[pageName.toLowerCase()] = hash;
+        }
+      }
+
+      // 2. Scan doc directories for .md files
+      for (const dir of config.docDirs) {
+        const entries = await provider.getDirectoryContents(project.fullPath, dir);
+        if (!entries) continue;
+
+        const mdFiles = entries.filter(
+          (e) => e.type === 'file' && e.name.toLowerCase().endsWith('.md'),
+        );
+        for (const mdFile of mdFiles) {
+          let content = await provider.getFileContent(project.fullPath, mdFile.path);
+          if (content) {
+            content = stripRedundantHeading(content, bookName);
+            const tagsStr = tags.map((t) => t.name).join(',');
+            const hash = await sha256(content + '\0' + tagsStr);
+            const pageName = `${mdFile.path} for ${bookName}`;
+            pagesToSync.push({ name: pageName, content, hash });
+            newHashes[pageName.toLowerCase()] = hash;
+          }
+        }
+      }
+
+      // Skip repos that have no docs at all
+      if (pagesToSync.length === 0) return null;
+
+      // 3. Find or create book for this repo
+      const book = await this.findOrCreateBook(bookstack, bookName, project.description || '');
+
+      // 4. Get existing pages in this book
+      const existingPages = await book.getPages();
+      const pageMap = new Map<string, plugins.bookstackClient.BookStackPage>();
+      for (const p of existingPages) {
+        pageMap.set(p.name.toLowerCase(), p);
+      }
+
+      // 5. Create or update pages (only if content changed)
+      let created = 0;
+      let updated = 0;
+      let skipped = 0;
+      const syncedPageNames = new Set<string>();
+
+      for (const page of pagesToSync) {
+        syncedPageNames.add(page.name.toLowerCase());
+        const existing = pageMap.get(page.name.toLowerCase());
+
+        if (existing) {
+          // Check hash — skip update if content hasn't changed
+          const oldHash = oldHashes[page.name.toLowerCase()];
+          if (oldHash === page.hash) {
+            skipped++;
+            continue;
+          }
+          await existing.update({ markdown: page.content, tags });
+          updated++;
+        } else {
+          await bookstack.createPage({
+            book_id: book.id,
+            name: page.name,
+            markdown: page.content,
+            tags,
+          });
+          created++;
+        }
+      }
+
+      // 6. Delete stale pages within this book
+      if (config.propagateDeletes) {
+        for (const [name, page] of pageMap) {
+          if (!syncedPageNames.has(name)) {
+            await page.delete();
+            logger.jobLog('warn', `Deleted stale page "${page.name}" from book "${bookName}"`, 'bookstack');
+          }
+        }
+      }
+
+      // 7. Sync visibility — only update permissions if visibility state changed
+      if (config.syncVisibility) {
+        const isPrivate = project.visibility === 'private' || project.visibility === 'internal'
+          || groupVisibility === 'private' || groupVisibility === 'internal';
+        const visKey = `__vis__${bookName.toLowerCase()}`;
+        const desiredVis = isPrivate ? `private:${config.privateRoleId || ''}` : 'public';
+        const oldVis = oldHashes[visKey];
+        newHashes[visKey] = desiredVis;
+
+        if (oldVis !== desiredVis) {
+          if (isPrivate) {
+            const permUpdate: any = {
+              fallback_permissions: { inheriting: false, view: false, create: false, update: false, delete: false },
+            };
+            if (config.privateRoleId) {
+              permUpdate.role_permissions = [{ role_id: config.privateRoleId, view: true, create: false, update: false, delete: false }];
+            }
+            await bookstack.updateContentPermissions('book', book.id, permUpdate);
+            logger.jobLog('info', `Restricted book "${bookName}" (private)`, 'bookstack');
+          } else {
+            await bookstack.updateContentPermissions('book', book.id, {
+              fallback_permissions: { inheriting: true },
+            });
+            logger.jobLog('info', `Set book "${bookName}" to public`, 'bookstack');
+          }
+        }
+      }
+
+      if (created > 0 || updated > 0) {
+        logger.jobLog('info', `${bookName}: ${created} created, ${updated} updated, ${skipped} unchanged`, 'bookstack');
+      }
+      return { bookId: book.id, created, updated, skipped };
+    } catch (err) {
+      logger.jobLog('warn', `Failed to sync repo ${bookName}: ${err}`, 'bookstack');
+      return null;
+    }
+  }
+
+  /**
+   * Removes stale BookStack shelves and orphaned books.
+   */
+  private async reconcileDeletes(
+    bookstack: plugins.bookstackClient.BookStackAccount,
+    shelfMap: Map<string, plugins.bookstackClient.BookStackShelf>,
+    syncedShelfNames: Set<string>,
+    syncedBookNames: Map<string, Set<string>>,
+  ): Promise<void> {
+    // Build flat set of all desired book names
+    const allDesiredBookNames = new Set<string>();
+    for (const bookNames of syncedBookNames.values()) {
+      for (const name of bookNames) {
+        allDesiredBookNames.add(name);
+      }
+    }
+
+    // Delete stale shelves
+    for (const [shelfKey, shelf] of shelfMap) {
+      if (!syncedShelfNames.has(shelfKey)) {
+        await shelf.delete();
+        logger.jobLog('warn', `Deleted stale shelf "${shelf.name}"`, 'bookstack');
+      }
+    }
+
+    // Delete all orphaned books
+    const allBooks = await bookstack.getBooks();
+    for (const book of allBooks) {
+      if (!allDesiredBookNames.has(book.name.toLowerCase())) {
+        await book.delete();
+        logger.jobLog('warn', `Deleted orphaned book "${book.name}"`, 'bookstack');
+      }
+    }
+  }
+
+  /**
+   * Collects tags from package.json keywords + git repo topics, deduplicated.
+   */
+  private async collectTags(
+    provider: BaseProvider,
+    project: interfaces.data.IProject,
+  ): Promise<{ name: string; value: string }[]> {
+    const tagNames = new Set<string>();
+
+    // Add git repo topics
+    if (project.topics) {
+      for (const topic of project.topics) {
+        if (topic) tagNames.add(topic.toLowerCase());
+      }
+    }
+
+    // Fetch package.json and extract keywords
+    try {
+      const pkgJson = await provider.getFileContent(project.fullPath, 'package.json');
+      if (pkgJson) {
+        const pkg = JSON.parse(pkgJson);
+        if (Array.isArray(pkg.keywords)) {
+          for (const kw of pkg.keywords) {
+            if (typeof kw === 'string' && kw) tagNames.add(kw.toLowerCase());
+          }
+        }
+      }
+    } catch {
+      // package.json missing or invalid — no keywords
+    }
+
+    return Array.from(tagNames).map((name) => ({ name, value: '' }));
+  }
+
+  private async findOrCreateBook(
+    bookstack: plugins.bookstackClient.BookStackAccount,
+    bookName: string,
+    description: string,
+  ): Promise<plugins.bookstackClient.BookStackBook> {
+    const books = await bookstack.getBooks();
+    const existing = books.find((b) => b.name.toLowerCase() === bookName.toLowerCase());
+    if (existing) return existing;
+    return bookstack.createBook({
+      name: bookName,
+      description: description || `Documentation for ${bookName}`,
+    });
+  }
+}

package/ts/classes/jobrunners/base.jobrunner.ts

@@ -0,0 +1,16 @@
+import type * as interfaces from '../../../ts_interfaces/index.js';
+import type { ConnectionManager } from '../connectionmanager.js';
+import type { StorageManager } from '../../storage/index.js';
+import type * as plugins from '../../plugins.js';
+
+export interface IJobRunContext {
+  jobConfig: interfaces.data.IJobConfig;
+  connectionManager: ConnectionManager;
+  storageManager: StorageManager;
+  smartSecret: plugins.smartsecret.SmartSecret;
+}
+
+export abstract class BaseJobRunner {
+  abstract readonly jobType: interfaces.data.TJobType;
+  abstract execute(context: IJobRunContext): Promise<void>;
+}

package/ts/classes/jobrunners/index.ts

@@ -0,0 +1,17 @@
+import type * as interfaces from '../../../ts_interfaces/index.js';
+import { BaseJobRunner } from './base.jobrunner.js';
+import { AutoBookstackDocsRunner } from './autobookstackdocs.runner.js';
+
+export { BaseJobRunner, type IJobRunContext } from './base.jobrunner.js';
+export { AutoBookstackDocsRunner } from './autobookstackdocs.runner.js';
+
+const runners = new Map<interfaces.data.TJobType, BaseJobRunner>();
+runners.set('autobookstackdocs', new AutoBookstackDocsRunner());
+
+export function getRunner(jobType: interfaces.data.TJobType): BaseJobRunner {
+  const runner = runners.get(jobType);
+  if (!runner) {
+    throw new Error(`No runner registered for job type: ${jobType}`);
+  }
+  return runner;
+}