collabmd 0.1.29 → 0.1.31

package/src/server/create-app-server.js

@@ -2,7 +2,9 @@ import { createServer } from 'http';
 
 import { loadConfig } from './config/env.js';
 import { createAuthService } from './auth/create-auth-service.js';
+import { logPerfEvent } from './config/perf-logging.js';
 import { BacklinkIndex } from './domain/backlink-index.js';
+import { BaseQueryService } from './domain/bases/base-query-service.js';
 import { CollaborationDocumentStore } from './domain/collaboration/collaboration-document-store.js';
 import { CollaborationRoom } from './domain/collaboration/collaboration-room.js';
 import { DocxExporter } from './domain/docx-exporter.js';
@@ -38,10 +40,47 @@ function closeHttpServer(httpServer) {
   });
 }
 
+function workspaceStateMetadataEqual(left = null, right = null) {
+  const leftMetadata = left?.metadata;
+  const rightMetadata = right?.metadata;
+  if (!(leftMetadata instanceof Map) || !(rightMetadata instanceof Map)) {
+    return false;
+  }
+
+  if (leftMetadata.size !== rightMetadata.size) {
+    return false;
+  }
+
+  for (const [pathValue, leftEntry] of leftMetadata.entries()) {
+    const rightEntry = rightMetadata.get(pathValue);
+    if (!rightEntry) {
+      return false;
+    }
+
+    if (
+      leftEntry.type !== rightEntry.type
+      || leftEntry.size !== rightEntry.size
+      || leftEntry.inode !== rightEntry.inode
+      || leftEntry.mtimeMs !== rightEntry.mtimeMs
+    ) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 export function createAppServer(config = loadConfig()) {
   const authService = createAuthService(config);
   const vaultFileStore = new VaultFileStore({ vaultDir: config.vaultDir });
   const backlinkIndex = new BacklinkIndex({ vaultFileStore });
+  let fileSystemSyncService = null;
+  let workspaceMutationCoordinator = null;
+  const baseQueryService = new BaseQueryService({
+    vaultFileStore,
+    workspaceStateProvider: () => workspaceMutationCoordinator?.workspaceState ?? null,
+    workspaceStateSynchronizer: () => fileSystemSyncService?.flushPendingChanges?.(),
+  });
   const docxExporter = new DocxExporter();
   const plantUmlRenderer = new PlantUmlRenderer({
     serverUrl: config.plantumlServerUrl,
@@ -54,7 +93,6 @@ export function createAppServer(config = loadConfig()) {
   const testControls = {
     wsRoomHydrateDelayMs: Math.max(0, Number(config.testWsRoomHydrateDelayMs || 0)),
   };
-  let workspaceMutationCoordinator = null;
   const roomRegistry = new RoomRegistry({
     createRoom: ({ name, onEmpty }) => {
       const isTransientRoom = name === '__lobby__' || name === WORKSPACE_ROOM_NAME || isDrawioLeaseRoom(name);
@@ -69,6 +107,7 @@ export function createAppServer(config = loadConfig()) {
         maxBufferedAmountBytes: config.wsMaxBufferedAmountBytes,
         name,
         onEmpty,
+        perfLoggingEnabled: config.perfLoggingEnabled,
       });
 
       if (name === WORKSPACE_ROOM_NAME && workspaceMutationCoordinator?.workspaceState) {
@@ -82,12 +121,14 @@ export function createAppServer(config = loadConfig()) {
   });
   workspaceMutationCoordinator = new WorkspaceMutationCoordinator({
     backlinkIndex,
+    baseQueryService,
     roomRegistry,
     vaultFileStore,
   });
   vaultFileStore.setManagedWriteTracker(workspaceMutationCoordinator);
-  const fileSystemSyncService = new FileSystemSyncService({
+  fileSystemSyncService = new FileSystemSyncService({
     mutationCoordinator: workspaceMutationCoordinator,
+    perfLoggingEnabled: config.perfLoggingEnabled,
     vaultFileStore,
   });
   const requestHandler = createRequestHandler(
@@ -95,6 +136,7 @@ export function createAppServer(config = loadConfig()) {
     authService,
     vaultFileStore,
     backlinkIndex,
+    baseQueryService,
     docxExporter,
     roomRegistry,
     plantUmlRenderer,
@@ -129,21 +171,84 @@ export function createAppServer(config = loadConfig()) {
   let vaultFileCount = 0;
 
   async function listen() {
-    vaultFileCount = await vaultFileStore.countVaultFiles();
-    await backlinkIndex.build();
-    await workspaceMutationCoordinator.initialize();
-    await fileSystemSyncService.start();
+    const startupStartedAt = Date.now();
+
+    const initialWorkspaceScanStartedAt = Date.now();
+    const initialWorkspaceSnapshot = await vaultFileStore.scanWorkspaceState();
+    vaultFileCount = initialWorkspaceSnapshot.vaultFileCount ?? 0;
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      durationMs: Date.now() - initialWorkspaceScanStartedAt,
+      phase: 'workspace-scan',
+      vaultFileCount,
+    });
+
+    const backlinkBuildStartedAt = Date.now();
+    await backlinkIndex.build({ workspaceState: initialWorkspaceSnapshot });
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      durationMs: Date.now() - backlinkBuildStartedAt,
+      markdownFileCount: initialWorkspaceSnapshot.markdownPaths?.length ?? 0,
+      phase: 'backlink-build',
+    });
+
+    const liveWorkspaceScanStartedAt = Date.now();
+    const liveWorkspaceSnapshot = await vaultFileStore.scanWorkspaceState();
+    const workspaceChangedDuringStartup = !workspaceStateMetadataEqual(initialWorkspaceSnapshot, liveWorkspaceSnapshot);
+    vaultFileCount = liveWorkspaceSnapshot.vaultFileCount ?? vaultFileCount;
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      changedDuringStartup: workspaceChangedDuringStartup,
+      durationMs: Date.now() - liveWorkspaceScanStartedAt,
+      phase: 'workspace-rescan',
+      vaultFileCount,
+    });
+
+    if (workspaceChangedDuringStartup) {
+      const backlinkRebuildStartedAt = Date.now();
+      await backlinkIndex.build({ workspaceState: liveWorkspaceSnapshot });
+      logPerfEvent(config.perfLoggingEnabled, 'startup', {
+        durationMs: Date.now() - backlinkRebuildStartedAt,
+        markdownFileCount: liveWorkspaceSnapshot.markdownPaths?.length ?? 0,
+        phase: 'backlink-rebuild',
+      });
+    }
+
+    const workspaceInitStartedAt = Date.now();
+    await workspaceMutationCoordinator.initialize({ snapshot: liveWorkspaceSnapshot });
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      durationMs: Date.now() - workspaceInitStartedAt,
+      phase: 'workspace-init',
+    });
+
+    const watcherStartStartedAt = Date.now();
+    await fileSystemSyncService.start({ snapshot: liveWorkspaceSnapshot });
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      durationMs: Date.now() - watcherStartStartedAt,
+      phase: 'watcher-start',
+    });
 
     return new Promise((resolve, reject) => {
+      const listenStartedAt = Date.now();
       httpServer.once('error', reject);
       httpServer.listen(config.port, config.host, () => {
         httpServer.off('error', reject);
         const address = httpServer.address();
-        resolve({
+        const result = {
           address,
           host: getDisplayHost(config.host),
           port: typeof address === 'object' && address ? address.port : config.port,
           wsPath: `${config.basePath || ''}${config.wsBasePath}/:file`,
+        };
+        logPerfEvent(config.perfLoggingEnabled, 'startup', {
+          durationMs: Date.now() - listenStartedAt,
+          phase: 'listen',
+          port: result.port,
+        });
+        logPerfEvent(config.perfLoggingEnabled, 'startup-total', {
+          durationMs: Date.now() - startupStartedAt,
+          markdownFileCount: liveWorkspaceSnapshot.markdownPaths?.length ?? 0,
+          vaultFileCount,
+        });
+        resolve({
+          ...result,
         });
       });
     });
@@ -176,6 +281,7 @@ export function createAppServer(config = loadConfig()) {
     roomRegistry,
     workspaceMutationCoordinator,
     authService,
+    backlinkIndex,
     fileSystemSyncService,
     gitService,
     setTestHydrateDelayMs(delayMs = 0) {

package/src/server/domain/backlink-index.js

@@ -11,8 +11,11 @@
  */
 
 import { createWikiTargetIndex, resolveWikiTargetWithIndex } from '../../domain/wiki-link-resolver.js';
+import { isMarkdownFilePath } from '../../domain/file-kind.js';
+import { mapWithConcurrency } from '../shared/async-utils.js';
 
 const WIKI_LINK_RE = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
+const BACKLINK_BUILD_CONCURRENCY = 8;
 
 function createDeferred() {
   let resolve;
@@ -41,8 +44,10 @@ export class BacklinkIndex {
     this.reverse = new Map();
     /** @type {Map<string, Map<string, string[]>>} sourcePath → targetPath → contexts[] */
     this.contextsBySource = new Map();
-    /** @type {string[]} cached flat file list for link resolution */
+    /** @type {string[]} cached flat target file list for link resolution */
     this._fileList = [];
+    /** @type {string[]} cached markdown source file list for content scans */
+    this._sourceFileList = [];
     /** @type {Set<string>} file path membership set */
     this._fileSet = new Set();
     this._wikiTargetIndex = createWikiTargetIndex(this._fileList);
@@ -58,13 +63,17 @@ export class BacklinkIndex {
    * Build the full index by scanning every markdown file in the vault.
    * Called once at server startup.
    */
-  async build() {
+  async build({ workspaceState = null } = {}) {
     this._requestedBuildVersion += 1;
+    this._latestRequestedWorkspaceState = workspaceState;
     return this.flushScheduledBuild();
   }
 
-  scheduleBuild({ delayMs = this.rebuildDelayMs } = {}) {
+  scheduleBuild({ delayMs = this.rebuildDelayMs, workspaceState } = {}) {
     this._requestedBuildVersion += 1;
+    if (workspaceState !== undefined) {
+      this._latestRequestedWorkspaceState = workspaceState;
+    }
     if (!this._scheduledBuildDeferred) {
       this._scheduledBuildDeferred = createDeferred();
     }
@@ -116,8 +125,9 @@ export class BacklinkIndex {
     }
 
     const targetVersion = this._requestedBuildVersion;
+    const workspaceState = this._latestRequestedWorkspaceState;
     this._buildPromise = (async () => {
-      await this._performBuild();
+      await this._performBuild(workspaceState);
       this._completedBuildVersion = Math.max(this._completedBuildVersion, targetVersion);
     })().finally(() => {
       this._buildPromise = null;
@@ -126,39 +136,69 @@ export class BacklinkIndex {
     return this._buildPromise;
   }
 
-  async _performBuild() {
+  async _performBuild(workspaceState = null) {
     this.forward.clear();
     this.reverse.clear();
     this.contextsBySource.clear();
 
-    const tree = await this.vaultFileStore.tree();
-    this._fileList = flattenTree(tree);
+    const snapshot = workspaceState ?? await this._resolveWorkspaceState();
+    this._fileList = Array.from(snapshot?.filePaths ?? snapshot?.markdownPaths ?? []);
+    this._sourceFileList = Array.from(
+      snapshot?.markdownPaths ?? this._fileList.filter((filePath) => isMarkdownFilePath(filePath)),
+    ).filter((filePath) => this._fileList.includes(filePath));
     this._fileSet = new Set(this._fileList);
     this._refreshWikiTargetIndex();
 
-    for (const filePath of this._fileList) {
-      const content = await this.vaultFileStore.readMarkdownFile(filePath);
+    const fileContents = await mapWithConcurrency(
+      this._sourceFileList,
+      BACKLINK_BUILD_CONCURRENCY,
+      async (filePath) => ({
+        content: await this.vaultFileStore.readMarkdownFile(filePath),
+        filePath,
+      }),
+    );
+
+    for (const { content, filePath } of fileContents) {
       if (content !== null) {
         this._indexFile(filePath, content);
       }
     }
 
     this._built = true;
-    console.log(`[backlinks] Index built: ${this._fileList.length} files, ${this.reverse.size} targets with backlinks`);
+    console.log(`[backlinks] Index built: ${this._sourceFileList.length} markdown sources, ${this._fileList.length} targets, ${this.reverse.size} targets with backlinks`);
+  }
+
+  async _resolveWorkspaceState() {
+    if (typeof this.vaultFileStore.scanWorkspaceState === 'function') {
+      return this.vaultFileStore.scanWorkspaceState();
+    }
+
+    const tree = await this.vaultFileStore.tree();
+    const filePaths = flattenTree(tree);
+    return {
+      filePaths,
+      markdownPaths: filePaths.filter((filePath) => isMarkdownFilePath(filePath)),
+    };
   }
 
   /**
    * Incrementally update the index when a file's content changes.
    * Call this after every persist / write.
    */
-  updateFile(filePath, content) {
+  updateFile(filePath, content, { refreshIndex = true } = {}) {
     // Remove old forward links for this file
     this._removeForwardLinks(filePath);
 
     if (!this._fileSet.has(filePath)) {
       this._fileSet.add(filePath);
       this._fileList.push(filePath);
-      this._refreshWikiTargetIndex();
+      if (refreshIndex) {
+        this._refreshWikiTargetIndex();
+      }
+    }
+
+    if (!isMarkdownFilePath(filePath)) {
+      return;
     }
 
     // Re-index with new content
@@ -168,36 +208,47 @@ export class BacklinkIndex {
   /**
    * Handle file creation — add to file list and index its content.
    */
-  onFileCreated(filePath, content = '') {
+  onFileCreated(filePath, content = '', { refreshIndex = true } = {}) {
+    let membershipChanged = false;
     if (!this._fileSet.has(filePath)) {
       this._fileSet.add(filePath);
       this._fileList.push(filePath);
-      this._refreshWikiTargetIndex();
+      membershipChanged = true;
+      if (refreshIndex) {
+        this._refreshWikiTargetIndex();
+      }
     }
 
-    if (content) {
+    if (content && isMarkdownFilePath(filePath)) {
       this._indexFile(filePath, content);
     }
+
+    return membershipChanged;
   }
 
   /**
    * Handle file deletion — remove from file list and all index entries.
    */
-  onFileDeleted(filePath) {
+  onFileDeleted(filePath, { refreshIndex = true } = {}) {
     this._removeForwardLinks(filePath);
+    let membershipChanged = false;
     if (this._fileSet.delete(filePath)) {
       this._fileList = this._fileList.filter((f) => f !== filePath);
-      this._refreshWikiTargetIndex();
+      membershipChanged = true;
+      if (refreshIndex) {
+        this._refreshWikiTargetIndex();
+      }
     }
 
     // Also remove this file as a reverse entry (no one can link to a deleted file)
     this.reverse.delete(filePath);
+    return membershipChanged;
   }
 
   /**
    * Handle file rename — re-map all index entries from oldPath to newPath.
    */
-  onFileRenamed(oldPath, newPath) {
+  onFileRenamed(oldPath, newPath, { refreshIndex = true } = {}) {
     // Move cached contexts for the renamed source file.
     if (this.contextsBySource.has(oldPath)) {
       const sourceContexts = this.contextsBySource.get(oldPath);
@@ -248,9 +299,68 @@ export class BacklinkIndex {
     }
 
     // Update file list
+    let membershipChanged = false;
     if (this._fileSet.delete(oldPath)) {
       this._fileSet.add(newPath);
       this._fileList = this._fileList.map((f) => (f === oldPath ? newPath : f));
+      membershipChanged = true;
+      if (refreshIndex) {
+        this._refreshWikiTargetIndex();
+      }
+    }
+
+    return membershipChanged;
+  }
+
+  async applyWorkspaceChange(workspaceChange = {}, {
+    previousState = null,
+    nextState = null,
+  } = {}) {
+    let refreshIndex = false;
+    const previousEntries = previousState?.entries ?? new Map();
+    const nextEntries = nextState?.entries ?? new Map();
+
+    for (const pathValue of workspaceChange.deletedPaths ?? []) {
+      refreshIndex = this.onFileDeleted(pathValue, { refreshIndex: false }) || refreshIndex;
+    }
+
+    for (const entry of workspaceChange.renamedPaths ?? []) {
+      if (!entry?.oldPath || !entry?.newPath) {
+        continue;
+      }
+      refreshIndex = this.onFileRenamed(entry.oldPath, entry.newPath, { refreshIndex: false }) || refreshIndex;
+    }
+
+    for (const pathValue of new Set(workspaceChange.changedPaths ?? [])) {
+      const existsNow = nextEntries.has(pathValue);
+      const existedBefore = previousEntries.has(pathValue);
+      if (!existsNow) {
+        if (existedBefore) {
+          refreshIndex = this.onFileDeleted(pathValue, { refreshIndex: false }) || refreshIndex;
+        }
+        continue;
+      }
+
+      if (!isMarkdownFilePath(pathValue)) {
+        if (!existedBefore) {
+          refreshIndex = this.onFileCreated(pathValue, '', { refreshIndex: false }) || refreshIndex;
+        }
+        continue;
+      }
+
+      const content = await this.vaultFileStore.readMarkdownFile(pathValue);
+      if (content === null) {
+        continue;
+      }
+
+      if (existedBefore) {
+        this.updateFile(pathValue, content, { refreshIndex: false });
+      } else {
+        refreshIndex = this.onFileCreated(pathValue, content, { refreshIndex: false }) || refreshIndex;
+      }
+    }
+
+    if (refreshIndex) {
       this._refreshWikiTargetIndex();
     }
   }
@@ -374,7 +484,7 @@ export class BacklinkIndex {
 function flattenTree(nodes) {
   const files = [];
   for (const node of nodes) {
-    if (node.type === 'file') {
+    if (node.type && node.type !== 'directory') {
       files.push(node.path);
     } else if (node.children) {
       files.push(...flattenTree(node.children));