collabmd 0.1.29 → 0.1.30

package/src/server/create-app-server.js

@@ -2,6 +2,7 @@ import { createServer } from 'http';
 
 import { loadConfig } from './config/env.js';
 import { createAuthService } from './auth/create-auth-service.js';
+import { logPerfEvent } from './config/perf-logging.js';
 import { BacklinkIndex } from './domain/backlink-index.js';
 import { CollaborationDocumentStore } from './domain/collaboration/collaboration-document-store.js';
 import { CollaborationRoom } from './domain/collaboration/collaboration-room.js';
@@ -38,6 +39,36 @@ function closeHttpServer(httpServer) {
   });
 }
 
+function workspaceStateMetadataEqual(left = null, right = null) {
+  const leftMetadata = left?.metadata;
+  const rightMetadata = right?.metadata;
+  if (!(leftMetadata instanceof Map) || !(rightMetadata instanceof Map)) {
+    return false;
+  }
+
+  if (leftMetadata.size !== rightMetadata.size) {
+    return false;
+  }
+
+  for (const [pathValue, leftEntry] of leftMetadata.entries()) {
+    const rightEntry = rightMetadata.get(pathValue);
+    if (!rightEntry) {
+      return false;
+    }
+
+    if (
+      leftEntry.type !== rightEntry.type
+      || leftEntry.size !== rightEntry.size
+      || leftEntry.inode !== rightEntry.inode
+      || leftEntry.mtimeMs !== rightEntry.mtimeMs
+    ) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 export function createAppServer(config = loadConfig()) {
   const authService = createAuthService(config);
   const vaultFileStore = new VaultFileStore({ vaultDir: config.vaultDir });
@@ -69,6 +100,7 @@ export function createAppServer(config = loadConfig()) {
         maxBufferedAmountBytes: config.wsMaxBufferedAmountBytes,
         name,
         onEmpty,
+        perfLoggingEnabled: config.perfLoggingEnabled,
       });
 
       if (name === WORKSPACE_ROOM_NAME && workspaceMutationCoordinator?.workspaceState) {
@@ -88,6 +120,7 @@ export function createAppServer(config = loadConfig()) {
   vaultFileStore.setManagedWriteTracker(workspaceMutationCoordinator);
   const fileSystemSyncService = new FileSystemSyncService({
     mutationCoordinator: workspaceMutationCoordinator,
+    perfLoggingEnabled: config.perfLoggingEnabled,
     vaultFileStore,
   });
   const requestHandler = createRequestHandler(
@@ -129,21 +162,84 @@ export function createAppServer(config = loadConfig()) {
   let vaultFileCount = 0;
 
   async function listen() {
-    vaultFileCount = await vaultFileStore.countVaultFiles();
-    await backlinkIndex.build();
-    await workspaceMutationCoordinator.initialize();
-    await fileSystemSyncService.start();
+    const startupStartedAt = Date.now();
+
+    const initialWorkspaceScanStartedAt = Date.now();
+    const initialWorkspaceSnapshot = await vaultFileStore.scanWorkspaceState();
+    vaultFileCount = initialWorkspaceSnapshot.vaultFileCount ?? 0;
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      durationMs: Date.now() - initialWorkspaceScanStartedAt,
+      phase: 'workspace-scan',
+      vaultFileCount,
+    });
+
+    const backlinkBuildStartedAt = Date.now();
+    await backlinkIndex.build({ workspaceState: initialWorkspaceSnapshot });
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      durationMs: Date.now() - backlinkBuildStartedAt,
+      markdownFileCount: initialWorkspaceSnapshot.markdownPaths?.length ?? 0,
+      phase: 'backlink-build',
+    });
+
+    const liveWorkspaceScanStartedAt = Date.now();
+    const liveWorkspaceSnapshot = await vaultFileStore.scanWorkspaceState();
+    const workspaceChangedDuringStartup = !workspaceStateMetadataEqual(initialWorkspaceSnapshot, liveWorkspaceSnapshot);
+    vaultFileCount = liveWorkspaceSnapshot.vaultFileCount ?? vaultFileCount;
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      changedDuringStartup: workspaceChangedDuringStartup,
+      durationMs: Date.now() - liveWorkspaceScanStartedAt,
+      phase: 'workspace-rescan',
+      vaultFileCount,
+    });
+
+    if (workspaceChangedDuringStartup) {
+      const backlinkRebuildStartedAt = Date.now();
+      await backlinkIndex.build({ workspaceState: liveWorkspaceSnapshot });
+      logPerfEvent(config.perfLoggingEnabled, 'startup', {
+        durationMs: Date.now() - backlinkRebuildStartedAt,
+        markdownFileCount: liveWorkspaceSnapshot.markdownPaths?.length ?? 0,
+        phase: 'backlink-rebuild',
+      });
+    }
+
+    const workspaceInitStartedAt = Date.now();
+    await workspaceMutationCoordinator.initialize({ snapshot: liveWorkspaceSnapshot });
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      durationMs: Date.now() - workspaceInitStartedAt,
+      phase: 'workspace-init',
+    });
+
+    const watcherStartStartedAt = Date.now();
+    await fileSystemSyncService.start({ snapshot: liveWorkspaceSnapshot });
+    logPerfEvent(config.perfLoggingEnabled, 'startup', {
+      durationMs: Date.now() - watcherStartStartedAt,
+      phase: 'watcher-start',
+    });
 
     return new Promise((resolve, reject) => {
+      const listenStartedAt = Date.now();
       httpServer.once('error', reject);
       httpServer.listen(config.port, config.host, () => {
         httpServer.off('error', reject);
         const address = httpServer.address();
-        resolve({
+        const result = {
           address,
           host: getDisplayHost(config.host),
           port: typeof address === 'object' && address ? address.port : config.port,
           wsPath: `${config.basePath || ''}${config.wsBasePath}/:file`,
+        };
+        logPerfEvent(config.perfLoggingEnabled, 'startup', {
+          durationMs: Date.now() - listenStartedAt,
+          phase: 'listen',
+          port: result.port,
+        });
+        logPerfEvent(config.perfLoggingEnabled, 'startup-total', {
+          durationMs: Date.now() - startupStartedAt,
+          markdownFileCount: liveWorkspaceSnapshot.markdownPaths?.length ?? 0,
+          vaultFileCount,
+        });
+        resolve({
+          ...result,
         });
       });
     });
@@ -176,6 +272,7 @@ export function createAppServer(config = loadConfig()) {
     roomRegistry,
     workspaceMutationCoordinator,
     authService,
+    backlinkIndex,
     fileSystemSyncService,
     gitService,
     setTestHydrateDelayMs(delayMs = 0) {

package/src/server/domain/backlink-index.js

@@ -11,8 +11,10 @@
  */
 
 import { createWikiTargetIndex, resolveWikiTargetWithIndex } from '../../domain/wiki-link-resolver.js';
+import { mapWithConcurrency } from '../shared/async-utils.js';
 
 const WIKI_LINK_RE = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
+const BACKLINK_BUILD_CONCURRENCY = 8;
 
 function createDeferred() {
   let resolve;
@@ -58,13 +60,15 @@ export class BacklinkIndex {
    * Build the full index by scanning every markdown file in the vault.
    * Called once at server startup.
    */
-  async build() {
+  async build({ workspaceState = null } = {}) {
     this._requestedBuildVersion += 1;
+    this._latestRequestedWorkspaceState = workspaceState;
     return this.flushScheduledBuild();
   }
 
   scheduleBuild({ delayMs = this.rebuildDelayMs } = {}) {
     this._requestedBuildVersion += 1;
+    this._latestRequestedWorkspaceState = null;
     if (!this._scheduledBuildDeferred) {
       this._scheduledBuildDeferred = createDeferred();
     }
@@ -116,8 +120,9 @@ export class BacklinkIndex {
     }
 
     const targetVersion = this._requestedBuildVersion;
+    const workspaceState = this._latestRequestedWorkspaceState;
     this._buildPromise = (async () => {
-      await this._performBuild();
+      await this._performBuild(workspaceState);
       this._completedBuildVersion = Math.max(this._completedBuildVersion, targetVersion);
     })().finally(() => {
       this._buildPromise = null;
@@ -126,18 +131,26 @@ export class BacklinkIndex {
     return this._buildPromise;
   }
 
-  async _performBuild() {
+  async _performBuild(workspaceState = null) {
     this.forward.clear();
     this.reverse.clear();
     this.contextsBySource.clear();
 
-    const tree = await this.vaultFileStore.tree();
-    this._fileList = flattenTree(tree);
+    const snapshot = workspaceState ?? await this._resolveWorkspaceState();
+    this._fileList = Array.from(snapshot?.markdownPaths ?? []);
     this._fileSet = new Set(this._fileList);
     this._refreshWikiTargetIndex();
 
-    for (const filePath of this._fileList) {
-      const content = await this.vaultFileStore.readMarkdownFile(filePath);
+    const fileContents = await mapWithConcurrency(
+      this._fileList,
+      BACKLINK_BUILD_CONCURRENCY,
+      async (filePath) => ({
+        content: await this.vaultFileStore.readMarkdownFile(filePath),
+        filePath,
+      }),
+    );
+
+    for (const { content, filePath } of fileContents) {
       if (content !== null) {
         this._indexFile(filePath, content);
       }
@@ -147,6 +160,17 @@ export class BacklinkIndex {
     console.log(`[backlinks] Index built: ${this._fileList.length} files, ${this.reverse.size} targets with backlinks`);
   }
 
+  async _resolveWorkspaceState() {
+    if (typeof this.vaultFileStore.scanWorkspaceState === 'function') {
+      return this.vaultFileStore.scanWorkspaceState();
+    }
+
+    const tree = await this.vaultFileStore.tree();
+    return {
+      markdownPaths: flattenTree(tree),
+    };
+  }
+
   /**
    * Incrementally update the index when a file's content changes.
    * Call this after every persist / write.

package/src/server/domain/collaboration/collaboration-room.js

@@ -8,6 +8,7 @@ import { MSG_AWARENESS, MSG_SYNC } from './protocol.js';
 import { CollaborationDocumentStore } from './collaboration-document-store.js';
 import { RoomClientStateStore } from './room-client-state-store.js';
 import { RoomPersistenceController } from './room-persistence-controller.js';
+import { logPerfEvent } from '../../config/perf-logging.js';
 import { populateCommentThreads, serializeCommentThreads } from '../../../domain/comment-threads.js';
 import {
   isExcalidrawRoomDocStructured,
@@ -131,6 +132,7 @@ export class CollaborationRoom {
     name,
     idleGraceMs = 0,
     maxBufferedAmountBytes,
+    perfLoggingEnabled = false,
     vaultFileStore,
     backlinkIndex,
     onEmpty,
@@ -138,6 +140,7 @@ export class CollaborationRoom {
     this.name = name;
     this.idleGraceMs = idleGraceMs;
     this.maxBufferedAmountBytes = maxBufferedAmountBytes;
+    this.perfLoggingEnabled = perfLoggingEnabled;
     this.documentStore = documentStore ?? new CollaborationDocumentStore({
       backlinkIndex,
       name,
@@ -187,6 +190,8 @@ export class CollaborationRoom {
         let snapshotExists = false;
         let snapshotValid = false;
         let commentThreadCount = 0;
+        let snapshotReadDurationMs = 0;
+        let contentFallbackDurationMs = 0;
         const hydrateDelayMs = Math.max(0, Number(this.getHydrateDelayMs?.() || 0));
 
         try {
@@ -195,7 +200,9 @@ export class CollaborationRoom {
           }
 
           if (this.documentStore?.hasPersistence()) {
+            const snapshotReadStartedAt = Date.now();
             const snapshot = await this.documentStore.readSnapshot();
+            snapshotReadDurationMs = Date.now() - snapshotReadStartedAt;
             if (snapshot) {
               snapshotExists = true;
               try {
@@ -227,10 +234,12 @@ export class CollaborationRoom {
               }
             }
 
+            const fallbackStartedAt = Date.now();
             const [content, commentThreads] = await Promise.all([
               this.documentStore.readContent(),
               this.documentStore.readCommentThreads(),
             ]);
+            contentFallbackDurationMs = Date.now() - fallbackStartedAt;
             commentThreadCount = commentThreads.length;
             if (content !== null || commentThreads.length > 0) {
               hydrateSource = 'content';
@@ -262,14 +271,23 @@ export class CollaborationRoom {
             this.debugMetrics.hydrateCount += 1;
             this.debugMetrics.lastHydrate = {
               commentThreadCount,
+              contentFallbackDurationMs,
               durationMs: Date.now() - startedAt,
+              snapshotReadDurationMs,
               snapshotExists,
               snapshotValid,
               source: hydrateSource,
             };
-            console.info(
-              `[perf][room:${this.name}] hydrate durationMs=${this.debugMetrics.lastHydrate.durationMs} source=${this.debugMetrics.lastHydrate.source} snapshotExists=${this.debugMetrics.lastHydrate.snapshotExists} snapshotValid=${this.debugMetrics.lastHydrate.snapshotValid} commentThreads=${this.debugMetrics.lastHydrate.commentThreadCount}`,
-            );
+            logPerfEvent(this.perfLoggingEnabled, `room:${this.name}`, {
+              commentThreads: this.debugMetrics.lastHydrate.commentThreadCount,
+              contentFallbackDurationMs: this.debugMetrics.lastHydrate.contentFallbackDurationMs,
+              durationMs: this.debugMetrics.lastHydrate.durationMs,
+              event: 'hydrate',
+              snapshotExists: this.debugMetrics.lastHydrate.snapshotExists,
+              snapshotReadDurationMs: this.debugMetrics.lastHydrate.snapshotReadDurationMs,
+              snapshotValid: this.debugMetrics.lastHydrate.snapshotValid,
+              source: this.debugMetrics.lastHydrate.source,
+            });
           }
         }
       })().catch((error) => {
@@ -522,8 +540,13 @@ export class CollaborationRoom {
   sendInitialSync(ws) {
     this.debugMetrics.initialSyncCount += 1;
     this.debugMetrics.lastInitialSyncAt = Date.now();
-    console.info(`[perf][room:${this.name}] send-initial-sync count=${this.debugMetrics.initialSyncCount}`);
-    return sendMessage.call(this, ws, this.getInitialSyncMessage(), this);
+    const initialSyncMessage = this.getInitialSyncMessage();
+    logPerfEvent(this.perfLoggingEnabled, `room:${this.name}`, {
+      bytes: initialSyncMessage.byteLength,
+      count: this.debugMetrics.initialSyncCount,
+      event: 'initial-sync',
+    });
+    return sendMessage.call(this, ws, initialSyncMessage, this);
   }
 
   async addClient(ws, { sendInitialSync: shouldSendInitialSync = true } = {}) {

package/src/server/infrastructure/git/history-service.js

@@ -430,7 +430,7 @@ export class GitHistoryService {
     }
 
     return this.runRequest(cacheKey, async () => {
-      let content = '';
+      let content;
       try {
         content = await this.commandRunner.execGit(['show', `${normalizedHash}:${normalizedPath}`]);
       } catch {

package/src/server/infrastructure/persistence/pull-backup-store.js

@@ -240,7 +240,7 @@ export class PullBackupStore {
   }
 
   async listBackups() {
-    let entries = [];
+    let entries;
     try {
       entries = await readdir(this.getStorageRoot(), { withFileTypes: true });
     } catch (error) {

package/src/server/infrastructure/persistence/vault-file-store.js

@@ -7,6 +7,7 @@ import {
   isMarkdownFilePath,
   isVaultFilePath,
 } from '../../../domain/file-kind.js';
+import { mapWithConcurrency } from '../../shared/async-utils.js';
 import { getVaultContentAdapter } from './vault-content-adapter.js';
 import {
   INVALID_VAULT_FILE_PATH_ERROR,
@@ -36,6 +37,7 @@ const MIME_TYPE_TO_IMAGE_EXTENSION = Object.freeze({
   'image/svg+xml': '.svg',
   'image/webp': '.webp',
 });
+const WORKSPACE_SCAN_CONCURRENCY = 8;
 
 function createTransactionalPath(targetPath, label) {
   return `${targetPath}.collabmd-${label}-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2, 10)}`;
@@ -74,6 +76,16 @@ function createWorkspaceEntry(relativePath, type) {
   };
 }
 
+function createWorkspaceMetadata(pathValue, type, info) {
+  return {
+    inode: Number(info.ino || 0),
+    mtimeMs: Number(info.mtimeMs || 0),
+    path: pathValue,
+    size: type === 'directory' ? 0 : Number(info.size || 0),
+    type,
+  };
+}
+
 function replacePathPrefix(pathValue, oldPrefix, newPrefix) {
   if (pathValue === oldPrefix) {
     return newPrefix;
@@ -94,6 +106,14 @@ function sortWorkspacePaths(paths = [], direction = 'asc') {
   });
 }
 
+function sortDirectoryEntries(entries = []) {
+  return [...entries].sort((left, right) => {
+    if (left.isDirectory() && !right.isDirectory()) return -1;
+    if (!left.isDirectory() && right.isDirectory()) return 1;
+    return left.name.localeCompare(right.name, undefined, { sensitivity: 'base' });
+  });
+}
+
 async function cleanupPaths(paths = []) {
   await Promise.allSettled(paths.filter(Boolean).map((pathValue) => rm(pathValue, { force: true })));
 }
@@ -398,8 +418,8 @@ export class VaultFileStore {
     const timestamp = createAttachmentTimestamp(now);
     const baseFileName = `${attachmentStem}-${timestamp}`;
     let collisionIndex = 0;
-    let storedPath = '';
-    let absolutePath = '';
+    let storedPath;
+    let absolutePath;
 
     do {
       const suffix = collisionIndex > 0 ? `-${collisionIndex + 1}` : '';
@@ -765,7 +785,8 @@ export class VaultFileStore {
   }
 
   async countVaultFiles() {
-    return this.countFilesInDir(this.vaultDir);
+    const snapshot = await this.scanWorkspaceState();
+    return snapshot.vaultFileCount;
   }
 
   async reconcileSidecars({
@@ -838,6 +859,8 @@ export class VaultFileStore {
   async scanWorkspaceState() {
     const entries = new Map();
     const metadata = new Map();
+    const markdownPaths = [];
+    let vaultFileCount = 0;
 
     const visitDirectory = async (dirPath) => {
       let dirEntries;
@@ -847,64 +870,84 @@ export class VaultFileStore {
         return;
       }
 
-      const sorted = dirEntries.sort((left, right) => {
-        if (left.isDirectory() && !right.isDirectory()) return -1;
-        if (!left.isDirectory() && right.isDirectory()) return 1;
-        return left.name.localeCompare(right.name, undefined, { sensitivity: 'base' });
-      });
+      const sorted = sortDirectoryEntries(dirEntries);
 
-      for (const entry of sorted) {
+      await mapWithConcurrency(sorted, WORKSPACE_SCAN_CONCURRENCY, async (entry) => {
         if (isIgnoredVaultEntry(entry.name)) {
-          continue;
+          return;
         }
 
         const fullPath = join(dirPath, entry.name);
         const relativePath = toVaultRelativePath(this.vaultDir, fullPath).replace(/\\/g, '/');
+        const direntKind = entry.isDirectory()
+          ? 'directory'
+          : (entry.isFile() ? 'file' : null);
 
-        if (entry.isDirectory()) {
+        if (direntKind === 'directory') {
           entries.set(relativePath, createWorkspaceEntry(relativePath, 'directory'));
           try {
             const info = await stat(fullPath);
-            metadata.set(relativePath, {
-              inode: Number(info.ino || 0),
-              mtimeMs: Number(info.mtimeMs || 0),
-              path: relativePath,
-              size: 0,
-              type: 'directory',
-            });
+            metadata.set(relativePath, createWorkspaceMetadata(relativePath, 'directory', info));
           } catch {
             // Ignore transient directories that disappear during scans.
           }
           await visitDirectory(fullPath);
-          continue;
+          return;
         }
 
-        if (!isVaultFilePath(entry.name)) {
-          continue;
+        if (direntKind === 'file') {
+          if (!isVaultFilePath(entry.name)) {
+            return;
+          }
+
+          entries.set(relativePath, createWorkspaceEntry(relativePath, 'file'));
+          vaultFileCount += 1;
+          if (isMarkdownFilePath(relativePath)) {
+            markdownPaths.push(relativePath);
+          }
+          try {
+            const info = await stat(fullPath);
+            metadata.set(relativePath, createWorkspaceMetadata(relativePath, 'file', info));
+          } catch {
+            // Ignore transient files that disappear during scans.
+          }
+          return;
         }
 
-        entries.set(relativePath, createWorkspaceEntry(relativePath, 'file'));
         try {
           const info = await stat(fullPath);
-          metadata.set(relativePath, {
-            inode: Number(info.ino || 0),
-            mtimeMs: Number(info.mtimeMs || 0),
-            path: relativePath,
-            size: Number(info.size || 0),
-            type: 'file',
-          });
+          if (info.isDirectory()) {
+            entries.set(relativePath, createWorkspaceEntry(relativePath, 'directory'));
+            metadata.set(relativePath, createWorkspaceMetadata(relativePath, 'directory', info));
+            await visitDirectory(fullPath);
+            return;
+          }
+
+          if (!info.isFile() || !isVaultFilePath(entry.name)) {
+            return;
+          }
+
+          entries.set(relativePath, createWorkspaceEntry(relativePath, 'file'));
+          metadata.set(relativePath, createWorkspaceMetadata(relativePath, 'file', info));
+          vaultFileCount += 1;
+          if (isMarkdownFilePath(relativePath)) {
+            markdownPaths.push(relativePath);
+          }
         } catch {
-          // Ignore transient files that disappear during scans.
+          // Ignore transient entries that disappear during scans.
         }
-      }
+      });
     };
 
     await visitDirectory(this.vaultDir);
+    markdownPaths.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: 'base' }));
 
     return {
       entries,
       metadata,
+      markdownPaths,
       scannedAt: Date.now(),
+      vaultFileCount,
     };
   }
 }