collabmd 0.1.29 → 0.1.31

package/src/server/domain/collaboration/collaboration-document-store.js

@@ -1,9 +1,10 @@
 import {
+  isBaseFilePath,
   isDrawioFilePath,
   isExcalidrawFilePath,
+  isMarkdownFilePath,
   isMermaidFilePath,
   isPlantUmlFilePath,
-  supportsBacklinksForFilePath,
 } from '../../../domain/file-kind.js';
 
 function assertWriteSucceeded(result, operation, filePath) {
@@ -54,6 +55,10 @@ export class CollaborationDocumentStore {
       return this.vaultFileStore.readExcalidrawFile(this.name);
     }
 
+    if (isBaseFilePath(this.name) && typeof this.vaultFileStore.readBaseFile === 'function') {
+      return this.vaultFileStore.readBaseFile(this.name);
+    }
+
     if (isDrawioFilePath(this.name) && typeof this.vaultFileStore.readDrawioFile === 'function') {
       return this.vaultFileStore.readDrawioFile(this.name);
     }
@@ -99,7 +104,7 @@ export class CollaborationDocumentStore {
       await this.writeSnapshot(snapshot);
     }
 
-    if (this.backlinkIndex && supportsBacklinksForFilePath(this.name)) {
+    if (this.backlinkIndex && isMarkdownFilePath(this.name)) {
       this.backlinkIndex.updateFile(this.name, content);
     }
   }
@@ -122,6 +127,12 @@ export class CollaborationDocumentStore {
       return;
     }
 
+    if (isBaseFilePath(this.name) && typeof this.vaultFileStore.writeBaseFile === 'function') {
+      const result = await this.vaultFileStore.writeBaseFile(this.name, content, options);
+      assertWriteSucceeded(result, 'write content', this.name);
+      return;
+    }
+
     if (isMermaidFilePath(this.name) && typeof this.vaultFileStore.writeMermaidFile === 'function') {
       const result = await this.vaultFileStore.writeMermaidFile(this.name, content, options);
       assertWriteSucceeded(result, 'write content', this.name);

package/src/server/domain/collaboration/collaboration-room.js

@@ -8,6 +8,7 @@ import { MSG_AWARENESS, MSG_SYNC } from './protocol.js';
 import { CollaborationDocumentStore } from './collaboration-document-store.js';
 import { RoomClientStateStore } from './room-client-state-store.js';
 import { RoomPersistenceController } from './room-persistence-controller.js';
+import { logPerfEvent } from '../../config/perf-logging.js';
 import { populateCommentThreads, serializeCommentThreads } from '../../../domain/comment-threads.js';
 import {
   isExcalidrawRoomDocStructured,
@@ -131,6 +132,7 @@ export class CollaborationRoom {
     name,
     idleGraceMs = 0,
     maxBufferedAmountBytes,
+    perfLoggingEnabled = false,
     vaultFileStore,
     backlinkIndex,
     onEmpty,
@@ -138,6 +140,7 @@ export class CollaborationRoom {
     this.name = name;
     this.idleGraceMs = idleGraceMs;
     this.maxBufferedAmountBytes = maxBufferedAmountBytes;
+    this.perfLoggingEnabled = perfLoggingEnabled;
     this.documentStore = documentStore ?? new CollaborationDocumentStore({
       backlinkIndex,
       name,
@@ -187,6 +190,8 @@ export class CollaborationRoom {
         let snapshotExists = false;
         let snapshotValid = false;
         let commentThreadCount = 0;
+        let snapshotReadDurationMs = 0;
+        let contentFallbackDurationMs = 0;
         const hydrateDelayMs = Math.max(0, Number(this.getHydrateDelayMs?.() || 0));
 
         try {
@@ -195,7 +200,9 @@ export class CollaborationRoom {
           }
 
           if (this.documentStore?.hasPersistence()) {
+            const snapshotReadStartedAt = Date.now();
             const snapshot = await this.documentStore.readSnapshot();
+            snapshotReadDurationMs = Date.now() - snapshotReadStartedAt;
             if (snapshot) {
               snapshotExists = true;
               try {
@@ -227,10 +234,12 @@ export class CollaborationRoom {
               }
             }
 
+            const fallbackStartedAt = Date.now();
             const [content, commentThreads] = await Promise.all([
               this.documentStore.readContent(),
               this.documentStore.readCommentThreads(),
             ]);
+            contentFallbackDurationMs = Date.now() - fallbackStartedAt;
             commentThreadCount = commentThreads.length;
             if (content !== null || commentThreads.length > 0) {
               hydrateSource = 'content';
@@ -262,14 +271,23 @@ export class CollaborationRoom {
             this.debugMetrics.hydrateCount += 1;
             this.debugMetrics.lastHydrate = {
               commentThreadCount,
+              contentFallbackDurationMs,
               durationMs: Date.now() - startedAt,
+              snapshotReadDurationMs,
               snapshotExists,
               snapshotValid,
               source: hydrateSource,
             };
-            console.info(
-              `[perf][room:${this.name}] hydrate durationMs=${this.debugMetrics.lastHydrate.durationMs} source=${this.debugMetrics.lastHydrate.source} snapshotExists=${this.debugMetrics.lastHydrate.snapshotExists} snapshotValid=${this.debugMetrics.lastHydrate.snapshotValid} commentThreads=${this.debugMetrics.lastHydrate.commentThreadCount}`,
-            );
+            logPerfEvent(this.perfLoggingEnabled, `room:${this.name}`, {
+              commentThreads: this.debugMetrics.lastHydrate.commentThreadCount,
+              contentFallbackDurationMs: this.debugMetrics.lastHydrate.contentFallbackDurationMs,
+              durationMs: this.debugMetrics.lastHydrate.durationMs,
+              event: 'hydrate',
+              snapshotExists: this.debugMetrics.lastHydrate.snapshotExists,
+              snapshotReadDurationMs: this.debugMetrics.lastHydrate.snapshotReadDurationMs,
+              snapshotValid: this.debugMetrics.lastHydrate.snapshotValid,
+              source: this.debugMetrics.lastHydrate.source,
+            });
           }
         }
       })().catch((error) => {
@@ -522,8 +540,13 @@ export class CollaborationRoom {
   sendInitialSync(ws) {
     this.debugMetrics.initialSyncCount += 1;
     this.debugMetrics.lastInitialSyncAt = Date.now();
-    console.info(`[perf][room:${this.name}] send-initial-sync count=${this.debugMetrics.initialSyncCount}`);
-    return sendMessage.call(this, ws, this.getInitialSyncMessage(), this);
+    const initialSyncMessage = this.getInitialSyncMessage();
+    logPerfEvent(this.perfLoggingEnabled, `room:${this.name}`, {
+      bytes: initialSyncMessage.byteLength,
+      count: this.debugMetrics.initialSyncCount,
+      event: 'initial-sync',
+    });
+    return sendMessage.call(this, ws, initialSyncMessage, this);
   }
 
   async addClient(ws, { sendInitialSync: shouldSendInitialSync = true } = {}) {

package/src/server/domain/workspace-state.js

@@ -0,0 +1,52 @@
+import { isMarkdownFilePath, isVaultFilePath } from '../../domain/file-kind.js';
+
+function compareWorkspacePaths(left = '', right = '') {
+  return String(left).localeCompare(String(right), undefined, { sensitivity: 'base' });
+}
+
+export function collectWorkspaceStateStats(entries = new Map(), metadata = new Map()) {
+  const filePaths = [];
+  const markdownPaths = [];
+  let vaultFileCount = 0;
+
+  entries.forEach((entry, pathValue) => {
+    const entryType = entry?.type ?? entry?.nodeType ?? metadata.get(pathValue)?.type ?? '';
+    if (entryType === 'directory') {
+      return;
+    }
+
+    vaultFileCount += 1;
+    if (isVaultFilePath(pathValue)) {
+      filePaths.push(pathValue);
+    }
+    if (isMarkdownFilePath(pathValue)) {
+      markdownPaths.push(pathValue);
+    }
+  });
+
+  filePaths.sort(compareWorkspacePaths);
+  markdownPaths.sort(compareWorkspacePaths);
+
+  return {
+    filePaths,
+    markdownPaths,
+    vaultFileCount,
+  };
+}
+
+export function createWorkspaceStateSnapshot(entries = new Map(), metadata = new Map(), {
+  scannedAt = Date.now(),
+} = {}) {
+  const normalizedEntries = entries instanceof Map ? entries : new Map(entries ?? []);
+  const normalizedMetadata = metadata instanceof Map ? metadata : new Map(metadata ?? []);
+  const stats = collectWorkspaceStateStats(normalizedEntries, normalizedMetadata);
+
+  return {
+    entries: normalizedEntries,
+    filePaths: stats.filePaths,
+    markdownPaths: stats.markdownPaths,
+    metadata: normalizedMetadata,
+    scannedAt,
+    vaultFileCount: stats.vaultFileCount,
+  };
+}

package/src/server/infrastructure/git/history-service.js

@@ -430,7 +430,7 @@ export class GitHistoryService {
     }
 
     return this.runRequest(cacheKey, async () => {
-      let content = '';
+      let content;
       try {
         content = await this.commandRunner.execGit(['show', `${normalizedHash}:${normalizedPath}`]);
       } catch {

package/src/server/infrastructure/http/create-request-handler.js

@@ -40,6 +40,7 @@ export function createRequestHandler(
   authService,
   vaultFileStore,
   backlinkIndex,
+  baseQueryService = null,
   docxExporter = null,
   roomRegistry = null,
   plantUmlRenderer = null,
@@ -60,6 +61,7 @@ export function createRequestHandler(
     workspaceMutationCoordinator,
   });
   const handleVaultApi = createVaultApiHandler({
+    baseQueryService,
     backlinkIndex,
     docxExporter,
     plantUmlRenderer,

package/src/server/infrastructure/http/create-vault-api-command-handler.js

@@ -1,4 +1,5 @@
 import {
+  isBaseFilePath,
   isDrawioFilePath,
   isExcalidrawFilePath,
   isMermaidFilePath,
@@ -55,6 +56,10 @@ function selectWriteOperation(vaultFileStore, filePath, content) {
     return vaultFileStore.writeExcalidrawFile(filePath, content);
   }
 
+  if (isBaseFilePath(filePath)) {
+    return vaultFileStore.writeBaseFile(filePath, content);
+  }
+
   if (isDrawioFilePath(filePath)) {
     return vaultFileStore.writeDrawioFile(filePath, content);
   }

package/src/server/infrastructure/http/create-vault-api-handler.js

@@ -3,6 +3,7 @@ import { createVaultApiCommandHandler } from './create-vault-api-command-handler
 import { createVaultApiQueryHandler } from './create-vault-api-query-handler.js';
 
 export function createVaultApiHandler({
+  baseQueryService = null,
   backlinkIndex,
   docxExporter = null,
   plantUmlRenderer = null,
@@ -11,6 +12,7 @@ export function createVaultApiHandler({
   workspaceMutationCoordinator = null,
 }) {
   const handleVaultApiQuery = createVaultApiQueryHandler({
+    baseQueryService,
     backlinkIndex,
     vaultFileStore,
     workspaceMutationCoordinator,

package/src/server/infrastructure/http/create-vault-api-query-handler.js

@@ -1,12 +1,14 @@
 import { basename } from 'node:path';
 
 import {
+  isBaseFilePath,
   isDrawioFilePath,
   isExcalidrawFilePath,
   isImageAttachmentFilePath,
   isMermaidFilePath,
   isPlantUmlFilePath,
 } from '../../../domain/file-kind.js';
+import { parseJsonBody } from './request-body.js';
 import { jsonResponse, sendResponse } from './http-response.js';
 
 const SVG_MIME_TYPE = 'image/svg+xml';
@@ -46,6 +48,10 @@ function selectReadOperation(vaultFileStore, filePath) {
     return vaultFileStore.readExcalidrawFile(filePath);
   }
 
+  if (isBaseFilePath(filePath)) {
+    return vaultFileStore.readBaseFile(filePath);
+  }
+
   if (isDrawioFilePath(filePath)) {
     return vaultFileStore.readDrawioFile(filePath);
   }
@@ -62,11 +68,71 @@ function selectReadOperation(vaultFileStore, filePath) {
 }
 
 export function createVaultApiQueryHandler({
+  baseQueryService = null,
   backlinkIndex,
   vaultFileStore,
   workspaceMutationCoordinator = null,
 }) {
   return async function handleVaultApiQuery(req, res, requestUrl) {
+    if (requestUrl.pathname === '/api/base/query' && req.method === 'POST') {
+      try {
+        if (!baseQueryService?.query) {
+          jsonResponse(req, res, 503, { error: 'Bases query service is unavailable' });
+          return true;
+        }
+
+        const body = await parseJsonBody(req);
+        const result = await baseQueryService.query({
+          activeFilePath: body?.activeFilePath ?? '',
+          basePath: body?.path ?? '',
+          search: body?.search ?? '',
+          source: typeof body?.source === 'string' ? body.source : null,
+          sourcePath: body?.sourcePath ?? '',
+          view: body?.view ?? '',
+        });
+
+        jsonResponse(req, res, 200, { ok: true, result });
+      } catch (error) {
+        console.error('[api] Failed to query base:', error.message);
+        jsonResponse(req, res, 400, { error: error.message || 'Failed to query base' });
+      }
+      return true;
+    }
+
+    if (requestUrl.pathname === '/api/base/export' && req.method === 'POST') {
+      try {
+        if (!baseQueryService?.query) {
+          jsonResponse(req, res, 503, { error: 'Bases query service is unavailable' });
+          return true;
+        }
+
+        const body = await parseJsonBody(req);
+        const result = await baseQueryService.query({
+          activeFilePath: body?.activeFilePath ?? '',
+          basePath: body?.path ?? '',
+          search: body?.search ?? '',
+          source: typeof body?.source === 'string' ? body.source : null,
+          sourcePath: body?.sourcePath ?? '',
+          view: body?.view ?? '',
+        });
+        const fileName = basename(String(body?.path || body?.sourcePath || 'base')).replace(/\.[^.]+$/u, '') || 'base';
+        sendResponse(req, res, {
+          body: result.csv,
+          headers: {
+            'Cache-Control': 'no-store',
+            'Content-Disposition': `attachment; filename="${createSafeAsciiFilename(`${fileName}.csv`)}"; filename*=UTF-8''${encodeContentDispositionFilename(`${fileName}.csv`)}`,
+            'Content-Type': 'text/csv; charset=utf-8',
+            'X-Content-Type-Options': 'nosniff',
+          },
+          statusCode: 200,
+        });
+      } catch (error) {
+        console.error('[api] Failed to export base CSV:', error.message);
+        jsonResponse(req, res, 400, { error: error.message || 'Failed to export base CSV' });
+      }
+      return true;
+    }
+
     if (requestUrl.pathname === '/api/files' && req.method === 'GET') {
       try {
         const tree = workspaceMutationCoordinator?.getWorkspaceTree?.() ?? await vaultFileStore.tree();

package/src/server/infrastructure/persistence/path-utils.js

@@ -3,7 +3,7 @@ import { isAbsolute, normalize, relative, resolve } from 'path';
 import { isVaultFilePath } from '../../../domain/file-kind.js';
 
 export const IGNORED_DIRECTORIES = new Set(['.git', '.obsidian', '.trash', 'node_modules', '.DS_Store']);
-export const VAULT_FILE_PATH_REQUIREMENT = '.md, .excalidraw, .drawio, .mmd, .mermaid, .puml, .plantuml, .png, .jpg, .jpeg, .webp, .gif, or .svg';
+export const VAULT_FILE_PATH_REQUIREMENT = '.md, .base, .excalidraw, .drawio, .mmd, .mermaid, .puml, .plantuml, .png, .jpg, .jpeg, .webp, .gif, or .svg';
 export const INVALID_VAULT_FILE_PATH_ERROR = `Invalid file path — must end in ${VAULT_FILE_PATH_REQUIREMENT}`;
 export const INVALID_DIRECTORY_PATH_ERROR = 'Invalid directory path';
 

package/src/server/infrastructure/persistence/pull-backup-store.js

@@ -240,7 +240,7 @@ export class PullBackupStore {
   }
 
   async listBackups() {
-    let entries = [];
+    let entries;
     try {
       entries = await readdir(this.getStorageRoot(), { withFileTypes: true });
     } catch (error) {

package/src/server/infrastructure/persistence/vault-content-adapter.js

@@ -1,4 +1,5 @@
 import {
+  isBaseFilePath,
   isDrawioFilePath,
   isExcalidrawFilePath,
   isMarkdownFilePath,
@@ -28,6 +29,11 @@ const adapters = [
     kind: 'markdown',
     matches: isMarkdownFilePath,
   }),
+  new VaultContentAdapter({
+    invalidPathError: 'Invalid file path — must end in .base',
+    kind: 'base',
+    matches: isBaseFilePath,
+  }),
   new VaultContentAdapter({
     invalidPathError: 'Invalid file path — must end in .excalidraw',
     kind: 'excalidraw',

package/src/server/infrastructure/persistence/vault-file-store.js

@@ -7,6 +7,7 @@ import {
   isMarkdownFilePath,
   isVaultFilePath,
 } from '../../../domain/file-kind.js';
+import { mapWithConcurrency } from '../../shared/async-utils.js';
 import { getVaultContentAdapter } from './vault-content-adapter.js';
 import {
   INVALID_VAULT_FILE_PATH_ERROR,
@@ -36,6 +37,7 @@ const MIME_TYPE_TO_IMAGE_EXTENSION = Object.freeze({
   'image/svg+xml': '.svg',
   'image/webp': '.webp',
 });
+const WORKSPACE_SCAN_CONCURRENCY = 8;
 
 function createTransactionalPath(targetPath, label) {
   return `${targetPath}.collabmd-${label}-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2, 10)}`;
@@ -74,6 +76,17 @@ function createWorkspaceEntry(relativePath, type) {
   };
 }
 
+function createWorkspaceMetadata(pathValue, type, info) {
+  return {
+    ctimeMs: Number(info.ctimeMs || 0),
+    inode: Number(info.ino || 0),
+    mtimeMs: Number(info.mtimeMs || 0),
+    path: pathValue,
+    size: type === 'directory' ? 0 : Number(info.size || 0),
+    type,
+  };
+}
+
 function replacePathPrefix(pathValue, oldPrefix, newPrefix) {
   if (pathValue === oldPrefix) {
     return newPrefix;
@@ -94,6 +107,14 @@ function sortWorkspacePaths(paths = [], direction = 'asc') {
   });
 }
 
+function sortDirectoryEntries(entries = []) {
+  return [...entries].sort((left, right) => {
+    if (left.isDirectory() && !right.isDirectory()) return -1;
+    if (!left.isDirectory() && right.isDirectory()) return 1;
+    return left.name.localeCompare(right.name, undefined, { sensitivity: 'base' });
+  });
+}
+
 async function cleanupPaths(paths = []) {
   await Promise.allSettled(paths.filter(Boolean).map((pathValue) => rm(pathValue, { force: true })));
 }
@@ -315,6 +336,10 @@ export class VaultFileStore {
     return this.readContentFile(filePath, 'markdown');
   }
 
+  async readBaseFile(filePath) {
+    return this.readContentFile(filePath, 'base');
+  }
+
   async readExcalidrawFile(filePath) {
     return this.readContentFile(filePath, 'excalidraw');
   }
@@ -357,6 +382,10 @@ export class VaultFileStore {
     return this.writeContentFile(filePath, content, 'markdown', options);
   }
 
+  async writeBaseFile(filePath, content, options = {}) {
+    return this.writeContentFile(filePath, content, 'base', options);
+  }
+
   async writeExcalidrawFile(filePath, content, options = {}) {
     return this.writeContentFile(filePath, content, 'excalidraw', options);
   }
@@ -398,8 +427,8 @@ export class VaultFileStore {
     const timestamp = createAttachmentTimestamp(now);
     const baseFileName = `${attachmentStem}-${timestamp}`;
     let collisionIndex = 0;
-    let storedPath = '';
-    let absolutePath = '';
+    let storedPath;
+    let absolutePath;
 
     do {
       const suffix = collisionIndex > 0 ? `-${collisionIndex + 1}` : '';
@@ -765,7 +794,8 @@ export class VaultFileStore {
   }
 
   async countVaultFiles() {
-    return this.countFilesInDir(this.vaultDir);
+    const snapshot = await this.scanWorkspaceState();
+    return snapshot.vaultFileCount;
   }
 
   async reconcileSidecars({
@@ -837,7 +867,10 @@ export class VaultFileStore {
 
   async scanWorkspaceState() {
     const entries = new Map();
+    const filePaths = [];
     const metadata = new Map();
+    const markdownPaths = [];
+    let vaultFileCount = 0;
 
     const visitDirectory = async (dirPath) => {
       let dirEntries;
@@ -847,64 +880,88 @@ export class VaultFileStore {
         return;
       }
 
-      const sorted = dirEntries.sort((left, right) => {
-        if (left.isDirectory() && !right.isDirectory()) return -1;
-        if (!left.isDirectory() && right.isDirectory()) return 1;
-        return left.name.localeCompare(right.name, undefined, { sensitivity: 'base' });
-      });
+      const sorted = sortDirectoryEntries(dirEntries);
 
-      for (const entry of sorted) {
+      await mapWithConcurrency(sorted, WORKSPACE_SCAN_CONCURRENCY, async (entry) => {
         if (isIgnoredVaultEntry(entry.name)) {
-          continue;
+          return;
         }
 
         const fullPath = join(dirPath, entry.name);
         const relativePath = toVaultRelativePath(this.vaultDir, fullPath).replace(/\\/g, '/');
+        const direntKind = entry.isDirectory()
+          ? 'directory'
+          : (entry.isFile() ? 'file' : null);
 
-        if (entry.isDirectory()) {
+        if (direntKind === 'directory') {
           entries.set(relativePath, createWorkspaceEntry(relativePath, 'directory'));
           try {
             const info = await stat(fullPath);
-            metadata.set(relativePath, {
-              inode: Number(info.ino || 0),
-              mtimeMs: Number(info.mtimeMs || 0),
-              path: relativePath,
-              size: 0,
-              type: 'directory',
-            });
+            metadata.set(relativePath, createWorkspaceMetadata(relativePath, 'directory', info));
           } catch {
             // Ignore transient directories that disappear during scans.
           }
           await visitDirectory(fullPath);
-          continue;
+          return;
         }
 
-        if (!isVaultFilePath(entry.name)) {
-          continue;
+        if (direntKind === 'file') {
+          if (!isVaultFilePath(entry.name)) {
+            return;
+          }
+
+          entries.set(relativePath, createWorkspaceEntry(relativePath, 'file'));
+          filePaths.push(relativePath);
+          vaultFileCount += 1;
+          if (isMarkdownFilePath(relativePath)) {
+            markdownPaths.push(relativePath);
+          }
+          try {
+            const info = await stat(fullPath);
+            metadata.set(relativePath, createWorkspaceMetadata(relativePath, 'file', info));
+          } catch {
+            // Ignore transient files that disappear during scans.
+          }
+          return;
         }
 
-        entries.set(relativePath, createWorkspaceEntry(relativePath, 'file'));
         try {
           const info = await stat(fullPath);
-          metadata.set(relativePath, {
-            inode: Number(info.ino || 0),
-            mtimeMs: Number(info.mtimeMs || 0),
-            path: relativePath,
-            size: Number(info.size || 0),
-            type: 'file',
-          });
+          if (info.isDirectory()) {
+            entries.set(relativePath, createWorkspaceEntry(relativePath, 'directory'));
+            metadata.set(relativePath, createWorkspaceMetadata(relativePath, 'directory', info));
+            await visitDirectory(fullPath);
+            return;
+          }
+
+          if (!info.isFile() || !isVaultFilePath(entry.name)) {
+            return;
+          }
+
+          entries.set(relativePath, createWorkspaceEntry(relativePath, 'file'));
+          metadata.set(relativePath, createWorkspaceMetadata(relativePath, 'file', info));
+          filePaths.push(relativePath);
+          vaultFileCount += 1;
+          if (isMarkdownFilePath(relativePath)) {
+            markdownPaths.push(relativePath);
+          }
         } catch {
-          // Ignore transient files that disappear during scans.
+          // Ignore transient entries that disappear during scans.
         }
-      }
+      });
     };
 
     await visitDirectory(this.vaultDir);
+    filePaths.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: 'base' }));
+    markdownPaths.sort((left, right) => left.localeCompare(right, undefined, { sensitivity: 'base' }));
 
     return {
       entries,
+      filePaths,
       metadata,
+      markdownPaths,
       scannedAt: Date.now(),
+      vaultFileCount,
     };
   }
 }