collabmd 0.1.52 → 0.1.55

package/src/server/domain/bases/base-query-service.js

@@ -56,32 +56,6 @@ function sortRows(rows, sortChain = []) {
   });
 }
 
-function compareRowsBySortChain(left, right, sortChain = []) {
-  for (const sortConfig of sortChain) {
-    const delta = compareValues(left.rawCells[sortConfig.property], right.rawCells[sortConfig.property]);
-    if (delta !== 0) {
-      return sortConfig.direction === 'desc' ? -delta : delta;
-    }
-  }
-
-  return String(left.file.path ?? '').localeCompare(String(right.file.path ?? ''));
-}
-
-function insertSortedBounded(rows, row, {
-  limit,
-  sortChain = [],
-}) {
-  let insertIndex = rows.findIndex((existingRow) => compareRowsBySortChain(row, existingRow, sortChain) < 0);
-  if (insertIndex === -1) {
-    insertIndex = rows.length;
-  }
-
-  rows.splice(insertIndex, 0, row);
-  if (rows.length > limit) {
-    rows.pop();
-  }
-}
-
 function pruneFilterNodeForProperty(filterNode, propertyId = '') {
   if (!propertyId || !filterNode) {
     return filterNode;
@@ -225,21 +199,12 @@ export class BaseQueryService {
     columns = [],
     definition,
     evaluatedPropertyIds,
-    limit = null,
-    maxRows = this.maxResultRows,
     snapshot,
-    sortChain = [],
     search = '',
     thisFile,
   }) {
+    // ponytail: retain matching rows for one native sort; use bounded selection if heap profiles require it.
     const rows = [];
-    const effectiveMaxRows = Number.isInteger(maxRows) && maxRows > 0 ? maxRows : Infinity;
-    const effectiveLimit = Number.isInteger(limit) && limit > 0 ? limit : Infinity;
-    const canBoundDuringCollection = Number.isFinite(Math.min(effectiveLimit, effectiveMaxRows))
-      && sortChain.length > 0
-      && !activeView.groupBy?.property
-      && (activeView.summaries?.length ?? 0) === 0;
-    const hardLimit = Math.min(effectiveLimit, effectiveMaxRows);
 
     const filePaths = snapshot.filePaths ?? [];
     for (let chunkStart = 0; chunkStart < filePaths.length; chunkStart += BASE_QUERY_ROW_CHUNK_SIZE) {
@@ -275,14 +240,6 @@ export class BaseQueryService {
           continue;
         }
 
-        if (canBoundDuringCollection) {
-          insertSortedBounded(rows, candidateRow, {
-            limit: hardLimit,
-            sortChain,
-          });
-          continue;
-        }
-
         rows.push(candidateRow);
       }
 
@@ -390,10 +347,7 @@ export class BaseQueryService {
     let rows = await this.collectCandidateRows({
       ...context,
       astCache,
-      limit: requestedLimit,
-      maxRows: this.maxResultRows,
       search,
-      sortChain,
     });
     rows = sortRows(rows, sortChain);
 

package/src/server/domain/collaboration/collaboration-room.js

@@ -14,7 +14,21 @@ import {
 import { CollaborationDocumentStore } from './collaboration-document-store.js';
 import { logPerfEvent } from '../../config/perf-logging.js';
 import { populateCommentThreads, serializeCommentThreads } from '../../../domain/comment-threads.js';
-import { getVaultFileKind } from '../../../domain/file-kind.js';
+import { getVaultFileKind, isCanvasFilePath } from '../../../domain/file-kind.js';
+import {
+  buildCanvasRoomDocument,
+  CANVAS_EDGES_KEY,
+  CANVAS_EDGE_ORDER_KEY,
+  CANVAS_META_KEY,
+  CANVAS_NODES_KEY,
+  CANVAS_NODE_ORDER_KEY,
+  isCanvasRoomDocStructured,
+  mergeCanvasDocuments,
+  parseCanvasJson,
+  replaceCanvasRoomDocument,
+  serializeCanvasDocument,
+  serializeCanvasRoomDocument,
+} from '../../../domain/canvas-room-codec.js';
 import {
   applySceneDiffToExcalidrawRoom,
   EXCALIDRAW_APP_STATE_KEY,
@@ -292,7 +306,28 @@ export class CollaborationRoom {
             if (snapshot) {
               snapshotExists = true;
               try {
-                if (isExcalidrawRoom(this.name)) {
+                if (isCanvasFilePath(this.name)) {
+                  const validationDoc = new Y.Doc({ gc: true });
+                  try {
+                    Y.applyUpdate(validationDoc, snapshot, 'hydrate');
+                    const stored = serializeCanvasRoomDocument(validationDoc);
+                    const content = await this.documentStore.readContent();
+                    const meta = validationDoc.getMap(CANVAS_META_KEY);
+                    if (meta.has('externalConflict')) {
+                      let invalid = false;
+                      try { parseCanvasJson(content); } catch { invalid = true; }
+                      meta.set('externalConflict', { content, invalid });
+                    } else {
+                      const external = parseCanvasJson(content);
+                      if (stored !== serializeCanvasDocument(external)) {
+                        replaceCanvasRoomDocument(validationDoc, external);
+                      }
+                    }
+                    Y.applyUpdate(this.doc, Y.encodeStateAsUpdate(validationDoc), 'hydrate');
+                  } finally {
+                    validationDoc.destroy();
+                  }
+                } else if (isExcalidrawRoom(this.name)) {
                   const validationDoc = new Y.Doc({ gc: true });
                   try {
                     Y.applyUpdate(validationDoc, snapshot, 'hydrate');
@@ -332,7 +367,13 @@ export class CollaborationRoom {
               const ytext = this.doc.getText('codemirror');
               const comments = this.doc.getArray('comments');
               this.doc.transact(() => {
-                if (isExcalidrawRoom(this.name)) {
+                if (isCanvasFilePath(this.name)) {
+                  try {
+                    replaceCanvasRoomDocument(this.doc, parseCanvasJson(content));
+                  } catch {
+                    this.doc.getMap(CANVAS_META_KEY).set('invalidContent', true);
+                  }
+                } else if (isExcalidrawRoom(this.name)) {
                   const parsedScene = tryParseExcalidrawSceneJson(content);
                   if (parsedScene) {
                     migrateLegacyExcalidrawRoomData(this.doc, parsedScene);
@@ -346,7 +387,7 @@ export class CollaborationRoom {
               }, 'hydrate');
 
               const replacementSnapshot = Y.encodeStateAsUpdate(this.doc);
-              if (snapshotNeedsReplacement) {
+              if (snapshotNeedsReplacement || isCanvasFilePath(this.name)) {
                 await this.documentStore.writeSnapshot(replacementSnapshot);
               } else {
                 void this.documentStore.writeSnapshot(replacementSnapshot).catch((error) => {
@@ -462,6 +503,13 @@ export class CollaborationRoom {
       this.contentDirty = true;
     };
 
+    if (isCanvasFilePath(this.name)) {
+      for (const key of [CANVAS_NODES_KEY, CANVAS_EDGES_KEY, CANVAS_NODE_ORDER_KEY, CANVAS_EDGE_ORDER_KEY, CANVAS_META_KEY]) {
+        this.doc.getMap(key).observeDeep((_events, transaction) => markFromTransaction(transaction));
+      }
+      return;
+    }
+
     if (isExcalidrawRoom(this.name)) {
       this.doc.getMap(EXCALIDRAW_ELEMENTS_KEY).observeDeep((_events, transaction) => {
         markFromTransaction(transaction);
@@ -482,6 +530,12 @@ export class CollaborationRoom {
 
   resetContentBaseline() {
     this.persistedContentBaseline = this.getPersistedContent();
+    if (isCanvasFilePath(this.name)) {
+      const conflict = this.doc.getMap(CANVAS_META_KEY).get('externalConflict');
+      if (conflict) {
+        this.persistedContentBaseline = conflict.invalid ? null : serializeCanvasDocument(parseCanvasJson(conflict.content));
+      }
+    }
     this.contentDirty = false;
   }
 
@@ -555,15 +609,38 @@ export class CollaborationRoom {
         return;
       }
 
-      const content = this.getPersistedContent();
+      let content = this.getPersistedContent();
       if (content === null) {
-        console.warn(`[room:${this.name}] Skipping persist because the Excalidraw scene is invalid`);
+        console.warn(`[room:${this.name}] Skipping persist because the document is invalid`);
         return;
       }
+      if (isCanvasFilePath(this.name) && content !== this.persistedContentBaseline && !this.doc.getMap(CANVAS_META_KEY).has('externalConflict')) {
+        const externalContent = await this.documentStore.readContent();
+        if (this.deleted || this.destroyed) return;
+        if (externalContent === null) {
+          this.markDeleted();
+          return;
+        }
+        let externalBaseline = null;
+        let invalid = false;
+        try { externalBaseline = serializeCanvasDocument(parseCanvasJson(externalContent)); } catch { invalid = true; }
+        content = this.getPersistedContent();
+        if (content === null) return;
+        // A filesystem observation may still be queued when the save timer fires.
+        if (externalBaseline !== this.persistedContentBaseline) {
+          this.persistedContentBaseline = externalBaseline;
+          if (externalBaseline !== content) {
+            this.doc.transact(() => {
+              this.doc.getMap(CANVAS_META_KEY).set('externalConflict', { content: externalContent, invalid });
+            }, 'workspace-reconcile');
+          }
+        }
+      }
       // Compare inline: refreshContentDirty() would serialize the document
       // a second time for the same check.
-      const includeContent = content !== this.persistedContentBaseline;
-      this.contentDirty = includeContent;
+      const conflict = isCanvasFilePath(this.name) && this.doc.getMap(CANVAS_META_KEY).has('externalConflict');
+      this.contentDirty = content !== this.persistedContentBaseline;
+      const includeContent = this.contentDirty && !conflict;
       const commentThreads = serializeCommentThreads(this.doc.getArray('comments'));
       await this.documentStore.persistState({
         commentThreads,
@@ -616,6 +693,46 @@ export class CollaborationRoom {
     }
 
     const comments = this.doc.getArray('comments');
+    if (isCanvasFilePath(this.name)) {
+      let external;
+      try {
+        external = parseCanvasJson(content);
+      } catch {
+        this.doc.transact(() => {
+          this.doc.getMap(CANVAS_META_KEY).set('externalConflict', { content, invalid: true });
+        }, 'workspace-reconcile');
+        this.persistedContentBaseline = null;
+        // Preserve the pending document in a sidecar while the invalid file remains untouched.
+        await this.persist();
+        return { ok: false, reason: 'invalid-canvas' };
+      }
+
+      const meta = this.doc.getMap(CANVAS_META_KEY);
+      const current = this.getPersistedContent();
+      const hasPendingChanges = current !== null && current !== this.persistedContentBaseline;
+      const reconciliation = hasPendingChanges && this.persistedContentBaseline !== null
+        ? mergeCanvasDocuments(parseCanvasJson(this.persistedContentBaseline), buildCanvasRoomDocument(this.doc), external)
+        : { conflict: hasPendingChanges, document: external };
+      const hadConflict = meta.has('externalConflict');
+      this.doc.transact(() => {
+        if (reconciliation.conflict || hadConflict) {
+          meta.set('externalConflict', { content, invalid: false });
+        } else {
+          replaceCanvasRoomDocument(this.doc, reconciliation.document);
+          meta.delete('invalidContent');
+        }
+        if (replaceCommentThreads) {
+          if (comments.length > 0) comments.delete(0, comments.length);
+          populateCommentThreads(comments, commentThreads);
+        }
+      }, 'workspace-reconcile');
+      this.persistedContentBaseline = serializeCanvasDocument(external);
+      this.refreshContentDirty();
+      if (this.contentDirty && !meta.has('externalConflict')) this.schedulePersist();
+      else await this.persist();
+      return { ok: true, highlightRange: null, conflict: meta.has('externalConflict') };
+    }
+
     if (isExcalidrawRoom(this.name)) {
       const parsedExcalidrawScene = tryParseExcalidrawSceneJson(content);
       if (!parsedExcalidrawScene) {
@@ -712,10 +829,6 @@ export class CollaborationRoom {
     }
   }
 
-  unmarkDeleted() {
-    this.deleted = false;
-  }
-
   isDeleted() {
     return this.deleted;
   }
@@ -917,14 +1030,14 @@ export class CollaborationRoom {
   }
 
   readEditableContent() {
-    if (!this.hydrated || this.deleted || this.destroyed || isExcalidrawRoom(this.name)) {
+    if (!this.hydrated || this.deleted || this.destroyed || isExcalidrawRoom(this.name) || isCanvasFilePath(this.name)) {
       return null;
     }
     return this.doc.getText('codemirror').toString();
   }
 
   applyExactTextChanges(changes, { origin = 'agent' } = {}) {
-    if (!this.hydrated || this.deleted || this.destroyed || isExcalidrawRoom(this.name)) {
+    if (!this.hydrated || this.deleted || this.destroyed || isExcalidrawRoom(this.name) || isCanvasFilePath(this.name)) {
       throw new Error('The collaborative document is unavailable');
     }
     const ytext = this.doc.getText('codemirror');
@@ -962,6 +1075,14 @@ export class CollaborationRoom {
       return null;
     }
 
+    if (isCanvasFilePath(this.name)) {
+      try {
+        return isCanvasRoomDocStructured(this.doc) ? serializeCanvasRoomDocument(this.doc) : null;
+      } catch {
+        return null;
+      }
+    }
+
     if (!isExcalidrawRoom(this.name)) {
       return this.doc.getText('codemirror').toString();
     }
@@ -1027,7 +1148,20 @@ export class CollaborationRoom {
           const encoder = encoding.createEncoder();
           encoding.writeVarUint(encoder, MSG_AGENT_FLUSH_ACK);
           encoding.writeVarString(encoder, requestId);
-          sendMessage.call(this, ws, encoding.toUint8Array(encoder), this);
+          const acknowledge = () => sendMessage.call(this, ws, encoding.toUint8Array(encoder), this);
+          if (isCanvasFilePath(this.name)) {
+            const available = () => this.hydrated && !this.deleted && !this.destroyed
+              && this.documentStore?.hasPersistence() && this.getPersistedContent() !== null;
+            if (available()) {
+              void this.persist().then(() => {
+                if (available()) acknowledge();
+              }).catch((error) => {
+                console.error(`[room:${this.name}] Failed to persist canvas before flush acknowledgement:`, error.message);
+              });
+            }
+          } else {
+            acknowledge();
+          }
           break;
         }
 

package/src/server/domain/collaboration/room-registry.js

@@ -1,3 +1,5 @@
+import { getVaultFileKind } from '../../../domain/file-kind.js';
+
 export class RoomRegistry {
   constructor({ createRoom }) {
     this.createRoom = createRoom;
@@ -31,6 +33,10 @@ export class RoomRegistry {
       return false;
     }
 
+    const oldKind = getVaultFileKind(oldName);
+    const newKind = getVaultFileKind(newName);
+    if (oldKind !== newKind && [oldKind, newKind].includes('canvas')) return false;
+
     const room = this.rooms.get(oldName);
     if (!room) {
       return false;
@@ -61,12 +67,6 @@ export class RoomRegistry {
     return Array.from(this.rooms.entries());
   }
 
-  async reloadAllFromDisk() {
-    await Promise.allSettled(
-      Array.from(this.rooms.values(), (room) => room.reloadFromDisk?.()),
-    );
-  }
-
   async reconcileWorkspaceChange(workspaceChange = {}) {
     const deletedPaths = new Set(workspaceChange.deletedPaths ?? []);
     const renamedPaths = Array.isArray(workspaceChange.renamedPaths) ? workspaceChange.renamedPaths : [];
@@ -128,7 +128,7 @@ export class RoomRegistry {
           }
 
           const result = await room.reloadFromDisk?.();
-          if (result && result.ok === false && result.reason === 'invalid-excalidraw') {
+          if (result && result.ok === false && ['invalid-excalidraw', 'invalid-canvas'].includes(result.reason)) {
             reloadRequiredPaths.push(pathValue);
           } else if (result?.highlightRange) {
             highlightRanges.push({

package/src/server/domain/hosted-workspace.js

@@ -102,6 +102,24 @@ function publicAuditEvent(record = null) {
   };
 }
 
+function parseAuditCursor(cursor) {
+  if (!cursor) return null;
+  try {
+    if (typeof cursor !== 'string' || cursor.length > 256 || !/^[A-Za-z0-9_-]+$/u.test(cursor)) {
+      throw new Error('Invalid cursor');
+    }
+    const value = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
+    if (!Array.isArray(value) || value.length !== 2
+      || !Number.isSafeInteger(value[0]) || value[0] < 0
+      || typeof value[1] !== 'string' || !value[1] || value[1].length > 128) {
+      throw new Error('Invalid cursor');
+    }
+    return { createdAt: value[0], id: value[1] };
+  } catch {
+    throw createHostedError(400, 'Invalid access history cursor.', 'HOSTED_AUDIT_CURSOR_INVALID');
+  }
+}
+
 function publicVaultSource(record = null) {
   if (!record) {
     return null;
@@ -734,8 +752,17 @@ export class HostedWorkspaceService {
     }));
   }
 
-  async listAuditEvents(user) {
+  async listAuditEvents(user, { cursor = '', limit = 50 } = {}) {
     await this.requireAdmin(user);
-    return (await this.store.listAuditEvents()).map(publicAuditEvent);
+    const pageSize = Math.min(100, Math.max(1, Number.parseInt(limit, 10) || 50));
+    const rows = await this.store.listAuditEvents({ before: parseAuditCursor(cursor), limit: pageSize + 1 });
+    const events = rows.slice(0, pageSize).map(publicAuditEvent);
+    const last = events.at(-1);
+    return {
+      events,
+      nextCursor: rows.length > pageSize
+        ? Buffer.from(JSON.stringify([last.createdAt, last.id])).toString('base64url')
+        : null,
+    };
   }
 }

package/src/server/domain/plantuml-encoder.js

@@ -1,49 +1,13 @@
 import { deflateRawSync } from 'node:zlib';
 
-function encode6bit(value) {
-  if (value < 10) {
-    return String.fromCharCode(48 + value);
-  }
-
-  if (value < 36) {
-    return String.fromCharCode(55 + value);
-  }
-
-  if (value < 62) {
-    return String.fromCharCode(61 + value);
-  }
-
-  if (value === 62) {
-    return '-';
-  }
-
-  if (value === 63) {
-    return '_';
-  }
-
-  return '?';
-}
-
-function append3bytes(byte1, byte2, byte3) {
-  const c1 = byte1 >> 2;
-  const c2 = ((byte1 & 0x03) << 4) | (byte2 >> 4);
-  const c3 = ((byte2 & 0x0f) << 2) | (byte3 >> 6);
-  const c4 = byte3 & 0x3f;
-
-  return `${encode6bit(c1 & 0x3f)}${encode6bit(c2 & 0x3f)}${encode6bit(c3 & 0x3f)}${encode6bit(c4 & 0x3f)}`;
-}
+const BASE64URL_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_';
+const PLANTUML_ALPHABET = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_';
 
 export function encodePlantUmlText(source = '') {
   const compressed = deflateRawSync(Buffer.from(String(source), 'utf-8'));
-  let encoded = '';
-
-  for (let index = 0; index < compressed.length; index += 3) {
-    encoded += append3bytes(
-      compressed[index] ?? 0,
-      compressed[index + 1] ?? 0,
-      compressed[index + 2] ?? 0,
-    );
-  }
-
-  return encoded;
+  const padded = Buffer.alloc(Math.ceil(compressed.length / 3) * 3);
+  compressed.copy(padded);
+  return padded.toString('base64url').replace(/./gu, (character) => (
+    PLANTUML_ALPHABET[BASE64URL_ALPHABET.indexOf(character)]
+  ));
 }

package/src/server/infrastructure/git/diff-service.js

@@ -107,7 +107,6 @@ export class GitDiffService {
   }
 
   getScopedFiles(status, scope = 'working-tree', path = null) {
-    const orderedFiles = [];
     const fileMap = new Map();
     const sectionMap = createSectionMap(status.sections);
     const candidateSections = scope === 'staged'
@@ -126,13 +125,10 @@ export class GitDiffService {
         const existing = fileMap.get(file.path) ?? null;
         const merged = mergeScopedFile(existing, file, scope);
         fileMap.set(file.path, merged);
-        if (!existing) {
-          orderedFiles.push(merged);
-        }
       }
     }
 
-    return orderedFiles;
+    return Array.from(fileMap.values());
   }
 
   async getScopeSummary({

package/src/server/infrastructure/git/history-service.js

@@ -1,7 +1,7 @@
 import { createRequestError as createGitRequestError } from '../http/http-errors.js';
 import { getVaultFileKind, isImageAttachmentFilePath } from '../../../domain/file-kind.js';
 import { normalizeRelativeGitPath } from './path-utils.js';
-import { getImageMimeType } from './image-mime.js';
+import { getImageMimeType } from '../../shared/image-mime.js';
 import {
   countPatchLines,
   parseGitLogHeader,
@@ -16,6 +16,8 @@ import {
 const EMPTY_TREE_HASH = '4b825dc642cb6eb9a060e54bf8d69288fbee4904';
 const DEFAULT_HISTORY_LIMIT = 30;
 const HISTORY_LIMIT_CAP = 50;
+const RESPONSE_CACHE_LIMIT = 50;
+const MAX_CACHED_RESPONSE_BYTES = 256 * 1024;
 
 function clampHistoryLimit(limit) {
   const parsedLimit = Number.parseInt(String(limit ?? DEFAULT_HISTORY_LIMIT), 10);
@@ -77,8 +79,22 @@ function getCachedValue(cache, key) {
 }
 
 function setCachedValue(cache, key, value, ttlMs) {
+  const now = Date.now();
+  for (const [cachedKey, cached] of cache) {
+    if (now >= cached.expiresAt) cache.delete(cachedKey);
+  }
+  cache.delete(key);
+  const payloadBytes = Buffer.isBuffer(value.content)
+    ? value.content.byteLength
+    : Buffer.byteLength(JSON.stringify(value));
+  if (ttlMs <= 0 || payloadBytes + Buffer.byteLength(key) > MAX_CACHED_RESPONSE_BYTES) {
+    return value;
+  }
+  while (cache.size >= RESPONSE_CACHE_LIMIT) {
+    cache.delete(cache.keys().next().value);
+  }
   cache.set(key, {
-    expiresAt: Date.now() + ttlMs,
+    expiresAt: now + ttlMs,
     value,
   });
   return value;

package/src/server/infrastructure/git/untracked-files.js

@@ -7,7 +7,7 @@ import { getVaultFileKind, isImageAttachmentFilePath } from '../../../domain/fil
 import { mapWithConcurrency } from '../../shared/async-utils.js';
 import { createEmptyStats } from './responses.js';
 import { splitContentLines } from './parsers.js';
-import { getImageMimeType } from './image-mime.js';
+import { getImageMimeType } from '../../shared/image-mime.js';
 
 async function countFileLines(filePath) {
   if (isImageAttachmentFilePath(filePath)) {