collabmd 0.1.32 → 0.1.34

package/src/server/domain/bases/base-query-service.js

@@ -1,6 +1,7 @@
 import {
   buildColumns,
   collectEvaluatedPropertyIds,
+  findView,
   normalizeBaseDefinition,
 } from './base-definition.js';
 import {
@@ -8,12 +9,97 @@ import {
   createEvaluationRootContext,
   evaluateFilterNode,
   getPropertyValue,
+  serializeBaseValue,
 } from './base-expression-runtime.js';
 import { BaseIndexSnapshotStore } from './base-index-snapshot-store.js';
+import {
+  buildAvailableProperties,
+  collectDistinctPropertyValues,
+  createPropertyCatalogSnapshot,
+  inferValueType,
+} from './base-query-metadata.js';
 import { buildQueryResultPayload, rowMatchesSearch } from './base-query-results.js';
+import { transformBaseSource } from './base-transform.js';
 
 export { serializeBaseDefinition } from './base-definition.js';
 
+function buildSortChain(activeView) {
+  const entries = [];
+  (activeView.sort ?? []).forEach((sortConfig) => {
+    if (
+      sortConfig?.property
+      && !entries.some((entry) => entry.property === sortConfig.property)
+    ) {
+      entries.push(sortConfig);
+    }
+  });
+  return entries;
+}
+
+function sortRows(rows, sortChain = []) {
+  if (!Array.isArray(sortChain) || sortChain.length === 0) {
+    return rows;
+  }
+
+  return rows.slice().sort((left, right) => {
+    for (const sortConfig of sortChain) {
+      const delta = compareValues(left.rawCells[sortConfig.property], right.rawCells[sortConfig.property]);
+      if (delta !== 0) {
+        return sortConfig.direction === 'desc' ? -delta : delta;
+      }
+    }
+
+    return String(left.file.path ?? '').localeCompare(String(right.file.path ?? ''));
+  });
+}
+
+function escapeRegExp(value = '') {
+  return String(value ?? '').replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+}
+
+function pruneFilterNodeForProperty(filterNode, propertyId = '') {
+  if (!propertyId || !filterNode) {
+    return filterNode;
+  }
+
+  if (typeof filterNode === 'string') {
+    const escapedPropertyId = escapeRegExp(propertyId);
+    const propertyFilterPattern = new RegExp(
+      `^!?${escapedPropertyId}(?:\\.(?:contains|startsWith|endsWith|isEmpty)\\(.*\\)|\\s*(?:==|!=|>=|<=|>|<)\\s*.+)$`,
+      'u',
+    );
+    return propertyFilterPattern.test(String(filterNode ?? '').trim()) ? null : filterNode;
+  }
+
+  if (Array.isArray(filterNode?.and)) {
+    const children = filterNode.and
+      .map((child) => pruneFilterNodeForProperty(child, propertyId))
+      .filter(Boolean);
+    return children.length > 0 ? { and: children } : null;
+  }
+
+  if (Array.isArray(filterNode?.or)) {
+    const children = filterNode.or
+      .map((child) => pruneFilterNodeForProperty(child, propertyId))
+      .filter(Boolean);
+    return children.length > 0 ? { or: children } : null;
+  }
+
+  if (filterNode?.not != null) {
+    if (Array.isArray(filterNode.not)) {
+      const children = filterNode.not
+        .map((child) => pruneFilterNodeForProperty(child, propertyId))
+        .filter(Boolean);
+      return children.length > 0 ? { not: children } : null;
+    }
+
+    const child = pruneFilterNodeForProperty(filterNode.not, propertyId);
+    return child ? { not: child } : null;
+  }
+
+  return filterNode;
+}
+
 export class BaseQueryService {
   constructor({
     vaultFileStore,
@@ -26,6 +112,7 @@ export class BaseQueryService {
       workspaceStateProvider,
       workspaceStateSynchronizer,
     });
+    this.propertyCatalogCache = new Map();
   }
 
   get workspaceStateProvider() {
@@ -105,13 +192,27 @@ export class BaseQueryService {
   }
 
   async applyWorkspaceChange(...args) {
-    return this.snapshotStore.applyWorkspaceChange(...args);
+    const result = await this.snapshotStore.applyWorkspaceChange(...args);
+    const scannedAt = this.snapshotStore.indexSnapshot?.scannedAt;
+    if (scannedAt) {
+      this.invalidatePropertyCatalogsExcept(scannedAt);
+    } else {
+      this.propertyCatalogCache.clear();
+    }
+    return result;
   }
 
-  async query({
+  invalidatePropertyCatalogsExcept(scannedAt = '') {
+    Array.from(this.propertyCatalogCache.keys()).forEach((cacheKey) => {
+      if (cacheKey !== scannedAt) {
+        this.propertyCatalogCache.delete(cacheKey);
+      }
+    });
+  }
+
+  async resolveQueryContext({
     activeFilePath = '',
     basePath = '',
-    search = '',
     source = null,
     sourcePath = '',
     view: requestedView = '',
@@ -126,74 +227,195 @@ export class BaseQueryService {
       basePath,
       sourcePath,
     });
+    this.invalidatePropertyCatalogsExcept(snapshot?.scannedAt ?? '');
+
     const thisFilePath = sourcePath || activeFilePath || basePath || '';
     const thisFile = snapshot.rowsByPath.get(thisFilePath)?.file ?? null;
-    const activeView = definition.views.find((entry) => entry.name === requestedView || entry.id === requestedView) ?? definition.views[0];
+    const activeView = findView(definition, requestedView);
     const columns = buildColumns(definition, activeView);
     const evaluatedPropertyIds = collectEvaluatedPropertyIds(columns, activeView);
 
-    let rows = snapshot.filePaths
-      .map((filePath) => snapshot.rowsByPath.get(filePath))
-      .filter(Boolean)
-      .filter((row) => {
-        const globalContext = createEvaluationRootContext({
-          currentRow: row,
-          definition,
-          snapshot,
-          thisFile,
-        });
-        if (!evaluateFilterNode(definition.filters, globalContext)) {
-          return false;
-        }
+    return {
+      activeView,
+      basePath,
+      baseSource,
+      columns,
+      definition,
+      evaluatedPropertyIds,
+      snapshot,
+      sourcePath,
+      thisFile,
+    };
+  }
 
-        const viewContext = createEvaluationRootContext({
-          currentRow: row,
-          definition,
-          snapshot,
-          thisFile,
-        });
-        return evaluateFilterNode(activeView.filters, viewContext);
-      })
-      .map((row) => {
-        const rawCells = {};
-        evaluatedPropertyIds.forEach((propertyId) => {
-          rawCells[propertyId] = getPropertyValue(propertyId, row, definition, snapshot, thisFile);
-        });
-        return {
-          file: row.file,
-          rawCells,
-        };
+  collectCandidateRows({
+    activeView,
+    astCache = new Map(),
+    definition,
+    evaluatedPropertyIds,
+    snapshot,
+    thisFile,
+  }) {
+    const rows = [];
+
+    snapshot.filePaths.forEach((filePath) => {
+      const row = snapshot.rowsByPath.get(filePath);
+      if (!row) {
+        return;
+      }
+
+      const rowContext = this.createRowQueryContext({
+        astCache,
+        definition,
+        row,
+        snapshot,
+        thisFile,
       });
+      if (!evaluateFilterNode(definition.filters, rowContext) || !evaluateFilterNode(activeView.filters, rowContext)) {
+        return;
+      }
 
-    activeView.sort.forEach((sortConfig) => {
-      rows = rows.slice().sort((left, right) => {
-        const delta = compareValues(left.rawCells[sortConfig.property], right.rawCells[sortConfig.property]);
-        return sortConfig.direction === 'desc' ? -delta : delta;
+      const rawCells = {};
+      evaluatedPropertyIds.forEach((propertyId) => {
+        rawCells[propertyId] = getPropertyValue(propertyId, row, definition, snapshot, thisFile, rowContext);
+      });
+
+      rows.push({
+        file: row.file,
+        rawCells,
       });
     });
 
-    if (activeView.limit != null) {
-      rows = rows.slice(0, activeView.limit);
+    return rows;
+  }
+
+  createRowQueryContext({
+    astCache = new Map(),
+    definition,
+    row,
+    snapshot,
+    thisFile,
+  }) {
+    return createEvaluationRootContext({
+      astCache,
+      currentRow: row,
+      definition,
+      snapshot,
+      thisFile,
+    });
+  }
+
+  getPropertyCatalog(snapshot) {
+    const cacheKey = snapshot?.scannedAt ?? 'unknown';
+    if (!this.propertyCatalogCache.has(cacheKey)) {
+      this.propertyCatalogCache.set(cacheKey, createPropertyCatalogSnapshot(snapshot));
     }
 
-    rows = rows.filter((row) => rowMatchesSearch(row, columns, search));
+    return this.propertyCatalogCache.get(cacheKey);
+  }
+
+  inferFormulaValueTypes(definition, rows = []) {
+    const valueTypes = {};
+
+    Object.keys(definition.formulas ?? {}).forEach((propertyId) => {
+      let inferred = 'unknown';
+      for (const row of rows) {
+        inferred = inferValueType(row?.rawCells?.[propertyId]);
+        if (inferred !== 'unknown') {
+          break;
+        }
+      }
+      valueTypes[propertyId] = inferred;
+    });
+
+    return valueTypes;
+  }
+
+  buildMeta({
+    activeView,
+    basePath = '',
+    columns,
+    definition,
+    rows,
+    snapshot,
+  }) {
+    const propertyCatalog = this.getPropertyCatalog(snapshot);
+    const formulaValueTypes = this.inferFormulaValueTypes(definition, rows);
 
     return {
-      columns,
-      definition,
-      ...buildQueryResultPayload({
+      activeViewConfig: {
+        filters: activeView.filters ?? null,
+        groupBy: activeView.groupBy ?? null,
+        order: [...(activeView.order ?? [])],
+        sort: (activeView.sort ?? []).map((entry) => ({ ...entry })),
+      },
+      availableProperties: buildAvailableProperties({
         activeView,
         columns,
         definition,
+        formulaValueTypes,
+        propertyCatalog,
+      }),
+      editable: Boolean(basePath),
+    };
+  }
+
+  async query({
+    activeFilePath = '',
+    basePath = '',
+    includeCsv = false,
+    search = '',
+    source = null,
+    sourcePath = '',
+    view: requestedView = '',
+  } = {}) {
+    const context = await this.resolveQueryContext({
+      activeFilePath,
+      basePath,
+      source,
+      sourcePath,
+      view: requestedView,
+    });
+
+    const astCache = new Map();
+    let rows = this.collectCandidateRows({
+      ...context,
+      astCache,
+    });
+    rows = rows.filter((row) => rowMatchesSearch(row, context.columns, search));
+    rows = sortRows(rows, buildSortChain(context.activeView));
+
+    if (context.activeView.limit != null) {
+      rows = rows.slice(0, context.activeView.limit);
+    }
+
+    const payload = buildQueryResultPayload({
+      activeView: context.activeView,
+      columns: context.columns,
+      definition: context.definition,
+      includeCsv,
+      rows,
+      snapshot: context.snapshot,
+      thisFile: context.thisFile,
+    });
+
+    return {
+      columns: context.columns,
+      definition: context.definition,
+      meta: this.buildMeta({
+        activeView: context.activeView,
+        basePath,
+        columns: context.columns,
+        definition: context.definition,
         rows,
-        snapshot,
-        thisFile,
+        snapshot: context.snapshot,
       }),
+      ...payload,
       view: {
-        ...activeView,
-        supported: activeView.supported,
+        ...context.activeView,
+        supported: context.activeView.supported,
       },
-      views: definition.views.map((view) => ({
+      views: context.definition.views.map((view) => ({
         id: view.id,
         name: view.name,
         supported: view.supported,
@@ -201,4 +423,93 @@ export class BaseQueryService {
       })),
     };
   }
+
+  async propertyValues({
+    activeFilePath = '',
+    basePath = '',
+    propertyId = '',
+    query = '',
+    source = null,
+    sourcePath = '',
+    view: requestedView = '',
+  } = {}) {
+    if (!propertyId) {
+      throw new Error('Missing propertyId');
+    }
+
+    const context = await this.resolveQueryContext({
+      activeFilePath,
+      basePath,
+      source,
+      sourcePath,
+      view: requestedView,
+    });
+    const propertyValueContext = {
+      ...context,
+      activeView: {
+        ...context.activeView,
+        filters: pruneFilterNodeForProperty(context.activeView?.filters, propertyId),
+      },
+      definition: {
+        ...context.definition,
+        filters: pruneFilterNodeForProperty(context.definition?.filters, propertyId),
+      },
+    };
+    const astCache = new Map();
+    const rows = this.collectCandidateRows({
+      ...propertyValueContext,
+      astCache,
+      evaluatedPropertyIds: [...new Set([
+        ...(context.evaluatedPropertyIds ?? []),
+        propertyId,
+      ])],
+    });
+    const values = collectDistinctPropertyValues(rows, propertyId, query).map((entry) => ({
+      count: entry.count,
+      text: entry.text,
+      value: serializeBaseValue(entry.value, {
+        snapshot: context.snapshot,
+        sourcePath: context.thisFile?.path ?? '',
+      }),
+    }));
+
+    return {
+      ok: true,
+      propertyId,
+      values,
+    };
+  }
+
+  async transform({
+    activeFilePath = '',
+    basePath = '',
+    mutation = null,
+    source = null,
+    sourcePath = '',
+    view = '',
+  } = {}) {
+    if (!basePath) {
+      throw new Error('Only .base files can be transformed');
+    }
+
+    const baseSource = source ?? await this.vaultFileStore.readBaseFile(basePath);
+    if (typeof baseSource !== 'string') {
+      throw new Error('Base source not found');
+    }
+
+    const nextSource = transformBaseSource(baseSource, mutation);
+    const result = await this.query({
+      activeFilePath,
+      basePath,
+      source: nextSource,
+      sourcePath,
+      view,
+    });
+
+    return {
+      meta: result.meta,
+      result,
+      source: nextSource,
+    };
+  }
 }

package/src/server/domain/bases/base-transform.js

@@ -0,0 +1,116 @@
+import yaml from 'js-yaml';
+
+import {
+  findView,
+  normalizeBaseDefinition,
+  normalizeRawDefinitionForWrite,
+} from './base-definition.js';
+
+function isPlainObject(value) {
+  return Object.prototype.toString.call(value) === '[object Object]';
+}
+
+function cloneRawSource(source = '') {
+  const parsed = yaml.load(String(source ?? '')) ?? {};
+  return isPlainObject(parsed) ? parsed : {};
+}
+
+function ensureViews(rawDefinition) {
+  if (!Array.isArray(rawDefinition.views) || rawDefinition.views.length === 0) {
+    rawDefinition.views = [{ name: 'Table', type: 'table' }];
+  }
+
+  return rawDefinition.views;
+}
+
+function findViewIndex(definition, requestedView = '') {
+  const activeView = findView(definition, requestedView);
+  return definition.views.findIndex((view) => view.id === activeView.id);
+}
+
+function serializeGroupBy(groupBy) {
+  if (!groupBy?.property) {
+    return null;
+  }
+
+  return {
+    direction: String(groupBy.direction ?? 'asc').toUpperCase(),
+    property: groupBy.property,
+  };
+}
+
+function serializeSort(sort = []) {
+  return sort
+    .filter((entry) => entry?.property)
+    .map((entry) => ({
+      direction: String(entry.direction ?? 'asc').toUpperCase(),
+      property: entry.property,
+    }));
+}
+
+function applySetViewConfig(rawDefinition, definition, mutation = {}) {
+  const viewIndex = findViewIndex(definition, mutation.view ?? '');
+  if (viewIndex < 0) {
+    throw new Error('Base view not found');
+  }
+
+  const rawViews = ensureViews(rawDefinition);
+  const nextView = isPlainObject(rawViews[viewIndex])
+    ? { ...rawViews[viewIndex] }
+    : {};
+  const nextConfig = isPlainObject(mutation.config) ? mutation.config : {};
+
+  if (Object.hasOwn(nextConfig, 'order')) {
+    nextView.order = Array.isArray(nextConfig.order)
+      ? nextConfig.order.filter((entry) => typeof entry === 'string' && entry.trim())
+      : [];
+  }
+
+  if (Object.hasOwn(nextConfig, 'sort')) {
+    const sort = serializeSort(nextConfig.sort);
+    if (sort.length > 0) {
+      nextView.sort = sort;
+    } else {
+      delete nextView.sort;
+      delete nextView.sorts;
+    }
+  }
+
+  if (Object.hasOwn(nextConfig, 'groupBy')) {
+    const groupBy = serializeGroupBy(nextConfig.groupBy);
+    if (groupBy) {
+      nextView.groupBy = groupBy;
+    } else {
+      delete nextView.groupBy;
+      delete nextView.group_by;
+    }
+  }
+
+  if (Object.hasOwn(nextConfig, 'filters')) {
+    if (nextConfig.filters == null || nextConfig.filters === '') {
+      delete nextView.filters;
+    } else {
+      nextView.filters = nextConfig.filters;
+    }
+  }
+
+  rawViews[viewIndex] = nextView;
+}
+
+export function transformBaseSource(source = '', mutation = {}) {
+  const definition = normalizeBaseDefinition(source);
+  const rawDefinition = normalizeRawDefinitionForWrite({
+    raw: cloneRawSource(source),
+  });
+  const mutationType = String(mutation?.type ?? '');
+
+  switch (mutationType) {
+    case 'set-view-config':
+      applySetViewConfig(rawDefinition, definition, mutation);
+      break;
+    default:
+      throw new Error(`Unsupported base mutation: ${mutationType || 'unknown'}`);
+  }
+
+  return `${yaml.dump(normalizeRawDefinitionForWrite(rawDefinition), { lineWidth: -1, noRefs: true }).trim()}\n`;
+}