@andespindola/brainlink 0.1.0-beta.84 → 0.1.0-beta.86

package/README.md

@@ -84,10 +84,11 @@ Legacy `.jsonl.gz` packs are upgraded to `.blpk` automatically on first search/c
 - Graph renderer optimized for large datasets with viewport-driven node culling and edge lookup by visible nodes.
 - Canvas graph rendering uses the same batched node and edge pipeline for every graph size, reducing per-frame draw calls while keeping selected and hovered items highlighted.
 - WebGL acceleration is used when available for dense node and edge drawing, with Canvas 2D preserved as the interaction and fallback layer.
+- Graph zoom-out renders hierarchical ecosystem subgraphs: macro clusters of up to 1000 notes expand near the user's focus into smaller graph meshes before individual notes are rendered.
 - Large graph layout API automatically uses compact payload encoding with link-coverage-aware edge selection to reduce initial client load without hiding major relationships.
 - Large-segment layout spacing now grows logarithmically to keep initial visual density consistent between medium and very large vaults (for example, ~1k vs ~50k notes).
 - Graph coordinates are visually compacted across graph sizes so reset starts from a stable macro mass and zoom-in progressively expands toward local detail.
-- Zoomed-out graph LOD clusters dense regions and progressively expands the focused viewport as zoom increases, including very large vaults.
+- Zoomed-out graph LOD renders nested subgraphs and progressively expands only the focused cluster as zoom increases, including very large vaults.
 - Graph reset starts in macro "galaxy" overview mode and progressively reveals nearby nodes as zoom increases, including smaller vaults.
 - Graph filtering runs in a dedicated browser worker to keep the UI thread responsive during heavy datasets.
 - Edge rendering budgets adapt to zoom level to prevent frame spikes on large graph panoramas.
@@ -601,7 +602,7 @@ The graph UI shows:
 - WebGL node and edge acceleration when supported, falling back to Canvas 2D without changing graph behavior
 - compact macro-to-micro density progression so reset keeps the graph mass oriented and zoom-in separates local neighborhoods progressively
 - graph camera treats hub-centered navigation as structural only when the hub is dominant; diffuse stress graphs reset and zoom around the full graph mass
-- massive-graph LOD progression: very low zoom uses spatial overview sampling plus hub-neighborhood edge previews to preserve whole-vault shape and orientation, then progressively raises the focused node budget as zoom increases so dense local areas keep nearby notes and links visible
+- graph LOD progression: very low zoom uses connected ecosystem clusters of up to 1000 notes, zoom-in expands only focused clusters into 250-note and 60-note subgraphs with aggregated real links, then progressively raises the focused node budget so local areas keep nearby notes and links visible
 
 The server indexes before starting by default. Use `--no-index` to skip that step:
 

package/dist/application/frontend/client-js.js

@@ -21,14 +21,19 @@ const viewportPaddingPx = 280
 const worldCoordinateLimit = 5_000_000
 const transformCoordinateLimit = 20_000_000
 const hoverHitTestIntervalMs = 64
-const overviewClusterMaxCount = 1400
+const ecosystemGroupSize = 1000
+const ecosystemGroupSizes = [1000, 250, 60]
+const ecosystemClusterEdgeLimit = 520
+const ecosystemClusterScaleThreshold = 0.32
+const ecosystemSubgraphScaleThreshold = 0.18
+const ecosystemMicroScaleThreshold = 0.08
+const ecosystemFocusedParentLimit = 3
 const zoomRecoveryGuardMs = 4200
 const zoomCapTargetViewportShare = 0.72
 const meshEdgeScaleThreshold = 0.09
 const meshEdgeMinBudget = 140
 const meshEdgeMaxBudget = 1400
 const layeredCoreScaleThreshold = 0.55
-const massiveOverviewClusterScaleThreshold = 0.035
 const dragNeighborhoodMaxAffected = 180
 const dragSettleRounds = 3
 const wheelZoomExponent = 0.0018
@@ -44,6 +49,7 @@ const state = {
   renderNodes: [],
   renderEdges: [],
   renderClusters: [],
+  renderClusterEdges: [],
   nodeDegrees: new Map(),
   selected: null,
   hovered: null,
@@ -65,7 +71,8 @@ const state = {
   lastViewportKey: '',
   visibleNodeSpatial: { cellSize: 220, minX: 0, minY: 0, maxX: 0, maxY: 0, buckets: new Map() },
   visibleEdgeByNode: new Map(),
-  overviewClusters: [],
+  ecosystemClusters: [],
+  ecosystemClustersBySize: new Map(),
   macroCenter: { x: 0, y: 0 },
   macroRepresentative: null,
   primaryHub: null,
@@ -562,7 +569,11 @@ const recomputeVisibility = () => {
   state.visibleEdges = limitedEdges
   state.visibleNodeSpatial = createSpatialIndex(nodes)
   state.visibleEdgeByNode = createVisibleEdgeLookup(limitedEdges)
-  state.overviewClusters = nodes.length > massiveGraphNodeThreshold ? buildOverviewClusters(nodes) : []
+  const ecosystemGraph = nodes.length > 1
+    ? buildEcosystemGraph(nodes)
+    : { clusters: [], clustersBySize: new Map() }
+  state.ecosystemClusters = ecosystemGraph.clusters
+  state.ecosystemClustersBySize = ecosystemGraph.clustersBySize
   const primaryHub = rankedHubNodes()[0] ?? null
   state.primaryHub = primaryHub
   state.hubNeighborDistance = nearestHubNeighborDistance(primaryHub, nodes)
@@ -679,68 +690,219 @@ const createVisibleEdgeLookup = edges => {
   return lookup
 }
 
-const buildOverviewClusters = nodes => {
-  if (nodes.length === 0) {
-    return []
+const ecosystemKeyForNode = node => {
+  if (typeof node.segment === 'string' && node.segment.trim()) {
+    return node.segment.trim()
   }
-
-  const bounds = graphBounds(nodes)
-  if (!bounds) {
-    return []
+  if (typeof node.group === 'string' && node.group.trim()) {
+    return node.group.trim()
   }
+  const pathParts = String(node.path || '')
+    .split('/')
+    .filter(part => part.trim())
+    .slice(0, 2)
+  return pathParts.length > 0 ? pathParts.join('/') : 'root'
+}
 
-  const longest = Math.max(bounds.width, bounds.height, 1)
-  const cellSize = Math.max(longest / 56, 900)
-  const buckets = new Map()
+const compareNodesForEcosystem = (left, right) => {
+  const keyComparison = ecosystemKeyForNode(left).localeCompare(ecosystemKeyForNode(right))
+  if (keyComparison !== 0) return keyComparison
+  const leftDegree = state.nodeDegrees.get(left.id) ?? 0
+  const rightDegree = state.nodeDegrees.get(right.id) ?? 0
+  if (leftDegree !== rightDegree) return rightDegree - leftDegree
+  return String(left.title || left.id).localeCompare(String(right.title || right.id))
+}
+
+const selectEcosystemRepresentative = nodes => {
+  let representative = nodes[0] ?? null
+  let representativeScore = Number.NEGATIVE_INFINITY
 
   for (let index = 0; index < nodes.length; index += 1) {
     const node = nodes[index]
-    const keyX = Math.floor((node.x - bounds.minX) / cellSize)
-    const keyY = Math.floor((node.y - bounds.minY) / cellSize)
-    const key = keyX + ':' + keyY
-    const degree = state.nodeDegrees.get(node.id) ?? 0
-    const current = buckets.get(key)
+    const score = (state.nodeDegrees.get(node.id) ?? 0) + hubNodeScore(node) * 1000
+    if (score > representativeScore) {
+      representative = node
+      representativeScore = score
+    }
+  }
+
+  return representative
+}
+
+const buildEcosystemCluster = (nodes, index) => {
+  const count = Math.max(nodes.length, 1)
+  const sum = nodes.reduce((accumulator, node) => ({
+    x: accumulator.x + node.x,
+    y: accumulator.y + node.y
+  }), { x: 0, y: 0 })
+  const representative = selectEcosystemRepresentative(nodes)
+
+  return {
+    id: 'ecosystem-' + index,
+    x: sum.x / count,
+    y: sum.y / count,
+    count,
+    nodeIds: nodes.map(node => node.id),
+    representative,
+    label: ecosystemKeyForNode(nodes[0] ?? representative ?? { path: '' })
+  }
+}
+
+const buildEcosystemLevel = (sortedNodes, size, parentLookup) => {
+  const clusters = []
+  const clusterByNodeId = new Map()
+
+  for (let offset = 0; offset < sortedNodes.length; offset += size) {
+    const clusterNodes = sortedNodes.slice(offset, offset + size)
+    const parentCluster = parentLookup?.get(clusterNodes[0]?.id)
+    const cluster = {
+      ...buildEcosystemCluster(clusterNodes, clusters.length),
+      id: 'ecosystem-' + size + '-' + clusters.length,
+      size,
+      parentId: parentCluster?.id ?? null
+    }
+    clusters.push(cluster)
+    for (let index = 0; index < clusterNodes.length; index += 1) {
+      clusterByNodeId.set(clusterNodes[index].id, cluster)
+    }
+  }
+
+  return { clusters, clusterByNodeId }
+}
+
+const buildEcosystemGraph = (nodes) => {
+  if (nodes.length === 0) {
+    return { clusters: [], clustersBySize: new Map() }
+  }
+
+  const sortedNodes = [...nodes].sort(compareNodesForEcosystem)
+  const clustersBySize = new Map()
+  let parentLookup = null
+
+  for (let index = 0; index < ecosystemGroupSizes.length; index += 1) {
+    const size = ecosystemGroupSizes[index]
+    const level = buildEcosystemLevel(sortedNodes, size, parentLookup)
+    clustersBySize.set(size, level.clusters)
+    parentLookup = level.clusterByNodeId
+  }
+
+  return {
+    clusters: clustersBySize.get(ecosystemGroupSize) ?? [],
+    clustersBySize
+  }
+}
+
+const isClusterInViewport = (cluster, viewport) =>
+  cluster.x >= viewport.minX &&
+  cluster.x <= viewport.maxX &&
+  cluster.y >= viewport.minY &&
+  cluster.y <= viewport.maxY
+
+const filterEcosystemClustersByViewport = (clusters, viewport) => {
+  const visible = clusters.filter(cluster => isClusterInViewport(cluster, viewport))
+  return visible.length > 0 ? visible : [...clusters]
+}
+
+const ecosystemFocusPoint = () => {
+  const now = performance.now()
+  if (now - state.lastZoomFocus.at <= 1800) {
+    return { x: state.lastZoomFocus.x, y: state.lastZoomFocus.y }
+  }
+  return viewportCenterWorldPoint()
+}
+
+const nearestEcosystemParentIds = (clusters, focusPoint, limit) =>
+  clusters
+    .map(cluster => ({
+      cluster,
+      distance: Math.hypot(cluster.x - focusPoint.x, cluster.y - focusPoint.y)
+    }))
+    .sort((left, right) => left.distance - right.distance)
+    .slice(0, limit)
+    .map(item => item.cluster.id)
+
+const ecosystemPlanForScale = scale => {
+  if (scale <= ecosystemMicroScaleThreshold) {
+    return { baseSize: 1000, childSize: null }
+  }
+  if (scale <= ecosystemSubgraphScaleThreshold) {
+    return { baseSize: 1000, childSize: 250 }
+  }
+  return { baseSize: 250, childSize: 60 }
+}
+
+const selectHierarchicalEcosystemClusters = viewport => {
+  const plan = ecosystemPlanForScale(state.transform.scale)
+  const baseClusters = state.ecosystemClustersBySize.get(plan.baseSize) ?? state.ecosystemClusters
+  const visibleBaseClusters = filterEcosystemClustersByViewport(baseClusters, viewport)
+
+  if (!plan.childSize) {
+    return visibleBaseClusters
+  }
+
+  const focusPoint = ecosystemFocusPoint()
+  const expandedParentIds = new Set(nearestEcosystemParentIds(
+    visibleBaseClusters,
+    focusPoint,
+    ecosystemFocusedParentLimit
+  ))
+  const childClusters = state.ecosystemClustersBySize.get(plan.childSize) ?? []
+  const visibleChildClusters = childClusters.filter(cluster =>
+    expandedParentIds.has(cluster.parentId) &&
+    isClusterInViewport(cluster, viewport)
+  )
+
+  if (visibleChildClusters.length === 0) {
+    return visibleBaseClusters
+  }
+
+  return [
+    ...visibleBaseClusters.filter(cluster => !expandedParentIds.has(cluster.id)),
+    ...visibleChildClusters
+  ]
+}
+
+const ecosystemEdgesForClusters = clusters => {
+  const clusterByNodeId = new Map()
+  for (let clusterIndex = 0; clusterIndex < clusters.length; clusterIndex += 1) {
+    const cluster = clusters[clusterIndex]
+    for (let nodeIndex = 0; nodeIndex < cluster.nodeIds.length; nodeIndex += 1) {
+      clusterByNodeId.set(cluster.nodeIds[nodeIndex], cluster)
+    }
+  }
+
+  const edgeByClusterPair = new Map()
+  for (let index = 0; index < state.visibleEdges.length; index += 1) {
+    const edge = state.visibleEdges[index]
+    const sourceCluster = clusterByNodeId.get(edge.source)
+    const targetCluster = clusterByNodeId.get(edge.target)
+    if (!sourceCluster || !targetCluster || sourceCluster.id === targetCluster.id) {
+      continue
+    }
+
+    const orderedIds = sourceCluster.id < targetCluster.id
+      ? [sourceCluster.id, targetCluster.id]
+      : [targetCluster.id, sourceCluster.id]
+    const key = orderedIds.join(':')
+    const current = edgeByClusterPair.get(key)
     if (current) {
-      current.count += 1
-      current.sumX += node.x
-      current.sumY += node.y
-      if (degree > current.degree) {
-        current.representative = node
-        current.degree = degree
-      }
+      current.weight += edgeWeight(edge)
       continue
     }
 
-    buckets.set(key, {
+    edgeByClusterPair.set(key, {
       id: key,
-      count: 1,
-      sumX: node.x,
-      sumY: node.y,
-      representative: node,
-      degree
+      sourceCluster,
+      targetCluster,
+      weight: edgeWeight(edge)
     })
   }
 
-  return Array.from(buckets.values())
-    .sort((left, right) => right.count - left.count)
-    .slice(0, overviewClusterMaxCount)
-    .map((cluster) => ({
-      id: cluster.id,
-      x: cluster.sumX / Math.max(cluster.count, 1),
-      y: cluster.sumY / Math.max(cluster.count, 1),
-      count: cluster.count,
-      representative: cluster.representative
-    }))
+  return Array.from(edgeByClusterPair.values())
+    .sort((left, right) => right.weight - left.weight)
+    .slice(0, ecosystemClusterEdgeLimit)
 }
 
-const filterOverviewClustersByViewport = viewport =>
-  state.overviewClusters.filter((cluster) =>
-    cluster.x >= viewport.minX &&
-    cluster.x <= viewport.maxX &&
-    cluster.y >= viewport.minY &&
-    cluster.y <= viewport.maxY
-  )
-
 const edgeBudgetForCurrentFrame = () => {
   const zoom = state.transform.scale
   if (zoom < 0.12) return 380
@@ -775,14 +937,6 @@ const nodeBudgetForScale = (scale) => {
   return renderNodeBudget
 }
 
-const massiveLowZoomNodeBudgetForScale = (scale) => {
-  if (scale < 0.004) return 780
-  if (scale < 0.01) return 860
-  if (scale < 0.02) return 900
-  if (scale < 0.035) return 900
-  return renderNodeBudget
-}
-
 const layerFocusForScale = (scale) => {
   const normalized = Math.max(0, Math.min(1, (scale - 0.06) / 0.94))
   const shellCenter = Math.max(0.08, 0.96 - normalized * 0.86)
@@ -2316,6 +2470,7 @@ const computeRenderVisibility = () => {
   }
   state.lastViewportKey = viewportKey
   state.renderVisibilityDirty = false
+  state.renderClusterEdges = []
 
   const shouldRenderMacroGalaxy = shouldRenderMacroGalaxyView()
 
@@ -2339,12 +2494,24 @@ const computeRenderVisibility = () => {
       state.renderNodes = []
     }
     state.renderEdges = []
+    state.renderClusterEdges = []
+    return
+  }
+
+  if (state.transform.scale <= ecosystemClusterScaleThreshold && state.ecosystemClusters.length > 0) {
+    const clusters = selectHierarchicalEcosystemClusters(viewport)
+      .sort((left, right) => right.count - left.count)
+    state.renderClusters = clusters
+    state.renderClusterEdges = ecosystemEdgesForClusters(clusters)
+    state.renderNodes = []
+    state.renderEdges = []
     return
   }
 
   if (state.visibleNodes.length <= 2000) {
     state.renderNodes = state.visibleNodes
     state.renderClusters = []
+    state.renderClusterEdges = []
     const ids = new Set(state.renderNodes.map((node) => node.id))
     state.renderEdges = withMeshEdges(state.renderNodes, collectVisibleEdgesForNodes(ids))
     return
@@ -2352,30 +2519,6 @@ const computeRenderVisibility = () => {
 
   if (state.visibleNodes.length > massiveGraphNodeThreshold) {
     const viewportNodes = viewportNodesFromSpatialIndex(viewport)
-    if (state.transform.scale <= massiveOverviewClusterScaleThreshold) {
-      const overviewLimit = Math.min(renderNodeBudget, massiveLowZoomNodeBudgetForScale(state.transform.scale))
-      const overviewClusters = filterOverviewClustersByViewport(viewport)
-        .sort((left, right) => right.count - left.count)
-        .slice(0, overviewLimit)
-      if (overviewClusters.length > 0) {
-        const overviewNodes = representativeNodesFromClusters(
-          overviewClusters,
-          overviewLimit
-        )
-        const anchoredNodes = includeHubPreviewNeighborhood(
-          overviewNodes,
-          Math.min(renderNodeBudget, overviewLimit)
-        )
-        const enriched = enrichSampleWithNeighbors(anchoredNodes)
-        const previewNodes = ensureHubNodesInRenderedSet(enriched.nodes)
-        const previewIds = new Set(previewNodes.map((node) => node.id))
-        const previewEdges = collectVisibleEdgesForNodes(previewIds)
-        state.renderClusters = []
-        state.renderNodes = previewNodes
-        state.renderEdges = previewEdges
-        return
-      }
-    }
     const sourceNodes = viewportNodes.length > 0 ? viewportNodes : state.visibleNodes
     const sampleLimit = nodeBudgetForScale(state.transform.scale)
     const carryMargin = Math.max(240, Math.min(1200, 340 / Math.max(state.transform.scale, 0.0001)))
@@ -2421,6 +2564,7 @@ const computeRenderVisibility = () => {
     }
 
     state.renderClusters = []
+    state.renderClusterEdges = []
     state.renderNodes = sampledNodes
     state.renderEdges = withMeshEdges(sampledNodes, sampledEdges)
     return
@@ -2430,6 +2574,7 @@ const computeRenderVisibility = () => {
     const sampled = sampleVisibleNodes(Math.min(renderNodeBudget, 900))
     const sampledIds = new Set(sampled.map((node) => node.id))
     state.renderClusters = []
+    state.renderClusterEdges = []
     state.renderNodes = sampled
     state.renderEdges = withMeshEdges(sampled, collectVisibleEdgesForNodes(sampledIds))
     return
@@ -2439,11 +2584,13 @@ const computeRenderVisibility = () => {
   const clusters = clusterViewportNodes(viewportNodes)
   if (clusters.length > 0) {
     state.renderClusters = []
+    state.renderClusterEdges = []
     state.renderNodes = representativeNodesFromClusters(clusters, Math.min(renderNodeBudget, 900))
     state.renderEdges = []
     return
   }
   state.renderClusters = []
+  state.renderClusterEdges = []
   const stride = viewportNodeStride()
   const picked = []
 
@@ -2467,6 +2614,7 @@ const computeRenderVisibility = () => {
     const fallbackIds = new Set(fallbackNodes.map((node) => node.id))
     state.renderNodes = fallbackNodes
     state.renderClusters = []
+    state.renderClusterEdges = []
     state.renderEdges = withMeshEdges(fallbackNodes, collectVisibleEdgesForNodes(fallbackIds))
     return
   }
@@ -2482,6 +2630,7 @@ const computeRenderVisibility = () => {
     const fallbackNodes = sampleVisibleNodes(Math.min(renderNodeBudget, 260))
     const fallbackIds = new Set(fallbackNodes.map((node) => node.id))
     state.renderClusters = []
+    state.renderClusterEdges = []
     state.renderNodes = fallbackNodes
     state.renderEdges = withMeshEdges(fallbackNodes, collectVisibleEdgesForNodes(fallbackIds))
   }
@@ -2598,6 +2747,17 @@ const render = now => {
     ctx.translate(state.transform.x, state.transform.y)
     ctx.scale(state.transform.scale, state.transform.scale)
     const safeScale = Math.max(state.transform.scale, 0.0001)
+    if (state.renderClusterEdges.length > 0) {
+      ctx.beginPath()
+      for (let index = 0; index < state.renderClusterEdges.length; index += 1) {
+        const edge = state.renderClusterEdges[index]
+        ctx.moveTo(edge.sourceCluster.x, edge.sourceCluster.y)
+        ctx.lineTo(edge.targetCluster.x, edge.targetCluster.y)
+      }
+      ctx.lineWidth = 1.2 / safeScale
+      ctx.strokeStyle = 'rgba(153, 165, 181, 0.22)'
+      ctx.stroke()
+    }
     state.renderClusters.forEach(cluster => {
       const isMacro = cluster.id === 'macro-galaxy'
       const radiusPx = isMacro

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andespindola/brainlink",
-  "version": "0.1.0-beta.84",
+  "version": "0.1.0-beta.86",
   "description": "Local-first knowledge memory for agents with Markdown, backlinks, indexing and context retrieval.",
   "type": "module",
   "license": "MIT",