@grafema/util 0.3.17 → 0.3.20

package/src/federation/FederatedRouter.ts

@@ -0,0 +1,440 @@
+/**
+ * FederatedRouter — orchestrates cross-shard graph queries.
+ *
+ * Two query patterns:
+ * 1. Traversal with frontier: trace_dataflow, trace_alias
+ *    - Start in local shard, follow edges until boundary
+ *    - Frontier edges grouped by target shard, batch-resolved via SUBGRAPH
+ *    - Repeat until no more frontier or depth exhausted
+ *
+ * 2. Scatter-gather: find_calls, find_nodes
+ *    - Broadcast query to all shards, merge results
+ *
+ * Key properties:
+ * - Global visited set across all hops (cycle prevention / INV-3)
+ * - Cost budget: max connections, max total nodes (fan-out protection / INV-6)
+ * - Frontier grouping: N edges to same shard → 1 SUBGRAPH call (not N)
+ */
+
+import type { WireNode, WireEdge } from '@grafema/types';
+import type { ShardDiscovery, ShardRegistration } from './ShardDiscovery.js';
+import type { ManifestResolver } from '../manifest/index.js';
+import type { ResolveResult } from '../manifest/resolver.js';
+
+// ── Types ──────────────────────────────────────────────────────
+
+export interface FrontierEdge {
+  /** Source node semantic ID (in the source shard) */
+  src: string;
+  /** Target node ID (hash — target doesn't exist in source shard) */
+  dst: string;
+  /** Edge type */
+  edgeType: string;
+  /** Edge metadata (JSON string, may contain "source" for IMPORTS_FROM) */
+  metadata?: string;
+  /** Absolute file path of the target (extracted from semantic ID) */
+  targetFile?: string;
+  /** How this edge was resolved: "graph" | "manifest" | "unresolved" */
+  resolvedVia?: 'graph' | 'manifest' | 'unresolved';
+}
+
+export interface SubgraphResponse {
+  ok: boolean;
+  nodes: WireNode[];
+  edges: WireEdge[];
+  frontier: FrontierEdge[];
+}
+
+export interface FederatedTraceHop {
+  /** Which shard this hop came from */
+  shard: ShardRegistration | null;
+  /** Nodes discovered in this hop */
+  nodes: WireNode[];
+  /** Edges discovered in this hop */
+  edges: WireEdge[];
+  /** Unresolved frontier (shard not found or unavailable) */
+  unresolvedFrontier: FrontierEdge[];
+}
+
+/** Node resolved from manifest (no full graph, just export surface) */
+export interface ManifestResolvedNode {
+  /** Package name (e.g., "@grafema/util") */
+  packageName: string;
+  /** Symbol name */
+  symbolName: string;
+  /** Export kind: FUNCTION, CLASS, etc. */
+  kind: string;
+  /** Known effects */
+  effects: string[];
+  /** Semantic ID from manifest (if available) */
+  semanticId?: string;
+}
+
+export interface FederatedTraceResult {
+  /** All nodes from all hops, merged */
+  nodes: WireNode[];
+  /** All edges from all hops, merged */
+  edges: WireEdge[];
+  /** Individual hops for debugging/visualization */
+  hops: FederatedTraceHop[];
+  /** Nodes resolved via manifest (partial info, no subgraph) */
+  manifestNodes: ManifestResolvedNode[];
+  /** Unresolved frontier across all hops */
+  unresolvedFrontier: FrontierEdge[];
+  /** Whether the traversal was cut short by cost budget */
+  truncated: boolean;
+}
+
+export interface FederatedRouterOptions {
+  /** Max total nodes across all hops (default: 10000) */
+  maxNodes?: number;
+  /** Max shard connections per round (default: 10) */
+  maxShardsPerRound?: number;
+  /** Max traversal rounds (default: 5) */
+  maxRounds?: number;
+}
+
+/** Function that sends a SUBGRAPH command to an rfdb-server via its socket */
+type SubgraphSender = (
+  socket: string,
+  entries: string[],
+  direction: string,
+  edgeTypes: string[],
+  maxDepth: number,
+) => Promise<SubgraphResponse>;
+
+// ── Router ─────────────────────────────────────────────────────
+
+export class FederatedRouter {
+  private discovery: ShardDiscovery;
+  private sendSubgraph: SubgraphSender;
+  private manifestResolver: ManifestResolver | null;
+  private options: Required<FederatedRouterOptions>;
+
+  constructor(
+    discovery: ShardDiscovery,
+    sendSubgraph: SubgraphSender,
+    options: FederatedRouterOptions = {},
+    manifestResolver?: ManifestResolver,
+  ) {
+    this.discovery = discovery;
+    this.sendSubgraph = sendSubgraph;
+    this.manifestResolver = manifestResolver ?? null;
+    this.options = {
+      maxNodes: options.maxNodes ?? 10_000,
+      maxShardsPerRound: options.maxShardsPerRound ?? 10,
+      maxRounds: options.maxRounds ?? 5,
+    };
+  }
+
+  /**
+   * Federated traversal: start from entry points, hop across shards via frontier.
+   *
+   * 1. Send SUBGRAPH to the shard owning the first entry point
+   * 2. Collect frontier, group by target shard
+   * 3. Send SUBGRAPH to each target shard (parallel)
+   * 4. Repeat until no more frontier or budget exhausted
+   */
+  async trace(
+    entries: string[],
+    direction: 'forward' | 'backward' | 'both',
+    edgeTypes: string[],
+    maxDepth: number,
+  ): Promise<FederatedTraceResult> {
+    const allNodes: WireNode[] = [];
+    const allEdges: WireEdge[] = [];
+    const allHops: FederatedTraceHop[] = [];
+    const manifestNodes: ManifestResolvedNode[] = [];
+    const globalVisited = new Set<string>(); // INV-3: global across all hops
+    let unresolvedFrontier: FrontierEdge[] = [];
+    let truncated = false;
+
+    // Initial frontier: the entry points themselves, grouped by shard
+    let currentFrontier: Array<{ shard: ShardRegistration; entries: string[] }> = [];
+
+    // Group entry points by shard
+    const entryGroups = this.groupByShard(
+      entries.map(e => ({ src: '', dst: e, edgeType: '' })),
+    );
+    for (const [, group] of entryGroups) {
+      currentFrontier.push({
+        shard: group.shard,
+        entries: group.edges.map(e => e.dst),
+      });
+    }
+
+    let remainingDepth = maxDepth;
+
+    for (let round = 0; round < this.options.maxRounds; round++) {
+      if (currentFrontier.length === 0 || remainingDepth <= 0) break;
+
+      // Cost check: limit shards per round
+      const roundShards = currentFrontier.slice(0, this.options.maxShardsPerRound);
+      if (roundShards.length < currentFrontier.length) {
+        truncated = true;
+      }
+
+      // Query all shards in parallel
+      const hopResults = await Promise.all(
+        roundShards.map(async ({ shard, entries: entryIds }) => {
+          // Filter out already-visited entries
+          const newEntries = entryIds.filter(e => !globalVisited.has(e));
+          if (newEntries.length === 0) {
+            return null;
+          }
+
+          try {
+            const result = await this.sendSubgraph(
+              shard.socket,
+              newEntries,
+              direction,
+              edgeTypes,
+              remainingDepth,
+            );
+
+            // Mark visited
+            for (const node of result.nodes) {
+              const nodeId = node.semanticId || node.id;
+              globalVisited.add(nodeId);
+            }
+
+            return { shard, result };
+          } catch {
+            // Shard unavailable — all its entries become unresolved
+            return {
+              shard,
+              result: {
+                ok: false,
+                nodes: [],
+                edges: [],
+                frontier: newEntries.map(e => ({
+                  src: '',
+                  dst: e,
+                  edgeType: 'UNRESOLVED',
+                })),
+              } as SubgraphResponse,
+            };
+          }
+        }),
+      );
+
+      // Collect results
+      const nextFrontierEdges: FrontierEdge[] = [];
+
+      for (const hopResult of hopResults) {
+        if (!hopResult) continue;
+
+        const { shard, result } = hopResult;
+        const hop: FederatedTraceHop = {
+          shard,
+          nodes: result.nodes,
+          edges: result.edges,
+          unresolvedFrontier: [],
+        };
+
+        allNodes.push(...result.nodes);
+        allEdges.push(...result.edges);
+
+        // Check cost budget
+        if (allNodes.length >= this.options.maxNodes) {
+          truncated = true;
+          allHops.push(hop);
+          break;
+        }
+
+        // Process frontier
+        for (const edge of result.frontier) {
+          if (globalVisited.has(edge.dst)) continue; // Already visited
+          nextFrontierEdges.push(edge);
+        }
+
+        allHops.push(hop);
+      }
+
+      if (truncated) break;
+
+      // Group next frontier by target shard
+      const nextGroups = this.groupByShard(nextFrontierEdges);
+      currentFrontier = [];
+      const newUnresolved: FrontierEdge[] = [];
+
+      for (const [, group] of nextGroups) {
+        if (group.shard) {
+          currentFrontier.push({
+            shard: group.shard,
+            entries: group.edges.map(e => e.dst),
+          });
+        } else {
+          // No shard found — try manifest fallback
+          for (const edge of group.edges) {
+            const resolved = this.tryManifestFallback(edge);
+            if (resolved) {
+              manifestNodes.push(resolved);
+              edge.resolvedVia = 'manifest';
+            } else {
+              edge.resolvedVia = 'unresolved';
+              newUnresolved.push(edge);
+            }
+          }
+        }
+      }
+
+      unresolvedFrontier = newUnresolved;
+      remainingDepth = Math.max(0, remainingDepth - 1);
+    }
+
+    return {
+      nodes: allNodes,
+      edges: allEdges,
+      hops: allHops,
+      manifestNodes,
+      unresolvedFrontier,
+      truncated,
+    };
+  }
+
+  /**
+   * Scatter-gather: broadcast a query to all known shards, merge results.
+   * Used for find_nodes, find_calls when no file filter is specified.
+   */
+  async scatterGather<T>(
+    queryFn: (socket: string) => Promise<T[]>,
+  ): Promise<{ results: T[]; queriedShards: number }> {
+    const shards = this.discovery.all();
+    const allResults: T[] = [];
+
+    const responses = await Promise.all(
+      shards.map(async (shard) => {
+        try {
+          return await queryFn(shard.socket);
+        } catch {
+          return [];
+        }
+      }),
+    );
+
+    for (const results of responses) {
+      allResults.push(...results);
+    }
+
+    return { results: allResults, queriedShards: shards.length };
+  }
+
+  // ── Private ────────────────────────────────────────────────────
+
+  /**
+   * Try to resolve a frontier edge via ManifestResolver.
+   * Works for IMPORTS_FROM edges that carry "source" in metadata.
+   * Returns ManifestResolvedNode if found, null otherwise.
+   */
+  private tryManifestFallback(edge: FrontierEdge): ManifestResolvedNode | null {
+    if (!this.manifestResolver) return null;
+    if (edge.edgeType !== 'IMPORTS_FROM') return null;
+
+    // Extract package source and symbol from edge metadata
+    const meta = parseEdgeMetadata(edge.metadata);
+    if (!meta.source) return null;
+
+    // Try to find the symbol in the manifest
+    // Symbol name might be in metadata or extractable from src semantic ID
+    const symbolName = meta.specifier || meta.localName || extractSymbolFromId(edge.src);
+    if (!symbolName) {
+      // No specific symbol — check if package exists in manifests at all
+      if (this.manifestResolver.has(meta.source)) {
+        return {
+          packageName: meta.source,
+          symbolName: '*',
+          kind: 'VARIABLE',
+          effects: ['UNKNOWN'],
+        };
+      }
+      return null;
+    }
+
+    const result: ResolveResult | null = this.manifestResolver.resolve(meta.source, symbolName);
+    if (!result) return null;
+
+    return {
+      packageName: meta.source,
+      symbolName,
+      kind: result.export.kind,
+      effects: result.export.effects,
+      semanticId: result.export.semanticId,
+    };
+  }
+
+  /**
+   * Group frontier edges by target shard using ShardDiscovery.
+   * Edges whose target shard can't be found go into a null group.
+   */
+  private groupByShard(
+    edges: FrontierEdge[],
+  ): Map<string, { shard: ShardRegistration; edges: FrontierEdge[] }> {
+    const groups = new Map<string, { shard: ShardRegistration; edges: FrontierEdge[] }>();
+
+    for (const edge of edges) {
+      // Try to extract file path from the target ID (dst)
+      // Semantic IDs contain file path as first segment: "path/to/file.ts->FUNCTION->name"
+      const filePath = extractFilePath(edge.dst);
+
+      const shard = filePath ? this.discovery.resolve(filePath) : null;
+      const key = shard ? shard.root : '__unresolved__';
+
+      if (!groups.has(key)) {
+        groups.set(key, { shard: shard!, edges: [] });
+      }
+      groups.get(key)!.edges.push(edge);
+    }
+
+    return groups;
+  }
+}
+
+/** Parse JSON edge metadata, returning an object with known fields. */
+function parseEdgeMetadata(metadata?: string): Record<string, string> {
+  if (!metadata) return {};
+  try {
+    return JSON.parse(metadata);
+  } catch {
+    return {};
+  }
+}
+
+/** Extract symbol name from the last segment of a semantic ID. */
+function extractSymbolFromId(semanticId: string): string | null {
+  if (!semanticId) return null;
+  // "file.ts->FUNCTION->myFunc" → "myFunc"
+  const parts = semanticId.split('->');
+  return parts.length >= 3 ? parts[parts.length - 1] : null;
+}
+
+/**
+ * Extract absolute file path from a semantic ID or grafema URI.
+ * Handles both formats:
+ * - Arrow format: "path/to/file.ts->FUNCTION->name"
+ * - URI format: "grafema://path/to/file.ts#FUNCTION%3Aname"
+ */
+function extractFilePath(semanticId: string): string | null {
+  if (!semanticId) return null;
+
+  // URI format: grafema://path/to/file.ts#fragment
+  if (semanticId.startsWith('grafema://')) {
+    const hashIdx = semanticId.indexOf('#');
+    return hashIdx > 0
+      ? semanticId.slice('grafema://'.length, hashIdx)
+      : semanticId.slice('grafema://'.length);
+  }
+
+  // Arrow format: path/to/file.ts->FUNCTION->name
+  const arrowIdx = semanticId.indexOf('->');
+  if (arrowIdx > 0) {
+    return semanticId.slice(0, arrowIdx);
+  }
+
+  // Might be just a file path or hash — return as-is if it looks like a path
+  if (semanticId.includes('/') && !semanticId.includes(' ')) {
+    return semanticId;
+  }
+
+  return null;
+}

package/src/federation/ShardDiscovery.ts

@@ -0,0 +1,130 @@
+/**
+ * ShardDiscovery — discovers registered RFDB shards via /tmp/rfdb-shards/.
+ *
+ * Each rfdb-server running with --federate writes a JSON registration file
+ * containing its root path, socket, port, and PID. ShardDiscovery scans
+ * these files and resolves file paths to the appropriate shard.
+ *
+ * Discovery is by file path prefix: if a target file's absolute path
+ * starts with a shard's root, that shard owns the file.
+ */
+
+import { readFileSync, readdirSync, existsSync } from 'fs';
+import { join } from 'path';
+
+export interface ShardRegistration {
+  /** Absolute path of the project root this shard covers */
+  root: string;
+  /** Unix socket path for this shard */
+  socket: string;
+  /** WebSocket port (if available) */
+  wsPort?: number;
+  /** Process ID of the rfdb-server */
+  pid: number;
+  /** Epoch timestamp when the server started */
+  started: number;
+  /** Server version */
+  serverVersion: string;
+}
+
+const SHARDS_DIR = '/tmp/rfdb-shards';
+
+export class ShardDiscovery {
+  /** root path → registration (sorted by longest root first for prefix matching) */
+  private shards = new Map<string, ShardRegistration>();
+
+  /** Number of discovered shards */
+  get size(): number {
+    return this.shards.size;
+  }
+
+  /** All registered shard roots */
+  get roots(): string[] {
+    return [...this.shards.keys()];
+  }
+
+  /**
+   * Scan /tmp/rfdb-shards/ for registered shards.
+   * Validates each registration: checks PID is alive, parses JSON.
+   * Call this periodically or before federated queries.
+   */
+  scan(): number {
+    this.shards.clear();
+
+    if (!existsSync(SHARDS_DIR)) {
+      return 0;
+    }
+
+    const files = readdirSync(SHARDS_DIR).filter(f => f.endsWith('.json'));
+
+    for (const file of files) {
+      try {
+        const raw = readFileSync(join(SHARDS_DIR, file), 'utf-8');
+        const reg = JSON.parse(raw) as ShardRegistration;
+
+        if (!reg.root || !reg.socket || !reg.pid) continue;
+
+        // Check if process is still alive
+        if (!isProcessAlive(reg.pid)) continue;
+
+        // Normalize root path (ensure no trailing slash)
+        const root = reg.root.replace(/\/$/, '');
+        this.shards.set(root, { ...reg, root });
+      } catch {
+        // Skip malformed registration files
+      }
+    }
+
+    return this.shards.size;
+  }
+
+  /**
+   * Find the shard that owns a given absolute file path.
+   * Uses longest-prefix matching: /repo/packages/api/ wins over /repo/.
+   */
+  resolve(absoluteFilePath: string): ShardRegistration | null {
+    let bestMatch: ShardRegistration | null = null;
+    let bestLength = 0;
+
+    for (const [root, reg] of this.shards) {
+      if (absoluteFilePath.startsWith(root) && root.length > bestLength) {
+        bestMatch = reg;
+        bestLength = root.length;
+      }
+    }
+
+    return bestMatch;
+  }
+
+  /**
+   * Get all known shards (for scatter-gather queries).
+   */
+  all(): ShardRegistration[] {
+    return [...this.shards.values()];
+  }
+
+  /**
+   * Register a shard programmatically (for testing or remote shards).
+   */
+  register(reg: ShardRegistration): void {
+    const root = reg.root.replace(/\/$/, '');
+    this.shards.set(root, { ...reg, root });
+  }
+
+  /**
+   * Remove a shard by root path.
+   */
+  unregister(root: string): void {
+    this.shards.delete(root.replace(/\/$/, ''));
+  }
+}
+
+/** Check if a process with given PID is alive */
+function isProcessAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/federation/index.ts

@@ -0,0 +1,35 @@
+/**
+ * Federation module — cross-shard query routing for RFDB.
+ *
+ * Enables multiple RFDB shards to discover each other and resolve
+ * cross-shard references at query time. Within a shard, the graph
+ * is fully materialized (analysis-time). Between shards, references
+ * are resolved lazily (execution-time) via the SUBGRAPH primitive.
+ *
+ * ## Architecture
+ *
+ * - ShardDiscovery: scans /tmp/rfdb-shards/ for registered shards
+ * - FederatedRouter: orchestrates cross-shard traversal and scatter-gather
+ * - Each shard is a separate rfdb-server process with its own graph
+ * - Discovery is by file path prefix matching (zero-config)
+ *
+ * ## Usage
+ *
+ *   const discovery = new ShardDiscovery();
+ *   await discovery.scan();
+ *
+ *   const router = new FederatedRouter(discovery, localClient);
+ *   const result = await router.traceDataflow('src/app.ts->userInput', 'forward', 10);
+ */
+
+export { ShardDiscovery } from './ShardDiscovery.js';
+export type { ShardRegistration } from './ShardDiscovery.js';
+
+export { FederatedRouter } from './FederatedRouter.js';
+export type {
+  FederatedTraceResult,
+  FederatedTraceHop,
+  FrontierEdge,
+  SubgraphResponse,
+  ManifestResolvedNode,
+} from './FederatedRouter.js';

package/src/index.ts

@@ -257,8 +257,19 @@ export type {
   TraceNarrativeOptions,
 } from './notation/index.js';
 
+// Federation — cross-shard query routing
+export { ShardDiscovery, FederatedRouter } from './federation/index.js';
+export type {
+  ShardRegistration,
+  FederatedTraceResult,
+  FederatedTraceHop,
+  FrontierEdge,
+  SubgraphResponse,
+  ManifestResolvedNode,
+} from './federation/index.js';
+
 // Manifest generation & resolution (federation)
-export { ManifestGenerator, ManifestResolver } from './manifest/index.js';
+export { ManifestGenerator, ManifestResolver, RegistryBuilder, resolvePackageDir, detectSourceType, resolveEntryPoint } from './manifest/index.js';
 export type {
   Manifest,
   ManifestExport,
@@ -269,6 +280,10 @@ export type {
   ExportKind,
   ResolveResult,
   ManifestSummary,
+  RegistryEntry,
+  RegistryIndex,
+  RegistryBuilderOptions,
+  BuildResult,
 } from './manifest/index.js';
 
 // Re-export types for convenience

package/src/manifest/generator.ts

@@ -174,9 +174,13 @@ export class ManifestGenerator {
       // Graph-based approach:
       // EXPORT(named) --EXPORTS--> EXPORT_BINDING(name, source) --> definition in source file
       await this.collectExportsViaBindings(entryFile, exports, seen, new Set());
-    } else {
-      // Fallback: all exported definitions from the package
+    }
+
+    // Fallback: if entry-file mode found 0 exports (e.g., CJS barrel,
+    // broken re-export chain in compiled_js), scan all exported definitions
+    if (exports.length === 0) {
       const prefix = this.options.packagePrefix ?? '';
+      // Check standard definition types (FUNCTION, CLASS, CONSTANT, INTERFACE)
       for (const type of ManifestGenerator.DEF_TYPES) {
         for await (const node of this.backend.queryNodes({ type: type as never })) {
           if (prefix && !node.file?.startsWith(prefix)) continue;
@@ -187,6 +191,21 @@ export class ManifestGenerator {
           await this.addExportFromDefinition(node, exports);
         }
       }
+      // Also check EXPORT_BINDING nodes (from CJS exports.foo = ...)
+      if (exports.length === 0) {
+        for await (const node of this.backend.queryNodes({ type: 'EXPORT_BINDING' as never })) {
+          if (prefix && !node.file?.startsWith(prefix)) continue;
+          if (!node.name || node.name === 'named' || node.name === 'default') continue;
+          if (seen.has(node.name)) continue;
+          seen.add(node.name);
+          exports.push({
+            name: node.name,
+            kind: 'VARIABLE',
+            semanticId: node.id,
+            effects: ['UNKNOWN'],
+          });
+        }
+      }
     }
 
     exports.sort((a, b) => a.name.localeCompare(b.name));
@@ -276,7 +295,8 @@ export class ManifestGenerator {
     const dir = fromFile.substring(0, fromFile.lastIndexOf('/'));
     let resolved = source.startsWith('.') ? `${dir}/${source}` : source;
     resolved = resolved.replace(/\/\.\//g, '/');
-    if (resolved.endsWith('.js')) {
+    // Only replace .js → .ts for source TypeScript code, not compiled_js
+    if (this.options.sourceType !== 'compiled_js' && resolved.endsWith('.js')) {
       resolved = resolved.replace(/\.js$/, '.ts');
     }
     return resolved;
@@ -330,7 +350,8 @@ export class ManifestGenerator {
   /** Enrich all exports with computed effects (transitive call graph analysis) */
   private async enrichEffects(exports: ManifestExport[]): Promise<void> {
     for (const entry of exports) {
-      if (entry.kind !== 'FUNCTION' && entry.kind !== 'CLASS') continue;
+      // Enrich FUNCTION, CLASS, and VARIABLE (CJS exports may be VARIABLE kind)
+      if (entry.kind !== 'FUNCTION' && entry.kind !== 'CLASS' && entry.kind !== 'VARIABLE') continue;
 
       const effects = new Set<EffectType>();
       const visited = new Set<string>();

package/src/manifest/index.ts

@@ -1,6 +1,8 @@
 export { ManifestGenerator } from './generator.js';
 export { ManifestResolver } from './resolver.js';
 export type { ResolveResult, ManifestSummary } from './resolver.js';
+export { RegistryBuilder, resolvePackageDir, detectSourceType, resolveEntryPoint } from './registry.js';
+export type { RegistryEntry, RegistryIndex, RegistryBuilderOptions, BuildResult } from './registry.js';
 export type {
   Manifest,
   ManifestExport,