@grafema/rfdb-client 0.2.5-beta → 0.2.7

package/ts/client.ts

@@ -8,6 +8,7 @@
 import { createConnection, Socket } from 'net';
 import { encode, decode } from '@msgpack/msgpack';
 import { EventEmitter } from 'events';
+import { StreamQueue } from './stream-queue.js';
 
 import type {
   RFDBCommand,
@@ -25,6 +26,15 @@ import type {
   OpenDatabaseResponse,
   ListDatabasesResponse,
   CurrentDatabaseResponse,
+  SnapshotRef,
+  SnapshotDiff,
+  SnapshotInfo,
+  DiffSnapshotsResponse,
+  FindSnapshotResponse,
+  ListSnapshotsResponse,
+  CommitDelta,
+  CommitBatchResponse,
+  NodesChunkResponse,
 } from '@grafema/types';
 
 interface PendingRequest {
@@ -40,6 +50,17 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
   private reqId: number;
   private buffer: Buffer;
 
+  // Batch state
+  private _batching: boolean = false;
+  private _batchNodes: WireNode[] = [];
+  private _batchEdges: WireEdge[] = [];
+  private _batchFiles: Set<string> = new Set();
+
+  // Streaming state
+  private _supportsStreaming: boolean = false;
+  private _pendingStreams: Map<number, StreamQueue<WireNode>> = new Map();
+  private _streamTimers: Map<number, ReturnType<typeof setTimeout>> = new Map();
+
   constructor(socketPath: string = '/tmp/rfdb.sock') {
     super();
     this.socketPath = socketPath;
@@ -50,6 +71,14 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
     this.buffer = Buffer.alloc(0);
   }
 
+  /**
+   * Whether the connected server supports streaming responses.
+   * Set after calling hello(). Defaults to false.
+   */
+  get supportsStreaming(): boolean {
+    return this._supportsStreaming;
+  }
+
   /**
    * Connect to RFDB server
    */
@@ -82,6 +111,15 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
           reject(new Error('Connection closed'));
         }
         this.pending.clear();
+        // Fail all pending streams
+        for (const [, stream] of this._pendingStreams) {
+          stream.fail(new Error('Connection closed'));
+        }
+        this._pendingStreams.clear();
+        for (const [, timer] of this._streamTimers) {
+          clearTimeout(timer);
+        }
+        this._streamTimers.clear();
       });
 
       this.socket.on('data', (chunk: Buffer) => {
@@ -154,16 +192,48 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
   }
 
   /**
-   * Handle decoded response
+   * Handle decoded response — match by requestId, route streaming chunks
+   * to StreamQueue or resolve single-response Promise.
    */
   private _handleResponse(response: RFDBResponse): void {
-    if (this.pending.size === 0) {
+    if (this.pending.size === 0 && this._pendingStreams.size === 0) {
       this.emit('error', new Error('Received response with no pending request'));
       return;
     }
 
-    // Get the oldest pending request (FIFO)
-    const [id, { resolve, reject }] = this.pending.entries().next().value as [number, PendingRequest];
+    let id: number;
+
+    if (response.requestId) {
+      const parsed = this._parseRequestId(response.requestId);
+      if (parsed === null) {
+        this.emit('error', new Error(`Received response for unknown requestId: ${response.requestId}`));
+        return;
+      }
+      id = parsed;
+    } else {
+      // FIFO fallback for servers that don't echo requestId
+      if (this.pending.size > 0) {
+        id = (this.pending.entries().next().value as [number, PendingRequest])[0];
+      } else {
+        this.emit('error', new Error('Received response with no pending request'));
+        return;
+      }
+    }
+
+    // Route to streaming handler if this requestId has a StreamQueue
+    const streamQueue = this._pendingStreams.get(id);
+    if (streamQueue) {
+      this._handleStreamingResponse(id, response, streamQueue);
+      return;
+    }
+
+    // Non-streaming response — existing behavior
+    if (!this.pending.has(id)) {
+      this.emit('error', new Error(`Received response for unknown requestId: ${response.requestId}`));
+      return;
+    }
+
+    const { resolve, reject } = this.pending.get(id)!;
     this.pending.delete(id);
 
     if (response.error) {
@@ -173,6 +243,88 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
     }
   }
 
+  /**
+   * Handle a response for a streaming request.
+   * Routes chunk data to StreamQueue and manages stream lifecycle.
+   * Resets per-chunk timeout on each successful chunk arrival.
+   */
+  private _handleStreamingResponse(
+    id: number,
+    response: RFDBResponse,
+    streamQueue: StreamQueue<WireNode>,
+  ): void {
+    // Error response — fail the stream
+    if (response.error) {
+      this._cleanupStream(id);
+      streamQueue.fail(new Error(response.error));
+      return;
+    }
+
+    // Streaming chunk (has `done` field)
+    if ('done' in response) {
+      const chunk = response as unknown as NodesChunkResponse;
+      const nodes = chunk.nodes || [];
+      for (const node of nodes) {
+        streamQueue.push(node);
+      }
+
+      if (chunk.done) {
+        this._cleanupStream(id);
+        streamQueue.end();
+      } else {
+        // Reset per-chunk timeout
+        this._resetStreamTimer(id, streamQueue);
+      }
+      return;
+    }
+
+    // Auto-fallback: server sent a non-streaming Nodes response
+    // (server doesn't support streaming or result was below threshold)
+    const nodesResponse = response as unknown as { nodes?: WireNode[] };
+    const nodes = nodesResponse.nodes || [];
+    for (const node of nodes) {
+      streamQueue.push(node);
+    }
+    this._cleanupStream(id);
+    streamQueue.end();
+  }
+
+  /**
+   * Reset the per-chunk timeout for a streaming request.
+   */
+  private _resetStreamTimer(id: number, streamQueue: StreamQueue<WireNode>): void {
+    const existing = this._streamTimers.get(id);
+    if (existing) clearTimeout(existing);
+
+    const timer = setTimeout(() => {
+      this._cleanupStream(id);
+      streamQueue.fail(new Error(
+        `RFDB queryNodesStream timed out after ${RFDBClient.DEFAULT_TIMEOUT_MS}ms (no chunk received)`
+      ));
+    }, RFDBClient.DEFAULT_TIMEOUT_MS);
+
+    this._streamTimers.set(id, timer);
+  }
+
+  /**
+   * Clean up all state for a completed/failed streaming request.
+   */
+  private _cleanupStream(id: number): void {
+    this._pendingStreams.delete(id);
+    this.pending.delete(id);
+    const timer = this._streamTimers.get(id);
+    if (timer) {
+      clearTimeout(timer);
+      this._streamTimers.delete(id);
+    }
+  }
+
+  private _parseRequestId(requestId: string): number | null {
+    if (!requestId.startsWith('r')) return null;
+    const num = parseInt(requestId.slice(1), 10);
+    return Number.isNaN(num) ? null : num;
+  }
+
   /**
    * Default timeout for operations (60 seconds)
    * Flush/compact may take time for large graphs, but should not hang indefinitely
@@ -191,11 +343,10 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
       throw new Error('Not connected to RFDB server');
     }
 
-    const request = { cmd, ...payload };
-    const msgBytes = encode(request);
-
     return new Promise((resolve, reject) => {
       const id = this.reqId++;
+      const request = { requestId: `r${id}`, cmd, ...payload };
+      const msgBytes = encode(request);
 
       // Setup timeout
       const timer = setTimeout(() => {
@@ -246,13 +397,13 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
       const nodeRecord = n as Record<string, unknown>;
 
       // Extract known wire format fields, rest goes to metadata
-      const { id, type, node_type, nodeType, name, file, exported, metadata, ...rest } = nodeRecord;
+      const { id, type, node_type, nodeType, name, file, exported, metadata, semanticId, semantic_id, ...rest } = nodeRecord;
 
       // Merge explicit metadata with extra properties
       const existingMeta = typeof metadata === 'string' ? JSON.parse(metadata as string) : (metadata || {});
       const combinedMeta = { ...existingMeta, ...rest };
 
-      return {
+      const wire: WireNode = {
         id: String(id),
         nodeType: (node_type || nodeType || type || 'UNKNOWN') as NodeType,
         name: (name as string) || '',
@@ -260,8 +411,24 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
         exported: (exported as boolean) || false,
         metadata: JSON.stringify(combinedMeta),
       };
+
+      // Preserve semanticId as top-level field for v3 protocol
+      const sid = semanticId || semantic_id;
+      if (sid) {
+        (wire as WireNode & { semanticId: string }).semanticId = String(sid);
+      }
+
+      return wire;
     });
 
+    if (this._batching) {
+      this._batchNodes.push(...wireNodes);
+      for (const node of wireNodes) {
+        if (node.file) this._batchFiles.add(node.file);
+      }
+      return { ok: true } as RFDBResponse;
+    }
+
     return this._send('addNodes', { nodes: wireNodes });
   }
 
@@ -292,6 +459,11 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
       };
     });
 
+    if (this._batching) {
+      this._batchEdges.push(...wireEdges);
+      return { ok: true } as RFDBResponse;
+    }
+
     return this._send('addEdges', { edges: wireEdges, skipValidation });
   }
 
@@ -521,18 +693,87 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
    * Query nodes (async generator)
    */
   async *queryNodes(query: AttrQuery): AsyncGenerator<WireNode, void, unknown> {
+    // When server supports streaming (protocol v3+), delegate to streaming handler
+    // to correctly handle chunked NodesChunk responses for large result sets.
+    if (this._supportsStreaming) {
+      yield* this.queryNodesStream(query);
+      return;
+    }
+
+    const serverQuery = this._buildServerQuery(query);
+    const response = await this._send('queryNodes', { query: serverQuery });
+    const nodes = (response as { nodes?: WireNode[] }).nodes || [];
+
+    for (const node of nodes) {
+      yield node;
+    }
+  }
+
+  /**
+   * Build a server query object from an AttrQuery.
+   */
+  private _buildServerQuery(query: AttrQuery): Record<string, unknown> {
     const serverQuery: Record<string, unknown> = {};
     if (query.nodeType) serverQuery.nodeType = query.nodeType;
     if (query.type) serverQuery.nodeType = query.type;
     if (query.name) serverQuery.name = query.name;
     if (query.file) serverQuery.file = query.file;
     if (query.exported !== undefined) serverQuery.exported = query.exported;
+    return serverQuery;
+  }
 
-    const response = await this._send('queryNodes', { query: serverQuery });
-    const nodes = (response as { nodes?: WireNode[] }).nodes || [];
+  /**
+   * Stream nodes matching query with true streaming support.
+   *
+   * Behavior depends on server capabilities:
+   * - Server supports streaming (protocol v3): receives chunked NodesChunk
+   *   responses via StreamQueue. Nodes are yielded as they arrive.
+   * - Server does NOT support streaming (fallback): delegates to queryNodes()
+   *   which yields nodes one by one from bulk response.
+   *
+   * The generator can be aborted by breaking out of the loop or calling .return().
+   */
+  async *queryNodesStream(query: AttrQuery): AsyncGenerator<WireNode, void, unknown> {
+    if (!this._supportsStreaming) {
+      yield* this.queryNodes(query);
+      return;
+    }
 
-    for (const node of nodes) {
-      yield node;
+    if (!this.connected || !this.socket) {
+      throw new Error('Not connected to RFDB server');
+    }
+
+    const serverQuery = this._buildServerQuery(query);
+    const id = this.reqId++;
+    const streamQueue = new StreamQueue<WireNode>();
+    this._pendingStreams.set(id, streamQueue);
+
+    // Build and send request manually (can't use _send which expects single response)
+    const request = { requestId: `r${id}`, cmd: 'queryNodes', query: serverQuery };
+    const msgBytes = encode(request);
+    const header = Buffer.alloc(4);
+    header.writeUInt32BE(msgBytes.length);
+
+    // Register in pending map for error routing
+    this.pending.set(id, {
+      resolve: () => { this._cleanupStream(id); },
+      reject: (error) => {
+        this._cleanupStream(id);
+        streamQueue.fail(error);
+      },
+    });
+
+    // Start per-chunk timeout (resets on each chunk in _handleStreamingResponse)
+    this._resetStreamTimer(id, streamQueue);
+
+    this.socket!.write(Buffer.concat([header, Buffer.from(msgBytes)]));
+
+    try {
+      for await (const node of streamQueue) {
+        yield node;
+      }
+    } finally {
+      this._cleanupStream(id);
     }
   }
 
@@ -639,6 +880,15 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
     return (response as { violations?: DatalogResult[] }).violations || [];
   }
 
+  /**
+   * Execute unified Datalog — handles both direct queries and rule-based programs.
+   * Auto-detects the head predicate instead of hardcoding violation(X).
+   */
+  async executeDatalog(source: string): Promise<DatalogResult[]> {
+    const response = await this._send('executeDatalog', { source });
+    return (response as { results?: DatalogResult[] }).results || [];
+  }
+
   /**
    * Ping the server
    */
@@ -656,9 +906,11 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
    * @param protocolVersion - Protocol version to negotiate (default: 2)
    * @returns Server capabilities including protocolVersion, serverVersion, features
    */
-  async hello(protocolVersion: number = 2): Promise<HelloResponse> {
+  async hello(protocolVersion: number = 3): Promise<HelloResponse> {
     const response = await this._send('hello' as RFDBCommand, { protocolVersion });
-    return response as HelloResponse;
+    const hello = response as HelloResponse;
+    this._supportsStreaming = hello.features?.includes('streaming') ?? false;
+    return hello;
   }
 
   /**
@@ -712,6 +964,159 @@ export class RFDBClient extends EventEmitter implements IRFDBClient {
     return response as CurrentDatabaseResponse;
   }
 
+  // ===========================================================================
+  // Snapshot Operations
+  // ===========================================================================
+
+  /**
+   * Convert a SnapshotRef to wire format payload fields.
+   *
+   * - number -> { version: N }
+   * - { tag, value } -> { tagKey, tagValue }
+   */
+  private _resolveSnapshotRef(ref: SnapshotRef): Record<string, unknown> {
+    if (typeof ref === 'number') return { version: ref };
+    return { tagKey: ref.tag, tagValue: ref.value };
+  }
+
+  /**
+   * Compute diff between two snapshots.
+   * @param from - Source snapshot (version number or tag reference)
+   * @param to - Target snapshot (version number or tag reference)
+   * @returns SnapshotDiff with added/removed segments and stats
+   */
+  async diffSnapshots(from: SnapshotRef, to: SnapshotRef): Promise<SnapshotDiff> {
+    const response = await this._send('diffSnapshots', {
+      from: this._resolveSnapshotRef(from),
+      to: this._resolveSnapshotRef(to),
+    });
+    return (response as DiffSnapshotsResponse).diff;
+  }
+
+  /**
+   * Tag a snapshot with key-value metadata.
+   * @param version - Snapshot version to tag
+   * @param tags - Key-value pairs to apply (e.g. { "release": "v1.0" })
+   */
+  async tagSnapshot(version: number, tags: Record<string, string>): Promise<void> {
+    await this._send('tagSnapshot', { version, tags });
+  }
+
+  /**
+   * Find a snapshot by tag key/value pair.
+   * @param tagKey - Tag key to search for
+   * @param tagValue - Tag value to match
+   * @returns Snapshot version number, or null if not found
+   */
+  async findSnapshot(tagKey: string, tagValue: string): Promise<number | null> {
+    const response = await this._send('findSnapshot', { tagKey, tagValue });
+    return (response as FindSnapshotResponse).version;
+  }
+
+  /**
+   * List snapshots, optionally filtered by tag key.
+   * @param filterTag - Optional tag key to filter by (only snapshots with this tag)
+   * @returns Array of SnapshotInfo objects
+   */
+  async listSnapshots(filterTag?: string): Promise<SnapshotInfo[]> {
+    const payload: Record<string, unknown> = {};
+    if (filterTag !== undefined) payload.filterTag = filterTag;
+    const response = await this._send('listSnapshots', payload);
+    return (response as ListSnapshotsResponse).snapshots;
+  }
+
+  // ===========================================================================
+  // Batch Operations
+  // ===========================================================================
+
+  /**
+   * Begin a batch operation.
+   * While batching, addNodes/addEdges buffer locally instead of sending to server.
+   * Call commitBatch() to send all buffered data atomically.
+   */
+  beginBatch(): void {
+    if (this._batching) throw new Error('Batch already in progress');
+    this._batching = true;
+    this._batchNodes = [];
+    this._batchEdges = [];
+    this._batchFiles = new Set();
+  }
+
+  /**
+   * Commit the current batch to the server.
+   * Sends all buffered nodes/edges with the list of changed files.
+   * Server atomically replaces old data for changed files with new data.
+   */
+  async commitBatch(tags?: string[]): Promise<CommitDelta> {
+    if (!this._batching) throw new Error('No batch in progress');
+    const response = await this._send('commitBatch', {
+      changedFiles: [...this._batchFiles],
+      nodes: this._batchNodes,
+      edges: this._batchEdges,
+      tags,
+    });
+
+    this._batching = false;
+    this._batchNodes = [];
+    this._batchEdges = [];
+    this._batchFiles = new Set();
+
+    return (response as CommitBatchResponse).delta;
+  }
+
+  /**
+   * Abort the current batch, discarding all buffered data.
+   */
+  abortBatch(): void {
+    this._batching = false;
+    this._batchNodes = [];
+    this._batchEdges = [];
+    this._batchFiles = new Set();
+  }
+
+  /**
+   * Check if a batch is currently in progress.
+   */
+  isBatching(): boolean {
+    return this._batching;
+  }
+
+  /**
+   * Find files that depend on the given changed files.
+   * Uses backward reachability to find dependent modules.
+   *
+   * Note: For large result sets, each reachable node requires a separate
+   * getNode RPC. A future server-side optimization could return file paths
+   * directly from the reachability query.
+   */
+  async findDependentFiles(changedFiles: string[]): Promise<string[]> {
+    const nodeIds: string[] = [];
+    for (const file of changedFiles) {
+      const ids = await this.findByAttr({ file });
+      nodeIds.push(...ids);
+    }
+
+    if (nodeIds.length === 0) return [];
+
+    const reachable = await this.reachability(
+      nodeIds,
+      2,
+      ['IMPORTS_FROM', 'DEPENDS_ON', 'CALLS'] as EdgeType[],
+      true,
+    );
+
+    const changedSet = new Set(changedFiles);
+    const files = new Set<string>();
+    for (const id of reachable) {
+      const node = await this.getNode(id);
+      if (node?.file && !changedSet.has(node.file)) {
+        files.add(node.file);
+      }
+    }
+
+    return [...files];
+  }
+
   /**
    * Unref the socket so it doesn't keep the process alive.
    *

package/ts/index.ts

@@ -10,6 +10,7 @@
 
 // Client
 export { RFDBClient } from './client.js';
+export { StreamQueue } from './stream-queue.js';
 
 // Protocol types (re-exported from @grafema/types for convenience)
 export type {
@@ -21,4 +22,13 @@ export type {
   AttrQuery,
   DatalogResult,
   IRFDBClient,
+  // Snapshot types
+  SnapshotRef,
+  SnapshotStats,
+  SegmentInfo,
+  SnapshotDiff,
+  SnapshotInfo,
+  DiffSnapshotsResponse,
+  FindSnapshotResponse,
+  ListSnapshotsResponse,
 } from './protocol.js';

package/ts/protocol.ts

@@ -43,12 +43,23 @@ export type {
   CountResponse,
   CountsByTypeResponse,
   PingResponse,
+  NodesChunkResponse,
 
   // Query types
   AttrQuery,
   DatalogBinding,
   DatalogResult,
 
+  // Snapshot types
+  SnapshotRef,
+  SnapshotStats,
+  SegmentInfo,
+  SnapshotDiff,
+  SnapshotInfo,
+  DiffSnapshotsResponse,
+  FindSnapshotResponse,
+  ListSnapshotsResponse,
+
   // Client interface
   IRFDBClient,
 } from '@grafema/types';