@majkapp/plugin-kit 3.5.4 → 3.6.0

package/docs/RPC_CALLBACKS.md

@@ -0,0 +1,518 @@
+# RPC Callbacks
+
+RPC callbacks allow you to pass callback functions across plugin boundaries. The MAJK RPC system automatically handles callback serialization, invocation, and cleanup.
+
+## Overview
+
+When building plugins that communicate via RPC, you can now pass callback functions as arguments. The system automatically:
+- **Wraps callbacks** into serializable handles
+- **Registers them** as temporary RPC services
+- **Invokes them** when called from the remote side
+- **Cleans them up** based on your chosen strategy
+
+## Basic Usage (Inline Callbacks)
+
+The simplest way to use callbacks is to pass them directly as function arguments. These have **no automatic cleanup** by default.
+
+```typescript
+// Plugin A: Register a service that accepts callbacks
+const plugin = definePlugin('file-processor', 'File Processor', '1.0.0')
+  .pluginRoot(__dirname)
+  .onLoad(async (ctx) => {
+    // Register service with callback parameter
+    await ctx.rpc.registerService('processor', {
+      async processFile(
+        filePath: string,
+        onProgress: (percent: number, message: string) => void
+      ): Promise<string> {
+        // Call the callback as the file processes
+        await onProgress(0, 'Starting...');
+        await onProgress(25, 'Reading file...');
+        await onProgress(50, 'Processing...');
+        await onProgress(75, 'Writing results...');
+        await onProgress(100, 'Complete!');
+
+        return `Processed: ${filePath}`;
+      }
+    });
+  })
+  .build();
+```
+
+```typescript
+// Plugin B: Use the service with inline callback
+const plugin = definePlugin('file-consumer', 'File Consumer', '1.0.0')
+  .pluginRoot(__dirname)
+  .onLoad(async (ctx) => {
+    // Create proxy to the remote service
+    const processor = await ctx.rpc.createProxy<{
+      processFile(path: string, onProgress: (p: number, m: string) => void): Promise<string>
+    }>('processor');
+
+    // Pass callback directly - it just works!
+    const result = await processor.processFile('/data/large-file.txt', (percent, message) => {
+      console.log(`Progress: ${percent}% - ${message}`);
+    });
+
+    console.log(result); // "Processed: /data/large-file.txt"
+  })
+  .build();
+```
+
+**Key Point**: Inline callbacks live forever (no automatic cleanup). This is perfect for:
+- One-time operations (file processing, data fetching)
+- Short-lived interactions
+- Callbacks that will be called a known number of times
+
+## Explicit Callbacks with Cleanup Strategies
+
+For long-lived callbacks (subscriptions, event listeners, etc.), use `createCallback()` to manage cleanup explicitly.
+
+### Max Calls Strategy
+
+Auto-cleanup after N invocations:
+
+```typescript
+// Plugin A: Event emitter service
+await ctx.rpc.registerService('events', {
+  async subscribe(callback: (event: string) => void): Promise<void> {
+    // callback will be called multiple times
+    setInterval(() => callback('tick'), 1000);
+  }
+});
+```
+
+```typescript
+// Plugin B: Subscribe with maxCalls limit
+const events = await ctx.rpc.createProxy<{
+  subscribe(cb: (event: string) => void): Promise<void>
+}>('events');
+
+// Create callback that self-destructs after 10 calls
+const callback = await ctx.rpc.createCallback!(
+  (event: string) => {
+    console.log('Event:', event);
+  },
+  { maxCalls: 10 } // Auto-cleanup after 10 invocations
+);
+
+await events.subscribe(callback);
+// After 10 events, the callback is automatically cleaned up
+```
+
+### Timeout Strategy
+
+Auto-cleanup after time expires:
+
+```typescript
+// Subscribe for only 5 seconds
+const callback = await ctx.rpc.createCallback!(
+  (data: string) => {
+    console.log('Received:', data);
+  },
+  { timeout: 5000 } // Auto-cleanup after 5 seconds
+);
+
+await dataStream.subscribe(callback);
+// After 5 seconds, callback is cleaned up automatically
+```
+
+### Manual Cleanup
+
+For complete control, clean up manually:
+
+```typescript
+// Create callback without auto-cleanup strategies
+const callback = await ctx.rpc.createCallback!((message: string) => {
+  console.log('Message:', message);
+});
+
+await messageService.subscribe(callback);
+
+// Later... manually clean up when done
+ctx.rpc.cleanupCallback!(callback);
+```
+
+## Real-World Patterns
+
+### Pattern 1: File Processing with Progress
+
+```typescript
+// Service side
+await ctx.rpc.registerService('fileOps', {
+  async batchProcess(
+    files: string[],
+    onFileComplete: (file: string, result: string) => void,
+    onError: (file: string, error: string) => void
+  ): Promise<void> {
+    for (const file of files) {
+      try {
+        const result = await processFile(file);
+        await onFileComplete(file, result);
+      } catch (error) {
+        await onError(file, error.message);
+      }
+    }
+  }
+});
+
+// Client side
+const fileOps = await ctx.rpc.createProxy<typeof FileOpsService>('fileOps');
+
+await fileOps.batchProcess(
+  ['/file1.txt', '/file2.txt', '/file3.txt'],
+  (file, result) => {
+    console.log(`✓ ${file}: ${result}`);
+  },
+  (file, error) => {
+    console.error(`✗ ${file}: ${error}`);
+  }
+);
+```
+
+### Pattern 2: Real-Time Data Subscriptions
+
+```typescript
+// Service side: Data stream
+await ctx.rpc.registerService('dataStream', {
+  async subscribe(
+    filter: string,
+    onData: (data: any) => void
+  ): Promise<() => void> {
+    const subscription = dataSource.on(filter, onData);
+
+    // Return unsubscribe function (also a callback!)
+    return () => subscription.close();
+  }
+});
+
+// Client side
+const stream = await ctx.rpc.createProxy<{
+  subscribe(filter: string, onData: (data: any) => void): Promise<() => void>
+}>('dataStream');
+
+// Subscribe with 1-minute timeout
+const callback = await ctx.rpc.createCallback!(
+  (data: any) => {
+    console.log('New data:', data);
+  },
+  { timeout: 60000 }
+);
+
+const unsubscribe = await stream.subscribe('temperature > 100', callback);
+
+// When done early, clean up both callback and subscription
+unsubscribe(); // Stop receiving data
+ctx.rpc.cleanupCallback!(callback); // Clean up callback
+```
+
+### Pattern 3: Filtering with Predicates
+
+```typescript
+// Service side
+await ctx.rpc.registerService('dataService', {
+  async getFiltered(
+    items: any[],
+    predicate: (item: any) => boolean | Promise<boolean>
+  ): Promise<any[]> {
+    const filtered = [];
+    for (const item of items) {
+      if (await predicate(item)) {
+        filtered.push(item);
+      }
+    }
+    return filtered;
+  }
+});
+
+// Client side
+const dataService = await ctx.rpc.createProxy<typeof DataService>('dataService');
+
+const filtered = await dataService.getFiltered(
+  allUsers,
+  (user) => user.age > 18 && user.active
+);
+```
+
+### Pattern 4: Event Hooks
+
+```typescript
+// Service side: Task runner with hooks
+await ctx.rpc.registerService('taskRunner', {
+  async runTask(
+    taskName: string,
+    hooks: {
+      onStart?: () => void,
+      onProgress?: (percent: number) => void,
+      onComplete?: (result: any) => void,
+      onError?: (error: string) => void
+    }
+  ): Promise<void> {
+    try {
+      await hooks.onStart?.();
+
+      // Execute task with progress updates
+      for (let i = 0; i <= 100; i += 10) {
+        await hooks.onProgress?.(i);
+        await simulateWork();
+      }
+
+      const result = await finalizeTask();
+      await hooks.onComplete?.(result);
+    } catch (error) {
+      await hooks.onError?.(error.message);
+    }
+  }
+});
+
+// Client side
+const taskRunner = await ctx.rpc.createProxy<typeof TaskRunner>('taskRunner');
+
+await taskRunner.runTask('compile-project', {
+  onStart: () => console.log('Starting compilation...'),
+  onProgress: (p) => console.log(`Progress: ${p}%`),
+  onComplete: (result) => console.log('Done!', result),
+  onError: (error) => console.error('Failed:', error)
+});
+```
+
+## Error Handling
+
+### Callback Cleaned Up Error
+
+If a callback is invoked after cleanup, a `CallbackCleanedUpError` is thrown:
+
+```typescript
+const callback = await ctx.rpc.createCallback!(
+  (data) => console.log(data),
+  { maxCalls: 1 }
+);
+
+await service.subscribe(callback);
+// After 1 call, callback is cleaned up
+
+// If service tries to call it again:
+// Error: Callback xyz has been cleaned up and is no longer available.
+//        This can happen if: 1) maxCalls limit was reached,
+//                            2) timeout expired,
+//                            3) cleanupCallback() was called manually.
+```
+
+### Handling Remote Errors
+
+Errors thrown in callbacks propagate back to the caller:
+
+```typescript
+// Service side
+await service.process(
+  files,
+  async (file) => {
+    if (!file.exists()) {
+      throw new Error(`File not found: ${file.path}`);
+    }
+  }
+);
+// Error will be received by service with the message
+```
+
+### Robust Error Handling Pattern
+
+```typescript
+// Service side
+async processWithCallback(
+  data: any[],
+  callback: (item: any) => Promise<void>
+): Promise<{ success: any[], errors: any[] }> {
+  const success = [];
+  const errors = [];
+
+  for (const item of data) {
+    try {
+      await callback(item);
+      success.push(item);
+    } catch (error) {
+      errors.push({ item, error: error.message });
+    }
+  }
+
+  return { success, errors };
+}
+```
+
+## Monitoring and Debugging
+
+### Get Callback Statistics
+
+```typescript
+const stats = ctx.rpc.getCallbackStats!();
+console.log('Active callbacks:', stats.activeCallbacks);
+console.log('Inline callbacks:', stats.inlineCallbacks);
+console.log('Explicit callbacks:', stats.explicitCallbacks);
+console.log('Total created:', stats.totalCreated);
+console.log('Total cleaned:', stats.totalCleaned);
+```
+
+**Statistics breakdown**:
+- `activeCallbacks`: Currently registered callbacks
+- `inlineCallbacks`: Auto-wrapped inline callbacks (no cleanup)
+- `explicitCallbacks`: Explicitly created via `createCallback()`
+- `totalCreated`: Lifetime count of callbacks created
+- `totalCleaned`: Lifetime count of callbacks cleaned up
+
+### Debugging Tips
+
+1. **Use timeouts for testing**: When developing, use short timeouts to test cleanup behavior:
+   ```typescript
+   const callback = await ctx.rpc.createCallback!(fn, { timeout: 5000 });
+   ```
+
+2. **Monitor stats**: Check stats periodically to detect callback leaks:
+   ```typescript
+   setInterval(() => {
+     const stats = ctx.rpc.getCallbackStats!();
+     if (stats.activeCallbacks > 100) {
+       console.warn('High callback count - possible leak');
+     }
+   }, 60000);
+   ```
+
+3. **Log cleanup**: Add logging to understand cleanup timing:
+   ```typescript
+   const callback = await ctx.rpc.createCallback!(
+     (data) => {
+       console.log('Callback invoked:', data);
+     },
+     { maxCalls: 10 }
+   );
+   console.log('Created callback:', callback.callbackId);
+   ```
+
+## Best Practices
+
+### ✅ DO
+
+- **Use inline callbacks for simple, one-time operations**
+  ```typescript
+  await service.processFile(file, (progress) => console.log(progress));
+  ```
+
+- **Use `createCallback()` with `maxCalls` for subscriptions**
+  ```typescript
+  const cb = await ctx.rpc.createCallback!(handler, { maxCalls: 100 });
+  ```
+
+- **Use `createCallback()` with `timeout` for time-limited operations**
+  ```typescript
+  const cb = await ctx.rpc.createCallback!(handler, { timeout: 30000 });
+  ```
+
+- **Clean up manually when you control the lifecycle**
+  ```typescript
+  const cb = await ctx.rpc.createCallback!(handler);
+  // ... use callback ...
+  ctx.rpc.cleanupCallback!(cb);
+  ```
+
+- **Handle errors in callbacks gracefully**
+  ```typescript
+  (data) => {
+    try {
+      processData(data);
+    } catch (error) {
+      console.error('Callback error:', error);
+    }
+  }
+  ```
+
+### ❌ DON'T
+
+- **Don't use inline callbacks for long-lived subscriptions**
+  ```typescript
+  // ❌ Bad: No cleanup, callback lives forever
+  await eventStream.subscribe((event) => console.log(event));
+
+  // ✅ Good: Use timeout or manual cleanup
+  const cb = await ctx.rpc.createCallback!(
+    (event) => console.log(event),
+    { timeout: 60000 }
+  );
+  await eventStream.subscribe(cb);
+  ```
+
+- **Don't forget to clean up when lifecycle is known**
+  ```typescript
+  // ❌ Bad: No cleanup on plugin unload
+  await ctx.rpc.createCallback!(handler);
+
+  // ✅ Good: Track and clean up
+  const callbacks = [];
+  callbacks.push(await ctx.rpc.createCallback!(handler));
+
+  // In onUnload:
+  callbacks.forEach(cb => ctx.rpc.cleanupCallback!(cb));
+  ```
+
+- **Don't pass complex objects through callbacks**
+  ```typescript
+  // ❌ Bad: Large objects, circular references
+  callback({ largeData: hugeArray, circular: someObjectWithCircularRefs });
+
+  // ✅ Good: Pass only necessary data
+  callback({ id: item.id, status: item.status });
+  ```
+
+## Type Safety
+
+All callbacks maintain full TypeScript type safety:
+
+```typescript
+// Service definition with typed callbacks
+interface FileService {
+  processFile(
+    path: string,
+    onProgress: (percent: number, message: string) => void,
+    onError: (error: Error) => void
+  ): Promise<string>;
+}
+
+// Client gets full type checking
+const service = await ctx.rpc.createProxy<FileService>('fileService');
+
+await service.processFile(
+  '/file.txt',
+  (percent, message) => {
+    // 'percent' is number, 'message' is string - fully typed!
+  },
+  (error) => {
+    // 'error' is Error - fully typed!
+  }
+);
+```
+
+## Backward Compatibility
+
+The callback feature is **100% backward compatible**:
+- All callback methods are optional (`createCallback?`, `cleanupCallback?`, `getCallbackStats?`)
+- Existing plugins work unchanged
+- RPC calls without callbacks work exactly as before
+- No breaking changes to existing APIs
+
+## Limitations
+
+1. **Serialization**: Callbacks must not capture complex objects or closures that reference non-serializable data
+2. **Network boundaries**: Callbacks work within the same MAJK process (in-process RPC)
+3. **Performance**: Each callback creates a temporary RPC service - avoid creating thousands at once
+4. **Memory**: Inline callbacks without cleanup will remain in memory until the bridge is destroyed
+
+## Summary
+
+RPC callbacks make inter-plugin communication natural and intuitive:
+
+| Use Case | Strategy | Example |
+|----------|----------|---------|
+| One-time operations | Inline callback | `process(file, (progress) => ...)` |
+| N-limited subscriptions | `maxCalls` | `createCallback(fn, { maxCalls: 10 })` |
+| Time-limited operations | `timeout` | `createCallback(fn, { timeout: 5000 })` |
+| Controlled lifecycle | Manual cleanup | `createCallback(fn)` + `cleanupCallback()` |
+
+Choose the right strategy for your use case, and let the RPC system handle the rest!

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@majkapp/plugin-kit",
-  "version": "3.5.4",
+  "version": "3.6.0",
   "description": "Pure plugin definition library for MAJK - outputs plugin definitions, not HTTP servers",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -33,6 +33,7 @@
     "promptable:hooks": "node ./bin/promptable-cli.js --hooks",
     "promptable:context": "node ./bin/promptable-cli.js --context",
     "promptable:services": "node ./bin/promptable-cli.js --services",
+    "promptable:rpc": "node ./bin/promptable-cli.js --rpc",
     "promptable:lifecycle": "node ./bin/promptable-cli.js --lifecycle",
     "promptable:testing": "node ./bin/promptable-cli.js --testing",
     "promptable:config": "node ./bin/promptable-cli.js --config",