@nonstrict/recordkit 0.51.1 → 0.53.0

package/src/Recorder.ts

@@ -3,6 +3,54 @@ import { NSRPC } from "./NonstrictRPC.js";
 import { EventEmitter } from "stream";
 import { AppleDevice, Camera, Display, Microphone, RunningApplication, Window } from "./RecordKit.js";
 
+/**
+ * Converts RPC audio buffer data to AudioStreamBuffer format
+ * @internal
+ */
+function convertRPCParamsToAudioStreamBuffer(params: any): AudioStreamBuffer | null {
+  try {
+    // params is the AudioBufferData directly from Swift
+    const rawAudioBuffer = params as any;
+    
+    if (!rawAudioBuffer || !Array.isArray(rawAudioBuffer.channelData)) {
+      console.error('RecordKit: Invalid audio buffer received from RPC');
+      return null;
+    }
+    
+    const channelData: Float32Array[] = [];
+    
+    for (const base64Data of rawAudioBuffer.channelData) {
+      if (typeof base64Data !== 'string') {
+        console.error('RecordKit: Invalid base64 data received');
+        return null;
+      }
+      
+      // Decode base64 to binary data
+      const binaryString = atob(base64Data);
+      const bytes = new Uint8Array(binaryString.length);
+      for (let i = 0; i < binaryString.length; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+      }
+      
+      // Convert bytes to Float32Array
+      const float32Array = new Float32Array(bytes.buffer);
+      channelData.push(float32Array);
+    }
+    
+    const audioStreamBuffer: AudioStreamBuffer = {
+      sampleRate: rawAudioBuffer.sampleRate,
+      numberOfChannels: rawAudioBuffer.numberOfChannels,
+      numberOfFrames: rawAudioBuffer.numberOfFrames,
+      channelData: channelData
+    };
+    
+    return audioStreamBuffer;
+  } catch (error) {
+    console.error('RecordKit: Error processing audio stream buffer:', error);
+    return null;
+  }
+}
+
 /**
  * @group Recording
  */
@@ -70,6 +118,19 @@ export class Recorder extends EventEmitter {
             lifecycle: object
           });
         }
+        if (item.output == 'stream' && item.streamCallback) {
+          const streamHandler = item.streamCallback;
+          (item as any).streamCallback = rpc.registerClosure({
+            handler: (params) => {
+              const audioBuffer = convertRPCParamsToAudioStreamBuffer(params);
+              if (audioBuffer) {
+                streamHandler(audioBuffer);
+              }
+            },
+            prefix: 'SystemAudioStream.onAudioBuffer',
+            lifecycle: object
+          });
+        }
       }
       if (item.type == 'applicationAudio') {
         if (item.output == 'segmented' && item.segmentCallback) {
@@ -80,6 +141,45 @@ export class Recorder extends EventEmitter {
             lifecycle: object
           });
         }
+        if (item.output == 'stream' && item.streamCallback) {
+          const streamHandler = item.streamCallback;
+          (item as any).streamCallback = rpc.registerClosure({
+            handler: (params) => {
+              const audioBuffer = convertRPCParamsToAudioStreamBuffer(params);
+              if (audioBuffer) {
+                streamHandler(audioBuffer);
+              }
+            },
+            prefix: 'ApplicationAudioStream.onAudioBuffer',
+            lifecycle: object
+          });
+        }
+      }
+      if (item.type == 'microphone') {
+        if (typeof item.microphone != 'string') {
+          item.microphone = item.microphone.id
+        }
+        if (item.output == 'segmented' && item.segmentCallback) {
+          const segmentHandler = item.segmentCallback;
+          (item as any).segmentCallback = rpc.registerClosure({
+            handler: (params) => { segmentHandler(params.path as string) },
+            prefix: 'Microphone.onSegment',
+            lifecycle: object
+          });
+        }
+        if (item.output == 'stream' && item.streamCallback) {
+          const streamHandler = item.streamCallback;
+          (item as any).streamCallback = rpc.registerClosure({
+            handler: (params) => {
+              const audioBuffer = convertRPCParamsToAudioStreamBuffer(params);
+              if (audioBuffer) {
+                streamHandler(audioBuffer);
+              }
+            },
+            prefix: 'MicrophoneStream.onAudioBuffer',
+            lifecycle: object
+          });
+        }
       }
     })
 
@@ -134,8 +234,11 @@ export type RecorderSchemaItem =
   | AppleDeviceStaticOrientationSchema
   | SystemAudioSchema
   | ApplicationAudioSchema
+  | MicrophoneSchema
 
 /**
+ * Creates a recorder item for a webcam movie file, using the provided microphone and camera. Output is stored in a RecordKit bundle.
+ * 
  * @group Recording Schemas
  */
 export interface WebcamSchema {
@@ -146,6 +249,8 @@ export interface WebcamSchema {
 }
 
 /**
+ * Creates a recorder item for recording a single display. Output is stored in a RecordKit bundle.
+ * 
  * @group Recording Schemas
  */
 export type DisplaySchema = {
@@ -170,6 +275,8 @@ export type DisplaySchema = {
 }
 
 /**
+ * Creates a recorder item for recording the initial crop of a window on a display. Output is stored in a RecordKit bundle.
+ * 
  * @group Recording Schemas
  */
 export type WindowBasedCropSchema = {
@@ -192,6 +299,8 @@ export type WindowBasedCropSchema = {
 }
 
 /**
+ * Creates a recorder item for an Apple device screen recording, using the provided deviceID. Output is stored in a RecordKit bundle.
+ * 
  * @group Recording Schemas
  */
 export interface AppleDeviceStaticOrientationSchema {
@@ -206,6 +315,11 @@ export interface AppleDeviceStaticOrientationSchema {
 export type SystemAudioMode = 'exclude' | 'include'
 
 /**
+ * Enumeration specifying the backend to use for system audio recording.
+ * 
+ * - `screenCaptureKit`: Use ScreenCaptureKit for system audio recording.
+ * - `_beta_coreAudio`: This a beta feature, it is not fully implemented yet. Do not use in production. Currently only records single files in .caf format.
+ * 
  * @group Recording Schemas
  */
 export type SystemAudioBackend = 'screenCaptureKit' | '_beta_coreAudio'
@@ -213,9 +327,19 @@ export type SystemAudioBackend = 'screenCaptureKit' | '_beta_coreAudio'
 /**
  * @group Recording Schemas
  */
-export type AudioOutputOptionsType = 'singleFile' | 'segmented'
+export type AudioOutputOptionsType = 'singleFile' | 'segmented' | 'stream'
+
+/**
+ * @group Recording Schemas
+ */
+export type MicrophoneOutputOptionsType = 'singleFile' | 'segmented' | 'stream'
 
 /**
+ * Creates a recorder item for recording system audio. By default current process audio is excluded. Output is stored in a RecordKit bundle.
+ * 
+ * When using `mode: 'exclude'`, all system audio is recorded except for excluded applications.
+ * When using `mode: 'include'`, only audio from specified applications is recorded.
+ * 
  * @group Recording Schemas
  */
 export type SystemAudioSchema = {
@@ -235,6 +359,15 @@ export type SystemAudioSchema = {
   output: 'segmented'
   filenamePrefix?: string
   segmentCallback?: (url: string) => void
+} | {
+  type: 'systemAudio'
+  mode: 'exclude'
+  backend?: SystemAudioBackend
+  excludeOptions?: ('currentProcess')[]
+  excludedProcessIDs?: number[] // Int32
+  output: 'stream'
+  /** Called with real-time audio buffer data compatible with Web Audio API. Requires _beta_coreAudio backend and macOS 14.2+ */
+  streamCallback?: (audioBuffer: AudioStreamBuffer) => void
 } | {
   type: 'systemAudio'
   mode: 'include'
@@ -250,9 +383,19 @@ export type SystemAudioSchema = {
   output: 'segmented'
   filenamePrefix?: string
   segmentCallback?: (url: string) => void
+} | {
+  type: 'systemAudio'
+  mode: 'include'
+  backend?: SystemAudioBackend
+  includedApplicationIDs?: number[] // Int32
+  output: 'stream'
+  /** Called with real-time audio buffer data compatible with Web Audio API. Requires _beta_coreAudio backend and macOS 14.2+ */
+  streamCallback?: (audioBuffer: AudioStreamBuffer) => void
 }
 
 /**
+ * Creates a recorder item for recording the audio of a single application. Output is stored in a RecordKit bundle.
+ * 
  * @group Recording Schemas
  */
 export type ApplicationAudioSchema = {
@@ -268,8 +411,60 @@ export type ApplicationAudioSchema = {
   output: 'segmented'
   filenamePrefix?: string
   segmentCallback?: (url: string) => void
+} | {
+  type: 'applicationAudio'
+  applicationID: number // Int32
+  backend?: SystemAudioBackend
+  output: 'stream'
+  /** Called with real-time audio buffer data compatible with Web Audio API. Requires _beta_coreAudio backend and macOS 14.2+ */
+  streamCallback?: (audioBuffer: AudioStreamBuffer) => void
+}
+
+/**
+ * Creates a recorder item for an audio file, using the provided microphone. Output is stored in a RecordKit bundle.
+ * 
+ * @group Recording Schemas
+ */
+export type MicrophoneSchema = {
+  type: 'microphone'
+  microphone: Microphone | string
+  leftChannelOnly?: boolean
+  audioDelay?: number
+  output?: 'singleFile'
+  filename?: string
+} | {
+  type: 'microphone'
+  microphone: Microphone | string
+  leftChannelOnly?: boolean
+  audioDelay?: number
+  output: 'segmented'
+  filenamePrefix?: string
+  segmentCallback?: (url: string) => void
+} | {
+  type: 'microphone'
+  microphone: Microphone | string
+  output: 'stream'
+  /** Called with real-time audio buffer data compatible with Web Audio API */
+  streamCallback?: (audioBuffer: AudioStreamBuffer) => void
+}
+
+/**
+ * Audio buffer compatible with Web Audio API
+ * 
+ * @group Recording
+ */
+export interface AudioStreamBuffer {
+  /** Sample rate in Hz (e.g., 44100, 48000) */
+  sampleRate: number
+  /** Number of audio channels */
+  numberOfChannels: number
+  /** Number of frames per channel */
+  numberOfFrames: number
+  /** Non-interleaved Float32 audio data - one array per channel */
+  channelData: Float32Array[]
 }
 
+
 /**
  * @group Recording
  */

package/src/WebAudioUtils.ts

@@ -0,0 +1,108 @@
+import { AudioStreamBuffer } from './Recorder.js';
+
+/**
+ * Creates a Web Audio API AudioBuffer from a RecordKit AudioStreamBuffer.
+ * 
+ * This utility converts RecordKit's streaming audio format to the standard Web Audio API format,
+ * handling various edge cases and IPC serialization issues that may occur in Electron environments.
+ * 
+ * @param audioStreamBuffer - The RecordKit AudioStreamBuffer to convert
+ * @param audioContext - The Web Audio API AudioContext to use for buffer creation
+ * @returns The created AudioBuffer, or null if conversion failed
+ * 
+ * @example
+ * ```typescript
+ * import { createWebAudioBuffer } from '@nonstrict/recordkit';
+ * 
+ * // In your stream callback
+ * const streamCallback = (audioBuffer: AudioStreamBuffer) => {
+ *   const audioContext = new AudioContext();
+ *   const webAudioBuffer = createWebAudioBuffer(audioBuffer, audioContext);
+ *   
+ *   if (webAudioBuffer) {
+ *     // Use the buffer with Web Audio API
+ *     const source = audioContext.createBufferSource();
+ *     source.buffer = webAudioBuffer;
+ *     source.connect(audioContext.destination);
+ *     source.start();
+ *   }
+ * };
+ * ```
+ */
+export function createWebAudioBuffer(
+  audioStreamBuffer: AudioStreamBuffer,
+  audioContext: AudioContext
+): AudioBuffer | null {
+  // Input validation
+  if (!audioStreamBuffer || typeof audioStreamBuffer !== 'object') {
+    return null;
+  }
+
+  if (!audioContext || typeof audioContext.createBuffer !== 'function') {
+    return null;
+  }
+
+  try {
+    const { sampleRate, numberOfChannels, numberOfFrames, channelData } = audioStreamBuffer;
+
+    // Validate required properties
+    if (typeof sampleRate !== 'number' || sampleRate <= 0 || sampleRate > 192000) {
+      return null;
+    }
+
+    if (typeof numberOfChannels !== 'number' || numberOfChannels <= 0 || numberOfChannels > 32) {
+      return null;
+    }
+
+    if (typeof numberOfFrames !== 'number' || numberOfFrames <= 0 || numberOfFrames > 1048576) {
+      return null;
+    }
+
+    if (!Array.isArray(channelData) || channelData.length !== numberOfChannels) {
+      return null;
+    }
+
+    // Validate channel data arrays
+    for (let i = 0; i < numberOfChannels; i++) {
+      const channel = channelData[i];
+      if (!channel || (!Array.isArray(channel) && !(channel instanceof Float32Array))) {
+        return null;
+      }
+      
+      // Check length matches expected frame count
+      if (channel.length !== numberOfFrames) {
+        return null;
+      }
+    }
+
+    // Create Web Audio AudioBuffer
+    const audioBuffer = audioContext.createBuffer(numberOfChannels, numberOfFrames, sampleRate);
+
+    // Copy channel data to AudioBuffer
+    for (let channel = 0; channel < numberOfChannels; channel++) {
+      const outputArray = audioBuffer.getChannelData(channel);
+      const inputArray = channelData[channel];
+
+      // Handle both Float32Array and regular arrays (from Electron IPC serialization)
+      if (inputArray instanceof Float32Array) {
+        // Direct copy for Float32Array
+        outputArray.set(inputArray);
+      } else if (Array.isArray(inputArray)) {
+        // Convert regular array to Float32Array for better performance
+        const float32Array = new Float32Array(inputArray);
+        outputArray.set(float32Array);
+      } else {
+        // Fallback: manual copy with type conversion
+        for (let i = 0; i < numberOfFrames; i++) {
+          const sample = inputArray[i];
+          outputArray[i] = typeof sample === 'number' && isFinite(sample) ? sample : 0;
+        }
+      }
+    }
+
+    return audioBuffer;
+  } catch (error) {
+    // Return null for any conversion failures - don't throw in streaming contexts
+    return null;
+  }
+}

package/src/browser.ts

@@ -0,0 +1,7 @@
+// Browser-only exports - safe for use in browser environments
+// This entry point excludes Node.js-specific functionality like EventEmitter, crypto, etc.
+
+export { createWebAudioBuffer } from './WebAudioUtils.js';
+
+// Type-only exports for browser use
+export type { AudioStreamBuffer } from './Recorder.js';

package/tsconfig.json

@@ -17,7 +17,8 @@
     /* Language and Environment */
     "target": "ES2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
     "lib": [
-      "ES2022"
+      "ES2022",
+      "DOM"
     ], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
     // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */