@motiadev/core 0.2.1-beta.47 → 0.2.1-beta.49

package/dist/src/python/communication_factory.py

@@ -0,0 +1,35 @@
+import os
+import platform
+from typing import Union
+from rpc_communication import RpcCommunication
+from ipc_communication import IpcCommunication
+
+def create_communication() -> Union[RpcCommunication, IpcCommunication]:
+    """
+    Create appropriate communication instance based on platform and environment.
+    
+    Logic:
+    - Python + Windows = RPC (stdin/stdout)
+    - Other cases with NODE_CHANNEL_FD = IPC
+    - Fallback = RPC
+    """
+    
+    # Check if we're on Windows
+    is_windows = platform.system() == 'Windows'
+    
+    # Check if IPC file descriptor is available
+    has_ipc_fd = "NODE_CHANNEL_FD" in os.environ
+    
+    if is_windows:
+        # On Windows, always use RPC
+        return RpcCommunication()
+    elif has_ipc_fd:
+        # On Unix with IPC FD available, use IPC
+        try:
+            return IpcCommunication()
+        except RuntimeError:
+            # Fallback to RPC if IPC fails to initialize
+            return RpcCommunication()
+    else:
+        # Fallback to RPC
+        return RpcCommunication() 

package/dist/src/python/get-config.py

@@ -2,6 +2,7 @@ import sys
 import json
 import importlib.util
 import os
+import platform
 
 # get the FD from ENV
 NODEIPCFD = int(os.environ["NODE_CHANNEL_FD"])
@@ -10,34 +11,36 @@ def sendMessage(text):
     'sends a Node IPC message to parent proccess'
     # encode message as json string + newline in bytes
     bytesMessage = (json.dumps(text) + "\n").encode('utf-8')
-    # send message
-    os.write(NODEIPCFD, bytesMessage)
+    
+    # Handle Windows differently
+    if platform.system() == 'Windows':
+        # On Windows, write to stdout
+        sys.stdout.buffer.write(bytesMessage)
+        sys.stdout.buffer.flush()
+    else:
+        # On Unix systems, use the file descriptor approach
+        NODEIPCFD = int(os.environ["NODE_CHANNEL_FD"])
+        os.write(NODEIPCFD, bytesMessage)
 
 async def run_python_module(file_path: str) -> None:
     try:
-        # Get the directory containing the module file
         module_dir = os.path.dirname(os.path.abspath(file_path))
         
-        # Add module directory to Python path
         if module_dir not in sys.path:
             sys.path.insert(0, module_dir)
             
-        # Get the flows directory (parent of steps)
         flows_dir = os.path.dirname(module_dir)
         if flows_dir not in sys.path:
             sys.path.insert(0, flows_dir)
 
-        # Load the module dynamically
         spec = importlib.util.spec_from_file_location("dynamic_module", file_path)
         if spec is None or spec.loader is None:
             raise ImportError(f"Could not load module from {file_path}")
             
         module = importlib.util.module_from_spec(spec)
-        # Add module's directory as its __package__
         module.__package__ = os.path.basename(module_dir)
         spec.loader.exec_module(module)
 
-        # Print the config if it exists
         if not hasattr(module, 'config'):
             raise AttributeError(f"No 'config' found in module {file_path}")
 

package/dist/src/python/ipc_communication.py

@@ -0,0 +1,145 @@
+import uuid
+import asyncio
+import json
+import sys
+import os
+from typing import Any, Dict, Optional, Callable
+
+def serialize_for_json(obj: Any) -> Any:
+    """Convert Python objects to JSON-serializable types"""
+    if hasattr(obj, '__dict__'):
+        return obj.__dict__
+    elif hasattr(obj, '_asdict'):
+        return obj._asdict()
+    elif isinstance(obj, (list, tuple)):
+        return [serialize_for_json(item) for item in obj]
+    elif isinstance(obj, dict):
+        return {k: serialize_for_json(v) for k, v in obj.items()}
+    else:
+        return obj
+
+class IpcCommunication:
+    """IPC communication using file descriptors"""
+    
+    def __init__(self):
+        self.executing = True
+        self.pending_requests: Dict[str, asyncio.Future] = {}
+        self.ipc_reader_task: Optional[asyncio.Task] = None
+        self.message_handlers: Dict[str, Callable] = {}
+        self.ipc_fd: Optional[int] = None
+        
+        # Get IPC file descriptor
+        if "NODE_CHANNEL_FD" in os.environ:
+            try:
+                self.ipc_fd = int(os.environ["NODE_CHANNEL_FD"])
+            except (ValueError, TypeError):
+                raise RuntimeError("Invalid NODE_CHANNEL_FD environment variable")
+        else:
+            raise RuntimeError("NODE_CHANNEL_FD environment variable not found")
+        
+    def send_no_wait(self, method: str, args: Any) -> None:
+        """Send IPC request without waiting for response"""
+        request = {
+            'type': 'rpc_request',
+            'method': method,
+            'args': args
+        }
+        
+        try:
+            json_str = json.dumps(request, default=serialize_for_json)
+            message_bytes = (json_str + "\n").encode('utf-8')
+            os.write(self.ipc_fd, message_bytes)
+        except Exception as e:
+            print(f"ERROR: Failed to send IPC request: {e}", file=sys.stderr)
+
+    async def send(self, method: str, args: Any) -> Any:
+        """Send IPC request and wait for response"""
+        request_id = str(uuid.uuid4())
+        future = asyncio.Future()
+        self.pending_requests[request_id] = future
+
+        request = {
+            'type': 'rpc_request',
+            'id': request_id,
+            'method': method,
+            'args': args
+        }
+        
+        try:
+            json_str = json.dumps(request, default=serialize_for_json)
+            message_bytes = (json_str + "\n").encode('utf-8')
+            os.write(self.ipc_fd, message_bytes)
+        except Exception as e:
+            future.set_exception(e)
+            return await future
+
+        return await future
+
+    def _handle_message(self, msg: Dict[str, Any]) -> None:
+        """Handle incoming message from Node.js"""
+        msg_type = msg.get('type')
+        
+        if msg_type == 'rpc_response':
+            request_id = msg.get('id')
+            if request_id in self.pending_requests:
+                future = self.pending_requests[request_id]
+                del self.pending_requests[request_id]
+                
+                error = msg.get('error')
+                if error is not None:
+                    future.set_exception(Exception(str(error)))
+                else:
+                    future.set_result(msg.get('result'))
+        
+        elif msg_type in self.message_handlers:
+            try:
+                self.message_handlers[msg_type](msg)
+            except Exception as e:
+                print(f"ERROR: Handler for {msg_type} failed: {e}", file=sys.stderr)
+
+    async def _read_ipc(self) -> None:
+        """Read messages from IPC file descriptor in background"""
+        loop = asyncio.get_event_loop()
+        buffer = ""
+        
+        while self.executing:
+            try:
+                data = await loop.run_in_executor(None, os.read, self.ipc_fd, 4096)
+                if not data:
+                    break
+                
+                buffer += data.decode('utf-8')
+                lines = buffer.split('\n')
+                buffer = lines[-1]  # Keep incomplete line in buffer
+                
+                for line in lines[:-1]:
+                    if line.strip():
+                        try:
+                            msg = json.loads(line.strip())
+                            self._handle_message(msg)
+                        except json.JSONDecodeError as e:
+                            print(f"WARNING: Failed to parse JSON: {e}", file=sys.stderr)
+                        
+            except OSError:
+                # IPC channel closed
+                break
+            except Exception as e:
+                print(f"ERROR: Reading IPC failed: {e}", file=sys.stderr)
+                await asyncio.sleep(0.1)
+
+    async def init(self) -> None:
+        """Initialize IPC communication"""
+        if not self.ipc_reader_task:
+            self.ipc_reader_task = asyncio.create_task(self._read_ipc())
+
+    def close(self) -> None:
+        """Close IPC communication"""
+        self.executing = False
+        
+        for future in self.pending_requests.values():
+            if not future.done():
+                future.set_exception(Exception("IPC connection closed"))
+        self.pending_requests.clear()
+        
+        if self.ipc_reader_task and not self.ipc_reader_task.done():
+            self.ipc_reader_task.cancel() 

package/dist/src/python/rpc.py

@@ -1,12 +1,13 @@
 import uuid
 import asyncio
-import os
 import json
 import sys
-from typing import Any, Dict, Tuple
-
-# get the FD from ENV
-NODEIPCFD = int(os.environ["NODE_CHANNEL_FD"])
+import os
+import platform
+from typing import Any, Dict, Optional, Callable, Union
+from communication_factory import create_communication
+from rpc_communication import RpcCommunication
+from ipc_communication import IpcCommunication
 
 def serialize_for_json(obj: Any) -> Any:
     """Convert Python objects to JSON-serializable types"""
@@ -19,87 +20,26 @@ def serialize_for_json(obj: Any) -> Any:
     elif isinstance(obj, dict):
         return {k: serialize_for_json(v) for k, v in obj.items()}
     else:
-        # Try to return the object as is, letting json handle basic types
         return obj
 
 class RpcSender:
+    """Unified communication interface that delegates to appropriate implementation"""
+    
     def __init__(self):
-        self.executing = True
-        self.pending_requests: Dict[str, Tuple[asyncio.Future, str, Any]] = {}
-
+        self._communication: Union[RpcCommunication, IpcCommunication] = create_communication()
+        
     def send_no_wait(self, method: str, args: Any) -> None:
-        request = {
-            'type': 'rpc_request',
-            'method': method,
-            'args': args
-        }
-        # encode message as json string + newline in bytes
-        bytesMessage = (json.dumps(request) + "\n").encode('utf-8')
-        # send message
-        os.write(NODEIPCFD, bytesMessage)
+        """Send request without waiting for response"""
+        return self._communication.send_no_wait(method, args)
 
-    async def send(self, method: str, args: Any) -> asyncio.Future[Any]:
-        future = asyncio.Future()
-        request_id = str(uuid.uuid4())
-        self.pending_requests[request_id] = future
+    async def send(self, method: str, args: Any) -> Any:
+        """Send request and wait for response"""
+        return await self._communication.send(method, args)
 
-        request = {
-            'type': 'rpc_request',
-            'id': request_id,
-            'method': method,
-            'args': args
-        }
-        # encode message as json string + newline in bytes
-        bytesMessage = (json.dumps(request) + "\n").encode('utf-8')
-        # send message
-        os.write(NODEIPCFD, bytesMessage)
-
-        return await future
-
-    async def init(self):
-        def on_message(msg: Dict[str, Any]):
-            if msg.get('type') == 'rpc_response':
-                request_id = msg['id']
-
-                if request_id in self.pending_requests:
-                    future = self.pending_requests[request_id]
-                    del self.pending_requests[request_id]
-                    
-                    if msg.get('error'):
-                        future.set_exception(msg['error'])
-                    elif msg.get('result'):
-                        future.set_result(msg['result'])
-                    else:
-                        # It's a void response
-                        future.set_result(None)
-            
-        # Read messages from Node IPC file descriptor
-        while self.executing:
-            # Use asyncio to read from fd without blocking
-            try:
-                message = await asyncio.get_event_loop().run_in_executor(
-                    None, lambda: os.read(NODEIPCFD, 4096).decode('utf-8')
-                )
-                if not message:
-                    await asyncio.sleep(0.01)  # Add small delay to prevent busy-waiting
-                    continue
-                    
-                # Parse messages (may be multiple due to buffering)
-                for line in message.splitlines():
-                    if line:
-                        try:
-                            msg = json.loads(line)
-                            on_message(msg)
-                        except json.JSONDecodeError:
-                            pass
-            except Exception as e:
-                print(f"Error reading from IPC: {e}", file=sys.stderr)
-                await asyncio.sleep(0.01)
+    async def init(self) -> None:
+        """Initialize communication"""
+        return await self._communication.init()
 
     def close(self) -> None:
-        self.executing = False
-        outstanding_requests = list(self.pending_requests.values())
-        
-        if len(outstanding_requests) > 0:
-            print("Process ended while there are some promises outstanding", file=sys.stderr)
-            sys.exit(1)
+        """Close communication"""
+        return self._communication.close()

package/dist/src/python/rpc_communication.py

@@ -0,0 +1,125 @@
+import uuid
+import asyncio
+import json
+import sys
+from typing import Any, Dict, Optional, Callable
+
+def serialize_for_json(obj: Any) -> Any:
+    """Convert Python objects to JSON-serializable types"""
+    if hasattr(obj, '__dict__'):
+        return obj.__dict__
+    elif hasattr(obj, '_asdict'):
+        return obj._asdict()
+    elif isinstance(obj, (list, tuple)):
+        return [serialize_for_json(item) for item in obj]
+    elif isinstance(obj, dict):
+        return {k: serialize_for_json(v) for k, v in obj.items()}
+    else:
+        return obj
+
+class RpcCommunication:
+    """RPC communication using stdin/stdout"""
+    
+    def __init__(self):
+        self.executing = True
+        self.pending_requests: Dict[str, asyncio.Future] = {}
+        self.stdin_reader_task: Optional[asyncio.Task] = None
+        self.message_handlers: Dict[str, Callable] = {}
+        
+    def send_no_wait(self, method: str, args: Any) -> None:
+        """Send RPC request without waiting for response"""
+        request = {
+            'type': 'rpc_request',
+            'method': method,
+            'args': args
+        }
+        
+        try:
+            json_str = json.dumps(request, default=serialize_for_json)
+            print(json_str, flush=True)
+        except Exception as e:
+            print(f"ERROR: Failed to send RPC request: {e}", file=sys.stderr)
+
+    async def send(self, method: str, args: Any) -> Any:
+        """Send RPC request and wait for response"""
+        request_id = str(uuid.uuid4())
+        future = asyncio.Future()
+        self.pending_requests[request_id] = future
+
+        request = {
+            'type': 'rpc_request',
+            'id': request_id,
+            'method': method,
+            'args': args
+        }
+        
+        try:
+            json_str = json.dumps(request, default=serialize_for_json)
+            print(json_str, flush=True)
+        except Exception as e:
+            future.set_exception(e)
+            return await future
+
+        return await future
+
+    def _handle_message(self, msg: Dict[str, Any]) -> None:
+        """Handle incoming message from Node.js"""
+        msg_type = msg.get('type')
+        
+        if msg_type == 'rpc_response':
+            request_id = msg.get('id')
+            if request_id in self.pending_requests:
+                future = self.pending_requests[request_id]
+                del self.pending_requests[request_id]
+                
+                error = msg.get('error')
+                if error is not None:
+                    future.set_exception(Exception(str(error)))
+                else:
+                    future.set_result(msg.get('result'))
+        
+        elif msg_type in self.message_handlers:
+            try:
+                self.message_handlers[msg_type](msg)
+            except Exception as e:
+                print(f"ERROR: Handler for {msg_type} failed: {e}", file=sys.stderr)
+
+    async def _read_stdin(self) -> None:
+        """Read messages from stdin in background"""
+        loop = asyncio.get_event_loop()
+        
+        while self.executing:
+            try:
+                line = await loop.run_in_executor(None, sys.stdin.readline)
+                
+                if not line:
+                    break
+                    
+                line = line.strip()
+                if line:
+                    try:
+                        msg = json.loads(line)
+                        self._handle_message(msg)
+                    except json.JSONDecodeError as e:
+                        print(f"WARNING: Failed to parse JSON: {e}", file=sys.stderr)
+                        
+            except Exception as e:
+                print(f"ERROR: Reading stdin failed: {e}", file=sys.stderr)
+                await asyncio.sleep(0.1)
+
+    async def init(self) -> None:
+        """Initialize RPC communication"""
+        if not self.stdin_reader_task:
+            self.stdin_reader_task = asyncio.create_task(self._read_stdin())
+
+    def close(self) -> None:
+        """Close RPC communication"""
+        self.executing = False
+        
+        for future in self.pending_requests.values():
+            if not future.done():
+                future.set_exception(Exception("RPC connection closed"))
+        self.pending_requests.clear()
+        
+        if self.stdin_reader_task and not self.stdin_reader_task.done():
+            self.stdin_reader_task.cancel() 

package/dist/src/ruby/rpc.rb

@@ -2,16 +2,12 @@ require 'securerandom'
 require 'json'
 require 'thread'
 
-# Get the file descriptor from ENV
-NODE_CHANNEL_FD = ENV['NODE_CHANNEL_FD'].to_i
-
 class RpcSender
   def initialize
     @closed = false
-    @writer = IO.new(NODE_CHANNEL_FD, 'w')
-    @reader = IO.new(NODE_CHANNEL_FD, 'r')
     @pending_requests = {}
     @mutex = Mutex.new
+    @background_thread = nil
   end
 
   def send(method, args)
@@ -25,46 +21,58 @@ class RpcSender
     end
 
     data = { type: 'rpc_request', id: id, method: method, args: args }
-    message = (JSON.dump(data) + "\n").encode('utf-8')
+    message = JSON.dump(data)
 
-    @writer.write(message)
-    @writer.flush
+    puts message
+    STDOUT.flush
 
     result = promise.pop
     raise result if result.is_a?(StandardError)
     result
+  rescue IOError, Errno::EPIPE
+    # Handle broken pipe errors gracefully
+    @closed = true
+    raise StandardError.new("RPC connection lost")
   end
 
   def send_no_wait(method, args)
     return if @closed
 
     data = { type: 'rpc_request', method: method, args: args }
-    message = (JSON.dump(data) + "\n").encode('utf-8')
+    message = JSON.dump(data)
 
-    @writer.write(message)
-    @writer.flush
-  rescue IOError, Errno::EBADF
-    # Ignore errors during shutdown
+    puts message
+    STDOUT.flush
+  rescue IOError, Errno::EPIPE
+    # Ignore errors during shutdown or if connection is lost
+    @closed = true
   end
 
   def init
     @background_thread = Thread.new do
       until @closed
         begin
-          line = @reader.gets
-          if line
-            msg = JSON.parse(line)
+          line = STDIN.gets
+          if line && !line.strip.empty?
+            msg = JSON.parse(line.strip)
             handle_response(msg) if msg['type'] == 'rpc_response'
+          elsif line.nil?
+            # EOF received, connection closed
+            break
           end
+        rescue JSON::ParserError => e
+          # Log JSON parsing errors but continue
+          STDERR.puts "Warning: Failed to parse JSON: #{e.message}"
+          STDERR.puts "Raw line: #{line}"
         rescue IOError, Errno::EBADF
           break if @closed
         rescue => e
-          puts "Error in RPC thread: #{e.message}"
+          STDERR.puts "Error in RPC thread: #{e.message}"
         end
       end
     end
     
-    @background_thread.abort_on_exception = true
+    @background_thread.abort_on_exception = false
   end
 
   def handle_response(msg)
@@ -102,22 +110,9 @@ class RpcSender
     # Check for pending requests
     @mutex.synchronize do
       if @pending_requests.any?
-        puts 'Process ended while there are some promises outstanding'
+        STDERR.puts 'Process ended while there are some promises outstanding'
         exit(1)
       end
     end
-
-    # Close file descriptors
-    begin
-      @writer.close unless @writer.closed?
-    rescue
-      # Ignore errors during shutdown
-    end
-
-    begin
-      @reader.close unless @reader.closed?
-    rescue
-      # Ignore errors during shutdown
-    end
   end
 end

package/dist/src/server.js

@@ -32,7 +32,7 @@ const createServer = async (lockedData, eventManager, state, config) => {
             const streams = lockedData.getStreams();
             const stream = streams[streamName];
             if (stream) {
-                const result = await stream(state).get(id);
+                const result = await stream().get(id);
                 delete result.__motia; // deleting because we don't need it in the socket
                 return result;
             }
@@ -41,15 +41,15 @@ const createServer = async (lockedData, eventManager, state, config) => {
             const streams = lockedData.getStreams();
             const stream = streams[streamName];
             if (stream) {
-                const result = stream ? await stream(state).getList(groupId) : [];
+                const result = stream ? await stream().getList(groupId) : [];
                 // eslint-disable-next-line @typescript-eslint/no-unused-vars
                 return result.map(({ __motia, ...rest }) => rest);
             }
         },
     });
-    lockedData.applyStreamWrapper((streamName, stream) => {
-        return (state) => {
-            const suuper = stream(state);
+    lockedData.applyStreamWrapper(state, (streamName, stream) => {
+        return () => {
+            const suuper = stream();
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const wrapObject = (id, object) => ({
                 ...object,
@@ -114,7 +114,7 @@ const createServer = async (lockedData, eventManager, state, config) => {
             baseConfig: { storageType: 'custom', factory: () => new logs_stream_1.LogsStream() },
             schema: null,
         },
-    })(state);
+    })();
     const allSteps = [...steps_1.systemSteps, ...lockedData.activeSteps];
     const loggerFactory = new logger_factory_1.LoggerFactory(config.isVerbose, logStream);
     const cronManager = (0, cron_handler_1.setupCronHandlers)(lockedData, eventManager, state, loggerFactory);
@@ -199,8 +199,8 @@ const createServer = async (lockedData, eventManager, state, config) => {
     allSteps.filter(guards_1.isApiStep).forEach(addRoute);
     app.use((0, cors_1.default)());
     app.use(router);
-    (0, api_endpoints_1.apiEndpoints)(lockedData, state);
-    (0, flows_endpoint_1.flowsEndpoint)(lockedData, app, state);
+    (0, api_endpoints_1.apiEndpoints)(lockedData);
+    (0, flows_endpoint_1.flowsEndpoint)(lockedData, app);
     (0, flows_config_endpoint_1.flowsConfigEndpoint)(app, process.cwd());
     server.on('error', (error) => {
         console.error('Server error:', error);

package/dist/src/state-stream.d.ts

@@ -1,13 +1,23 @@
-import { InternalStateManager, IStateStream, BaseStateStreamData } from './types';
-export type StateStreamFactory<TData extends BaseStateStreamData> = (state: InternalStateManager) => IStateStream<TData>;
-export declare class StateStream<TData extends BaseStateStreamData> implements IStateStream<TData> {
+import type { InternalStateManager } from './types';
+import type { BaseStreamItem, IStateStream, StateStreamEvent, StateStreamEventChannel } from './types-stream';
+export type StateStreamFactory<TData> = () => IStateStream<TData>;
+export declare abstract class StateStream<TData> implements IStateStream<TData> {
+    abstract get(id: string): Promise<BaseStreamItem<TData> | null>;
+    abstract update(id: string, data: TData): Promise<BaseStreamItem<TData> | null>;
+    abstract delete(id: string): Promise<BaseStreamItem<TData> | null>;
+    abstract create(id: string, data: TData): Promise<BaseStreamItem<TData>>;
+    abstract getGroupId(data: TData): string | null;
+    abstract getList(groupId: string): Promise<BaseStreamItem<TData>[]>;
+    send<T>(channel: StateStreamEventChannel, event: StateStreamEvent<T>): Promise<void>;
+}
+export declare class InternalStateStream<TData> extends StateStream<TData> {
     private readonly state;
     private readonly propertyName;
     constructor(state: InternalStateManager, propertyName: string);
-    get(id: string): Promise<TData | null>;
-    update(id: string, data: TData): Promise<TData>;
-    delete(id: string): Promise<TData | null>;
-    create(id: string, data: TData): Promise<TData>;
+    get(id: string): Promise<BaseStreamItem<TData> | null>;
+    update(id: string, data: TData): Promise<BaseStreamItem<TData> | null>;
+    delete(id: string): Promise<BaseStreamItem<TData> | null>;
+    create(id: string, data: TData): Promise<BaseStreamItem<TData>>;
     getGroupId(): null;
     getList(): Promise<never[]>;
     send(): Promise<void>;