@topolo/cli 0.12.32 → 0.13.0

package/agent-packs/src/topolo_agent_pack/live.py

@@ -0,0 +1,362 @@
+"""Owned desktop backend proxy and same-window reconnect adapter.
+
+The proxy replaces only its own idle native child. It never signals the desktop
+or another Codex profile. The renderer bridge is available only in the managed
+profile explicitly launched with local debugging enabled.
+"""
+import asyncio
+import base64
+import hashlib
+import json
+import os
+import pathlib
+import secrets
+import signal
+import re
+import sqlite3
+import socket
+import struct
+import subprocess
+import sys
+import time
+import urllib.request
+import urllib.parse
+import uuid
+
+from .storage import Library,PackError,atomic_json,canonical,read_json
+
+def socket_path(lib):
+    directory=pathlib.Path('/tmp')/('topolo-agent-pack-'+hashlib.sha256(str(lib.root).encode()).hexdigest()[:16])
+    if directory.exists() and (directory.is_symlink() or directory.stat().st_uid!=os.getuid()): raise PackError('Unowned live bridge directory')
+    directory.mkdir(mode=0o700,exist_ok=True)
+    return directory/'control.sock'
+
+def request(lib,operation):
+    record=read_json(lib.root/'live-bridge.json',{})
+    if not record: raise PackError('This desktop profile has no live bridge. Open it through topolo-agent-pack first.')
+    with socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) as connection:
+        connection.settimeout(3 if operation=='status' else 20)
+        try: connection.connect(record['socket'])
+        except OSError as exc: raise PackError('Managed live bridge is unavailable: '+str(exc)) from exc
+        connection.sendall(canonical({'token':record['token'],'operation':operation})+b'\n')
+        with connection.makefile('rb') as stream:
+            line=stream.readline(1024*1024)
+    if not line: raise PackError('Live bridge closed without a reply')
+    reply=json.loads(line)
+    if 'error' in reply: raise PackError(reply['error'])
+    return reply
+
+def available(lib):
+    try: return request(lib,'status')
+    except (PackError,OSError,ValueError): return None
+
+def quiesce(lib,timeout=10):
+    deadline=time.monotonic()+timeout
+    while True:
+        state=request(lib,'status')
+        if state.get('activeTurns'):raise PackError('Outgoing agents have active turns. Wait for their current turns; Codex can stay open.')
+        if not state.get('pendingRequests'):
+            try:return request(lib,'quiesce')
+            except PackError as exc:
+                if 'in flight' not in str(exc):raise
+        if time.monotonic()>=deadline:raise PackError('Backend requests have not settled; no switch committed. Retry with Codex still open.')
+        time.sleep(0.05)
+
+class BackendProxy:
+    def __init__(self,lib,real,args):
+        self.lib=lib;self.real=real;self.args=args;self.child=None
+        self.pending=set();self.active_turns=set();self.initialized=False
+        self.quiesced=False;self.output_task=None;self.closed=False
+        self.token=secrets.token_hex(32);self.path=socket_path(lib)
+
+    async def start_child(self):
+        active=self.lib.active()
+        if not active.get('runtime'): raise PackError('No selected managed runtime')
+        self.runtime=active['runtime'];env=os.environ.copy()
+        env['CODEX_HOME']=str(pathlib.Path(self.runtime)/'home');env['CODEX_CLI_PATH']=self.real
+        self.child=await asyncio.create_subprocess_exec(self.real,*self.args,env=env,stdin=asyncio.subprocess.PIPE,stdout=asyncio.subprocess.PIPE,limit=64*1024*1024)
+        self.output_task=asyncio.create_task(self.output())
+
+    async def output(self):
+        while line:=await self.child.stdout.readline():
+            value=json.loads(line)
+            if 'id' in value and ('result' in value or 'error' in value):
+                self.pending.discard(str(value['id']))
+                if 'result' in value and isinstance(value['result'],dict) and 'userAgent' in value['result']: self.initialized=True
+            method=value.get('method');params=value.get('params') or {}
+            if method=='turn/started': self.active_turns.add(params['threadId'])
+            if method=='turn/completed': self.active_turns.discard(params['threadId'])
+            if method=='thread/status/changed':
+                if params.get('status',{}).get('type')=='active':self.active_turns.add(params['threadId'])
+                elif params.get('status',{}).get('type') in {'idle','notLoaded','systemError'}:self.active_turns.discard(params['threadId'])
+            sys.stdout.buffer.write(line);sys.stdout.buffer.flush()
+
+    async def stop_child(self):
+        if self.child and self.child.returncode is None:
+            self.child.terminate()
+            try: await asyncio.wait_for(self.child.wait(),5)
+            except asyncio.TimeoutError:
+                # Never escalate to killing a child whose stop was not confirmed.
+                raise PackError('Owned backend did not stop; no storage switch committed')
+        if self.output_task: await self.output_task
+
+    def status(self):
+        return {'pid':os.getpid(),'childPid':self.child.pid if self.child else None,'desktopPid':os.getppid(),'runtime':self.runtime,
+                'quiesced':self.quiesced,'initialized':self.initialized,
+                'activeTurns':sorted(self.active_turns),'pendingRequests':len(self.pending)}
+
+    async def control(self,reader,writer):
+        try:
+            message=json.loads(await asyncio.wait_for(reader.readline(),5))
+            if not secrets.compare_digest(message.get('token',''),self.token): raise PackError('Live bridge authentication failed')
+            op=message.get('operation')
+            if op=='quiesce':
+                if self.active_turns: raise PackError('Outgoing agents have active turns. Wait for their current turns; Codex can stay open.')
+                if self.pending: raise PackError('Backend requests are in flight. Retry the switch; Codex can stay open.')
+                self.quiesced=True
+                await self.stop_child()
+            elif op=='resume':
+                if self.quiesced: raise PackError('Use the desktop reconnect action to resume a quiesced bridge')
+            elif op!='status': raise PackError('Unknown live bridge operation')
+            result=self.status()
+        except Exception as exc: result={'error':str(exc)}
+        writer.write(canonical(result)+b'\n');await writer.drain();writer.close();await writer.wait_closed()
+
+    async def run(self):
+        task=asyncio.current_task();loop=asyncio.get_running_loop()
+        loop.add_signal_handler(signal.SIGTERM,task.cancel)
+        loop.add_signal_handler(signal.SIGINT,task.cancel)
+        if self.path.exists():
+            previous=available(self.lib)
+            if previous: raise PackError('A managed desktop already owns this bridge')
+            if not self.path.is_socket() or self.path.stat().st_uid!=os.getuid(): raise PackError('Unowned socket path')
+            self.path.unlink()
+        server=await asyncio.start_unix_server(self.control,path=str(self.path),limit=1024*1024)
+        self.path.chmod(0o600)
+        atomic_json(self.lib.root/'live-bridge.json',{'pid':os.getpid(),'socket':str(self.path),'token':self.token})
+        try:
+            await self.start_child()
+            reader=asyncio.StreamReader(limit=64*1024*1024)
+            protocol=asyncio.StreamReaderProtocol(reader)
+            await asyncio.get_running_loop().connect_read_pipe(lambda:protocol,sys.stdin.buffer)
+            while line:=await reader.readline():
+                value=json.loads(line)
+                if self.quiesced:
+                    # No new model work may start between quiesce and reconnect.
+                    if 'id' in value:
+                        sys.stdout.buffer.write(canonical({'id':value['id'],'error':{'code':-32000,'message':'Pack switch in progress; reconnecting backend'}})+b'\n');sys.stdout.buffer.flush()
+                    continue
+                if 'id' in value and 'method' in value: self.pending.add(str(value['id']))
+                self.child.stdin.write(line);await self.child.stdin.drain()
+        finally:
+            await self.stop_child();server.close();await server.wait_closed()
+            record=read_json(self.lib.root/'live-bridge.json',{})
+            if record.get('pid')==os.getpid():
+                (self.lib.root/'live-bridge.json').unlink(missing_ok=True);self.path.unlink(missing_ok=True)
+
+def proxy_main():
+    lib=Library(os.environ['CODEX_PACK_HOME']);real=os.environ['CODEX_PACK_REAL_CLI']
+    args=sys.argv[1:]
+    if 'app-server' not in args: os.execv(real,[real,*args])
+    try: asyncio.run(BackendProxy(lib,real,args).run())
+    except asyncio.CancelledError: return 0
+    except Exception as exc:
+        print('topolo-agent-pack backend: '+str(exc),file=sys.stderr);return 2
+    return 0
+
+class WebSocket:
+    """Small masked RFC6455 client for the owned Electron debug endpoint."""
+    def __init__(self,url):
+        uri=urllib.parse.urlsplit(url)
+        if uri.scheme!='ws' or uri.hostname not in {'127.0.0.1','localhost'}: raise PackError('Debug endpoint must be local')
+        self.socket=socket.create_connection((uri.hostname,uri.port),timeout=20);self.buffer=b''
+        key=base64.b64encode(os.urandom(16)).decode()
+        headers=f'GET {uri.path} HTTP/1.1\r\nHost: {uri.hostname}:{uri.port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n'
+        self.socket.sendall(headers.encode())
+        while b'\r\n\r\n' not in self.buffer:
+            chunk=self.socket.recv(65536)
+            if not chunk:self.socket.close();raise PackError('Desktop debug handshake closed')
+            self.buffer+=chunk
+            if len(self.buffer)>65536:self.socket.close();raise PackError('Oversized debug handshake')
+        header,self.buffer=self.buffer.split(b'\r\n\r\n',1)
+        accept=base64.b64encode(hashlib.sha1((key+'258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest())
+        if not header.startswith(b'HTTP/1.1 101') or accept not in header: self.socket.close();raise PackError('Debug WebSocket handshake rejected')
+        self.seq=0
+    def read(self,n):
+        while len(self.buffer)<n:
+            chunk=self.socket.recv(max(65536,n-len(self.buffer)))
+            if not chunk: raise PackError('Desktop debug connection closed')
+            self.buffer+=chunk
+        result,self.buffer=self.buffer[:n],self.buffer[n:];return result
+    def send(self,payload,opcode=1):
+        mask=os.urandom(4);size=len(payload)
+        prefix=bytes([0x80|opcode,0x80|size]) if size<126 else bytes([0x80|opcode,0x80|126])+struct.pack('!H',size) if size<65536 else bytes([0x80|opcode,0x80|127])+struct.pack('!Q',size)
+        self.socket.sendall(prefix+mask+bytes(v^mask[i%4] for i,v in enumerate(payload)))
+    def receive(self):
+        pieces=[]
+        while True:
+            a,b=self.read(2);size=b&127
+            if size==126:size=struct.unpack('!H',self.read(2))[0]
+            elif size==127:size=struct.unpack('!Q',self.read(8))[0]
+            if size>64*1024*1024: raise PackError('Oversized debug reply')
+            mask=self.read(4) if b&128 else None;payload=self.read(size)
+            if mask:payload=bytes(v^mask[i%4] for i,v in enumerate(payload))
+            op=a&15
+            if op==8: raise PackError('Desktop debug connection closed')
+            if op==9:self.send(payload,10);continue
+            if op==10:continue
+            pieces.append(payload)
+            if a&128:return json.loads(b''.join(pieces))
+    def evaluate(self,expression):
+        self.seq+=1;self.send(canonical({'id':self.seq,'method':'Runtime.evaluate','params':{'expression':expression,'awaitPromise':True,'returnByValue':True}}))
+        while True:
+            reply=self.receive()
+            if reply.get('id')!=self.seq:continue
+            if 'error' in reply or reply.get('result',{}).get('exceptionDetails'): raise PackError('Desktop bridge expression failed: '+str(reply.get('error') or reply['result']['exceptionDetails'].get('text')))
+            return reply['result']['result'].get('value')
+    def close(self):self.socket.close()
+
+def renderer(lib):
+    file=lib.root/'desktop-data'/'DevToolsActivePort'
+    if not file.exists(): raise PackError('Managed window has no reconnect bridge; launch it through the updated topolo-agent-pack command')
+    port=int(file.read_text().splitlines()[0])
+    opener=urllib.request.build_opener(urllib.request.ProxyHandler({}))
+    with opener.open('http://127.0.0.1:'+str(port)+'/json/list',timeout=5) as response:targets=json.load(response)
+    for target in targets:
+        if target['type']=='page' and target.get('url','').startswith(('app://','file://')):
+            ws=WebSocket(target['webSocketDebuggerUrl'])
+            if ws.evaluate("typeof electronBridge !== 'undefined' && !!document.querySelector('[aria-label=\"App navigation\"]') && !!document.querySelector('[aria-label^=\"Switch mode\"]')"):
+                return ws,target['id']
+            ws.close()
+    raise PackError('Managed primary Codex window not found')
+
+def fetch(ws,method,body):
+    request_id='topolo-agent-pack-'+uuid.uuid4().hex
+    message={'type':'fetch','requestId':request_id,'url':'vscode://codex/'+method,'method':'POST','body':json.dumps(body),'headers':{}}
+    expression='''new Promise((resolve,reject)=>{const id=ID;const timer=setTimeout(()=>{window.removeEventListener('message',handler);reject(new Error('Desktop state request timed out'))},10000);function handler(event){const value=event.data;if(value?.type!=='fetch-response'||value.requestId!==id)return;clearTimeout(timer);window.removeEventListener('message',handler);if(value.responseType==='error')reject(new Error(value.error));else resolve(JSON.parse(value.bodyJsonString))}window.addEventListener('message',handler);electronBridge.sendMessageFromView(MESSAGE).catch(reject)})'''.replace('ID',json.dumps(request_id),1).replace('MESSAGE',json.dumps(message),1)
+    return ws.evaluate(expression)
+
+def adapter(app='/Applications/ChatGPT.app'):
+    # Resolve the app's exported host capability from its installed assets.
+    # Refuse changed layouts instead of guessing a private versioned symbol.
+    with (pathlib.Path(app)/'Contents/Resources/app.asar').open('rb') as stream:
+        header=struct.unpack('<4I',stream.read(16));tree=json.loads(stream.read(header[3]));base=8+header[1]
+        assets=tree['files']['webview']['files']['assets']['files']
+        for name,entry in assets.items():
+            if not name.startswith('app-initial-') or not name.endswith('.js'):continue
+            stream.seek(base+int(entry['offset']));source=stream.read(entry['size']).decode()
+            match=re.search(r'let ([\w$]+)=([\w$]+)\?\.localThreadCatalog',source)
+            if not match:continue
+            symbol=match[2]
+            for clause in re.finditer(r'import\{([^;]+?)\}from"([^\"]+)";',source[:150000]):
+                export=re.search(r'(?:^|,)([\w$]+) as '+re.escape(symbol)+r'(?:,|$)',clause[1])
+                if export:return {'module':'/assets/'+clause[2].removeprefix('./'),'export':export[1]}
+    raise PackError('This desktop build has no recognized catalog bridge; no switch started')
+
+def thread_ids(runtime):
+    connection=sqlite3.connect('file:'+str(pathlib.Path(runtime)/'home/state_5.sqlite')+'?mode=ro',uri=True)
+    try:return [r[0] for r in connection.execute('SELECT id FROM threads WHERE archived=0')]
+    finally:connection.close()
+
+def rendered_threads(ws):
+    return ws.evaluate('Array.from(document.querySelectorAll("[data-app-action-sidebar-thread-row][data-app-action-sidebar-thread-host-id=local]")).map(e=>({id:e.getAttribute("data-app-action-sidebar-thread-id").replace(/^local:/,""),title:e.getAttribute("data-app-action-sidebar-thread-title")}))')
+
+def checkpoint_ui(lib,runtime):
+    ws,_=renderer(lib)
+    try:
+        path=pathlib.Path(runtime)/'home/.codex-global-state.json';gs=read_json(path)
+        for key in list(gs):
+            value=fetch(ws,'get-global-state',{'key':key}).get('value')
+            if key=='electron-persisted-atom-state':
+                value={k:(value or {}).get(k,v) for k,v in gs[key].items()}
+            if value is not None:gs[key]=value
+        atomic_json(path,gs)
+        selected=ws.evaluate('document.querySelector("[data-app-action-sidebar-thread-row][data-app-action-sidebar-thread-selected=true]")?.getAttribute("data-app-action-sidebar-thread-id")?.replace(/^local:/,"") ?? null')
+        atomic_json(pathlib.Path(runtime)/'desktop-view.json',{'threadId':selected})
+    finally:ws.close()
+
+def reconcile(ws,capability,incoming,outgoing):
+    params=json.dumps({'adapter':capability,'incoming':incoming,'outgoing':outgoing})
+    return ws.evaluate('''(async()=>{const p=PARAMS;const c=(await import(p.adapter.module))[p.adapter.export]?.localThreadCatalog;if(!c)throw new Error('Catalog capability unavailable');for(const threadId of p.outgoing)await c.removeMissingEntry({hostId:'local',threadId});for(const threadId of p.incoming)await c.notifyThread({hostId:'local',threadId},'upsert');await c.invalidateSource('local');await c.requestSync(['local'],'immediate');let proof;const deadline=Date.now()+15000;do{const entries=await c.readEntries(p.incoming.map(threadId=>({hostId:'local',threadId})));const missing=await c.readEntries(p.outgoing.map(threadId=>({hostId:'local',threadId})));proof={incoming:entries.map(e=>e.threadId),titles:entries.map(e=>e.displayTitle),outgoing:missing.map(e=>e.threadId),count:await c.readThreadCount('local')};if(proof.incoming.length===p.incoming.length&&!proof.outgoing.length&&proof.count===p.incoming.length)return proof;await new Promise(r=>setTimeout(r,50))}while(Date.now()<deadline);return proof})()'''.replace('PARAMS',params,1))
+
+def reconnect(lib,runtime,previous_runtime=None,app='/Applications/ChatGPT.app'):
+    ws,target_id=renderer(lib)
+    try:
+        current_bridge=available(lib)
+        if current_bridge and not current_bridge['quiesced']:quiesce(lib)
+        capability=adapter(app);bind(lib,runtime)
+        gs=read_json(pathlib.Path(runtime)/'home'/'.codex-global-state.json')
+        current=fetch(ws,'get-global-state',{'key':'electron-persisted-atom-state'}).get('value') or {}
+        updates={**gs,'electron-persisted-atom-state':{**current,**gs.get('electron-persisted-atom-state',{})}}
+        for key,value in updates.items():
+            result=fetch(ws,'set-global-state',{'key':key,'value':value})
+            if not result.get('success'): raise PackError('Desktop refused state update: '+key)
+        ws.evaluate('electronBridge.sendMessageFromView({type:"codex-app-server-restart",hostId:"local",intent:"reconnect",killCodexProcess:false}).then(()=>true)')
+        incoming=thread_ids(runtime);outgoing=sorted(set(thread_ids(previous_runtime) if previous_runtime else [])-set(incoming))
+        deadline=time.monotonic()+30
+        while time.monotonic()<deadline:
+            bridge=available(lib)
+            if bridge and not bridge['quiesced'] and bridge['runtime']==str(runtime) and bridge['initialized']:break
+            time.sleep(0.05)
+        else:raise PackError('Managed backend did not reconnect; run topolo-agent-pack recover')
+        proof=reconcile(ws,capability,incoming,outgoing)
+        if set(proof['incoming'])!=set(incoming) or proof['outgoing'] or proof['count']!=len(incoming):raise PackError('Sidebar catalog did not match selected runtime; run recover')
+        # Soft reload clears thread/composer caches while retaining this exact
+        # renderer target and native window. No window or desktop process quits.
+        view=read_json(pathlib.Path(runtime)/'desktop-view.json',{})
+        selected=view.get('threadId') if view.get('threadId') in incoming else None
+        navigation={'path':'/local/'+selected if selected else '/','replace':True,'persistForReload':True}
+        ws.evaluate('(async()=>{const h=(await import('+json.dumps(capability['module'])+'))['+json.dumps(capability['export'])+'];const ok=await h.windowNavigation.navigateInOwnerWindow('+json.dumps(navigation)+');if(!ok)throw new Error("Owner window navigation unavailable");return true})()')
+        ws.evaluate('setTimeout(()=>location.reload(),0);true')
+        ws.close()
+        sidebar=[]
+        while time.monotonic()<deadline:
+            try:
+                ws,confirmed=renderer(lib)
+                if confirmed!=target_id:raise PackError('Desktop switched renderer targets instead of retaining the window')
+                rows=rendered_threads(ws);sidebar=[r['title'] for r in rows]
+                text=ws.evaluate('document.body.innerText')
+                shown={r['id'] for r in rows}
+                if text and shown and shown<=set(incoming) and not shown & set(outgoing):break
+                ws.close();time.sleep(0.05)
+            except (OSError,PackError):time.sleep(0.05)
+        else:raise PackError('Desktop did not render after reconnect; run recover')
+        return {'state':'same-window-verified','app':str(app),'targetId':target_id,'desktopPid':bridge['desktopPid'],'frontendVerified':True,'windowClosed':False,'catalog':proof,'renderedTitles':sidebar,'renderedThreadIds':sorted(shown),'selectedThreadId':selected}
+    finally:ws.close()
+
+def prepare(lib,real):
+    home=lib.root/'desktop-home'
+    if home.exists() and not (lib.root/'desktop-layout.json').exists():raise PackError('Existing desktop-home is unowned; choose a separate pack library')
+    folder=lib.root/'launchers';folder.mkdir(exist_ok=True,mode=0o700)
+    executable=folder/'topolo-agent-pack-backend'
+    source=pathlib.Path(__file__).resolve().parents[1]
+    executable.write_text('#!'+sys.executable+'\nimport sys\nsys.path.insert(0,'+repr(str(source))+')\nfrom topolo_agent_pack.live import proxy_main\nraise SystemExit(proxy_main())\n');executable.chmod(0o700)
+    home.mkdir(exist_ok=True,mode=0o700)
+    atomic_json(lib.root/'desktop-layout.json',{'owner':'topolo-agent-pack','schemaVersion':1})
+    bind(lib,lib.active()['runtime'])
+    if not (home/'.codex-global-state.json').exists():atomic_json(home/'.codex-global-state.json',read_json(pathlib.Path(lib.active()['runtime'])/'home/.codex-global-state.json'))
+    return executable,home
+
+def bind(lib,runtime):
+    if not (lib.root/'desktop-layout.json').exists():return
+    if read_json(lib.root/'desktop-layout.json').get('owner') not in {'topolo-agent-pack','codex-pack'}:raise PackError('Unowned desktop layout')
+    target=lib.root/'selected-runtime';temporary=lib.root/('.selected-'+uuid.uuid4().hex)
+    if target.exists() and not target.is_symlink():raise PackError('Selected-runtime path is unowned')
+    temporary.symlink_to(pathlib.Path(runtime));os.replace(temporary,target)
+    home=lib.root/'desktop-home'
+    paths=['state_5.sqlite','thread_history_1.sqlite','goals_1.sqlite','auth.json','config.toml','AGENTS.md','sessions','archived_sessions','skills','agents','worktrees']
+    for name in ['sessions','archived_sessions','skills','agents','worktrees']:
+        (pathlib.Path(runtime)/'home'/name).mkdir(exist_ok=True,mode=0o700)
+    for name in paths:
+        dest=home/name
+        expected=pathlib.Path('..')/'selected-runtime'/'home'/name
+        if dest.is_symlink():
+            if os.readlink(dest)!=str(expected):raise PackError('Unowned desktop route: '+name)
+            continue
+        if dest.exists():
+            # Earlier manager prototypes used ordinary control-home files.
+            # Keep them for inspection when migrating to the routed home.
+            backup=home/('legacy-'+name)
+            if backup.exists():raise PackError('Cannot preserve existing desktop-home file: '+name)
+            dest.rename(backup)
+        dest.symlink_to(expected)