@atmem/architect 0.0.0-stage → 0.2.0

package/server/codex_engineering_backend.py

@@ -0,0 +1,266 @@
+"""Codex adapter for portable engineering sessions; no public/browser endpoint.
+
+Workspace registrations and scope originate from the worker's trusted control plane.
+The native sandbox constrains writes/network, but this adapter alone is NOT a hosted
+multi-tenant isolation boundary. Hosted workers need separately qualified isolation.
+"""
+from pathlib import Path
+from .codex_app_server import CodexAppServer, ProtocolError
+from .engineering_sessions import BackendEvent, EmptySessionUnavailable, RecoveredStart, SessionError
+
+
+class CodexEngineeringBackend:
+    name='codex-app-server'
+
+    def __init__(self,workspaces,*,client_factory=CodexAppServer,instructions='',permission_profile=None,tools=None):
+        self.workspaces=dict(workspaces)
+        self.client_factory=client_factory
+        self.instructions=instructions
+        self.permission_profile=permission_profile
+        self.tools=tools
+        if permission_profile:
+            # The durable coordinator must refuse a legacy-policy fallback after
+            # restart, even though both implementations use the same model API.
+            self.name=f'codex-app-server:{permission_profile}'
+        self.clients={}
+
+    def _workspace(self,scope,workspace):
+        registered=self.workspaces.get((scope,workspace))
+        if registered is None:
+            raise SessionError('Workspace is not registered for this project')
+        root=Path(registered).resolve(strict=True)
+        if not root.is_dir():
+            raise SessionError('Registered workspace is unavailable')
+        return root
+
+    @staticmethod
+    def _policy(root):
+        return {'type':'workspaceWrite','writableRoots':[str(root)],'networkAccess':False,
+                'excludeSlashTmp':True,'excludeTmpdirEnvVar':True}
+
+    def _verify(self,result,root,selection):
+        policy=result.get('sandbox',{})
+        if result.get('cwd')!=str(root) or result.get('model')!=selection.model or result.get('reasoningEffort')!=selection.effort:
+            raise SessionError('Codex did not confirm the selected workspace, model and reasoning effort')
+        expected_network=bool(self.permission_profile)
+        if result.get('approvalPolicy')!='never' or policy.get('type')!='workspaceWrite' or policy.get('networkAccess') is not expected_network:
+            raise SessionError('Codex did not confirm the required local execution policy')
+        if policy.get('excludeSlashTmp') is not True or policy.get('excludeTmpdirEnvVar') is not True or set(policy.get('writableRoots',[]))-{str(root)}:
+            raise SessionError('Codex returned additional writable roots or temporary directories')
+        if self.permission_profile:
+            if result.get('activePermissionProfile',{}).get('id')!=self.permission_profile or result.get('runtimeWorkspaceRoots')!=[str(root)]:
+                raise SessionError('Codex did not confirm the selected worker profile and sole project root')
+            sources=result.get('instructionSources',[])
+            if not isinstance(sources,list) or any(not isinstance(source,str) or not Path(source).resolve().is_relative_to(root) for source in sources):
+                raise SessionError('The native worker inherited instructions outside this project workspace')
+        handle=result.get('thread',{}).get('id')
+        if not isinstance(handle,str) or not handle:
+            raise SessionError('Codex did not confirm a thread')
+        return handle
+
+    def _params(self,root,selection):
+        return {'cwd':str(root),'model':selection.model,'approvalPolicy':'never',
+                **({'permissions':self.permission_profile,'config':{'model_reasoning_effort':selection.effort}}
+                   if self.permission_profile else {'sandbox':'workspace-write','config':{
+                    'model_reasoning_effort':selection.effort,
+                    'sandbox_workspace_write.network_access':False,
+                    'sandbox_workspace_write.exclude_slash_tmp':True,
+                    'sandbox_workspace_write.exclude_tmpdir_env_var':True}}),
+                **({'developerInstructions':self.instructions} if self.instructions else {})}
+
+    def _owned(self,scope,handle,selection=None):
+        row=self.clients.get((scope,handle))
+        if not row or (selection is not None and row[2]!=selection):
+            raise SessionError('Codex session is unavailable for this project or model selection')
+        return row
+
+    def _bind_tools(self,client,scope,workspace,handle):
+        if self.tools is None:return
+        async def invoke(params):
+            self._owned(scope,handle)
+            if not isinstance(params,dict) or set(params)-{'threadId','turnId','callId','tool','arguments','namespace'} or params.get('threadId')!=handle or params.get('namespace') is not None:
+                raise SessionError('Native tool callback belongs to another thread or namespace')
+            return await self.tools.invoke(scope,workspace,params.get('turnId'),params.get('callId'),params.get('tool'),params.get('arguments'))
+        client.dynamic_tool_handler=invoke
+
+    async def create(self,scope,workspace,selection):
+        root=self._workspace(scope,workspace)
+        client=self.client_factory(root)
+        try:
+            if self.tools is not None:client.experimental=True
+            await client.start()
+            result=await client.request('thread/start',{**self._params(root,selection),
+                **({'dynamicTools':self.tools.specs} if self.tools is not None else {})})
+            handle=self._verify(result,root,selection)
+            if self.permission_profile:
+                await client.verify_worker_tools(self.permission_profile)
+            if (scope,handle) in self.clients:
+                raise SessionError('Codex returned a previously registered thread')
+            self.clients[scope,handle]=(client,root,selection)
+            self._bind_tools(client,scope,workspace,handle)
+            return handle
+        except BaseException:
+            await client.close();raise
+
+    async def resume(self,scope,handle,workspace,selection):
+        root=self._workspace(scope,workspace)
+        if (scope,handle) in self.clients:
+            row=self._owned(scope,handle,selection)
+            if row[1]!=root:
+                raise SessionError('A session cannot change its registered workspace')
+            client=row[0]
+            process=getattr(client,'process',None)
+            if not getattr(client,'failure',None) and (process is None or process.returncode is None):
+                return
+            # A dead cached transport cannot recover itself. Reopen the same
+            # persisted native thread; never start a replacement for accepted work.
+            self.clients.pop((scope,handle))
+            await client.close()
+        client=self.client_factory(root)
+        try:
+            if self.tools is not None:client.experimental=True
+            await client.start()
+            # App-server resume reloads runtime configuration. Reassert the exact
+            # project model/effort and execution policy instead of inheriting the
+            # worker's current defaults, then verify its acknowledgement below.
+            params=self._params(root,selection)
+            result=await client.request('thread/resume',{
+                **params,'threadId':handle,'excludeTurns':True})
+            if self._verify(result,root,selection)!=handle:
+                raise SessionError('Codex resumed a different thread')
+            if self.permission_profile:
+                await client.verify_worker_tools(self.permission_profile)
+            self.clients[scope,handle]=(client,root,selection)
+            self._bind_tools(client,scope,workspace,handle)
+        except ProtocolError as error:
+            await client.close()
+            if error.category=='missing_thread':
+                raise EmptySessionUnavailable('Native thread has no stored history') from None
+            raise
+        except BaseException:
+            await client.close();raise
+
+    async def start(self,scope,handle,text,selection,*,request_id=None,context=None):
+        client,root,_=self._owned(scope,handle,selection)
+        inputs = self._inputs(scope, text, context)
+        result=await client.request('turn/start',{'threadId':handle,
+            'input':inputs,'cwd':str(root),
+            'model':selection.model,'effort':selection.effort,
+            'approvalPolicy':'never',
+            **({'permissions':self.permission_profile} if self.permission_profile else {'sandboxPolicy':self._policy(root)}),
+            **({'clientUserMessageId':request_id} if request_id is not None else {})})
+        turn=result.get('turn',{})
+        if turn.get('status')!='inProgress' or not isinstance(turn.get('id'),str) or not turn['id']:
+            raise SessionError('Codex did not confirm an active engineering turn')
+        return turn['id']
+
+    async def recover_start(self,scope,handle,request_id):
+        """Match durable client message identity; never infer acceptance from text.
+
+        This is acknowledgement recovery, not an idempotent resubmission promise.
+        An absent/ambiguous match leaves the reservation unresolved.
+        """
+        client,_,_=self._owned(scope,handle)
+        cursor=None;seen=set();matches=[]
+        for _ in range(20):
+            params={'threadId':handle,'limit':100,'sortDirection':'desc','itemsView':'summary'}
+            if cursor is not None:params['cursor']=cursor
+            result=await client.request('thread/turns/list',params)
+            turns=result.get('data')
+            if not isinstance(turns,list):raise SessionError('Native start history is unavailable')
+            for candidate in turns:
+                if not isinstance(candidate,dict) or not isinstance(candidate.get('items'),list):
+                    raise SessionError('Native start history is malformed')
+                for item in candidate['items']:
+                    if not isinstance(item,dict):raise SessionError('Native message identity is malformed')
+                    if item.get('type')=='userMessage' and item.get('clientId')==request_id:
+                        status=candidate.get('status')
+                        matches.append(RecoveredStart(request_id,candidate.get('id'),'running' if status=='inProgress' else status))
+            cursor=result.get('nextCursor')
+            if cursor is None:
+                if len(matches)>1:raise SessionError('Native start identity is ambiguous; no work was replayed')
+                return matches[0] if matches else None
+            if not isinstance(cursor,str) or not cursor or cursor in seen:
+                raise SessionError('Native start history cursor is invalid')
+            seen.add(cursor)
+        raise SessionError('Native start history exceeded the recovery bound')
+
+    async def observe(self,scope,handle,turn):
+        """Recover terminal state without resubmitting work or loading its content.
+
+        Missing, malformed or excessively paginated history is uncertain, never
+        an idle session. The control-plane reservation remains in place.
+        """
+        client,_,_=self._owned(scope,handle)
+        cursor=None
+        seen=set()
+        for _ in range(20):
+            params={'threadId':handle,'limit':100,'sortDirection':'desc','itemsView':'notLoaded'}
+            if cursor is not None:
+                params['cursor']=cursor
+            result=await client.request('thread/turns/list',params)
+            turns=result.get('data')
+            if not isinstance(turns,list):
+                raise SessionError('Codex turn history is unavailable; recovery is required')
+            for candidate in turns:
+                if not isinstance(candidate,dict):
+                    raise SessionError('Codex returned malformed turn history')
+                if candidate.get('id')==turn:
+                    status=candidate.get('status')
+                    if status=='inProgress':
+                        return None
+                    if status in {'completed','failed','interrupted'}:
+                        return BackendEvent(turn,status)
+                    raise SessionError('Codex returned an unknown recovered turn status')
+            cursor=result.get('nextCursor')
+            if cursor is None:
+                break
+            if not isinstance(cursor,str) or not cursor or cursor in seen:
+                raise SessionError('Codex returned an invalid turn-history cursor')
+            seen.add(cursor)
+        raise SessionError('The owned turn was not found in bounded history; recovery is required')
+
+    @staticmethod
+    def _inputs(scope, text, context):
+        inputs = [{'type':'text','text':text}]
+        if context is not None:
+            from .engineering_contexts import context_input
+            if context.get('projectId') != scope.project:
+                raise SessionError('Project context does not belong to this engineering session')
+            inputs.append(context_input(context))
+        return inputs
+
+    async def steer(self,scope,handle,turn,text,*,request_id=None,context=None):
+        client,_,_=self._owned(scope,handle)
+        await client.request('turn/steer',{'threadId':handle,'expectedTurnId':turn,
+                                         'input':self._inputs(scope, text, context),
+                                         **({'clientUserMessageId':request_id} if request_id is not None else {})})
+
+    async def interrupt(self,scope,handle,turn):
+        client,_,_=self._owned(scope,handle)
+        await client.request('turn/interrupt',{'threadId':handle,'turnId':turn})
+
+    async def next_event(self,scope,handle):
+        client,_,_=self._owned(scope,handle)
+        while True:
+            message=await client.notification()
+            method=message.get('method');params=message.get('params',{})
+            if params.get('threadId') not in (None,handle):
+                raise SessionError('Codex notification belongs to another thread')
+            if method=='turn/completed':
+                turn=params.get('turn',{});status=turn.get('status')
+                if status not in {'completed','failed','interrupted'}:
+                    raise SessionError('Codex returned an unknown terminal turn status')
+                return BackendEvent(turn.get('id'),status)
+            if method=='item/agentMessage/delta':
+                return BackendEvent(params.get('turnId'),'message',params.get('delta',''))
+            if method in {'item/started','item/completed'}:
+                item=params.get('item',{})
+                if params.get('turnId'):
+                    # Tool summaries are progress, never successful validation.
+                    return BackendEvent(params['turnId'],'progress',str(item.get('type','work')))
+
+    async def close(self):
+        rows=list(self.clients.values());self.clients.clear()
+        for client,_,_ in rows:
+            await client.close()

package/server/codex_worker_policy.py

@@ -0,0 +1,193 @@
+"""Trusted local worker policy, separate from browser input and model selection.
+
+This constrains native local commands, not a hosted OS/tenant boundary. Only
+control-plane selected tool directories may be added as read roots. Do not pass
+client paths, source repositories or credential directories here.
+"""
+import json
+import os
+import re
+from pathlib import Path
+import shutil
+import tomllib
+import platform
+import sys
+
+from .codex_app_server import CodexAppServer
+
+PROFILE = 'atmem-engineering-v2'
+
+
+def configured_mcp_names():
+    """Inspect names only; never copy credentials or server definitions.
+
+    An empty table override merges with the user's existing table in current
+    Codex versions. Explicitly disable known entries; the live worker check must
+    still reject any server introduced by another effective configuration layer.
+    """
+    names=set()
+    paths=[Path('/etc/codex/config.toml'),Path(os.getenv('CODEX_HOME',str(Path.home()/'.codex')))/'config.toml']
+    def collect(value):
+        if not isinstance(value,dict):return
+        servers=value.get('mcp_servers',{})
+        if isinstance(servers,dict):names.update(servers)
+        for nested in value.values():
+            if isinstance(nested,dict):collect(nested)
+    for path in paths:
+        if path.exists():
+            with path.open('rb') as source:collect(tomllib.load(source))
+    return sorted(names)
+
+
+def configured_shell_override_names():
+    """Return keys to blank in the worker; never forward their original values."""
+    names=set()
+    paths=[Path('/etc/codex/config.toml'),Path(os.getenv('CODEX_HOME',str(Path.home()/'.codex')))/'config.toml']
+    def collect(value):
+        if not isinstance(value,dict):return
+        policy=value.get('shell_environment_policy',{})
+        if isinstance(policy,dict) and isinstance(policy.get('set',{}),dict):names.update(policy.get('set',{}))
+        for nested in value.values():
+            if isinstance(nested,dict):collect(nested)
+    for path in paths:
+        if path.exists():
+            with path.open('rb') as source:collect(tomllib.load(source))
+    return sorted(names)
+
+
+def native_codex(executable):
+    """Resolve the official npm native binary so sandbox helpers see their code tree."""
+    selected=Path(executable).resolve(strict=True)
+    package=selected.parent.parent
+    if selected.suffix.lower()=='.cmd':
+        package=selected.parent/'node_modules/@openai/codex'
+    metadata=package/'package.json'
+    try:
+        if json.loads(metadata.read_text(encoding='utf-8')).get('name')!='@openai/codex':return selected,selected.parent
+    except (OSError,ValueError):return selected,selected.parent
+    arch={'x86_64':'x64','AMD64':'x64','aarch64':'arm64','arm64':'arm64'}.get(platform.machine())
+    targets={('linux','x64'):('linux-x64','x86_64-unknown-linux-musl'),('linux','arm64'):('linux-arm64','aarch64-unknown-linux-musl'),('darwin','x64'):('darwin-x64','x86_64-apple-darwin'),('darwin','arm64'):('darwin-arm64','aarch64-apple-darwin'),('win32','x64'):('win32-x64','x86_64-pc-windows-msvc'),('win32','arm64'):('win32-arm64','aarch64-pc-windows-msvc')}
+    target=targets.get((sys.platform,arch))
+    if not target:raise RuntimeError('This Codex native target is unsupported')
+    suffix,triple=target
+    for vendor in [package.parent/('codex-'+suffix)/'vendor',package/'vendor']:
+        candidate=vendor/triple/'bin'/('codex.exe'if os.name=='nt'else'codex')
+        if candidate.is_file():return candidate.resolve(strict=True),candidate.parent.parent.resolve(strict=True)
+    raise RuntimeError('The Codex npm installation is missing its platform binary; reinstall the official client')
+
+
+def worker_command(read_roots=(),*,temporary_dir=None):
+    executable = shutil.which('codex')
+    if not executable:
+        raise RuntimeError('The selected native worker is unavailable')
+    binary, native_root = native_codex(executable)
+    # Include the symlink's directory and native package tree so the CLI's
+    # sandbox helper can execute. These are code paths, not its auth/state home.
+    roots = {Path(executable).parent.resolve(), native_root}
+    for root in read_roots:
+        path = Path(root).resolve(strict=True)
+        if not path.is_dir():
+            raise ValueError('A worker tool read root must be an existing directory')
+        secrets = [Path.home()/name for name in ['.codex','.aws','.ssh','.kube']]
+        if Path.home().is_relative_to(path) or any(path.is_relative_to(p) or p.is_relative_to(path) for p in secrets):
+            raise ValueError('Worker tools cannot expose a home or credential tree')
+        roots.add(path)
+    forbidden = {Path('/'), Path.home(), Path.home()/'.codex', Path.home()/'.aws', Path.home()/'.ssh'}
+    if roots & forbidden:
+        raise ValueError('Worker tool roots cannot grant filesystem, home or credential trees')
+    entries = {':root':'deny', ':minimal':'read', ':tmpdir':'deny', ':slash_tmp':'deny'}
+    entries.update({str(p):'read' for p in sorted(roots)})
+    filesystem = ','.join(f'{json.dumps(k)}={json.dumps(v)}' for k,v in entries.items())
+    # The native proxy must be active to propagate its Unix-socket allowlist.
+    # No domain is allowed, local binding is forbidden and raw IP access remains
+    # sandboxed. The profile's enabled flag is not an unrestricted network grant.
+    network='enabled=true,domains={},proxy_url="http://127.0.0.1:0",enable_socks5=false,allow_local_binding=false'
+    if temporary_dir is not None:
+        # Terraform's provider protocol binds random AF_UNIX sockets beneath its
+        # private temp directory. Permit only that subtree, never Docker/host IPC
+        # or IP networking. This must be requalified per native worker platform.
+        network+=',unix_sockets={'+json.dumps(str(temporary_dir))+'="allow"}'
+    definition = f'permissions.{PROFILE}={{extends=":workspace",filesystem={{{filesystem}}},network={{{network}}}}}'
+    command = [str(binary),'app-server','--listen','stdio://',
+            '-c','analytics.enabled=false','-c',definition,
+            '-c',f'default_permissions={json.dumps(PROFILE)}',
+            '-c','shell_environment_policy.inherit="core"',
+            '-c','shell_environment_policy.ignore_default_excludes=false',
+            '-c','allow_login_shell=false','-c','features.shell_snapshot=false',
+            '-c','features.network_proxy=true',
+            '-c','mcp_servers={}','-c','features.apps=false','-c','features.plugins=false',
+            '-c','web_search="disabled"']
+    if temporary_dir is not None:
+        command.extend(['-c',f'shell_environment_policy.set.TMPDIR={json.dumps(str(temporary_dir))}'])
+    for name in configured_mcp_names():
+        # CLI override dotted paths do not parse TOML quotes in key segments.
+        # Refuse unusual keys rather than creating a second, ineffective entry.
+        if not re.fullmatch(r'[A-Za-z0-9_-]+',name):
+            raise ValueError('An MCP configuration key cannot be disabled safely by this worker')
+        command.extend(['-c',f'mcp_servers.{name}.enabled=false'])
+    for name in configured_shell_override_names():
+        if name=='TMPDIR':continue
+        if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*',name):
+            raise ValueError('A shell override key cannot be disabled safely by this worker')
+        command.extend(['-c',f'shell_environment_policy.set.{name}=""'])
+    return command
+
+
+class CodexWorkerClient(CodexAppServer):
+    def __init__(self,workspace,*,read_roots=(),requires_unix_sockets=True):
+        root=Path(workspace).resolve(strict=True)
+        temporary=root/'.worker-tmp'
+        if requires_unix_sockets and os.name=='posix' and len(os.fsencode(temporary))>80:
+            raise ValueError('Register a shorter worker workspace path for Unix-socket validators')
+        if temporary.is_symlink():
+            raise ValueError('Worker temporary storage cannot be a symlink')
+        temporary.mkdir(exist_ok=True)
+        command=worker_command(read_roots,temporary_dir=temporary)
+        definition=next(value for value in command if value.startswith(f'permissions.{PROFILE}='))
+        self.expected_worker_profile=tomllib.loads(definition)['permissions'][PROFILE]
+        super().__init__(root,command=command,experimental=True)
+
+    async def verify_worker_tools(self,expected_profile=PROFILE):
+        if expected_profile!=PROFILE:
+            raise RuntimeError('This worker cannot attest a different named permission profile')
+        effective=await self.request('config/read',{'includeLayers':False,'cwd':str(self.workspace)})
+        config=effective.get('config')
+        if not isinstance(config,dict):
+            raise RuntimeError('Worker configuration acknowledgement is unavailable')
+        definitions=config.get('mcp_servers',{})
+        if not isinstance(definitions,dict) or any(not isinstance(v,dict) or v.get('enabled') is not False for v in definitions.values()):
+            raise RuntimeError('Unapproved MCP enablement is present in this local worker')
+        features=config.get('features',{})
+        if not isinstance(features,dict) or features.get('apps') is not False or features.get('plugins') is not False or config.get('web_search')!='disabled':
+            raise RuntimeError('External tools were not disabled in this local worker')
+        shell=config.get('shell_environment_policy',{})
+        if config.get('allow_login_shell') is not False or features.get('shell_snapshot') is not False or not isinstance(shell,dict) or shell.get('inherit')!='core' or shell.get('ignore_default_excludes') is not False or shell.get('include_only') not in (None,[]):
+            raise RuntimeError('The restricted worker shell environment was not confirmed')
+        overrides=shell.get('set')
+        if not isinstance(overrides,dict) or overrides.get('TMPDIR')!=str(self.workspace/'.worker-tmp') or any(k!='TMPDIR' and v!='' for k,v in overrides.items()):
+            raise RuntimeError('Unexpected inherited shell variable overrides were configured')
+        profile=config.get('permissions',{}).get(PROFILE,{})
+        network=profile.get('network',{})
+        if features.get('network_proxy') is not True or network.get('enabled') is not True or network.get('domains') not in ({},None) or network.get('allow_local_binding') is not False:
+            raise RuntimeError('The deny-all worker proxy was not confirmed')
+        filesystem=profile.get('filesystem')
+        if isinstance(filesystem,dict):
+            filesystem=dict(filesystem)
+            if filesystem.get('glob_scan_max_depth') is None:filesystem.pop('glob_scan_max_depth',None)
+        if profile.get('extends')!=':workspace' or filesystem!=self.expected_worker_profile['filesystem']:
+            raise RuntimeError('The worker filesystem policy changed in an effective configuration layer')
+        if network.get('unix_sockets')!=self.expected_worker_profile['network'].get('unix_sockets') or network.get('dangerously_allow_all_unix_sockets') is True or network.get('dangerously_allow_non_loopback_proxy') is True:
+            raise RuntimeError('Unexpected local socket access was added to the worker')
+        roots=profile.get('workspace_roots',{}) or {}
+        if not isinstance(roots,dict) or any(value is not False for value in roots.values()):
+            raise RuntimeError('Unexpected profile-owned workspace roots were added')
+        result=await self.request('mcpServerStatus/list',{'limit':100})
+        states=result.get('data')
+        if not isinstance(states,list) or result.get('nextCursor'):
+            raise RuntimeError('Unapproved MCP tools are configured in this local worker')
+        # The status API lists disabled definitions too. Require both effective
+        # disabled configuration and no initialized server/tool/resource surface.
+        for state in states:
+            if not isinstance(state,dict) or state.get('name') not in definitions or state.get('runtimeStatus') is not None or state.get('tools')!={} or state.get('resources')!=[] or state.get('resourceTemplates')!=[]:
+                raise RuntimeError('An MCP server exposed tools or resources in this local worker')
+        return {'configuredMcpDefinitions':len(definitions),'allDisabled':True,'exposedMcpTools':0,'apps':False,'plugins':False,'webSearch':'disabled'}