@atmem/architect 0.0.0-stage → 0.2.0-alpha.4.1

package/server/architecture_activity.py

@@ -0,0 +1,67 @@
+"""Project-scoped public activity, never model reasoning or raw protocol output."""
+from datetime import datetime,timezone
+import json
+import re
+import sqlite3
+import uuid
+
+KINDS={'context_started','context_ready','pass_started','model_started','model_message','model_checked','response_repair','tool_started','tool_finished','tool_cached','response_ready','failed','cancelled'}
+DETAILS={'round','attempt','provider','model','effort','skillIds','recordCount','knowledgeCount','tool','success','cached','encoding','proposal','requirementCount','decisionCount'}
+
+def now():return datetime.now(timezone.utc).isoformat()
+
+
+class ArchitectureActivity:
+    def __init__(self,path):
+        self.db=sqlite3.connect(path);self.db.row_factory=sqlite3.Row
+        self.db.execute('''CREATE TABLE IF NOT EXISTS architecture_turns (
+            id TEXT PRIMARY KEY, project TEXT NOT NULL, metadata TEXT NOT NULL,
+            state TEXT NOT NULL, started TEXT NOT NULL)''')
+        self.db.execute('''CREATE TABLE IF NOT EXISTS architecture_events (
+            sequence INTEGER PRIMARY KEY AUTOINCREMENT, turn TEXT NOT NULL,
+            payload TEXT NOT NULL)''')
+        with self.db:self.db.execute("UPDATE architecture_turns SET state='recovery_required' WHERE state='running'")
+
+    def close(self):self.db.close()
+
+    def owned(self,project,turn):
+        row=self.db.execute('SELECT * FROM architecture_turns WHERE id=? AND project=?',(turn,project)).fetchone()
+        if row is None:raise LookupError('Architecture activity is unavailable for this project')
+        return row
+
+    def start(self,project,turn,metadata):
+        if not isinstance(project,str) or not re.fullmatch(r'[A-Za-z0-9-]{8,64}',project) or not isinstance(turn,str) or str(uuid.UUID(turn))!=turn:
+            raise ValueError('Use an exact project and canonical activity UUID')
+        if set(metadata)-{'provider','model','effort','origin','revision'}:raise ValueError('Unsupported public turn metadata')
+        if self.db.execute('SELECT 1 FROM architecture_turns WHERE id=?',(turn,)).fetchone():
+            raise ValueError('This architecture request is already recorded; it was not replayed')
+        if self.db.execute('SELECT COUNT(*) FROM architecture_turns WHERE project=?',(project,)).fetchone()[0]>=1000:
+            raise ValueError('Architecture activity reached its history bound; no earlier turn was removed')
+        with self.db:self.db.execute('INSERT INTO architecture_turns VALUES (?,?,?,?,?)',(turn,project,json.dumps(metadata,allow_nan=False),'running',now()))
+
+    def append(self,project,turn,event):
+        row=self.owned(project,turn)
+        if row['state']!='running':raise ValueError('Activity is already settled; no work was replayed')
+        if set(event)-{'kind','text','details'} or event.get('kind') not in KINDS or not isinstance(event.get('text'),str) or len(event['text'])>16000:
+            raise ValueError('Unsupported public activity event')
+        details=event.get('details',{})
+        if not isinstance(details,dict) or set(details)-DETAILS:raise ValueError('Private or unsupported activity details are forbidden')
+        for value in details.values():
+            if value is not None and type(value) not in {str,int,bool,list}:raise ValueError('Unsupported activity detail')
+            if isinstance(value,list) and (len(value)>100 or any(not isinstance(v,str) or len(v)>200 for v in value)):raise ValueError('Invalid public activity references')
+        if self.db.execute('SELECT COUNT(*) FROM architecture_events WHERE turn=?',(turn,)).fetchone()[0]>=500:raise ValueError('Activity event bound reached; no events were discarded')
+        encoded=json.dumps({**event,'at':now()},ensure_ascii=False,allow_nan=False)
+        if len(encoded.encode())>80000:raise ValueError('Activity event is oversized; no text was clipped')
+        with self.db:self.db.execute('INSERT INTO architecture_events(turn,payload) VALUES (?,?)',(turn,encoded))
+
+    def finish(self,project,turn,state):
+        if state not in {'response_ready','failed','cancelled','details_unavailable'}:raise ValueError('Unsupported activity state')
+        self.owned(project,turn)
+        with self.db:self.db.execute("UPDATE architecture_turns SET state=? WHERE id=? AND project=? AND state='running'",(state,turn,project))
+
+    def read(self,project,turn,after=0):
+        row=self.owned(project,turn)
+        if type(after) is not int or after<0:raise ValueError('Invalid activity cursor')
+        events=[{'sequence':r['sequence'],**json.loads(r['payload'])} for r in self.db.execute('SELECT sequence,payload FROM architecture_events WHERE turn=? AND sequence>? ORDER BY sequence LIMIT 200',(turn,after))]
+        return {'id':turn,'projectId':project,'state':row['state'],'startedAt':row['started'],'selection':json.loads(row['metadata']),'events':events,
+                'boundary':'Recorded public model messages and observed host actions. No private reasoning, raw prompts, credentials or provider protocol dumps. Response readiness is not design/application acceptance.'}

package/server/attachments.py

@@ -0,0 +1,101 @@
+"""Bounded, ephemeral attachment parsing. Content is evidence, never instructions."""
+import base64,io,json,subprocess,sys,zipfile,zlib
+from urllib.parse import unquote_to_bytes
+from pathlib import Path
+from xml.etree import ElementTree
+MAX_FILE=4*1024*1024
+MAX_TOTAL=8*1024*1024
+
+def decode(item):
+    try:data=base64.b64decode(item['data'],validate=True)
+    except Exception:raise ValueError('Attachment encoding is invalid.')
+    if not data or len(data)>MAX_FILE:raise ValueError('Each attachment must be between 1 byte and 4 MB.')
+    return data
+
+def diagram_text(data):
+    """Decode draw.io XML locally; never fetch external resources or execute markup."""
+    def parse(raw):
+        if len(raw)>100000:raise ValueError('Diagram text exceeds 100,000 bytes. Use a smaller diagram.')
+        if b'<!DOCTYPE' in raw.upper() or b'<!ENTITY' in raw.upper():raise ValueError('Unsupported diagram XML declarations.')
+        try:return ElementTree.fromstring(raw)
+        except ElementTree.ParseError:raise ValueError('Cannot read diagram XML. Export a standard draw.io file or image.')
+    root=parse(data)
+    if root.tag=='mxGraphModel':return ElementTree.tostring(root,encoding='unicode')
+    if root.tag!='mxfile':raise ValueError('Use a draw.io mxfile or mxGraphModel diagram.')
+    pages=root.findall('diagram')
+    if not pages or len(pages)>20:raise ValueError('Use a diagram with between 1 and 20 pages.')
+    result=[]
+    for page in pages:
+        model=page.find('mxGraphModel')
+        if model is None:
+            try:
+                encoded=base64.b64decode((page.text or '').strip(),validate=True)
+                decoder=zlib.decompressobj(-15)
+                raw=decoder.decompress(encoded,100001)
+                if len(raw)>100000 or not decoder.eof or decoder.unused_data:raise ValueError('Diagram expansion exceeds the limit or is invalid.')
+                model=parse(unquote_to_bytes(raw.decode('utf-8')))
+            except (UnicodeError,zlib.error,ValueError) as exc:raise ValueError('Cannot decode this draw.io page within the supported text limit.') from exc
+        if model.tag!='mxGraphModel':raise ValueError('A draw.io page must contain an mxGraphModel.')
+        result.append('[Diagram: '+page.get('name','Untitled')+']\n'+ElementTree.tostring(model,encoding='unicode'))
+    text='\n\n'.join(result)
+    if len(text)>100000:raise ValueError('Combined diagram text exceeds 100,000 characters.')
+    return text
+
+def extract(items):
+    if len(items)>4:raise ValueError('Attach up to four files per message.')
+    decoded=[(item,decode(item)) for item in items]
+    if sum(len(data) for _,data in decoded)>MAX_TOTAL:raise ValueError('Attachments exceed the 8 MB total limit.')
+    result=[]
+    for item,data in decoded:
+        mime=item['mime'];name=item['name']
+        if mime.startswith('image/'):
+            from PIL import Image
+            Image.MAX_IMAGE_PIXELS=20_000_000
+            with Image.open(io.BytesIO(data)) as im:
+                if im.width*im.height>20_000_000:raise ValueError('Image exceeds 20 megapixels.')
+                if im.format not in {'PNG','JPEG','WEBP'} or Image.MIME[im.format]!=mime:raise ValueError('Image format does not match its content.')
+                im.verify()
+            result.append({'name':name,'mime':mime,'data':item['data']});continue
+        if mime=='application/pdf':
+            from pypdf import PdfReader,overwrite_configuration
+            overwrite_configuration(maximum_declared_stream_length=8_000_000,zlib_maximum_output_length=8_000_000,lzw_maximum_output_length=8_000_000,run_length_maximum_output_length=8_000_000)
+            reader=PdfReader(io.BytesIO(data))
+            if reader.is_encrypted:raise ValueError('Unlock the PDF before attaching it.')
+            if len(reader.pages)>50:raise ValueError('Use a PDF with at most 50 pages.')
+            pages=[page.extract_text() or '' for page in reader.pages]
+            text='\n\n'.join(f'[Page {i+1}]\n{value}' for i,value in enumerate(pages))
+            if not any(value.strip() for value in pages):raise ValueError('This PDF has no readable text. Attach screenshots of the relevant pages instead.')
+        elif mime=='application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+            with zipfile.ZipFile(io.BytesIO(data)) as archive:
+                if sum(i.file_size for i in archive.infolist())>16*1024*1024:raise ValueError('The expanded document exceeds 16 MB.')
+                xml=archive.read('word/document.xml')
+                if b'<!DOCTYPE' in xml or b'<!ENTITY' in xml:raise ValueError('Unsupported document XML.')
+                root=ElementTree.fromstring(xml)
+                text='\n'.join(''.join(p.itertext()) for p in root.iter('{http://schemas.openxmlformats.org/wordprocessingml/2006/main}p'))
+        elif mime=='text/plain' and name.lower().endswith(('.drawio','.xml')):text=diagram_text(data)
+        elif mime in {'text/plain','text/markdown','text/csv','application/json'}:text=data.decode('utf-8-sig')
+        else:raise ValueError('Use PNG, JPEG, WebP, PDF, DOCX, TXT, Markdown, CSV or JSON.')
+        if not text.strip():raise ValueError('The document has no readable text.')
+        if len(text)>100000:raise ValueError('Document text exceeds 100,000 characters. Attach a shorter excerpt.')
+        result.append({'name':name,'mime':mime,'text':text,'mode':'extracted text; embedded images and layout are not included'})
+    if sum(len(i.get('text','')) for i in result)>150000:raise ValueError('Combined document text exceeds 150,000 characters.')
+    return result
+
+def prepare(items):
+    if not items:return []
+    try:
+        run=subprocess.run([sys.executable,str(Path(__file__).resolve()),'--worker'],input=json.dumps(items),text=True,capture_output=True,timeout=20)
+        response=json.loads(run.stdout)
+        if 'error' in response:raise RuntimeError(response['error'])
+        return response['attachments']
+    except (subprocess.TimeoutExpired,json.JSONDecodeError):raise RuntimeError('Attachment processing stopped. Try a smaller or simpler file.')
+
+if __name__=='__main__':
+    try:
+        if sys.platform!='win32':
+            import resource
+            resource.setrlimit(resource.RLIMIT_CPU,(12,12))
+            if sys.platform!='darwin':resource.setrlimit(resource.RLIMIT_AS,(768*1024*1024,768*1024*1024))
+        print(json.dumps({'attachments':extract(json.load(sys.stdin))}))
+    except (ValueError,UnicodeError,zipfile.BadZipFile,KeyError) as exc:print(json.dumps({'error':str(exc)[:250]}))
+    except Exception:print(json.dumps({'error':'Cannot read this attachment. Export a standard PDF, DOCX, text file or image and try again.'}))

package/server/cloud-emulation-requirements.txt

@@ -0,0 +1,29 @@
+# Optional Azure/GCP clients; install after server/emulation-requirements.txt.
+# IBM structured-runtime dependencies are isolated by scripts/setup-structured-runtime.py.
+azure-core==1.41.0
+azure-storage-blob==12.31.1
+certifi==2026.10.23
+cffi==2.1.1
+charset-normalizer==3.5.2
+cryptography==50.0.2
+google-api-core==2.44.0
+google-auth==2.62.0
+google-cloud-core==2.8.0
+google-cloud-storage==3.17.1
+google-crc32c==1.9.0
+google-resumable-media==2.11.0
+googleapis-common-protos==1.76.0
+idna==3.20
+isodate==0.7.2
+opentelemetry-api==1.45.2
+proto-plus==1.29.0
+protobuf==7.36.2
+pyasn1==0.6.5
+pyasn1_modules==0.4.2
+pycparser==3.12
+requests==2.34.2
+typing_extensions==4.16.0
+urllib3==2.8.0
+
+# Public FAQ real HTTP API v2 event adapter (runtime wheel independently hash-pinned in emitted source).
+mangum==0.22.0

package/server/codex_app_server.py

@@ -0,0 +1,219 @@
+"""Persistent Codex protocol transport for the engineering-session foundation.
+
+Not a sandbox, execution policy or public endpoint. Callers must separately qualify
+workspace isolation and tools before exposing engineering turns to the product.
+Uses the official signed-in CLI; never reads or extracts account tokens.
+"""
+import asyncio
+import json
+import os
+from pathlib import Path
+import signal
+from .local_providers import account_environment
+
+
+class ProtocolError(RuntimeError):
+    def __init__(self, message, *, code=None, category=None):
+        super().__init__(message)
+        self.code = code
+        self.category = category
+
+
+class CodexAppServer:
+    def __init__(self, workspace, *, command=None, event_limit=4096, experimental=False):
+        self.workspace = Path(workspace).resolve(strict=True)
+        if not self.workspace.is_dir():
+            raise ValueError('Codex workspace must be an existing directory')
+        self.command = command or ['codex', 'app-server', '--listen', 'stdio://',
+                                   '-c', 'analytics.enabled=false']
+        self.experimental = experimental
+        self.events = asyncio.Queue(maxsize=event_limit)
+        self.process = None
+        self.pending = {}
+        self.serial = 0
+        self.failure = None
+        self.closed = asyncio.Event()
+        self.reader = self.stderr_reader = None
+        self.write_lock = asyncio.Lock()
+        self.dynamic_tool_handler = None
+        self.tool_tasks = set()
+
+    async def start(self):
+        if self.process is not None:
+            raise ProtocolError('Codex transport has already started')
+        env = account_environment()
+        # Authentication remains CLI-owned. Do not forward deployment credentials.
+        for key in list(env):
+            if key.startswith(('AWS_', 'AZURE_', 'GOOGLE_', 'GCLOUD_', 'IBM_', 'WATSONX_', 'TF_')) or key in {'KUBECONFIG', 'DOCKER_HOST', 'DOCKER_CONFIG', 'CLOUDSDK_CONFIG', 'GOOGLE_APPLICATION_CREDENTIALS'}:
+                env.pop(key, None)
+        self.process = await asyncio.create_subprocess_exec(
+            *self.command, cwd=self.workspace, env=env,
+            stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE, limit=8_000_000, start_new_session=True)
+        self.reader = asyncio.create_task(self._read())
+        self.stderr_reader = asyncio.create_task(self._drain_stderr())
+        try:
+            result = await self.request('initialize', {'clientInfo': {
+                'name': 'atmem_ai_architect', 'title': 'AtMem AI Architect',
+                'version': json.loads((Path(__file__).resolve().parents[1] / 'package.json').read_text())['version']},
+                **({'capabilities': {'experimentalApi': True}} if self.experimental else {})})
+            await self._write({'method': 'initialized', 'params': {}})
+            return result
+        except BaseException:
+            await self.close()
+            raise
+
+    async def _write(self, message):
+        if self.failure:
+            raise ProtocolError(self.failure)
+        if not self.process or self.process.returncode is not None:
+            raise ProtocolError('Codex transport is not running')
+        wire = json.dumps(message, allow_nan=False, separators=(',', ':')).encode() + b'\n'
+        if len(wire) > 8_000_000:
+            raise ValueError('Codex protocol message exceeds the explicit bound')
+        async with self.write_lock:
+            self.process.stdin.write(wire)
+            await self.process.stdin.drain()
+
+    async def request(self, method, params=None, *, timeout=30):
+        if not isinstance(method, str) or not method:
+            raise ValueError('A protocol method is required')
+        self.serial += 1
+        ident = self.serial
+        future = asyncio.get_running_loop().create_future()
+        self.pending[ident] = future
+        try:
+            await self._write({'id': ident, 'method': method, 'params': params or {}})
+            return await asyncio.wait_for(future, timeout)
+        except BaseException:
+            # _fail may settle this future while a write is failing. Consume its
+            # exception so cancellation/transport errors never leak orphan futures.
+            if future.done() and not future.cancelled():
+                future.exception()
+            raise
+        finally:
+            self.pending.pop(ident, None)
+            if not future.done():
+                future.cancel()
+        # A timeout/cancel only stops waiting; it does not cancel a Codex turn.
+        # Engineering callers must use turn/interrupt and observe its terminal event.
+
+    async def notification(self, *, timeout=30):
+        if not self.events.empty():
+            return self.events.get_nowait()
+        if self.failure:
+            raise ProtocolError(self.failure)
+        received = asyncio.create_task(self.events.get())
+        closed = asyncio.create_task(self.closed.wait())
+        try:
+            done, _ = await asyncio.wait({received, closed}, timeout=timeout,
+                                         return_when=asyncio.FIRST_COMPLETED)
+            if received in done:
+                return received.result()
+            if closed in done:
+                raise ProtocolError(self.failure)
+            raise asyncio.TimeoutError('No Codex event arrived within the observation window')
+        finally:
+            for task in (received, closed):
+                if not task.done():
+                    task.cancel()
+            await asyncio.gather(received, closed, return_exceptions=True)
+
+    def _fail(self, reason):
+        self.failure = reason
+        self.closed.set()
+        for future in self.pending.values():
+            if not future.done():
+                future.set_exception(ProtocolError(reason))
+
+    async def _read(self):
+        try:
+            while line := await self.process.stdout.readline():
+                message = json.loads(line)
+                if not isinstance(message, dict):
+                    raise ProtocolError('Codex returned an invalid protocol envelope')
+                if 'method' in message:
+                    if 'id' in message:
+                        if message['method']=='item/tool/call' and self.dynamic_tool_handler is not None and len(self.tool_tasks)<8:
+                            # Never block protocol responses/progress on a tool.
+                            # Only this exact callback is enabled; approvals stay denied.
+                            task=asyncio.create_task(self._tool_reply(message))
+                            self.tool_tasks.add(task)
+                            task.add_done_callback(self.tool_tasks.discard)
+                        else:
+                            await self._write({'id': message['id'], 'error': {
+                                'code': -32601, 'message': 'Client handler is not enabled'}})
+                    else:
+                        self.events.put_nowait(message)
+                elif 'id' in message:
+                    future = self.pending.get(message['id'])
+                    if future and not future.done():
+                        if 'error' in message:
+                            code = message['error'].get('code') if isinstance(message['error'], dict) else None
+                            # Retain only a narrow recovery classification; never
+                            # expose the server's raw diagnostics or local paths.
+                            diagnostic = message['error'].get('message', '') if isinstance(message['error'], dict) else ''
+                            category = 'missing_thread' if code == -32600 and isinstance(diagnostic, str) and diagnostic.startswith('no rollout found for thread id ') else None
+                            future.set_exception(ProtocolError(f'Codex request rejected (code {code})', code=code, category=category))
+                        elif 'result' in message:
+                            future.set_result(message['result'])
+                        else:
+                            raise ProtocolError('Codex response contains neither result nor error')
+                else:
+                    raise ProtocolError('Codex protocol envelope has no method or id')
+            self._fail('Codex transport closed; pending work is not confirmed complete')
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            self._fail('Codex protocol failed or event capacity was exceeded; no completion is inferred')
+            if self.process.returncode is None:
+                self.process.terminate()
+
+    async def _drain_stderr(self):
+        # Do not leak credentials, local paths or provider diagnostic bodies into
+        # public reports. Drain to avoid deadlock; qualification stores safe receipts.
+        while await self.process.stderr.read(65536):
+            pass
+
+    async def _tool_reply(self,message):
+        try:
+            result=await asyncio.wait_for(self.dynamic_tool_handler(message.get('params',{})),30)
+            if not isinstance(result,dict) or set(result)!={'success','contentItems'} or type(result['success']) is not bool:
+                raise ValueError('Invalid tool response')
+            if not isinstance(result['contentItems'],list) or any(not isinstance(item,dict) or set(item)!={'type','text'} or item['type']!='inputText' or not isinstance(item['text'],str) for item in result['contentItems']):
+                raise ValueError('Only bounded text evidence is enabled')
+            if len(json.dumps(result,allow_nan=False).encode())>64000:raise ValueError('Tool response exceeds bound')
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            result={'success':False,'contentItems':[{'type':'inputText','text':'The registered engineering tool could not confirm this request. No success or work replay is inferred.'}]}
+        try:await self._write({'id':message['id'],'result':result})
+        except Exception:pass  # Durable tool state survives a lost transport reply.
+
+    async def close(self):
+        if not self.process:
+            return
+        self._fail('Codex transport closed; pending work is not confirmed complete')
+        for task in self.tool_tasks:task.cancel()
+        await asyncio.gather(*list(self.tool_tasks),return_exceptions=True)
+        if self.process.returncode is None:
+            self.process.stdin.close()
+            try:
+                await asyncio.wait_for(self.process.wait(), 2)
+            except asyncio.TimeoutError:
+                try:
+                    os.killpg(self.process.pid, signal.SIGTERM)
+                except ProcessLookupError:
+                    pass
+                try:
+                    await asyncio.wait_for(self.process.wait(), 2)
+                except asyncio.TimeoutError:
+                    try:
+                        os.killpg(self.process.pid, signal.SIGKILL)
+                    except ProcessLookupError:
+                        pass
+                    await self.process.wait()
+        for task in (self.reader, self.stderr_reader):
+            if task:
+                task.cancel()
+        await asyncio.gather(*(t for t in (self.reader, self.stderr_reader) if t), return_exceptions=True)