@topolo/cli 0.12.33 → 0.13.1

package/agent-packs/src/topolo_agent_pack/native.py

@@ -0,0 +1,645 @@
+"""Versioned local Codex storage adapter. Never writes to capture sources."""
+import base64
+import copy
+import hashlib
+import json
+import os
+import pathlib
+import re
+import shutil
+import sqlite3
+import subprocess
+import sys
+import time
+import tomllib
+
+from .storage import PackError, atomic_json, canonical, component_key, now, select, slug
+
+DB_TABLES = {
+    'state_5.sqlite': {'_sqlx_migrations', 'threads', 'thread_sections', 'projects', 'project_roots', 'thread_attachments', 'thread_dynamic_tools', 'thread_spawn_edges', 'backfill_state', 'remote_control_enrollments', 'external_agent_config_imports', 'rollout_migration_state', 'rollout_migration_skipped_rollouts', 'project_idempotency_keys'},
+    'thread_history_1.sqlite': {'_sqlx_migrations', 'thread_turns', 'thread_items', 'thread_history_projection_state', 'thread_realtime_items'},
+    'goals_1.sqlite': {'_sqlx_migrations', 'thread_goals', 'thread_goal_continuation_deferrals'},
+}
+SKIP_DIRS = {'.git', 'node_modules', '.venv', 'venv', '__pycache__', '.cache', '.next', '.DS_Store'}
+SECRET_NAMES = {'auth.json', '.npmrc', '.pypirc', 'credentials.json', 'credentials', 'id_rsa', 'id_ed25519', '.netrc'}
+NEVER_COPY_ROWS = {'backfill_state','remote_control_enrollments','external_agent_config_imports','rollout_migration_state','rollout_migration_skipped_rollouts','project_idempotency_keys'}
+
+def quote(name):
+    return '"' + name.replace('"', '""') + '"'
+
+def ro(path):
+    return sqlite3.connect('file:' + str(pathlib.Path(path).resolve()) + '?mode=ro', uri=True)
+
+def encode(value):
+    if isinstance(value, bytes): return {'$blob': base64.b64encode(value).decode()}
+    return value
+
+def decode(value):
+    if isinstance(value, dict) and set(value) == {'$blob'}: return base64.b64decode(value['$blob'], validate=True)
+    return value
+
+def busy_writers(home):
+    home = pathlib.Path(home).resolve()
+    files = [str(p) for name in DB_TABLES for suffix in ['', '-wal'] if (p := home / (name + suffix)).exists()]
+    pids = set()
+    if sys.platform == 'darwin' and files:
+        result = subprocess.run(['/usr/sbin/lsof', '-F', 'p', '--', *files], capture_output=True, text=True, timeout=15)
+        if result.returncode not in (0, 1): raise PackError('Cannot establish that the source databases are idle (lsof failed).')
+        pids = {int(line[1:]) for line in result.stdout.splitlines() if line.startswith('p') and line[1:].isdigit()}
+    elif pathlib.Path('/proc').exists():
+        targets = set(files)
+        denied = False
+        for proc in pathlib.Path('/proc').iterdir():
+            if not proc.name.isdigit() or int(proc.name) == os.getpid(): continue
+            try:
+                for fd in (proc / 'fd').iterdir():
+                    try:
+                        if os.readlink(fd) in targets: pids.add(int(proc.name))
+                    except FileNotFoundError: pass
+            except (PermissionError, FileNotFoundError):
+                try:
+                    if proc.stat().st_uid == os.getuid(): denied = True
+                except FileNotFoundError: pass
+        if denied: raise PackError('Cannot inspect all same-user database holders; close Codex before retrying.')
+    else:
+        raise PackError('This platform has no verified idle-runtime detector. Capture/activation requires macOS or Linux.')
+    pids.discard(os.getpid())
+    return sorted(pids)
+
+def assert_idle(home):
+    pids = busy_writers(home)
+    if pids:
+        raise PackError(f'Codex databases are open in processes {pids}. This holder is not a quiesced managed backend. Use a topolo-agent-pack managed window for live switching. No process was stopped.')
+
+def signature(home):
+    home = pathlib.Path(home)
+    names = [name + suffix for name in DB_TABLES for suffix in ['', '-wal']] + ['.codex-global-state.json']
+    return {name: (p.stat().st_size, p.stat().st_mtime_ns) for name in names if (p := home / name).exists() and (not name.endswith('-wal') or p.stat().st_size>0)}
+
+def display_name(row, global_state):
+    names = global_state.get('thread-titles', {})
+    if isinstance(names.get('titles'), dict): names = names['titles']
+    return row.get('name') or names.get(row['id']) or row.get('title') or row['id']
+
+def schemas_and_rows(connection, tables, thread_ids):
+    ids = sorted(thread_ids)
+    marks = ','.join('?' for _ in ids)
+    schemas = []
+    rows = {}
+    for kind, name, table, sql in connection.execute("SELECT type,name,tbl_name,sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END"):
+        if table not in tables or kind not in {'table', 'index'}: continue
+        schemas.append({'type': kind, 'name': name, 'table': table, 'sql': sql})
+        if kind != 'table': continue
+        cols = [x[1] for x in connection.execute('PRAGMA table_info(' + quote(name) + ')')]
+        if name in NEVER_COPY_ROWS: query, args = 'SELECT * FROM ' + quote(name) + ' WHERE 0', []
+        elif name == '_sqlx_migrations': query, args = 'SELECT * FROM ' + quote(name), []
+        elif name == 'threads': query, args = f'SELECT * FROM threads WHERE id IN ({marks})', ids
+        elif name == 'thread_sections': query, args = f'SELECT * FROM thread_sections WHERE id IN (SELECT thread_section_id FROM threads WHERE id IN ({marks}))', ids
+        elif name == 'projects': query, args = f'SELECT * FROM projects WHERE id IN (SELECT project_id FROM threads WHERE id IN ({marks}))', ids
+        elif name == 'project_roots': query, args = f'SELECT * FROM project_roots WHERE project_id IN (SELECT project_id FROM threads WHERE id IN ({marks}))', ids
+        elif name == 'thread_spawn_edges': query, args = f'SELECT * FROM thread_spawn_edges WHERE parent_thread_id IN ({marks}) AND child_thread_id IN ({marks})', ids + ids
+        elif 'thread_id' in cols: query, args = f'SELECT * FROM {quote(name)} WHERE thread_id IN ({marks})', ids
+        else: query, args = 'SELECT * FROM ' + quote(name) + ' WHERE 0', []
+        rows[name] = {'columns': cols, 'rows': [[encode(v) for v in row] for row in connection.execute(query, args)]}
+    return {'schemas': schemas, 'tables': rows}
+
+def unregistered_threads(home,known):
+    db=ro(pathlib.Path(home)/'state_5.sqlite')
+    try:
+        children={r[0] for r in db.execute('SELECT child_thread_id FROM thread_spawn_edges')}
+        return [r[0] for r in db.execute('SELECT id FROM threads WHERE archived=0') if r[0] not in known and r[0] not in children]
+    finally:db.close()
+
+def capture(home, library, name, sets=None, agents=None, include=None, all_threads=False, template=None, workspace_mode='copy', live=False):
+    home = pathlib.Path(home).expanduser().resolve()
+    if not (home / 'state_5.sqlite').exists(): raise PackError(f'No Codex thread catalog in {home}')
+    if home == library.root or home in library.root.parents: raise PackError('Pack library must be outside the captured Codex home.')
+    if not live: assert_idle(home)
+    initial = signature(home)
+    gs = json.loads((home / '.codex-global-state.json').read_text()) if (home / '.codex-global-state.json').exists() else {}
+    connections={}
+    try:
+        for filename in DB_TABLES:
+            if (home/filename).exists():
+                connection=ro(home/filename);connections[filename]=connection
+                if live:
+                    connection.execute('BEGIN')
+                    connection.execute('SELECT name FROM sqlite_master LIMIT 1').fetchone()
+        db=connections['state_5.sqlite'];db.row_factory=sqlite3.Row
+        available = {row['id']: dict(row) for row in db.execute('SELECT * FROM threads')}
+        section_names = {row['id']: row['name'] for row in db.execute('SELECT * FROM thread_sections')}
+        visible = []
+        if template:
+            visible = copy.deepcopy(template['agents'])
+            for a in visible:
+                if a['threadId'] not in available: raise PackError('Active runtime lost an agent record: ' + a['threadId'])
+                a['title'] = display_name(available[a['threadId']], gs)
+            if all_threads:
+                known={a['threadId'] for a in visible}
+                children={r[0] for r in db.execute('SELECT child_thread_id FROM thread_spawn_edges')}
+                keys={component_key(a) for a in visible}
+                for tid,row in available.items():
+                    if tid in known or tid in children or row.get('archived'):continue
+                    section=section_names.get(row.get('thread_section_id'),'Recents');title=display_name(row,gs)
+                    set_id='intakes' if section.lower()=='pinned' else slug(re.sub('^'+re.escape(name)+r'\s+','',section,flags=re.I))
+                    agent_id=slug(title)
+                    if name+'/'+set_id+'/'+agent_id in keys:agent_id=agent_id[:48]+'-'+tid[-8:]
+                    a={'pack':name,'set':set_id,'id':agent_id,'aliases':[],'threadId':tid,'title':title,'sectionName':section,'pinned':section.lower()=='pinned','position':row.get('section_position') or 0}
+                    visible.append(a);keys.add(component_key(a))
+        else:
+            matching = {sid for sid, label in section_names.items() if label.lower().startswith(name.lower() + ' ')}
+            for tid, row in available.items():
+                if row.get('archived') or row.get('source', '').startswith('subagent'): continue
+                section = section_names.get(row.get('thread_section_id'), 'Tasks')
+                title = display_name(row, gs)
+                pinned = section.lower() == 'pinned'
+                if not all_threads and matching:
+                    if row.get('thread_section_id') not in matching and not (pinned and title.lower().startswith(name.lower())): continue
+                elif not all_threads:
+                    raise PackError(f'No sections belong to {name!r}. Supply --all for an explicit whole-home capture.')
+                set_id = 'intakes' if pinned else slug(re.sub('^' + re.escape(name) + r'\s+', '', section, flags=re.I))
+                agent_id = slug(re.sub('^' + re.escape(name) + r'\s+', '', title, flags=re.I))
+                aliases = [agent_id.split('-')[0]]
+                visible.append({'pack': name, 'set': set_id, 'id': agent_id, 'aliases': aliases, 'threadId': tid, 'title': title,
+                                'sectionName': section, 'pinned': pinned, 'position': row.get('section_position') or 0})
+        chosen = select({'agents': visible}, sets, agents)
+        if len({component_key(a) for a in chosen}) != len(chosen): raise PackError('Duplicate semantic identities in source; rename or capture scoped sets.')
+        closure = {a['threadId'] for a in chosen}; pending = list(closure); threads = {}
+        while pending:
+            tid = pending.pop()
+            if tid not in available: raise PackError('Required inherited/child history is missing: ' + tid)
+            row = available[tid]; path = pathlib.Path(row['rollout_path'])
+            if not path.is_file(): raise PackError('Required rollout is missing: ' + str(path))
+            with path.open('rb') as f: header = json.loads(f.readline()).get('payload', {})
+            parent = (header.get('history_base') or {}).get('thread_id')
+            children = [r[0] for r in db.execute('SELECT child_thread_id FROM thread_spawn_edges WHERE parent_thread_id=?', (tid,))]
+            threads[tid] = {'basename': path.name, 'cwd': row['cwd'], 'historyBase': parent, 'children': children}
+            for related in ([parent] if parent else []) + children:
+                if related not in closure: closure.add(related); pending.append(related)
+        cuts={}
+        if live:
+            history=connections.get('thread_history_1.sqlite')
+            for tid in closure:
+                path=pathlib.Path(available[tid]['rollout_path'])
+                if available[tid].get('history_mode')=='paginated':
+                    row=history.execute('SELECT next_rollout_byte_offset FROM thread_history_projection_state WHERE thread_id=?',(tid,)).fetchone() if history else None
+                    if not row: raise PackError('History projection is not ready for '+tid+'; retry after its current turn settles. Codex can stay open.')
+                    cuts[tid]=row[0]
+                else:
+                    with path.open('rb') as stream:
+                        end=path.stat().st_size
+                        while end>0:
+                            begin=max(0,end-65536);stream.seek(begin);chunk=stream.read(end-begin)
+                            index=chunk.rfind(b'\n')
+                            if index>=0: end=begin+index+1;break
+                            end=begin
+                        cuts[tid]=end
+            for tid in closure:
+                path=pathlib.Path(available[tid]['rollout_path'])
+                with path.open('rb') as stream: base=json.loads(stream.readline()).get('payload',{}).get('history_base')
+                if base and base.get('end_byte_offset') is not None: cuts[base['thread_id']]=max(cuts.get(base['thread_id'],0),base['end_byte_offset'])
+            for tid,cut in cuts.items():
+                path=pathlib.Path(available[tid]['rollout_path'])
+                with path.open('rb') as stream:
+                    header_end=len(stream.readline())
+                    if cut<header_end: raise PackError('History projection has not saved its header; retry capture')
+                    stream.seek(cut-1)
+                    if stream.read(1)!=b'\n': raise PackError('History snapshot has an incomplete rollout boundary; retry capture')
+        for tid in closure:
+            threads[tid]['rollout']=library.put(pathlib.Path(available[tid]['rollout_path']),limit=cuts.get(tid) if live else None)
+        catalog = {}
+        for filename, allowed in DB_TABLES.items():
+            if (home / filename).exists():
+                catalog[filename] = schemas_and_rows(connections[filename], allowed, closure)
+        if any(available[tid].get('history_mode') == 'paginated' for tid in closure) and 'thread_history_1.sqlite' not in catalog:
+            raise PackError('Paginated history database is missing; capture would be incomplete.')
+        resources = []; warnings = []; roots = {}
+        if live: warnings.append('Live capture uses read-only database snapshots and complete rollout prefixes. Work started after the snapshot remains in the source profile.')
+        if workspace_mode == 'copy':
+            for tid in closure:
+                cwd = pathlib.Path(available[tid]['cwd']).absolute()
+                if not cwd.is_dir(): raise PackError(f'Working directory missing: {cwd}; use --workspace-mode reference and supply a destination --map.')
+                if cwd.resolve() == pathlib.Path.home() or cwd == pathlib.Path('/') or cwd.resolve() == home or cwd.resolve() in home.parents:
+                    raise PackError(f'Refusing an unbounded/credential-containing workspace capture: {cwd}')
+                roots[str(cwd)] = 'workspaces/' + hashlib.sha256(str(cwd).encode()).hexdigest()[:16]
+        else:
+            warnings.append('Workspaces are external references; destination paths must exist or be supplied using --map.')
+        for location in list(include or [])+list((template or {}).get('resourcePaths',[])):
+            path = pathlib.Path(location).expanduser().resolve()
+            if not path.exists(): raise PackError('Included resource is missing: ' + str(path))
+            if path == pathlib.Path('/') or path == pathlib.Path.home() or path == home or path in home.parents:
+                raise PackError('Refusing unbounded resource inclusion')
+            roots.setdefault(str(path),'resources/' + hashlib.sha256(str(path).encode()).hexdigest()[:16] + '/' + path.name)
+        seen = set()
+        for origin in roots:
+            root = pathlib.Path(origin)
+            paths = []
+            if root.is_file(): paths = [root]
+            else:
+                for folder, dirs, files in os.walk(root, followlinks=False):
+                    folder = pathlib.Path(folder)
+                    omitted = [x for x in dirs if x in SKIP_DIRS or (folder / x).resolve() == library.root or (folder / x).is_symlink()]
+                    dirs[:] = [x for x in dirs if x not in omitted]
+                    if omitted: warnings.append('Excluded generated/symlink directories under ' + str(folder) + ': ' + ','.join(omitted))
+                    paths.extend(folder / x for x in files)
+            for path in paths:
+                if path.is_symlink():
+                    warnings.append('External symlink not packaged: ' + str(path)); continue
+                if path.name in SECRET_NAMES or path.name.startswith('.env') or path.suffix in {'.pem', '.key', '.p12', '.pfx'}:
+                    warnings.append('Credential/environment file excluded: ' + str(path)); continue
+                if not path.is_file() or str(path) in seen: continue
+                seen.add(str(path)); resources.append({'original': str(path), 'content': library.put(path), 'executable': bool(path.stat().st_mode & 0o111)})
+        # Local instruction/skill definitions are portable; credentials and plugin caches are not.
+        definitions = []
+        for folder in ['skills', 'agents']:
+            source = home / folder
+            if source.exists():
+                for path in source.rglob('*'):
+                    if path.is_file() and not path.is_symlink() and not any(p in SKIP_DIRS for p in path.parts) and path.name not in SECRET_NAMES and not path.name.startswith('.env') and path.suffix not in {'.key','.pem','.p12','.pfx'}:
+                        definitions.append({'relative': str(path.relative_to(home)), 'content': library.put(path)})
+        if (home/'AGENTS.md').is_file(): definitions.append({'relative':'AGENTS.md','content':library.put(home/'AGENTS.md')})
+        config = {}
+        dependencies = copy.deepcopy((template or {}).get('dependencies',{}))
+        config_path = home / 'config.toml'
+        if config_path.exists():
+            raw = tomllib.loads(config_path.read_text())
+            for key in ['model', 'model_reasoning_effort', 'personality']:
+                if isinstance(raw.get(key), str): config[key] = raw[key]
+            dependencies.setdefault('mcpServers',{}).update({k: {'transport': 'url' if 'url' in v else 'command', 'requiredEnvironment': sorted(v.get('env', {}))} for k,v in raw.get('mcp_servers', {}).items()})
+            dependencies['plugins'] = sorted(set(dependencies.get('plugins',[])) | set(raw.get('plugins', {})))
+            if dependencies['mcpServers'] or dependencies['plugins']:
+                warnings.append('Account-scoped MCP/plugin dependencies are declared, not copied with credentials; configure them locally before agent work.')
+        aliases=(template or {}).get('rootOrigins',{})
+        if aliases:
+            # Keep stable resource identities across checkpoints of relocated runtimes.
+            roots={remap(k,aliases):v for k,v in roots.items()}
+            for resource in resources: resource['original']=remap(resource['original'],aliases)
+            for thread in threads.values(): thread['cwd']=remap(thread['cwd'],aliases)
+        descriptions = gs.get('electron-persisted-atom-state', {}).get('thread-descriptions-v1', {})
+        manifest = {'format': 'topolo-agent-pack', 'schemaVersion': 1, 'packId': name, 'capturedAt': now(), 'agents': chosen,
+                    'threads': threads, 'catalog': library.put(canonical(catalog)), 'resourceRoots': roots, 'resources': resources,
+                    'definitions': definitions, 'resourceAliases':aliases, 'runtimeCheckpoint':bool(template), 'config': config, 'dependencies': dependencies, 'warnings': sorted(set(warnings)),
+                    'workspaceMode': workspace_mode, 'captureMode':'live-snapshot' if live else 'idle', 'ui': {'descriptions': {k:v for k,v in descriptions.items() if any(tid in k for tid in closure)}, 'sourceAccountIds': list(gs.get('electron-persisted-atom-state', {}).get('sidebar-custom-sections-v3', {}))}}
+    finally:
+        for connection in connections.values():connection.close()
+    if not live:
+        assert_idle(home)
+        if initial != signature(home): raise PackError('Codex storage changed during capture. No complete snapshot was published; retry after closing the source profile.')
+    revision = library.save(manifest)
+    return manifest, revision
+
+def remap(value, mappings):
+    if not isinstance(value, str): return value
+    if value.startswith('file://'):
+        from urllib.parse import urlsplit,unquote,quote as urlquote
+        uri=urlsplit(value)
+        if uri.netloc not in {'','localhost'}: return value
+        original=unquote(uri.path);changed=remap(original,mappings)
+        return 'file://'+urlquote(changed,safe='/') if changed!=original else value
+    for old, new in sorted(mappings.items(), key=lambda x: -len(x[0])):
+        if value == old or value.startswith(old + os.sep): return new + value[len(old):]
+    return value
+
+def path_fields(value, mappings, parent_key=''):
+    fields = {'cwd','path','rollout_path','workspace_roots','runtime_workspace_roots','writable_roots','working_directory','roots'}
+    if isinstance(value, dict): return {k:path_fields(v,mappings,k) for k,v in value.items()}
+    if isinstance(value, list): return [path_fields(x,mappings,parent_key) for x in value]
+    if parent_key in fields: return remap(value,mappings)
+    return value
+
+def runtime_record(obj):
+    return obj.get('type') in {'session_meta','turn_context','world_state'} or (obj.get('type')=='event_msg' and obj.get('payload',{}).get('type')=='thread_settings_applied')
+
+def read_catalog(library, manifest):
+    return json.loads(library.object(manifest['catalog']).read_text())
+
+def slice_manifest(library,manifest,chosen):
+    result=copy.deepcopy(manifest);closure=set();pending=[a['threadId'] for a in chosen]
+    while pending:
+        tid=pending.pop()
+        if tid in closure: continue
+        closure.add(tid);t=manifest['threads'][tid]
+        pending.extend(([t['historyBase']] if t.get('historyBase') else [])+t.get('children',[]))
+    result['agents']=chosen;result['threads']={k:v for k,v in manifest['threads'].items() if k in closure}
+    catalog=read_catalog(library,manifest)
+    records=catalog['state_5.sqlite']['tables']['threads']
+    rows=[dict(zip(records['columns'],r)) for r in records['rows'] if r[records['columns'].index('id')] in closure]
+    sections={r.get('thread_section_id') for r in rows};projects={r.get('project_id') for r in rows}
+    for data in catalog.values():
+        for table,entry in data['tables'].items():
+            cols=entry['columns']
+            def keep(values):
+                row=dict(zip(cols,values))
+                if table=='threads': return row['id'] in closure
+                if 'thread_id' in cols: return row['thread_id'] in closure
+                if table=='thread_sections': return row['id'] in sections
+                if table=='projects': return row['id'] in projects
+                if table=='project_roots': return row['project_id'] in projects
+                if table=='thread_spawn_edges': return row['parent_thread_id'] in closure and row['child_thread_id'] in closure
+                return True
+            entry['rows']=[r for r in entry['rows'] if keep(r)]
+    result['catalog']=library.put(canonical(catalog))
+    cwds=[pathlib.Path(t['cwd']) for t in result['threads'].values()]
+    result['resourceRoots']={k:v for k,v in result['resourceRoots'].items() if not v.startswith('workspaces/') or any(c.is_relative_to(k) for c in cwds)}
+    result['resources']=[r for r in result['resources'] if any(pathlib.Path(r['original']).is_relative_to(k) for k in result['resourceRoots'])]
+    result['ui']['descriptions']={k:v for k,v in result['ui']['descriptions'].items() if any(t in k for t in closure)}
+    result['selectionInstalledAt']=now()
+    return result
+
+def same_history(library, left, right):
+    """Relocated typed paths and inherited byte offsets do not change saved history."""
+    def normalize(value, key=''):
+        if isinstance(value,dict): return {k:normalize(v,k) for k,v in value.items() if k!='end_byte_offset'}
+        if isinstance(value,list): return [normalize(v,key) for v in value]
+        if key in {'cwd','path','rollout_path','workspace_roots','runtime_workspace_roots','writable_roots','working_directory','roots'}: return '<relocated-path>'
+        return value
+    from itertools import zip_longest
+    with library.object(left).open('rb') as a, library.object(right).open('rb') as b:
+        for x,y in zip_longest(a,b):
+            if x is None or y is None: return False
+            if x==y: continue
+            x=json.loads(x);y=json.loads(y)
+            if runtime_record(x): x['payload']=normalize(x.get('payload',{}))
+            if runtime_record(y): y['payload']=normalize(y.get('payload',{}))
+            if x!=y: return False
+    return True
+
+def materialize(library, components, runtime, mappings=None):
+    """Build a fresh, manager-owned runtime; byte offsets are recomputed after path rebasing."""
+    runtime = pathlib.Path(runtime).resolve(); home = runtime / 'home'
+    if runtime.exists(): raise PackError('Target runtime already exists; never overwrite it.')
+    loaded = {}
+    chosen = []; thread_sources = {}; manifests = []; visible = set()
+    for component in components:
+        source = (component['sourcePack'], component['revision'])
+        if source not in loaded: loaded[source] = library.load(*source)[0]
+        manifest = loaded[source]
+        agent = next((a for a in manifest['agents'] if component_key(a) == component['key']), None)
+        if agent is None: raise PackError('Composition references missing agent: ' + component['key'])
+        if agent['threadId'] in visible: raise PackError('Two active agents refer to the same native thread identity; create an independently namespaced pack first.')
+        visible.add(agent['threadId']); chosen.append(agent)
+        if manifest not in manifests: manifests.append(manifest)
+        pending = [agent['threadId']]
+        while pending:
+            tid = pending.pop()
+            thread = manifest['threads'][tid]
+            if tid in thread_sources:
+                prior = thread_sources[tid][1]
+                if prior['rollout'] != thread['rollout'] and not same_history(library,prior['rollout'],thread['rollout']): raise PackError('Conflicting revisions of shared history: ' + tid)
+                continue
+            thread_sources[tid] = (manifest,thread)
+            pending.extend(([thread['historyBase']] if thread.get('historyBase') else []) + thread.get('children', []))
+    required=sum(t['rollout']['bytes'] for _,t in thread_sources.values())
+    required+=sum(m['catalog']['bytes']*2+sum(r['content']['bytes'] for r in m['resources'])+sum(d['content']['bytes'] for d in m['definitions']) for m in manifests)
+    required+=128*1024*1024
+    if shutil.disk_usage(library.root).free<required:raise PackError('Insufficient local space to stage this composition while preserving its previous runtime; estimated requirement '+str(required)+' bytes')
+    runtime.mkdir(parents=True,mode=0o700);home.mkdir(mode=0o700)
+    mapping = dict(mappings or {})
+    resource_roots = {}
+    # Establish all shared root bindings before writing any pack's files.
+    for manifest in manifests:
+        for old, relative in manifest['resourceRoots'].items():
+            if pathlib.Path(relative).is_absolute() or '..' in pathlib.Path(relative).parts: raise PackError('Unsafe resource root')
+            if old not in mapping: mapping[old] = str(runtime / 'files' / relative)
+            resource_roots[old] = mapping[old]
+    for manifest in manifests:
+        for old,original in manifest.get('resourceAliases',{}).items(): mapping[old]=remap(original,mapping)
+        if manifest.get('startingContext',{}).get('kind')=='fresh':
+            seed_home=manifest['startingContext'].get('seedHome')
+            if seed_home:
+                mapping[seed_home]=str(home)
+                mapping[str(pathlib.Path(seed_home).resolve())]=str(home)
+    written_resources={};written_definitions={}
+    def checkpoint_owns(owner,donor):
+        return owner.get('runtimeCheckpoint') and bool({a['pack'] for a in owner['agents']} & {a['pack'] for a in donor['agents']})
+    def content(reference,rebase=False):
+        data=library.object(reference).read_bytes()
+        if rebase:
+            text=data.decode()
+            for old,new in sorted(mapping.items(),key=lambda x:-len(x[0])):text=text.replace(old,new)
+            data=text.encode()
+        return data
+    for manifest in manifests:
+        for resource in manifest['resources']:
+            target = pathlib.Path(remap(resource['original'],mapping))
+            if target == pathlib.Path(resource['original']) or not target.is_relative_to(runtime):
+                # Explicit external mappings are references, never write into them.
+                if target.exists(): continue
+                raise PackError('Resource target needs a manager-owned mapping: ' + str(target))
+            target.parent.mkdir(parents=True,exist_ok=True,mode=0o700)
+            data=content(resource['content'],target.name=='AGENTS.md')
+            if target.exists() and target.read_bytes()!=data:
+                if target in written_resources and checkpoint_owns(written_resources[target],manifest): continue
+                raise PackError('Conflicting shared resource: ' + str(target))
+            target.write_bytes(data); target.chmod(0o700 if resource['executable'] else 0o600)
+            written_resources[target]=manifest
+        for definition in manifest['definitions']:
+            relative = pathlib.Path(definition['relative'])
+            if relative.is_absolute() or '..' in relative.parts or (relative.parts[0] not in {'skills','agents'} and str(relative)!='AGENTS.md'): raise PackError('Invalid definition path')
+            target = home / relative; target.parent.mkdir(parents=True,exist_ok=True)
+            data=content(definition['content'],relative.name in {'AGENTS.md','SKILL.md'} or (relative.parts[0]=='agents' and relative.suffix=='.md'))
+            if target.exists() and target.read_bytes()!=data:
+                if target in written_definitions and checkpoint_owns(written_definitions[target],manifest): continue
+                raise PackError('Conflicting skill/agent definition: ' + str(relative))
+            target.write_bytes(data); target.chmod(0o600)
+            written_definitions[target]=manifest
+    for tid,(_,thread) in thread_sources.items():
+        cwd = pathlib.Path(remap(thread['cwd'],mapping))
+        if not cwd.is_dir():
+            if any(cwd.is_relative_to(runtime / 'files') for _ in [0]): cwd.mkdir(parents=True,exist_ok=True)
+            else: raise PackError('External workspace unavailable; supply --map OLD=NEW: ' + str(cwd))
+    # Preserve original historical prose. Rebase typed current-context paths only.
+    for old, target in resource_roots.items():
+        path = pathlib.Path(target)
+        instructions = list(path.rglob('AGENTS.md')) if path.is_dir() else ([path] if path.name == 'AGENTS.md' else [])
+        for f in instructions:
+            if not f.is_relative_to(runtime): continue
+            text = f.read_text()
+            for source,dest in sorted(mapping.items(), key=lambda x:-len(x[0])): text = text.replace(source,dest)
+            f.write_text(text)
+    for f in [home/'AGENTS.md',*home.glob('skills/**/SKILL.md'),*home.glob('agents/**/*.md')]:
+        if not f.is_file(): continue
+        text=f.read_text()
+        for source,dest in sorted(mapping.items(),key=lambda x:-len(x[0])): text=text.replace(source,dest)
+        f.write_text(text)
+    offset_maps = {}; rollout_paths = {}; transforming = set(); donor_maps = {}
+    def parent_offsets(manifest,tid):
+        reference=manifest['threads'][tid]['rollout']
+        if reference==thread_sources[tid][1]['rollout']: return offset_maps[tid]
+        key=(reference['sha256'],tid)
+        if key not in donor_maps:
+            # The shared history was proved semantically identical above. Align its
+            # original record boundaries with the selected parent's new boundaries.
+            starts=[];position=0
+            with library.object(reference).open('rb') as stream:
+                for line in stream: starts.append(position);position+=len(line)
+            starts.append(position)
+            ends=list(offset_maps[tid].values())
+            if len(starts)!=len(ends): raise PackError('Shared history boundary mismatch')
+            donor_maps[key]=dict(zip(starts,ends))
+        return donor_maps[key]
+    def restore_rollout(tid):
+        if tid in offset_maps: return
+        if tid in transforming: raise PackError('Inherited history cycle')
+        transforming.add(tid)
+        manifest,thread = thread_sources[tid]
+        if thread.get('historyBase'): restore_rollout(thread['historyBase'])
+        src = library.object(thread['rollout'])
+        if pathlib.Path(thread['basename']).name != thread['basename']: raise PackError('Unsafe rollout basename')
+        folder = home / ('sessions' if tid in visible else 'archived_sessions')
+        date = re.match(r'rollout-(\d{4})-(\d{2})-(\d{2})T',thread['basename'])
+        if date: folder = folder.joinpath(*date.groups())
+        target = folder / thread['basename']
+        target.parent.mkdir(parents=True,exist_ok=True)
+        old_pos = 0; new_pos = 0; positions = {}
+        with src.open('rb') as source, target.open('wb') as dest:
+            for line in source:
+                positions[old_pos] = new_pos
+                obj = json.loads(line)
+                if manifest.get('startingContext',{}).get('kind')=='fresh':
+                    # Only newly generated bootstrap context is rewritten. Historical
+                    # work snapshots and later checkpointed conversations stay verbatim.
+                    def rebind_bootstrap(value):
+                        if isinstance(value,dict):return {k:rebind_bootstrap(v) for k,v in value.items()}
+                        if isinstance(value,list):return [rebind_bootstrap(v) for v in value]
+                        if isinstance(value,str):
+                            for old,new in sorted(mapping.items(),key=lambda pair:-len(pair[0])):value=value.replace(old,new)
+                        return value
+                    obj=rebind_bootstrap(obj)
+                    encoded=canonical(obj)+b'\n'
+                elif runtime_record(obj):
+                    obj['payload'] = path_fields(obj.get('payload',{}),mapping)
+                    base = obj['payload'].get('history_base')
+                    if base and base.get('end_byte_offset') is not None:
+                        parent_map = parent_offsets(manifest,base['thread_id'])
+                        if base['end_byte_offset'] not in parent_map: raise PackError('Invalid inherited history boundary')
+                        base['end_byte_offset'] = parent_map[base['end_byte_offset']]
+                    encoded = canonical(obj) + b'\n'
+                else: encoded = line
+                dest.write(encoded); old_pos += len(line); new_pos += len(encoded)
+            positions[old_pos] = new_pos
+        target.chmod(0o600); offset_maps[tid] = positions; rollout_paths[tid] = str(target)
+        transforming.remove(tid)
+    for tid in thread_sources: restore_rollout(tid)
+    paused_goals = []; dbs = {}; row_keys = {}
+    try:
+        for manifest in manifests:
+            catalog = read_catalog(library,manifest)
+            for filename,data in catalog.items():
+                if filename not in DB_TABLES: raise PackError('Unknown database adapter')
+                if filename not in dbs:
+                    conn = sqlite3.connect(home / filename); dbs[filename] = conn; row_keys[filename] = {}
+                    for schema in data['schemas']:
+                        if schema['table'] not in DB_TABLES[filename]: raise PackError('Unexpected native schema table')
+                        prefix = r'^CREATE\s+' + ('TABLE' if schema['type']=='table' else r'(?:UNIQUE\s+)?INDEX') + r'\s+'
+                        if not re.match(prefix,schema['sql'],re.I): raise PackError('Unsafe database schema')
+                        conn.execute(schema['sql'])
+                else:
+                    conn = dbs[filename]
+                    existing = {r[0]:r[1] for r in conn.execute("SELECT name,sql FROM sqlite_master WHERE sql IS NOT NULL")}
+                    if any(existing.get(s['name']) != s['sql'] for s in data['schemas']): raise PackError('Packs use incompatible Codex database schemas')
+                for table,entry in data['tables'].items():
+                    if table not in DB_TABLES[filename]: raise PackError('Unexpected native table')
+                    cols = entry['columns']
+                    if cols != [r[1] for r in conn.execute('PRAGMA table_info('+quote(table)+')')]: raise PackError('Native schema columns mismatch')
+                    primary = [r[1] for r in sorted(conn.execute('PRAGMA table_info('+quote(table)+')'),key=lambda r:r[5]) if r[5]]
+                    rows_seen = row_keys[filename].setdefault(table,{})
+                    for encoded in entry['rows']:
+                        row = dict(zip(cols,[decode(v) for v in encoded]))
+                        tid = row.get('thread_id') or (row.get('id') if table=='threads' else None)
+                        if tid and tid not in thread_sources: continue
+                        if tid and thread_sources[tid][0] is not manifest: continue
+                        if table=='thread_spawn_edges' and (row['parent_thread_id'] not in thread_sources or row['child_thread_id'] not in thread_sources): continue
+                        if table=='threads':
+                            row['rollout_path'] = rollout_paths[tid]; row['cwd'] = remap(row['cwd'],mapping)
+                            if row.get('agent_path'): row['agent_path']=remap(row['agent_path'],mapping)
+                            row['archived'] = int(tid not in visible)
+                            if tid not in visible and 'thread_section_id' in row: row['thread_section_id'] = None
+                            if 'archived_at' in row and tid in visible: row['archived_at'] = None
+                            if 'sandbox_policy' in row:
+                                try: row['sandbox_policy'] = json.dumps(path_fields(json.loads(row['sandbox_policy']),mapping))
+                                except (json.JSONDecodeError,TypeError): pass
+                        if table in {'thread_turns','thread_history_projection_state'}:
+                            for field in ['rollout_byte_offset','rollout_end_byte_offset','next_rollout_byte_offset']:
+                                if row.get(field) is not None:
+                                    positions = offset_maps[tid]
+                                    if row[field] not in positions: raise PackError('Stored history offset has no matching rollout boundary')
+                                    row[field] = positions[row[field]]
+                        if table=='thread_goals' and row['status']=='active': row['status']='paused'; paused_goals.append(tid)
+                        if table=='project_roots':
+                            row = path_fields(row,mapping)
+                        if table=='thread_attachments':
+                            try: row['payload']=json.dumps(path_fields(json.loads(row['payload']),mapping))
+                            except (json.JSONDecodeError,TypeError): pass
+                            row['identity_key']=remap(row['identity_key'],mapping)
+                        key = tuple(row[c] for c in primary) if primary else tuple(str(row[c]) for c in cols)
+                        if key in rows_seen:
+                            if table=='_sqlx_migrations':
+                                old = rows_seen[key]
+                                if old.get('checksum') != row.get('checksum'): raise PackError('Migration version checksum mismatch')
+                            elif rows_seen[key] != row: raise PackError('Conflicting native shared records in ' + table)
+                            continue
+                        rows_seen[key]=row
+                        conn.execute('INSERT INTO '+quote(table)+' ('+','.join(quote(c) for c in cols)+') VALUES ('+','.join('?' for _ in cols)+')',[row[c] for c in cols])
+        for filename,conn in dbs.items():
+            conn.commit()
+            if conn.execute('PRAGMA integrity_check').fetchone()[0] != 'ok': raise PackError('Native database integrity failed')
+    finally:
+        for conn in dbs.values(): conn.close()
+    # Generate the dual desktop/server section mapping from the selected composition.
+    catalog_conn = sqlite3.connect(home / 'state_5.sqlite')
+    import uuid
+    groups = {}; pins = []; titles = {}; descriptions = {}; account_ids = set()
+    for agent in chosen:
+        groupkey = (agent['pack'],agent['set'])
+        if groupkey not in groups:
+            native_id = '01984de2-8f74-7c91-a3b2-5c5e937cf318' if agent.get('pinned') else str(uuid.uuid5(uuid.NAMESPACE_URL,'codex-pack-section:'+ '/'.join(groupkey)))
+            desktop_id = str(uuid.uuid5(uuid.NAMESPACE_URL,'codex-pack-desktop:'+ '/'.join(groupkey)))
+            groups[groupkey] = {'id':desktop_id,'name':agent['sectionName'],'hostSectionIds':{'local':native_id},'itemKeys':[],'appearance':None}
+            catalog_conn.execute('INSERT OR REPLACE INTO thread_sections (id,name) VALUES (?,?)',(native_id,agent['sectionName']))
+        g = groups[groupkey]; key = 'codex:thread:local:'+agent['threadId']; g['itemKeys'].append(key)
+        if agent.get('pinned'): pins.append(key)
+        titles[agent['threadId']] = agent['title']
+        catalog_conn.execute('UPDATE threads SET thread_section_id=?,section_position=? WHERE id=?',(g['hostSectionIds']['local'],agent.get('position',0),agent['threadId']))
+    catalog_conn.execute('DELETE FROM thread_sections WHERE id NOT IN (SELECT DISTINCT thread_section_id FROM threads WHERE thread_section_id IS NOT NULL)')
+    # The complete native catalog was restored above. Starting a legacy rollout
+    # backfill would overwrite captured timestamps and creator metadata from
+    # older, less complete session headers. Generate this target-local marker;
+    # never import another home's watermark or migration progress.
+    catalog_conn.execute('INSERT OR REPLACE INTO backfill_state (id,status,last_watermark,last_success_at,updated_at) VALUES (1,?,?,?,?)',
+                         ('complete',None,int(time.time()),int(time.time())))
+    catalog_conn.commit(); catalog_conn.close()
+    for manifest in manifests:
+        account_ids.update(manifest['ui'].get('sourceAccountIds',[])); descriptions.update(manifest['ui'].get('descriptions',{}))
+    groups_list = [g for (_,sid),g in groups.items() if sid!='intakes']
+    sections = {account:{'sections':groups_list,'sectionOrder':['pinned']+[g['id'] for g in groups_list]+['chats','threads'],'collapsedSectionIds':[]} for account in account_ids}
+    gs={'thread-titles':{'titles':titles,'order':list(titles)},'pinned-thread-ids':[k.rsplit(':',1)[1] for k in pins],
+        'thread-workspace-root-hints':{a['threadId']:remap(thread_sources[a['threadId']][1]['cwd'],mapping) for a in chosen},
+        'electron-persisted-atom-state':{'sidebar-custom-sections-v3':sections,'unified-sidebar-pinned-order-v1':pins,'thread-descriptions-v1':descriptions,'flat-project-sidebar-preferences-v1':{'initialized':True,'mode':'project','chatSortMode':'updated_at','projectSortMode':'updated_at'}}}
+    atomic_json(home / '.codex-global-state.json',gs)
+    config_text = '[analytics]\nenabled = false\n'
+    for setting in ['model','model_reasoning_effort','personality']:
+        values={m.get('config',{}).get(setting) for m in manifests}-{None}
+        if len(values)==1: config_text=setting+' = '+json.dumps(next(iter(values)))+'\n'+config_text
+    (home/'config.toml').write_text(config_text); (home/'config.toml').chmod(0o600)
+    dependencies = {m['packId']:m.get('dependencies',{}) for m in manifests}
+    atomic_json(runtime/'dependencies.json',dependencies)
+    atomic_json(runtime/'bindings.json',mapping)
+    # Capturable semantic identities live independently of desktop caches.
+    declarations={'mcpServers':{},'plugins':[]}
+    for m in manifests:
+        for field in ['applications','capabilities','integrationNotes']:
+            declarations[field]=sorted(set(declarations.get(field,[])) | set(m.get('dependencies',{}).get(field,[])))
+        for key,value in m.get('dependencies',{}).get('mcpServers',{}).items():
+            if key in declarations['mcpServers'] and declarations['mcpServers'][key]!=value: raise PackError('Conflicting MCP prerequisite declarations: '+key)
+            declarations['mcpServers'][key]=value
+        declarations['plugins']=sorted(set(declarations['plugins']) | set(m.get('dependencies',{}).get('plugins',[])))
+    runtime_manifest={'agents':chosen,'resourcePaths':sorted(set(resource_roots.values())),
+                      'rootOrigins':{target:original for original,target in resource_roots.items()}, 'dependencies':declarations}
+    atomic_json(runtime/'agents.json',runtime_manifest)
+    from .topolo import attach_runtime
+    attach_runtime(library,runtime)
+    result={'threads':len(thread_sources),'agents':len(chosen),'historyOffsetsRebased':True,'activeGoalsPaused':sorted(set(paused_goals)),
+            'home':str(home),'userData':str(runtime/'desktop-data'),'dependencies':dependencies,'mappings':mapping}
+    atomic_json(runtime/'verification.json',result)
+    return result