mlai-code 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -2
- package/bin/mlai.mjs +3 -0
- package/package.json +18 -3
- package/src/browser-setup.mjs +77 -0
- package/src/cli-ext.mjs +146 -0
- package/src/cli.mjs +534 -0
- package/src/cloud.mjs +29 -0
- package/src/completion.mjs +159 -0
- package/src/credentials.mjs +54 -0
- package/src/deploy.mjs +134 -0
- package/src/distribution.mjs +373 -0
- package/src/engine-key.mjs +4 -0
- package/src/engine.mjs +167 -0
- package/src/headless.mjs +196 -0
- package/src/home.mjs +22 -0
- package/src/host.mjs +123 -0
- package/src/local-models.mjs +64 -0
- package/src/modes.mjs +38 -0
- package/src/netbox.mjs +97 -0
- package/src/netjail.py +59 -0
- package/src/offline.mjs +98 -0
- package/src/parity.mjs +86 -0
- package/src/project.mjs +68 -0
- package/src/relay.mjs +140 -0
- package/src/render.mjs +73 -0
- package/src/restricted.mjs +47 -0
- package/src/review.mjs +70 -0
- package/src/runner.mjs +127 -0
- package/src/runtime.mjs +75 -0
- package/src/sandbox.mjs +135 -0
- package/src/sdk.mjs +11 -0
- package/src/server.mjs +110 -0
- package/src/session.mjs +290 -0
- package/src/settings.mjs +135 -0
- package/src/shared-data.mjs +3 -0
- package/src/tui/acp.mjs +145 -0
- package/src/tui/agents-cli.mjs +114 -0
- package/src/tui/agents-local.mjs +65 -0
- package/src/tui/ansi.mjs +123 -0
- package/src/tui/app.mjs +710 -0
- package/src/tui/approval.mjs +88 -0
- package/src/tui/bg.mjs +172 -0
- package/src/tui/bgterm.mjs +79 -0
- package/src/tui/chat.mjs +230 -0
- package/src/tui/clipboard.mjs +74 -0
- package/src/tui/cloud-cli.mjs +57 -0
- package/src/tui/cloud-run.mjs +123 -0
- package/src/tui/cmd-agents.mjs +130 -0
- package/src/tui/cmd-auto-review.mjs +94 -0
- package/src/tui/cmd-cloud.mjs +17 -0
- package/src/tui/cmd-config.mjs +104 -0
- package/src/tui/cmd-core.mjs +160 -0
- package/src/tui/cmd-distribution.mjs +15 -0
- package/src/tui/cmd-ecosystem.mjs +169 -0
- package/src/tui/cmd-extensibility.mjs +333 -0
- package/src/tui/cmd-remote.mjs +18 -0
- package/src/tui/cmd-review.mjs +58 -0
- package/src/tui/cmd-session-extra.mjs +152 -0
- package/src/tui/cmd-tui.mjs +200 -0
- package/src/tui/commands.mjs +160 -0
- package/src/tui/custom-commands.mjs +98 -0
- package/src/tui/diff.mjs +60 -0
- package/src/tui/editor.mjs +190 -0
- package/src/tui/ext-state.mjs +16 -0
- package/src/tui/extensions-local.mjs +140 -0
- package/src/tui/files.mjs +72 -0
- package/src/tui/handoff.mjs +110 -0
- package/src/tui/import-setups.mjs +151 -0
- package/src/tui/init.mjs +104 -0
- package/src/tui/keymap.mjs +70 -0
- package/src/tui/keys.mjs +101 -0
- package/src/tui/loop.mjs +75 -0
- package/src/tui/markdown.mjs +118 -0
- package/src/tui/mcp-config.mjs +225 -0
- package/src/tui/mcp-oauth.mjs +162 -0
- package/src/tui/permissions.mjs +189 -0
- package/src/tui/plain.mjs +32 -0
- package/src/tui/plugins-local.mjs +103 -0
- package/src/tui/remote.mjs +125 -0
- package/src/tui/review-cli.mjs +332 -0
- package/src/tui/review-github.mjs +59 -0
- package/src/tui/screen.mjs +60 -0
- package/src/tui/session-ops.mjs +104 -0
- package/src/tui/skills-local.mjs +92 -0
- package/src/tui/snapshots.mjs +87 -0
- package/src/tui/statusline.mjs +98 -0
- package/src/tui/transcript.mjs +116 -0
- package/src/two-words.mjs +36 -0
- package/src/wakeups.mjs +53 -0
- package/src/winenv.mjs +94 -0
- package/src/winsandbox.py +351 -0
- package/src/workspace-api.mjs +621 -0
- package/src/worktree.mjs +128 -0
- package/vendor/client-sdk/controller.cjs +115 -0
- package/vendor/client-sdk/files.cjs +38 -0
- package/vendor/client-sdk/index.cjs +133 -0
- package/vendor/client-sdk/login.cjs +52 -0
- package/vendor/client-sdk/updates.cjs +41 -0
- package/vendor/cloud-cli/mlai.mjs +394 -0
- package/worker/architecture_parse.cjs +23 -0
- package/worker/architecture_worker.py +394 -0
- package/worker/ask_first.py +96 -0
- package/worker/attachment_reader.py +66 -0
- package/worker/background_jobs.py +111 -0
- package/worker/brief_questions.py +56 -0
- package/worker/browser_local.py +49 -0
- package/worker/browser_pane.py +89 -0
- package/worker/browser_tool.py +101 -0
- package/worker/build_planner.py +187 -0
- package/worker/chat_fork.py +277 -0
- package/worker/checkpoints.py +133 -0
- package/worker/cloud_pty.py +351 -0
- package/worker/command_policy.py +39 -0
- package/worker/command_safety.py +717 -0
- package/worker/computer_card.sh +23 -0
- package/worker/computer_setup.sh +86 -0
- package/worker/computer_use.py +221 -0
- package/worker/connector_search.py +43 -0
- package/worker/context_manager.py +60 -0
- package/worker/conversation.py +144 -0
- package/worker/deploy_pack.py +116 -0
- package/worker/design_director.py +240 -0
- package/worker/dev_jobs.py +166 -0
- package/worker/edit_guard.py +99 -0
- package/worker/env_setup.py +129 -0
- package/worker/evidence.py +109 -0
- package/worker/fast_search.py +24 -0
- package/worker/git_sync.py +333 -0
- package/worker/goals.py +156 -0
- package/worker/handover.py +98 -0
- package/worker/harness_memory.py +194 -0
- package/worker/image_generation.py +173 -0
- package/worker/intelligence.py +149 -0
- package/worker/intent_model.py +78 -0
- package/worker/intent_weights.json +1 -0
- package/worker/ios_simulator.py +87 -0
- package/worker/kimi_bridge.py +997 -0
- package/worker/kimi_compact.py +93 -0
- package/worker/kimi_gates.py +304 -0
- package/worker/kimi_hook.py +617 -0
- package/worker/lifecycle.py +95 -0
- package/worker/lsp.py +77 -0
- package/worker/mcp_local.py +293 -0
- package/worker/memory.py +99 -0
- package/worker/mlai_auto_review.py +177 -0
- package/worker/mlai_hooks.py +966 -0
- package/worker/mlai_mcp.py +441 -0
- package/worker/mlai_permissions.py +283 -0
- package/worker/model_stream.py +75 -0
- package/worker/nested_repos.py +76 -0
- package/worker/offline_guard.py +17 -0
- package/worker/preview.py +806 -0
- package/worker/proc_watch.py +124 -0
- package/worker/project_context.py +72 -0
- package/worker/project_rules.py +99 -0
- package/worker/quality.py +180 -0
- package/worker/reply_language.py +56 -0
- package/worker/repo_hygiene.py +132 -0
- package/worker/repo_memory.py +101 -0
- package/worker/repo_policy.py +269 -0
- package/worker/roles.json +3173 -0
- package/worker/roles.py +303 -0
- package/worker/runner.py +1795 -0
- package/worker/runtime_control.py +205 -0
- package/worker/secret_scan.py +168 -0
- package/worker/session_hub.py +80 -0
- package/worker/skill_library.py +129 -0
- package/worker/skill_pack.py +533 -0
- package/worker/subagents.py +91 -0
- package/worker/task_controller.py +139 -0
- package/worker/task_skills.py +159 -0
- package/worker/team_checkpoint.py +33 -0
- package/worker/token_count.py +32 -0
- package/worker/tool_catalog.py +38 -0
- package/worker/upload_storage.py +531 -0
- package/worker/verification.py +725 -0
- package/worker/vision.py +84 -0
- package/worker/wakeups.py +63 -0
- package/worker/web_tools.py +203 -0
- package/worker/winproc.py +135 -0
- package/worker/work_ledger.py +117 -0
- package/worker/worktrees.py +79 -0
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""Copy chat text and optional current source files, never the running computer.
|
|
2
|
+
|
|
3
|
+
Archives are bounded, reject links, and contain no git/config/process state. The
|
|
4
|
+
source index and branch are never changed. Used only after server-side ownership.
|
|
5
|
+
"""
|
|
6
|
+
import fcntl
|
|
7
|
+
import hashlib
|
|
8
|
+
import io
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path, PurePosixPath
|
|
12
|
+
import re
|
|
13
|
+
import shutil
|
|
14
|
+
import stat
|
|
15
|
+
import subprocess
|
|
16
|
+
import tarfile
|
|
17
|
+
import tempfile
|
|
18
|
+
|
|
19
|
+
from secret_scan import scan_text
|
|
20
|
+
|
|
21
|
+
MAX_BYTES = 64 * 1024 * 1024
|
|
22
|
+
MAX_FILE = 8 * 1024 * 1024
|
|
23
|
+
MAX_TEXT = 300000
|
|
24
|
+
MAX_FILES = 10000
|
|
25
|
+
PRIVATE = {'.git', '.mini', '.ssh', '.aws', '.azure', '.config', '.vercel',
|
|
26
|
+
'.netrc', '.npmrc', '.pypirc', 'credentials', 'credentials.json',
|
|
27
|
+
'service-account.json', 'id_rsa', 'id_ed25519'}
|
|
28
|
+
GENERATED = {'node_modules', '.next', '.venv', 'venv', '__pycache__', 'dist', 'build'}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def excluded(name):
|
|
32
|
+
parts = PurePosixPath(name).parts
|
|
33
|
+
return any(p.lower() in PRIVATE | GENERATED or p.lower() == '.env' or
|
|
34
|
+
p.lower().startswith('.env.') or p.lower().endswith(('.pem', '.key', '.p12', '.pfx'))
|
|
35
|
+
for p in parts)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def clean_text(value):
|
|
39
|
+
text = str(value or '')
|
|
40
|
+
for _, _, secret, offset in reversed(scan_text(text)):
|
|
41
|
+
text = text[:offset] + '[redacted]' + text[offset + len(secret):]
|
|
42
|
+
return text
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def read_regular(base, name, limit):
|
|
46
|
+
"""Open relative to directory descriptors; no symlink component can escape."""
|
|
47
|
+
parts = PurePosixPath(name).parts
|
|
48
|
+
if not parts or name.startswith('/') or any(p in ('..', '.') for p in parts):
|
|
49
|
+
raise ValueError('Invalid fork file path')
|
|
50
|
+
fd = os.open(base, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
51
|
+
try:
|
|
52
|
+
for part in parts[:-1]:
|
|
53
|
+
next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
54
|
+
os.close(fd)
|
|
55
|
+
fd = next_fd
|
|
56
|
+
file_fd = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd)
|
|
57
|
+
with os.fdopen(file_fd, 'rb') as stream:
|
|
58
|
+
before = os.fstat(stream.fileno())
|
|
59
|
+
if not stat.S_ISREG(before.st_mode) or before.st_size > limit:
|
|
60
|
+
raise ValueError('Fork file is not regular or exceeds the size limit')
|
|
61
|
+
data = stream.read(limit + 1)
|
|
62
|
+
after = os.fstat(stream.fileno())
|
|
63
|
+
if len(data) > limit or (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
|
|
64
|
+
raise ValueError('A source file changed during the fork; wait and try again')
|
|
65
|
+
return data, before.st_mode & 0o111
|
|
66
|
+
finally:
|
|
67
|
+
os.close(fd)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def load_json(root, name, default=None):
|
|
71
|
+
try:
|
|
72
|
+
data, _ = read_regular(root, name, 16 * 1024 * 1024)
|
|
73
|
+
return json.loads(data)
|
|
74
|
+
except FileNotFoundError:
|
|
75
|
+
return default
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def chat_turns(root, point):
|
|
79
|
+
if point != 'latest' and not re.fullmatch(r'[0-9]{1,24}', point):
|
|
80
|
+
raise ValueError('Invalid conversation point')
|
|
81
|
+
history = root / 'history'
|
|
82
|
+
if history.is_symlink():
|
|
83
|
+
raise ValueError('Linked history is not supported')
|
|
84
|
+
names = sorted(p.stem for p in history.glob('*.json') if re.fullmatch(r'[0-9]{1,24}', p.stem))
|
|
85
|
+
if point != 'latest':
|
|
86
|
+
if point not in names:
|
|
87
|
+
raise ValueError('This message no longer exists. Refresh the chat.')
|
|
88
|
+
names = names[:names.index(point) + 1]
|
|
89
|
+
if len(names) > 100:
|
|
90
|
+
raise ValueError('This chat is too large to fork automatically. Export a smaller transcript.')
|
|
91
|
+
def records():
|
|
92
|
+
for name in names:
|
|
93
|
+
yield load_json(root, 'history/' + name + '.json')
|
|
94
|
+
rows = records()
|
|
95
|
+
latest = None
|
|
96
|
+
if point == 'latest':
|
|
97
|
+
request = load_json(root, 'request.json', {})
|
|
98
|
+
if request.get('task'):
|
|
99
|
+
events = []
|
|
100
|
+
try:
|
|
101
|
+
raw, _ = read_regular(root, 'events.jsonl', 16 * 1024 * 1024)
|
|
102
|
+
lines = raw.decode().splitlines()
|
|
103
|
+
for index, line in enumerate(lines):
|
|
104
|
+
if not line.strip():
|
|
105
|
+
continue
|
|
106
|
+
try:
|
|
107
|
+
events.append(json.loads(line))
|
|
108
|
+
except json.JSONDecodeError:
|
|
109
|
+
if index == len(lines) - 1 and not raw.endswith(b'\n'):
|
|
110
|
+
break # A crashed worker may leave only its final event incomplete.
|
|
111
|
+
raise ValueError('A conversation event is damaged; restore its log before forking')
|
|
112
|
+
except FileNotFoundError:
|
|
113
|
+
pass
|
|
114
|
+
latest = {'task': request['task'], 'events': events}
|
|
115
|
+
turns, total = [], 0
|
|
116
|
+
import itertools
|
|
117
|
+
for row in itertools.chain(rows, [latest] if latest else []):
|
|
118
|
+
if not isinstance(row, dict):
|
|
119
|
+
raise ValueError('Conversation record is invalid')
|
|
120
|
+
events = row.get('events', [])
|
|
121
|
+
answer = next((e.get('content', '') for e in reversed(events) if e.get('type') == 'answer'), '')
|
|
122
|
+
if not answer:
|
|
123
|
+
answer = next((e.get('content', '') for e in reversed(events) if e.get('type') == 'assistant'), '')
|
|
124
|
+
task = clean_text(row.get('task', ''))
|
|
125
|
+
steering = [clean_text(e.get('content', '')) for e in events if e.get('type') in ('steering', 'imported_user')]
|
|
126
|
+
answer = clean_text(answer)
|
|
127
|
+
total += len(task) + len(answer) + sum(map(len, steering))
|
|
128
|
+
if total > MAX_TEXT:
|
|
129
|
+
raise ValueError('This chat exceeds the fork text limit. Export a smaller transcript.')
|
|
130
|
+
if task or answer:
|
|
131
|
+
turns.append({'task': task, 'answer': answer, 'steering': steering})
|
|
132
|
+
if not turns:
|
|
133
|
+
raise ValueError('Send a message before forking this chat.')
|
|
134
|
+
return turns
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def add_bytes(archive, name, data, executable=False):
|
|
138
|
+
info = tarfile.TarInfo(name)
|
|
139
|
+
info.size = len(data)
|
|
140
|
+
info.mode = 0o755 if executable else 0o644
|
|
141
|
+
archive.addfile(info, io.BytesIO(data))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def export_chat(root, point='latest', include_files=True):
|
|
145
|
+
root = Path(root)
|
|
146
|
+
mini, repo = root / '.mini', root / 'repo'
|
|
147
|
+
with (mini / 'setup.lock').open('a') as lock:
|
|
148
|
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
149
|
+
if (mini / 'run.lock').exists():
|
|
150
|
+
raise ValueError('Wait for the current run to finish before forking.')
|
|
151
|
+
turns = chat_turns(mini, point)
|
|
152
|
+
directory = Path(tempfile.mkdtemp(prefix='mlai-fork-'))
|
|
153
|
+
archive_path = directory / 'bundle.tar'
|
|
154
|
+
count, total, skipped = 0, 0, []
|
|
155
|
+
try:
|
|
156
|
+
with tarfile.open(archive_path, 'w') as archive:
|
|
157
|
+
if include_files:
|
|
158
|
+
result = subprocess.run(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'],
|
|
159
|
+
cwd=repo, capture_output=True, check=True, timeout=20)
|
|
160
|
+
names = sorted(set(result.stdout.decode('utf-8').strip('\0').split('\0')) - {''})
|
|
161
|
+
if len(names) > MAX_FILES:
|
|
162
|
+
raise ValueError('Too many files to fork. Use Git or choose conversation only.')
|
|
163
|
+
for name in names:
|
|
164
|
+
if excluded(name):
|
|
165
|
+
skipped.append(name)
|
|
166
|
+
continue
|
|
167
|
+
try:
|
|
168
|
+
data, executable = read_regular(repo, name, MAX_FILE)
|
|
169
|
+
except FileNotFoundError:
|
|
170
|
+
continue # deleted tracked file, as in the current working tree
|
|
171
|
+
except OSError as error:
|
|
172
|
+
raise ValueError('Linked or unreadable files cannot be forked. Choose conversation only.') from error
|
|
173
|
+
total += len(data)
|
|
174
|
+
if total > MAX_BYTES:
|
|
175
|
+
raise ValueError('Source exceeds 64 MB. Use Git or choose conversation only.')
|
|
176
|
+
# Scan in overlapping windows; the scanner intentionally caps each call at 2 MB.
|
|
177
|
+
for start in range(0, len(data), 1000000):
|
|
178
|
+
if scan_text(data[max(0, start-4096):start+1004096].decode('utf-8', errors='replace')):
|
|
179
|
+
raise ValueError('A source file contains a credential. Move it to Vault or fork conversation only.')
|
|
180
|
+
add_bytes(archive, 'files/' + name, data, executable)
|
|
181
|
+
count += 1
|
|
182
|
+
metadata = {'version': 1, 'turns': turns, 'point': point, 'files': count,
|
|
183
|
+
'bytes': total, 'skipped': skipped[:100], 'skippedCount': len(skipped),
|
|
184
|
+
'includeFiles': include_files, 'fileState': 'current'}
|
|
185
|
+
add_bytes(archive, 'chat.json', json.dumps(metadata).encode())
|
|
186
|
+
digest = hashlib.sha256(archive_path.read_bytes()).hexdigest()
|
|
187
|
+
return {'path': str(archive_path), 'sha256': digest, 'size': archive_path.stat().st_size,
|
|
188
|
+
'files': count, 'turns': len(turns), 'skippedCount': len(skipped)}
|
|
189
|
+
except BaseException:
|
|
190
|
+
shutil.rmtree(directory)
|
|
191
|
+
raise
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def import_chat(root, archive_path):
|
|
195
|
+
root = Path(root)
|
|
196
|
+
repo, mini = root / 'repo', root / '.mini'
|
|
197
|
+
if Path(archive_path).stat().st_size > MAX_BYTES + 12 * 1024 * 1024:
|
|
198
|
+
raise ValueError('Fork archive is too large')
|
|
199
|
+
with tarfile.open(archive_path, 'r:') as archive:
|
|
200
|
+
members = archive.getmembers()
|
|
201
|
+
if len(members) > MAX_FILES + 1:
|
|
202
|
+
raise ValueError('Too many archive entries')
|
|
203
|
+
seen, total, metadata = set(), 0, None
|
|
204
|
+
files = []
|
|
205
|
+
for member in members:
|
|
206
|
+
name, parts = member.name, PurePosixPath(member.name).parts
|
|
207
|
+
if name in seen or name != '/'.join(parts) or not member.isfile() or name.startswith('/') or any(p in ('..', '.') for p in parts):
|
|
208
|
+
raise ValueError('Invalid fork archive entry')
|
|
209
|
+
seen.add(name)
|
|
210
|
+
if name == 'chat.json':
|
|
211
|
+
if member.size > MAX_TEXT * 8:
|
|
212
|
+
raise ValueError('Fork text is too large')
|
|
213
|
+
metadata = json.load(archive.extractfile(member))
|
|
214
|
+
elif name.startswith('files/') and len(parts) > 1 and not excluded('/'.join(parts[1:])):
|
|
215
|
+
total += member.size
|
|
216
|
+
if member.size > MAX_FILE or total > MAX_BYTES:
|
|
217
|
+
raise ValueError('Fork files exceed the size limit')
|
|
218
|
+
files.append(member)
|
|
219
|
+
else:
|
|
220
|
+
raise ValueError('Unexpected fork archive path')
|
|
221
|
+
if not isinstance(metadata, dict) or metadata.get('version') != 1 or not isinstance(metadata.get('turns'), list) or not 1 <= len(metadata['turns']) <= 101:
|
|
222
|
+
raise ValueError('Invalid fork conversation')
|
|
223
|
+
if not all(key in metadata for key in ('point', 'fileState', 'includeFiles', 'files', 'skippedCount')):
|
|
224
|
+
raise ValueError('Invalid fork metadata')
|
|
225
|
+
validated = []
|
|
226
|
+
text_size = 0
|
|
227
|
+
for turn in metadata['turns']:
|
|
228
|
+
if not isinstance(turn, dict) or not isinstance(turn.get('task'), str) or not isinstance(turn.get('answer'), str):
|
|
229
|
+
raise ValueError('Invalid fork turn')
|
|
230
|
+
steering = turn.get('steering', [])
|
|
231
|
+
if not isinstance(steering, list) or any(not isinstance(text, str) for text in steering):
|
|
232
|
+
raise ValueError('Invalid fork follow-up messages')
|
|
233
|
+
task = clean_text(turn['task'])
|
|
234
|
+
if steering:
|
|
235
|
+
task += '\n\nFurther user messages in this turn:\n' + '\n\n'.join(clean_text(text) for text in steering)
|
|
236
|
+
answer = clean_text(turn['answer'])
|
|
237
|
+
text_size += len(task) + len(answer)
|
|
238
|
+
if text_size > MAX_TEXT + 10000:
|
|
239
|
+
raise ValueError('Fork conversation exceeds the text limit')
|
|
240
|
+
validated.append((task, answer))
|
|
241
|
+
# Destination is a newly provisioned empty computer, never the source.
|
|
242
|
+
if (mini / 'request.json').exists() or (mini / 'history').exists():
|
|
243
|
+
raise ValueError('Refusing to replace an existing conversation')
|
|
244
|
+
if repo.is_symlink() or mini.is_symlink():
|
|
245
|
+
raise ValueError('Linked destination is not supported')
|
|
246
|
+
for child in repo.iterdir():
|
|
247
|
+
if child.name != '.git':
|
|
248
|
+
if child.is_dir() and not child.is_symlink():
|
|
249
|
+
shutil.rmtree(child)
|
|
250
|
+
else:
|
|
251
|
+
child.unlink()
|
|
252
|
+
for member in files:
|
|
253
|
+
target = repo.joinpath(*PurePosixPath(member.name).parts[1:])
|
|
254
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
255
|
+
with target.open('xb') as output:
|
|
256
|
+
shutil.copyfileobj(archive.extractfile(member), output)
|
|
257
|
+
target.chmod(0o755 if member.mode & 0o111 else 0o644)
|
|
258
|
+
history = mini / 'history'
|
|
259
|
+
history.mkdir()
|
|
260
|
+
for index, (task, answer) in enumerate(validated):
|
|
261
|
+
events = [{'type': 'answer', 'content': answer, 'time': 0}] if answer else []
|
|
262
|
+
# Imported text is context, never a claimed successful run or verification result.
|
|
263
|
+
record = {'task': task, 'status': {'state': 'imported'}, 'events': events}
|
|
264
|
+
(history / (str(1000000000000000000 + index) + '.json')).write_text(json.dumps(record))
|
|
265
|
+
(mini / 'fork.json').write_text(json.dumps({k: metadata[k] for k in ('point', 'fileState', 'includeFiles', 'files', 'skippedCount')}))
|
|
266
|
+
return {'files': len(files), 'turns': len(metadata['turns']), 'skippedCount': metadata['skippedCount']}
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
if __name__ == '__main__':
|
|
270
|
+
import sys
|
|
271
|
+
try:
|
|
272
|
+
action, root, argument = sys.argv[1:4]
|
|
273
|
+
result = export_chat(root, argument, sys.argv[4] == 'true') if action == 'export' else import_chat(root, argument)
|
|
274
|
+
print(json.dumps(result))
|
|
275
|
+
except Exception as error:
|
|
276
|
+
print(json.dumps({'code': 'fork_refused', 'error': str(error)[:500]}))
|
|
277
|
+
sys.exit(1)
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Step checkpoints: snapshot the whole working tree (tracked + untracked, honoring .gitignore) without touching
|
|
2
|
+
the user's index, branch or stash. Each snapshot is a commit whose parent is HEAD, stored at refs/mini-checkpoints/<ms>.
|
|
3
|
+
|
|
4
|
+
CLI (used by the app): python3 checkpoints.py compare <ref> -> JSON {stat, diff, files}"""
|
|
5
|
+
import json, os, re, shutil, subprocess, sys, tempfile, time
|
|
6
|
+
|
|
7
|
+
PREFIX = 'refs/mini-checkpoints/'
|
|
8
|
+
KEEP = 40
|
|
9
|
+
STEP_MARK = 'mini-step: '
|
|
10
|
+
IDENT = {'GIT_AUTHOR_NAME': 'ML AI Code', 'GIT_AUTHOR_EMAIL': 'checkpoints@ml.ai', 'GIT_COMMITTER_NAME': 'ML AI Code', 'GIT_COMMITTER_EMAIL': 'checkpoints@ml.ai'}
|
|
11
|
+
|
|
12
|
+
def _git(repo, *args, env=None, timeout=30):
|
|
13
|
+
r = subprocess.run(['git', *args], cwd=repo, capture_output=True, text=True, timeout=timeout, env={**os.environ, **IDENT, **(env or {})})
|
|
14
|
+
if r.returncode:
|
|
15
|
+
raise RuntimeError((r.stderr or r.stdout).strip()[-400:] or 'git failed')
|
|
16
|
+
return r.stdout.strip()
|
|
17
|
+
|
|
18
|
+
def tree_of_worktree(repo):
|
|
19
|
+
"""Tree object for the current working tree, built in a throwaway index."""
|
|
20
|
+
with tempfile.TemporaryDirectory() as d:
|
|
21
|
+
env = {'GIT_INDEX_FILE': os.path.join(d, 'index')}
|
|
22
|
+
real = os.path.join(repo, _git(repo, 'rev-parse', '--git-dir'), 'index')
|
|
23
|
+
if os.path.isfile(real):
|
|
24
|
+
shutil.copy2(real, env['GIT_INDEX_FILE']) # reuse stat cache: only changed files are re-hashed
|
|
25
|
+
else:
|
|
26
|
+
try:
|
|
27
|
+
_git(repo, 'read-tree', 'HEAD', env=env)
|
|
28
|
+
except RuntimeError:
|
|
29
|
+
pass # no commits yet
|
|
30
|
+
try:
|
|
31
|
+
_git(repo, 'add', '-A', env=env, timeout=60)
|
|
32
|
+
except RuntimeError:
|
|
33
|
+
# An empty nested repository (forge init, cargo new) cannot be indexed; absorb it and retry once.
|
|
34
|
+
import nested_repos
|
|
35
|
+
if not nested_repos.absorb(repo):
|
|
36
|
+
raise
|
|
37
|
+
_git(repo, 'add', '-A', env=env, timeout=60)
|
|
38
|
+
return _git(repo, 'write-tree', env=env)
|
|
39
|
+
|
|
40
|
+
def head(repo):
|
|
41
|
+
try:
|
|
42
|
+
return _git(repo, 'rev-parse', '--verify', '-q', 'HEAD')
|
|
43
|
+
except RuntimeError:
|
|
44
|
+
return ''
|
|
45
|
+
|
|
46
|
+
def refs(repo):
|
|
47
|
+
out = _git(repo, 'for-each-ref', '--sort=-refname', '--format=%(refname) %(objectname)', PREFIX)
|
|
48
|
+
return [line.split(' ', 1) for line in out.splitlines() if line]
|
|
49
|
+
|
|
50
|
+
def snapshot(repo, label):
|
|
51
|
+
"""Create a step checkpoint when the tree differs from the newest one. Returns the ref or None."""
|
|
52
|
+
tree = tree_of_worktree(repo)
|
|
53
|
+
existing = refs(repo)
|
|
54
|
+
if existing and _git(repo, 'rev-parse', existing[0][1] + '^{tree}') == tree:
|
|
55
|
+
return None
|
|
56
|
+
parent = head(repo)
|
|
57
|
+
if not existing and parent and _git(repo, 'rev-parse', parent + '^{tree}') == tree:
|
|
58
|
+
return None
|
|
59
|
+
args = ['commit-tree', tree, '-m', STEP_MARK + re.sub(r'\s+', ' ', str(label))[:160]]
|
|
60
|
+
if parent:
|
|
61
|
+
args[2:2] = ['-p', parent]
|
|
62
|
+
commit = _git(repo, *args)
|
|
63
|
+
stamp = int(time.time() * 1000)
|
|
64
|
+
while any(r == PREFIX + str(stamp) for r, _ in existing):
|
|
65
|
+
stamp += 1
|
|
66
|
+
ref = PREFIX + str(stamp)
|
|
67
|
+
_git(repo, 'update-ref', ref, commit)
|
|
68
|
+
for old, _ in existing[KEEP - 1:]:
|
|
69
|
+
subprocess.run(['git', 'update-ref', '-d', old], cwd=repo, capture_output=True, timeout=10)
|
|
70
|
+
return ref
|
|
71
|
+
|
|
72
|
+
def compare(repo, ref, limit=200000):
|
|
73
|
+
"""Diff from a checkpoint to the current working tree, including untracked files."""
|
|
74
|
+
if not re.fullmatch(r'(refs/)?mini-checkpoints/\d{13}', ref):
|
|
75
|
+
raise ValueError('Invalid checkpoint')
|
|
76
|
+
ref = ref if ref.startswith('refs/') else 'refs/' + ref
|
|
77
|
+
current = tree_of_worktree(repo)
|
|
78
|
+
base = _git(repo, 'rev-parse', '--verify', ref + '^{tree}')
|
|
79
|
+
stat = _git(repo, 'diff', '--stat', '--no-ext-diff', base, current)
|
|
80
|
+
names = _git(repo, 'diff', '--name-status', '--no-ext-diff', base, current)
|
|
81
|
+
diff = _git(repo, 'diff', '--no-ext-diff', base, current, timeout=60)
|
|
82
|
+
return {'checkpoint': ref[5:], 'stat': stat, 'files': [l.split('\t')[-1] for l in names.splitlines() if l], 'diff': diff[:limit], 'truncated': len(diff) > limit}
|
|
83
|
+
|
|
84
|
+
class StepCheckpoints:
|
|
85
|
+
"""Throttled snapshots after mutating agent steps."""
|
|
86
|
+
def __init__(self, repo, event=None, min_interval=8.0):
|
|
87
|
+
self.repo, self.event, self.min_interval = repo, event, min_interval
|
|
88
|
+
self.last, self.steps, self.disabled = float('-inf'), 0, os.environ.get('MINI_STEP_CHECKPOINTS') == '0'
|
|
89
|
+
self.pending = None
|
|
90
|
+
def after(self, label, force=False):
|
|
91
|
+
if self.disabled:
|
|
92
|
+
return None
|
|
93
|
+
self.steps += 1
|
|
94
|
+
self.pending = label
|
|
95
|
+
if not force and time.monotonic() - self.last < self.min_interval:
|
|
96
|
+
return None
|
|
97
|
+
return self.flush()
|
|
98
|
+
def flush(self):
|
|
99
|
+
if self.disabled or not self.pending:
|
|
100
|
+
return None
|
|
101
|
+
label, self.pending = self.pending, None
|
|
102
|
+
try:
|
|
103
|
+
ref = snapshot(self.repo, label)
|
|
104
|
+
except (RuntimeError, OSError, subprocess.SubprocessError) as exc:
|
|
105
|
+
if self.event:
|
|
106
|
+
self.event('checkpoint_error', content=str(exc)[:300], label=str(label)[:160])
|
|
107
|
+
return None
|
|
108
|
+
self.last = time.monotonic()
|
|
109
|
+
if ref and self.event:
|
|
110
|
+
self.event('checkpoint', id=ref[5:], label=str(label)[:160])
|
|
111
|
+
# Architecture is opt-in when its panel is first opened. Coalesced
|
|
112
|
+
# background analysis cannot block or fail the coding task.
|
|
113
|
+
architecture = os.path.join(os.path.dirname(__file__), 'architecture_worker.py')
|
|
114
|
+
if ref and os.path.isfile(architecture) and os.path.isfile(os.path.join(os.path.dirname(__file__), 'architecture-version')):
|
|
115
|
+
try:
|
|
116
|
+
subprocess.Popen(
|
|
117
|
+
[sys.executable, architecture, self.repo,
|
|
118
|
+
os.path.join(os.path.dirname(__file__), 'architecture'), 'refresh', '{}'],
|
|
119
|
+
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
|
120
|
+
stderr=subprocess.DEVNULL, start_new_session=True)
|
|
121
|
+
except OSError:
|
|
122
|
+
pass
|
|
123
|
+
return ref
|
|
124
|
+
|
|
125
|
+
if __name__ == '__main__':
|
|
126
|
+
if len(sys.argv) == 3 and sys.argv[1] == 'compare':
|
|
127
|
+
try:
|
|
128
|
+
print(json.dumps(compare(os.getcwd(), sys.argv[2])))
|
|
129
|
+
except (ValueError, RuntimeError) as exc:
|
|
130
|
+
print(json.dumps({'error': str(exc)}))
|
|
131
|
+
sys.exit(1)
|
|
132
|
+
else:
|
|
133
|
+
print('usage: checkpoints.py compare <ref>'); sys.exit(2)
|