claude-migrate 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +78 -2
- package/bin/claude-migrate.js +61 -0
- package/cm_bundle.py +398 -0
- package/cm_import.py +560 -0
- package/cm_layout.py +122 -0
- package/cm_paths.py +42 -0
- package/cm_rewrite.py +158 -0
- package/cm_secrets.py +257 -0
- package/cm_sqlite.py +185 -0
- package/cm_ui.py +354 -0
- package/cm_wizard.py +314 -0
- package/migrate.py +176 -0
- package/package.json +42 -4
package/cm_paths.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Project path encoding and old-path -> new-path mapping."""
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def encode_path(path):
|
|
7
|
+
# Claude Code names ~/.claude/projects/<dir> after the project path with
|
|
8
|
+
# every non-alphanumeric character turned into "-". Lossy: "AI Projects"
|
|
9
|
+
# and "AI_Projects" encode identically, so decode via transcripts instead.
|
|
10
|
+
return re.sub(r"[^A-Za-z0-9]", "-", path)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _under(path, prefix):
|
|
14
|
+
return path == prefix or path.startswith(prefix.rstrip("/") + "/")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class PathMapper:
|
|
18
|
+
def __init__(self, default, projects):
|
|
19
|
+
self.default = dict(default)
|
|
20
|
+
self.projects = dict(projects)
|
|
21
|
+
|
|
22
|
+
def map(self, path):
|
|
23
|
+
best = None
|
|
24
|
+
for table in (self.projects, self.default):
|
|
25
|
+
for src in table:
|
|
26
|
+
if _under(path, src) and (best is None or len(src) > len(best[0])):
|
|
27
|
+
best = (src, table[src])
|
|
28
|
+
if best:
|
|
29
|
+
break
|
|
30
|
+
if not best:
|
|
31
|
+
return path
|
|
32
|
+
src, dst = best
|
|
33
|
+
return dst.rstrip("/") + path[len(src.rstrip("/")):]
|
|
34
|
+
|
|
35
|
+
def to_dict(self):
|
|
36
|
+
return {"default": dict(self.default), "projects": dict(self.projects)}
|
|
37
|
+
|
|
38
|
+
@classmethod
|
|
39
|
+
def from_file(cls, p):
|
|
40
|
+
with open(p) as f:
|
|
41
|
+
d = json.load(f)
|
|
42
|
+
return cls(d.get("default", {}), d.get("projects", {}))
|
package/cm_rewrite.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Rewrite structured path fields; never touches free conversation text."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
from cm_paths import encode_path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def rewrite_home(value, old_home, new_home):
|
|
10
|
+
old = old_home.rstrip("/")
|
|
11
|
+
if value == old or value.startswith(old + "/"):
|
|
12
|
+
return new_home.rstrip("/") + value[len(old):]
|
|
13
|
+
return value
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _map_cwd(obj, mapper, keys=("cwd",)):
|
|
17
|
+
changed = False
|
|
18
|
+
if isinstance(obj, dict):
|
|
19
|
+
for k, v in obj.items():
|
|
20
|
+
if k in keys and isinstance(v, str):
|
|
21
|
+
new = mapper.map(v)
|
|
22
|
+
if new != v:
|
|
23
|
+
obj[k] = new
|
|
24
|
+
changed = True
|
|
25
|
+
elif _map_cwd(v, mapper, keys):
|
|
26
|
+
changed = True
|
|
27
|
+
elif isinstance(obj, list):
|
|
28
|
+
for v in obj:
|
|
29
|
+
if _map_cwd(v, mapper, keys):
|
|
30
|
+
changed = True
|
|
31
|
+
return changed
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def rewrite_jsonl(src, dst, mapper, keys=("cwd",)):
|
|
35
|
+
"""Stream src to dst line by line, mapping the given path-valued keys. Returns lines changed."""
|
|
36
|
+
needles = ['"%s"' % k for k in keys]
|
|
37
|
+
changed = 0
|
|
38
|
+
with open(src, encoding="utf-8", errors="surrogateescape") as fin, \
|
|
39
|
+
open(dst, "w", encoding="utf-8", errors="surrogateescape") as fout:
|
|
40
|
+
for line in fin:
|
|
41
|
+
if any(n in line for n in needles):
|
|
42
|
+
try:
|
|
43
|
+
obj = json.loads(line)
|
|
44
|
+
except ValueError:
|
|
45
|
+
obj = None
|
|
46
|
+
if obj is not None and _map_cwd(obj, mapper, keys):
|
|
47
|
+
nl = "\n" if line.endswith("\n") else ""
|
|
48
|
+
line = json.dumps(obj, separators=(",", ":"), ensure_ascii=False) + nl
|
|
49
|
+
changed += 1
|
|
50
|
+
fout.write(line)
|
|
51
|
+
return changed
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def rewrite_claude_json_projects(data, mapper):
|
|
55
|
+
out = dict(data)
|
|
56
|
+
out["projects"] = {
|
|
57
|
+
mapper.map(k): _map_worktree_session(dict(v), mapper) if isinstance(v, dict) else v
|
|
58
|
+
for k, v in data.get("projects", {}).items()
|
|
59
|
+
}
|
|
60
|
+
repos = data.get("githubRepoPaths")
|
|
61
|
+
if isinstance(repos, dict):
|
|
62
|
+
out["githubRepoPaths"] = {
|
|
63
|
+
repo: [mapper.map(p) if isinstance(p, str) else p for p in paths] if isinstance(paths, list) else paths
|
|
64
|
+
for repo, paths in repos.items()
|
|
65
|
+
}
|
|
66
|
+
return out
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
_WORKTREE_PATH_KEYS = ("originalCwd", "preEnterOriginalCwd", "worktreePath")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _map_worktree_session(entry, mapper):
|
|
73
|
+
wt = entry.get("activeWorktreeSession")
|
|
74
|
+
if isinstance(wt, dict):
|
|
75
|
+
entry["activeWorktreeSession"] = {
|
|
76
|
+
k: mapper.map(v) if k in _WORKTREE_PATH_KEYS and isinstance(v, str) else v for k, v in wt.items()
|
|
77
|
+
}
|
|
78
|
+
return entry
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def rebase_home_in_strings(obj, old_home, new_home):
|
|
82
|
+
"""Replace the old home inside any string (commands, args), on path boundaries."""
|
|
83
|
+
rx = re.compile(r"(?<![\w.\-])" + re.escape(old_home.rstrip("/")) + r"(?![\w.\-])")
|
|
84
|
+
new = new_home.rstrip("/")
|
|
85
|
+
|
|
86
|
+
def walk(o):
|
|
87
|
+
if isinstance(o, str):
|
|
88
|
+
return rx.sub(lambda _m: new, o)
|
|
89
|
+
if isinstance(o, dict):
|
|
90
|
+
return {k: walk(v) for k, v in o.items()}
|
|
91
|
+
if isinstance(o, list):
|
|
92
|
+
return [walk(v) for v in o]
|
|
93
|
+
return o
|
|
94
|
+
|
|
95
|
+
return walk(obj)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _rebase_strings(obj, old_home, new_home):
|
|
99
|
+
if isinstance(obj, str):
|
|
100
|
+
return rewrite_home(obj, old_home, new_home)
|
|
101
|
+
if isinstance(obj, dict):
|
|
102
|
+
return {k: _rebase_strings(v, old_home, new_home) for k, v in obj.items()}
|
|
103
|
+
if isinstance(obj, list):
|
|
104
|
+
return [_rebase_strings(v, old_home, new_home) for v in obj]
|
|
105
|
+
return obj
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def rewrite_plugin_files(plugins_dir, old_home, new_home):
|
|
109
|
+
for name in ("installed_plugins.json", "known_marketplaces.json"):
|
|
110
|
+
p = os.path.join(plugins_dir, name)
|
|
111
|
+
if not os.path.isfile(p):
|
|
112
|
+
continue
|
|
113
|
+
with open(p) as f:
|
|
114
|
+
data = json.load(f)
|
|
115
|
+
with open(p, "w") as f:
|
|
116
|
+
json.dump(_rebase_strings(data, old_home, new_home), f, indent=2)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def rewrite_sessions_index(data, mapper, old_home, new_home, folder_map=None):
|
|
120
|
+
out = dict(data)
|
|
121
|
+
if "originalPath" in out:
|
|
122
|
+
out["originalPath"] = mapper.map(out["originalPath"])
|
|
123
|
+
entries = []
|
|
124
|
+
for e in data.get("entries", []):
|
|
125
|
+
e = dict(e)
|
|
126
|
+
old_proj = e.get("projectPath")
|
|
127
|
+
if old_proj:
|
|
128
|
+
e["projectPath"] = mapper.map(old_proj)
|
|
129
|
+
if "fullPath" in e:
|
|
130
|
+
full = rewrite_home(e["fullPath"], old_home, new_home)
|
|
131
|
+
folder = os.path.basename(os.path.dirname(full))
|
|
132
|
+
if folder_map and folder in folder_map:
|
|
133
|
+
parent = os.path.dirname(os.path.dirname(full))
|
|
134
|
+
full = os.path.join(parent, folder_map[folder], os.path.basename(full))
|
|
135
|
+
elif old_proj:
|
|
136
|
+
if folder == encode_path(old_proj):
|
|
137
|
+
parent = os.path.dirname(os.path.dirname(full))
|
|
138
|
+
full = os.path.join(parent, encode_path(e["projectPath"]), os.path.basename(full))
|
|
139
|
+
e["fullPath"] = full
|
|
140
|
+
entries.append(e)
|
|
141
|
+
out["entries"] = entries
|
|
142
|
+
return out
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def rebase_claude_json_home(data, old_home, new_home):
|
|
146
|
+
"""MCP server commands/args (global and per project) often hold absolute home paths."""
|
|
147
|
+
out = dict(data)
|
|
148
|
+
if "mcpServers" in out:
|
|
149
|
+
out["mcpServers"] = rebase_home_in_strings(out["mcpServers"], old_home, new_home)
|
|
150
|
+
projects = {}
|
|
151
|
+
for k, v in out.get("projects", {}).items():
|
|
152
|
+
if isinstance(v, dict) and "mcpServers" in v:
|
|
153
|
+
v = dict(v)
|
|
154
|
+
v["mcpServers"] = rebase_home_in_strings(v["mcpServers"], old_home, new_home)
|
|
155
|
+
projects[k] = v
|
|
156
|
+
if "projects" in out:
|
|
157
|
+
out["projects"] = projects
|
|
158
|
+
return out
|
package/cm_secrets.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
"""Mask secrets before they enter a bundle."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
MASK = "<MASKED>"
|
|
5
|
+
|
|
6
|
+
# A name is sensitive if one of its WORDS says so (split on _ - . and camelCase), so
|
|
7
|
+
# API_KEY and apiKey match but ai_keyboard and keybindings do not. SESSION is deliberately
|
|
8
|
+
# absent: ~/.claude.json holds harmless lastSessionId pointers; cookies match "cookie".
|
|
9
|
+
_WORD_SPLIT_RE = re.compile(r"[A-Z]+(?![a-z])|[A-Z]?[a-z]+|\d+")
|
|
10
|
+
_SECRET_WORDS = {"key", "keys", "apikey", "privatekey", "accesskey", "secretkey", "pass", "pwd", "passwd",
|
|
11
|
+
"private", "auth", "authorization", "authentication", "oauth", "bearer"}
|
|
12
|
+
_SECRET_WORD_PREFIXES = ("secret", "token", "passw", "cred", "cookie", "authoriz")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Strong terms are unambiguous even when glued into one word (GITHUBTOKEN, DBPASSWORD), so they
|
|
16
|
+
# match anywhere in the separator-stripped name; only the weak terms above need word boundaries.
|
|
17
|
+
_STRONG_RE = re.compile(r"token|secret|passw|passphrase|passcode|pwd|apikey|privatekey|accesskey|credential|cookie|authoriz|bearer")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# Words that merely end in "key" (mykey, sshkey are secrets; these are not).
|
|
21
|
+
_BENIGN_KEY_WORDS = {"monkey", "turkey", "hockey", "donkey", "whiskey", "jockey", "lackey"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def is_secret_name(name):
|
|
25
|
+
if _STRONG_RE.search(re.sub(r"[^A-Za-z0-9]", "", str(name)).lower()):
|
|
26
|
+
return True
|
|
27
|
+
for w in _WORD_SPLIT_RE.findall(str(name)):
|
|
28
|
+
w = w.lower()
|
|
29
|
+
if w in _SECRET_WORDS or w.startswith(_SECRET_WORD_PREFIXES):
|
|
30
|
+
return True
|
|
31
|
+
if (w.endswith("key") and w not in _BENIGN_KEY_WORDS) or (w.endswith("auth") and len(w) > 4):
|
|
32
|
+
return True
|
|
33
|
+
return False
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# Shapes of well-known credentials, masked wherever they appear in a string.
|
|
37
|
+
_TOKEN_RES = [
|
|
38
|
+
re.compile(r"(?<![A-Za-z0-9])sk-[A-Za-z0-9_\-]{8,}"),
|
|
39
|
+
re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"),
|
|
40
|
+
re.compile(r"github_pat_[A-Za-z0-9_]{20,}"),
|
|
41
|
+
re.compile(r"xox[abprs]-[A-Za-z0-9\-]{6,}"),
|
|
42
|
+
re.compile(r"AKIA[0-9A-Z]{16}"),
|
|
43
|
+
re.compile(r"AIza[0-9A-Za-z_\-]{20,}"),
|
|
44
|
+
re.compile(r"eyJ[\w\-]+\.[\w\-]+\.[\w\-]+"),
|
|
45
|
+
re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/\-]{8,}=*"),
|
|
46
|
+
]
|
|
47
|
+
_URL_USERINFO_RE = re.compile(r"(?<=://)[^/\s:@]+:[^/\s@]+(?=@)")
|
|
48
|
+
_URL_QUERY_RE = re.compile(
|
|
49
|
+
r"(?i)([?&](?:[\w\-]*(?:token|key|secret|password|passwd|auth|sig|signature)[\w\-]*)=)[^&\s#'\"]+"
|
|
50
|
+
)
|
|
51
|
+
_FLAG_EQ_RE = re.compile(r"^(-{1,2}[\w\-]+)=(.+)$", re.S)
|
|
52
|
+
_ENV_ITEM_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)=(.+)$", re.S)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _scrub_string(s):
|
|
56
|
+
"""Replace credential shapes inside a free-form string. Returns (new, changed)."""
|
|
57
|
+
new = _URL_USERINFO_RE.sub(MASK, s)
|
|
58
|
+
new = _URL_QUERY_RE.sub(lambda m: m.group(1) + MASK, new)
|
|
59
|
+
for rx in _TOKEN_RES:
|
|
60
|
+
new = rx.sub(MASK, new)
|
|
61
|
+
return new, new != s
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def find_secret_shapes(text):
|
|
65
|
+
"""True if text contains a well-known credential shape (used to warn, not to edit)."""
|
|
66
|
+
return bool(_URL_USERINFO_RE.search(text)) or any(rx.search(text) for rx in _TOKEN_RES)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _mask_all_strings(o, trail, masked):
|
|
70
|
+
"""Everything below a secret-named key is secret: mask every string leaf."""
|
|
71
|
+
if isinstance(o, str):
|
|
72
|
+
masked.append(".".join(trail))
|
|
73
|
+
return MASK if o else o
|
|
74
|
+
if isinstance(o, dict):
|
|
75
|
+
return {k: _mask_all_strings(v, trail + [str(k)], masked) for k, v in o.items()}
|
|
76
|
+
if isinstance(o, list):
|
|
77
|
+
return [_mask_all_strings(v, trail + [str(i)], masked) for i, v in enumerate(o)]
|
|
78
|
+
return o
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _scrub_list(items, trail, masked):
|
|
82
|
+
res, prev_flag = [], False
|
|
83
|
+
for i, v in enumerate(items):
|
|
84
|
+
here = trail + [str(i)]
|
|
85
|
+
if isinstance(v, str):
|
|
86
|
+
if prev_flag:
|
|
87
|
+
res.append(MASK)
|
|
88
|
+
masked.append(".".join(here))
|
|
89
|
+
prev_flag = False
|
|
90
|
+
continue
|
|
91
|
+
prev_flag = v.startswith("-") and "=" not in v and is_secret_name(v.lstrip("-"))
|
|
92
|
+
m = _FLAG_EQ_RE.match(v)
|
|
93
|
+
if m and is_secret_name(m.group(1).lstrip("-")):
|
|
94
|
+
res.append(m.group(1) + "=" + MASK)
|
|
95
|
+
masked.append(".".join(here))
|
|
96
|
+
continue
|
|
97
|
+
m = _ENV_ITEM_RE.match(v)
|
|
98
|
+
if m and is_secret_name(m.group(1)):
|
|
99
|
+
res.append(m.group(1) + "=" + MASK)
|
|
100
|
+
masked.append(".".join(here))
|
|
101
|
+
continue
|
|
102
|
+
new, changed = _scrub_string(v)
|
|
103
|
+
if changed:
|
|
104
|
+
masked.append(".".join(here))
|
|
105
|
+
res.append(new)
|
|
106
|
+
else:
|
|
107
|
+
prev_flag = False
|
|
108
|
+
res.append(_walk(v, here, masked))
|
|
109
|
+
return res
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _walk(o, trail, masked):
|
|
113
|
+
if isinstance(o, dict):
|
|
114
|
+
out = {}
|
|
115
|
+
for k, v in o.items():
|
|
116
|
+
here = trail + [str(k)]
|
|
117
|
+
if is_secret_name(k) and isinstance(v, (str, dict, list)):
|
|
118
|
+
out[k] = _mask_all_strings(v, here, masked)
|
|
119
|
+
else:
|
|
120
|
+
out[k] = _walk(v, here, masked)
|
|
121
|
+
return out
|
|
122
|
+
if isinstance(o, list):
|
|
123
|
+
return _scrub_list(o, trail, masked)
|
|
124
|
+
if isinstance(o, str):
|
|
125
|
+
new, changed = _scrub_string(o)
|
|
126
|
+
if changed:
|
|
127
|
+
masked.append(".".join(trail))
|
|
128
|
+
return new
|
|
129
|
+
return o
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def mask_secrets(obj):
|
|
133
|
+
"""Return (deep copy with secrets replaced, dotted key paths that were masked)."""
|
|
134
|
+
masked = []
|
|
135
|
+
return _walk(obj, [], masked), masked
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
_CLAUDE_LINE_RE = re.compile(r"claude|CLAUDE_|ANTHROPIC_", re.I)
|
|
139
|
+
# One shell word: $(...) and ${...} groups, "double quoted" (with \\ escapes), 'single quoted',
|
|
140
|
+
# backslash-escaped chars and unquoted chars, so abc"d e"f or ${X:-a b} is a single value.
|
|
141
|
+
_WORD = (
|
|
142
|
+
r"""(?:`(?:[^`\\]|\\.)*`|\$\((?:[^()'"\\]|\\.|'[^']*'|"(?:[^"\\]|\\.)*"|\([^()]*\))*\)"""
|
|
143
|
+
r"""|\$\{[^}]*\}|"(?:[^"\\]|\\.)*"|'[^']*'|\\.|[^\s'"\\])"""
|
|
144
|
+
)
|
|
145
|
+
_ASSIGN_RE = re.compile(r"(?<![\w$])([A-Za-z_][A-Za-z0-9_]*)=(" + _WORD + r"+)?")
|
|
146
|
+
_PURE_REF_RE = re.compile(r"\$(?:[A-Za-z_]\w*|\{[A-Za-z_]\w*\})")
|
|
147
|
+
# $(manager word word ...): arguments must be plain path-like words, so no "=", quotes,
|
|
148
|
+
# redirects or comments can smuggle a literal secret in next to a trusted command name.
|
|
149
|
+
_ARG = r"-{0,2}[A-Za-z0-9_./:@+\-]+"
|
|
150
|
+
_CMD_SUB_RE = re.compile(r"\$\(\s*([A-Za-z0-9_.\-]+)((?:\s+" + _ARG + r")*)\s*\)")
|
|
151
|
+
# Commands whose output is a secret fetched at runtime, so the line holds no secret itself.
|
|
152
|
+
_SECRET_MANAGERS = {"pass", "gopass", "op", "security", "bw", "lpass", "vault", "aws", "gcloud",
|
|
153
|
+
"secret-tool", "doppler", "keyring"}
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _is_reference(value):
|
|
157
|
+
"""True only for values that point at a variable or a secret manager, never a literal."""
|
|
158
|
+
if value.startswith("'"):
|
|
159
|
+
return False
|
|
160
|
+
inner = value[1:-1] if value.startswith('"') and value.endswith('"') and len(value) >= 2 else value
|
|
161
|
+
if _PURE_REF_RE.fullmatch(inner):
|
|
162
|
+
return True
|
|
163
|
+
m = _CMD_SUB_RE.fullmatch(inner)
|
|
164
|
+
return bool(m) and m.group(1) in _SECRET_MANAGERS and not find_secret_shapes(m.group(2))
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _balanced(value):
|
|
168
|
+
stack, quote, i = [], None, 0
|
|
169
|
+
pairs = {")": "(", "}": "{"}
|
|
170
|
+
while i < len(value):
|
|
171
|
+
c = value[i]
|
|
172
|
+
if c == "\\" and quote != "'":
|
|
173
|
+
i += 2
|
|
174
|
+
continue
|
|
175
|
+
if quote:
|
|
176
|
+
if c == quote:
|
|
177
|
+
quote = None
|
|
178
|
+
elif c in "'\"`":
|
|
179
|
+
quote = c
|
|
180
|
+
elif c in "({":
|
|
181
|
+
stack.append(c)
|
|
182
|
+
elif c in pairs:
|
|
183
|
+
if not stack or stack.pop() != pairs[c]:
|
|
184
|
+
return False
|
|
185
|
+
i += 1
|
|
186
|
+
return not stack and quote is None
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _has_secret_signal(text):
|
|
190
|
+
if find_secret_shapes(text):
|
|
191
|
+
return True
|
|
192
|
+
return any(is_secret_name(m.group(1)) for m in _ASSIGN_RE.finditer(text))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _mask_assignments(text, notes, label):
|
|
196
|
+
out, pos = [], 0
|
|
197
|
+
for m in _ASSIGN_RE.finditer(text):
|
|
198
|
+
if m.start() < pos:
|
|
199
|
+
continue
|
|
200
|
+
name, value = m.group(1), m.group(2) or ""
|
|
201
|
+
out.append(text[pos:m.start()])
|
|
202
|
+
pos = m.end()
|
|
203
|
+
if re.search(r"\balias\s+$", text[:m.start()]):
|
|
204
|
+
# alias name: keep it, but scrub assignments inside its quoted body
|
|
205
|
+
simple = len(value) >= 2 and value[0] in "'\"" and value[-1] == value[0] and _balanced(value)
|
|
206
|
+
if simple:
|
|
207
|
+
out.append("%s=%s%s%s" % (name, value[0], _mask_assignments(value[1:-1], notes, label), value[-1]))
|
|
208
|
+
elif _has_secret_signal(value):
|
|
209
|
+
notes.append("%s (alias %s body)" % (label, name))
|
|
210
|
+
out.append("%s=%s" % (name, MASK))
|
|
211
|
+
else:
|
|
212
|
+
out.append(m.group(0))
|
|
213
|
+
continue
|
|
214
|
+
if not is_secret_name(name):
|
|
215
|
+
out.append(m.group(0))
|
|
216
|
+
continue
|
|
217
|
+
nxt = text[m.end():m.end() + 1]
|
|
218
|
+
unparsed = (bool(nxt) and not nxt.isspace() and nxt not in ";|&)") or not _balanced(value)
|
|
219
|
+
if value and _is_reference(value) and not unparsed:
|
|
220
|
+
out.append(m.group(0))
|
|
221
|
+
continue
|
|
222
|
+
if not value and not unparsed:
|
|
223
|
+
out.append(m.group(0))
|
|
224
|
+
continue
|
|
225
|
+
notes.append("%s (variable %s)" % (label, name))
|
|
226
|
+
out.append("%s=%s" % (name, MASK))
|
|
227
|
+
if unparsed: # quoting/grouping we cannot follow: drop the rest of the line rather than leak it
|
|
228
|
+
pos = len(text)
|
|
229
|
+
break
|
|
230
|
+
out.append(text[pos:])
|
|
231
|
+
return "".join(out)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _mask_line(line, notes, label):
|
|
235
|
+
out = _mask_assignments(line, notes, label)
|
|
236
|
+
out, changed = _scrub_string(out)
|
|
237
|
+
if changed:
|
|
238
|
+
notes.append("%s (credential-like value)" % label)
|
|
239
|
+
# Fail closed: a clean result is a fixed point. If masking again still changes
|
|
240
|
+
# the line, something slipped past the parser, so omit the line entirely.
|
|
241
|
+
again = _scrub_string(_mask_assignments(out, [], label))[0]
|
|
242
|
+
if again != out:
|
|
243
|
+
notes.append("%s (could not be parsed safely; omitted)" % label)
|
|
244
|
+
return "# [claude-migrate] %s omitted: could not be masked safely" % label
|
|
245
|
+
return out
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def extract_zsh_claude_lines(text):
|
|
249
|
+
"""Claude-related, non-comment lines from a zshrc, with secrets masked.
|
|
250
|
+
|
|
251
|
+
Notes use "zshrc line N" labels only, never text from the line itself."""
|
|
252
|
+
lines, notes = [], []
|
|
253
|
+
for n, line in enumerate(text.splitlines(), 1):
|
|
254
|
+
if line.lstrip().startswith("#") or not _CLAUDE_LINE_RE.search(line):
|
|
255
|
+
continue
|
|
256
|
+
lines.append(_mask_line(line, notes, "zshrc line %d" % n))
|
|
257
|
+
return lines, notes
|
package/cm_sqlite.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Copy a tool's SQLite database without its secrets."""
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import sqlite3
|
|
5
|
+
import tempfile
|
|
6
|
+
|
|
7
|
+
from cm_secrets import _STRONG_RE, MASK, is_secret_name, mask_secrets
|
|
8
|
+
|
|
9
|
+
_MAGIC = b"SQLite format 3\x00"
|
|
10
|
+
# api_key_id, token_hash, key_prefix: they point at or describe a credential without being one.
|
|
11
|
+
_REFERENCE_ENDINGS = ("id", "ids", "hash", "prefix", "name", "type", "count", "at")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def is_sqlite(path):
|
|
15
|
+
try:
|
|
16
|
+
with open(path, "rb") as f:
|
|
17
|
+
return f.read(16) == _MAGIC
|
|
18
|
+
except OSError:
|
|
19
|
+
return False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _q(name):
|
|
23
|
+
return '"%s"' % name.replace('"', '""')
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _secret_columns(cols):
|
|
27
|
+
"""Names of columns that hold a credential. `key` alone is a name in a key/value store."""
|
|
28
|
+
names = {c[1].lower() for c in cols}
|
|
29
|
+
out = []
|
|
30
|
+
for _cid, name, *_ in cols:
|
|
31
|
+
if name.lower() == "key" and "value" in names:
|
|
32
|
+
continue
|
|
33
|
+
if re.split(r"[^A-Za-z0-9]+", name.lower())[-1] in _REFERENCE_ENDINGS:
|
|
34
|
+
continue
|
|
35
|
+
if is_secret_name(name):
|
|
36
|
+
out.append(name)
|
|
37
|
+
return out
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _is_credential(col):
|
|
41
|
+
"""Names that are unambiguously credentials. window_key or account_key are identifiers that are blanked
|
|
42
|
+
to be safe but legitimately appear in other tables, so they are not used for the leak check."""
|
|
43
|
+
squeezed = re.sub(r"[^A-Za-z0-9]", "", col).lower()
|
|
44
|
+
return squeezed == "key" or bool(_STRONG_RE.search(squeezed))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _values(db, table, col):
|
|
48
|
+
"""Distinct text and blob values of a column, as bytes-comparable originals."""
|
|
49
|
+
rows = db.execute("SELECT %s FROM %s WHERE typeof(%s) IN ('text','blob')" % (_q(col), _q(table), _q(col))).fetchall()
|
|
50
|
+
return {r[0] for r in rows if len(r[0]) >= 6 and r[0] not in (MASK, MASK.encode())}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _file_contains(blob, value, encodings):
|
|
54
|
+
needles = [value] if isinstance(value, bytes) else [value.encode(e, "replace") for e in encodings]
|
|
55
|
+
return any(n in blob for n in needles)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _leaks_logically(db, tables, values):
|
|
59
|
+
"""True if any original value is still stored in a cell of any table (text or blob), whatever the file layout."""
|
|
60
|
+
for t in tables:
|
|
61
|
+
try:
|
|
62
|
+
cols = [c[1] for c in db.execute("PRAGMA table_info(%s)" % _q(t))]
|
|
63
|
+
except sqlite3.DatabaseError:
|
|
64
|
+
continue
|
|
65
|
+
for c in cols:
|
|
66
|
+
for v in values:
|
|
67
|
+
# text is compared as text, so SQLite converts the needle to the database encoding itself;
|
|
68
|
+
# bytes are compared as bytes. Both are tried on both kinds of cell.
|
|
69
|
+
probes = (("TEXT", v), ("BLOB", v.encode("utf-8"))) if isinstance(v, str) else (("BLOB", v),)
|
|
70
|
+
for kind, needle in probes:
|
|
71
|
+
q = "SELECT 1 FROM %s WHERE instr(CAST(%s AS %s), ?) > 0 LIMIT 1" % (_q(t), _q(c), kind)
|
|
72
|
+
if db.execute(q, (needle,)).fetchone():
|
|
73
|
+
return True
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _virtual_tables(db):
|
|
78
|
+
try:
|
|
79
|
+
return [r[0] for r in db.execute("SELECT name FROM pragma_table_list WHERE type='virtual'")]
|
|
80
|
+
except sqlite3.DatabaseError: # SQLite older than 3.37: read the schema text, ignoring case and spacing
|
|
81
|
+
out = []
|
|
82
|
+
for name, sql in db.execute("SELECT name, sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL"):
|
|
83
|
+
if re.match(r"\s*create\s+virtual\s+table", sql, re.I):
|
|
84
|
+
out.append(name)
|
|
85
|
+
return out
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _has_populated_search_index(db):
|
|
89
|
+
"""True if any virtual table, or any of its shadow tables (docs_data, docs_content, ...), holds rows."""
|
|
90
|
+
all_tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
|
91
|
+
for v in _virtual_tables(db):
|
|
92
|
+
for name in [v] + [t for t in all_tables if t.startswith(v + "_")]:
|
|
93
|
+
if name == v + "_config":
|
|
94
|
+
continue # settings only
|
|
95
|
+
# an empty FTS5 index still has two structure rows in <name>_data; content adds more
|
|
96
|
+
floor = 2 if name == v + "_data" else 0
|
|
97
|
+
try:
|
|
98
|
+
if db.execute("SELECT count(*) FROM (SELECT 1 FROM %s LIMIT %d)" % (_q(name), floor + 1)).fetchone()[0] > floor:
|
|
99
|
+
return True
|
|
100
|
+
except sqlite3.DatabaseError:
|
|
101
|
+
return True # cannot be read, so cannot be vouched for
|
|
102
|
+
return False
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _blank(db, table, col):
|
|
106
|
+
"""Replace every value in a column. A UNIQUE constraint gets a per-row mask so the copy stays valid."""
|
|
107
|
+
# SQLite stores any type in any column: judge by what is there, so counts and timestamps survive
|
|
108
|
+
where = " WHERE typeof(%s) IN ('text','blob') AND %s != ''" % (_q(col), _q(col))
|
|
109
|
+
try:
|
|
110
|
+
return db.execute("UPDATE %s SET %s = ?%s" % (_q(table), _q(col), where), (MASK,)).rowcount
|
|
111
|
+
except sqlite3.IntegrityError:
|
|
112
|
+
return db.execute("UPDATE %s SET %s = ? || rowid%s" % (_q(table), _q(col), where), (MASK,)).rowcount
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def sanitized_copy(path):
|
|
116
|
+
"""Return (bytes of a consistent copy with secrets blanked, notes naming what was masked)."""
|
|
117
|
+
notes = []
|
|
118
|
+
src = sqlite3.connect("file:%s?mode=ro" % path, uri=True)
|
|
119
|
+
fd, tmp = tempfile.mkstemp(suffix=".sqlite")
|
|
120
|
+
os.close(fd)
|
|
121
|
+
try:
|
|
122
|
+
dst = sqlite3.connect(tmp)
|
|
123
|
+
src.backup(dst) # includes anything still sitting in the write-ahead log
|
|
124
|
+
src.close()
|
|
125
|
+
triggers = dst.execute("SELECT name, sql FROM sqlite_master WHERE type='trigger'").fetchall()
|
|
126
|
+
for name, _sql in triggers: # a trigger would fire on the blanking updates and copy secrets elsewhere
|
|
127
|
+
dst.execute("DROP TRIGGER %s" % _q(name))
|
|
128
|
+
originals = set()
|
|
129
|
+
tables = [r[0] for r in dst.execute(
|
|
130
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")]
|
|
131
|
+
for t in tables:
|
|
132
|
+
try:
|
|
133
|
+
cols = dst.execute("PRAGMA table_info(%s)" % _q(t)).fetchall()
|
|
134
|
+
except sqlite3.DatabaseError:
|
|
135
|
+
continue # virtual tables whose module is missing here
|
|
136
|
+
for c in _secret_columns(cols):
|
|
137
|
+
if _is_credential(c):
|
|
138
|
+
originals |= _values(dst, t, c)
|
|
139
|
+
n = _blank(dst, t, c)
|
|
140
|
+
if n:
|
|
141
|
+
notes.append("%s.%s (%d rows)" % (t, c, n))
|
|
142
|
+
if {"key", "value"} <= {c[1].lower() for c in cols}:
|
|
143
|
+
_mask_key_value(dst, t, notes)
|
|
144
|
+
for _name, sql in triggers:
|
|
145
|
+
if sql:
|
|
146
|
+
dst.execute(sql)
|
|
147
|
+
dst.commit()
|
|
148
|
+
if originals:
|
|
149
|
+
# Fail closed. Full-text indexes keep their own tokenised copy of the content, which cannot
|
|
150
|
+
# be inspected or rebuilt generically, so a populated one is refused outright.
|
|
151
|
+
if _has_populated_search_index(dst):
|
|
152
|
+
raise ValueError("database has a search index that may still hold masked values")
|
|
153
|
+
if _leaks_logically(dst, tables, originals):
|
|
154
|
+
raise ValueError("a masked value is still present elsewhere in the database")
|
|
155
|
+
dst.execute("VACUUM")
|
|
156
|
+
encoding = dst.execute("PRAGMA encoding").fetchone()[0]
|
|
157
|
+
dst.close()
|
|
158
|
+
with open(tmp, "rb") as f:
|
|
159
|
+
blob = f.read()
|
|
160
|
+
codecs = {"UTF-8": ["utf-8"], "UTF-16le": ["utf-16-le"], "UTF-16be": ["utf-16-be"]}.get(encoding, ["utf-8", "utf-16-le", "utf-16-be"])
|
|
161
|
+
# second layer: the raw file, in the database's own text encoding
|
|
162
|
+
if any(_file_contains(blob, v, codecs) for v in originals):
|
|
163
|
+
raise ValueError("a masked value is still present in the database file")
|
|
164
|
+
return blob, notes
|
|
165
|
+
finally:
|
|
166
|
+
os.unlink(tmp)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _mask_key_value(db, table, notes):
|
|
170
|
+
import json
|
|
171
|
+
rows = db.execute("SELECT rowid, key, value FROM %s" % _q(table)).fetchall()
|
|
172
|
+
hit = 0
|
|
173
|
+
for rowid, key, value in rows:
|
|
174
|
+
if not isinstance(value, str):
|
|
175
|
+
continue
|
|
176
|
+
try:
|
|
177
|
+
obj = json.loads(value)
|
|
178
|
+
except ValueError:
|
|
179
|
+
obj = value
|
|
180
|
+
new, masked = mask_secrets({str(key): obj})
|
|
181
|
+
if masked:
|
|
182
|
+
db.execute("UPDATE %s SET value = ? WHERE rowid = ?" % _q(table), (json.dumps(new[str(key)]), rowid))
|
|
183
|
+
hit += 1
|
|
184
|
+
if hit:
|
|
185
|
+
notes.append("%s.value (%d rows)" % (table, hit))
|