claude-migrate 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/cm_paths.py ADDED
@@ -0,0 +1,42 @@
1
+ """Project path encoding and old-path -> new-path mapping."""
2
+ import json
3
+ import re
4
+
5
+
6
+ def encode_path(path):
7
+ # Claude Code names ~/.claude/projects/<dir> after the project path with
8
+ # every non-alphanumeric character turned into "-". Lossy: "AI Projects"
9
+ # and "AI_Projects" encode identically, so decode via transcripts instead.
10
+ return re.sub(r"[^A-Za-z0-9]", "-", path)
11
+
12
+
13
+ def _under(path, prefix):
14
+ return path == prefix or path.startswith(prefix.rstrip("/") + "/")
15
+
16
+
17
+ class PathMapper:
18
+ def __init__(self, default, projects):
19
+ self.default = dict(default)
20
+ self.projects = dict(projects)
21
+
22
+ def map(self, path):
23
+ best = None
24
+ for table in (self.projects, self.default):
25
+ for src in table:
26
+ if _under(path, src) and (best is None or len(src) > len(best[0])):
27
+ best = (src, table[src])
28
+ if best:
29
+ break
30
+ if not best:
31
+ return path
32
+ src, dst = best
33
+ return dst.rstrip("/") + path[len(src.rstrip("/")):]
34
+
35
+ def to_dict(self):
36
+ return {"default": dict(self.default), "projects": dict(self.projects)}
37
+
38
+ @classmethod
39
+ def from_file(cls, p):
40
+ with open(p) as f:
41
+ d = json.load(f)
42
+ return cls(d.get("default", {}), d.get("projects", {}))
package/cm_rewrite.py ADDED
@@ -0,0 +1,158 @@
1
+ """Rewrite structured path fields; never touches free conversation text."""
2
+ import json
3
+ import os
4
+ import re
5
+
6
+ from cm_paths import encode_path
7
+
8
+
9
+ def rewrite_home(value, old_home, new_home):
10
+ old = old_home.rstrip("/")
11
+ if value == old or value.startswith(old + "/"):
12
+ return new_home.rstrip("/") + value[len(old):]
13
+ return value
14
+
15
+
16
+ def _map_cwd(obj, mapper, keys=("cwd",)):
17
+ changed = False
18
+ if isinstance(obj, dict):
19
+ for k, v in obj.items():
20
+ if k in keys and isinstance(v, str):
21
+ new = mapper.map(v)
22
+ if new != v:
23
+ obj[k] = new
24
+ changed = True
25
+ elif _map_cwd(v, mapper, keys):
26
+ changed = True
27
+ elif isinstance(obj, list):
28
+ for v in obj:
29
+ if _map_cwd(v, mapper, keys):
30
+ changed = True
31
+ return changed
32
+
33
+
34
+ def rewrite_jsonl(src, dst, mapper, keys=("cwd",)):
35
+ """Stream src to dst line by line, mapping the given path-valued keys. Returns lines changed."""
36
+ needles = ['"%s"' % k for k in keys]
37
+ changed = 0
38
+ with open(src, encoding="utf-8", errors="surrogateescape") as fin, \
39
+ open(dst, "w", encoding="utf-8", errors="surrogateescape") as fout:
40
+ for line in fin:
41
+ if any(n in line for n in needles):
42
+ try:
43
+ obj = json.loads(line)
44
+ except ValueError:
45
+ obj = None
46
+ if obj is not None and _map_cwd(obj, mapper, keys):
47
+ nl = "\n" if line.endswith("\n") else ""
48
+ line = json.dumps(obj, separators=(",", ":"), ensure_ascii=False) + nl
49
+ changed += 1
50
+ fout.write(line)
51
+ return changed
52
+
53
+
54
+ def rewrite_claude_json_projects(data, mapper):
55
+ out = dict(data)
56
+ out["projects"] = {
57
+ mapper.map(k): _map_worktree_session(dict(v), mapper) if isinstance(v, dict) else v
58
+ for k, v in data.get("projects", {}).items()
59
+ }
60
+ repos = data.get("githubRepoPaths")
61
+ if isinstance(repos, dict):
62
+ out["githubRepoPaths"] = {
63
+ repo: [mapper.map(p) if isinstance(p, str) else p for p in paths] if isinstance(paths, list) else paths
64
+ for repo, paths in repos.items()
65
+ }
66
+ return out
67
+
68
+
69
+ _WORKTREE_PATH_KEYS = ("originalCwd", "preEnterOriginalCwd", "worktreePath")
70
+
71
+
72
+ def _map_worktree_session(entry, mapper):
73
+ wt = entry.get("activeWorktreeSession")
74
+ if isinstance(wt, dict):
75
+ entry["activeWorktreeSession"] = {
76
+ k: mapper.map(v) if k in _WORKTREE_PATH_KEYS and isinstance(v, str) else v for k, v in wt.items()
77
+ }
78
+ return entry
79
+
80
+
81
+ def rebase_home_in_strings(obj, old_home, new_home):
82
+ """Replace the old home inside any string (commands, args), on path boundaries."""
83
+ rx = re.compile(r"(?<![\w.\-])" + re.escape(old_home.rstrip("/")) + r"(?![\w.\-])")
84
+ new = new_home.rstrip("/")
85
+
86
+ def walk(o):
87
+ if isinstance(o, str):
88
+ return rx.sub(lambda _m: new, o)
89
+ if isinstance(o, dict):
90
+ return {k: walk(v) for k, v in o.items()}
91
+ if isinstance(o, list):
92
+ return [walk(v) for v in o]
93
+ return o
94
+
95
+ return walk(obj)
96
+
97
+
98
+ def _rebase_strings(obj, old_home, new_home):
99
+ if isinstance(obj, str):
100
+ return rewrite_home(obj, old_home, new_home)
101
+ if isinstance(obj, dict):
102
+ return {k: _rebase_strings(v, old_home, new_home) for k, v in obj.items()}
103
+ if isinstance(obj, list):
104
+ return [_rebase_strings(v, old_home, new_home) for v in obj]
105
+ return obj
106
+
107
+
108
+ def rewrite_plugin_files(plugins_dir, old_home, new_home):
109
+ for name in ("installed_plugins.json", "known_marketplaces.json"):
110
+ p = os.path.join(plugins_dir, name)
111
+ if not os.path.isfile(p):
112
+ continue
113
+ with open(p) as f:
114
+ data = json.load(f)
115
+ with open(p, "w") as f:
116
+ json.dump(_rebase_strings(data, old_home, new_home), f, indent=2)
117
+
118
+
119
+ def rewrite_sessions_index(data, mapper, old_home, new_home, folder_map=None):
120
+ out = dict(data)
121
+ if "originalPath" in out:
122
+ out["originalPath"] = mapper.map(out["originalPath"])
123
+ entries = []
124
+ for e in data.get("entries", []):
125
+ e = dict(e)
126
+ old_proj = e.get("projectPath")
127
+ if old_proj:
128
+ e["projectPath"] = mapper.map(old_proj)
129
+ if "fullPath" in e:
130
+ full = rewrite_home(e["fullPath"], old_home, new_home)
131
+ folder = os.path.basename(os.path.dirname(full))
132
+ if folder_map and folder in folder_map:
133
+ parent = os.path.dirname(os.path.dirname(full))
134
+ full = os.path.join(parent, folder_map[folder], os.path.basename(full))
135
+ elif old_proj:
136
+ if folder == encode_path(old_proj):
137
+ parent = os.path.dirname(os.path.dirname(full))
138
+ full = os.path.join(parent, encode_path(e["projectPath"]), os.path.basename(full))
139
+ e["fullPath"] = full
140
+ entries.append(e)
141
+ out["entries"] = entries
142
+ return out
143
+
144
+
145
+ def rebase_claude_json_home(data, old_home, new_home):
146
+ """MCP server commands/args (global and per project) often hold absolute home paths."""
147
+ out = dict(data)
148
+ if "mcpServers" in out:
149
+ out["mcpServers"] = rebase_home_in_strings(out["mcpServers"], old_home, new_home)
150
+ projects = {}
151
+ for k, v in out.get("projects", {}).items():
152
+ if isinstance(v, dict) and "mcpServers" in v:
153
+ v = dict(v)
154
+ v["mcpServers"] = rebase_home_in_strings(v["mcpServers"], old_home, new_home)
155
+ projects[k] = v
156
+ if "projects" in out:
157
+ out["projects"] = projects
158
+ return out
package/cm_secrets.py ADDED
@@ -0,0 +1,257 @@
1
+ """Mask secrets before they enter a bundle."""
2
+ import re
3
+
4
+ MASK = "<MASKED>"
5
+
6
+ # A name is sensitive if one of its WORDS says so (split on _ - . and camelCase), so
7
+ # API_KEY and apiKey match but ai_keyboard and keybindings do not. SESSION is deliberately
8
+ # absent: ~/.claude.json holds harmless lastSessionId pointers; cookies match "cookie".
9
+ _WORD_SPLIT_RE = re.compile(r"[A-Z]+(?![a-z])|[A-Z]?[a-z]+|\d+")
10
+ _SECRET_WORDS = {"key", "keys", "apikey", "privatekey", "accesskey", "secretkey", "pass", "pwd", "passwd",
11
+ "private", "auth", "authorization", "authentication", "oauth", "bearer"}
12
+ _SECRET_WORD_PREFIXES = ("secret", "token", "passw", "cred", "cookie", "authoriz")
13
+
14
+
15
+ # Strong terms are unambiguous even when glued into one word (GITHUBTOKEN, DBPASSWORD), so they
16
+ # match anywhere in the separator-stripped name; only the weak terms above need word boundaries.
17
+ _STRONG_RE = re.compile(r"token|secret|passw|passphrase|passcode|pwd|apikey|privatekey|accesskey|credential|cookie|authoriz|bearer")
18
+
19
+
20
+ # Words that merely end in "key" (mykey, sshkey are secrets; these are not).
21
+ _BENIGN_KEY_WORDS = {"monkey", "turkey", "hockey", "donkey", "whiskey", "jockey", "lackey"}
22
+
23
+
24
+ def is_secret_name(name):
25
+ if _STRONG_RE.search(re.sub(r"[^A-Za-z0-9]", "", str(name)).lower()):
26
+ return True
27
+ for w in _WORD_SPLIT_RE.findall(str(name)):
28
+ w = w.lower()
29
+ if w in _SECRET_WORDS or w.startswith(_SECRET_WORD_PREFIXES):
30
+ return True
31
+ if (w.endswith("key") and w not in _BENIGN_KEY_WORDS) or (w.endswith("auth") and len(w) > 4):
32
+ return True
33
+ return False
34
+
35
+
36
+ # Shapes of well-known credentials, masked wherever they appear in a string.
37
+ _TOKEN_RES = [
38
+ re.compile(r"(?<![A-Za-z0-9])sk-[A-Za-z0-9_\-]{8,}"),
39
+ re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"),
40
+ re.compile(r"github_pat_[A-Za-z0-9_]{20,}"),
41
+ re.compile(r"xox[abprs]-[A-Za-z0-9\-]{6,}"),
42
+ re.compile(r"AKIA[0-9A-Z]{16}"),
43
+ re.compile(r"AIza[0-9A-Za-z_\-]{20,}"),
44
+ re.compile(r"eyJ[\w\-]+\.[\w\-]+\.[\w\-]+"),
45
+ re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/\-]{8,}=*"),
46
+ ]
47
+ _URL_USERINFO_RE = re.compile(r"(?<=://)[^/\s:@]+:[^/\s@]+(?=@)")
48
+ _URL_QUERY_RE = re.compile(
49
+ r"(?i)([?&](?:[\w\-]*(?:token|key|secret|password|passwd|auth|sig|signature)[\w\-]*)=)[^&\s#'\"]+"
50
+ )
51
+ _FLAG_EQ_RE = re.compile(r"^(-{1,2}[\w\-]+)=(.+)$", re.S)
52
+ _ENV_ITEM_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)=(.+)$", re.S)
53
+
54
+
55
+ def _scrub_string(s):
56
+ """Replace credential shapes inside a free-form string. Returns (new, changed)."""
57
+ new = _URL_USERINFO_RE.sub(MASK, s)
58
+ new = _URL_QUERY_RE.sub(lambda m: m.group(1) + MASK, new)
59
+ for rx in _TOKEN_RES:
60
+ new = rx.sub(MASK, new)
61
+ return new, new != s
62
+
63
+
64
+ def find_secret_shapes(text):
65
+ """True if text contains a well-known credential shape (used to warn, not to edit)."""
66
+ return bool(_URL_USERINFO_RE.search(text)) or any(rx.search(text) for rx in _TOKEN_RES)
67
+
68
+
69
+ def _mask_all_strings(o, trail, masked):
70
+ """Everything below a secret-named key is secret: mask every string leaf."""
71
+ if isinstance(o, str):
72
+ masked.append(".".join(trail))
73
+ return MASK if o else o
74
+ if isinstance(o, dict):
75
+ return {k: _mask_all_strings(v, trail + [str(k)], masked) for k, v in o.items()}
76
+ if isinstance(o, list):
77
+ return [_mask_all_strings(v, trail + [str(i)], masked) for i, v in enumerate(o)]
78
+ return o
79
+
80
+
81
+ def _scrub_list(items, trail, masked):
82
+ res, prev_flag = [], False
83
+ for i, v in enumerate(items):
84
+ here = trail + [str(i)]
85
+ if isinstance(v, str):
86
+ if prev_flag:
87
+ res.append(MASK)
88
+ masked.append(".".join(here))
89
+ prev_flag = False
90
+ continue
91
+ prev_flag = v.startswith("-") and "=" not in v and is_secret_name(v.lstrip("-"))
92
+ m = _FLAG_EQ_RE.match(v)
93
+ if m and is_secret_name(m.group(1).lstrip("-")):
94
+ res.append(m.group(1) + "=" + MASK)
95
+ masked.append(".".join(here))
96
+ continue
97
+ m = _ENV_ITEM_RE.match(v)
98
+ if m and is_secret_name(m.group(1)):
99
+ res.append(m.group(1) + "=" + MASK)
100
+ masked.append(".".join(here))
101
+ continue
102
+ new, changed = _scrub_string(v)
103
+ if changed:
104
+ masked.append(".".join(here))
105
+ res.append(new)
106
+ else:
107
+ prev_flag = False
108
+ res.append(_walk(v, here, masked))
109
+ return res
110
+
111
+
112
+ def _walk(o, trail, masked):
113
+ if isinstance(o, dict):
114
+ out = {}
115
+ for k, v in o.items():
116
+ here = trail + [str(k)]
117
+ if is_secret_name(k) and isinstance(v, (str, dict, list)):
118
+ out[k] = _mask_all_strings(v, here, masked)
119
+ else:
120
+ out[k] = _walk(v, here, masked)
121
+ return out
122
+ if isinstance(o, list):
123
+ return _scrub_list(o, trail, masked)
124
+ if isinstance(o, str):
125
+ new, changed = _scrub_string(o)
126
+ if changed:
127
+ masked.append(".".join(trail))
128
+ return new
129
+ return o
130
+
131
+
132
+ def mask_secrets(obj):
133
+ """Return (deep copy with secrets replaced, dotted key paths that were masked)."""
134
+ masked = []
135
+ return _walk(obj, [], masked), masked
136
+
137
+
138
+ _CLAUDE_LINE_RE = re.compile(r"claude|CLAUDE_|ANTHROPIC_", re.I)
139
+ # One shell word: $(...) and ${...} groups, "double quoted" (with \\ escapes), 'single quoted',
140
+ # backslash-escaped chars and unquoted chars, so abc"d e"f or ${X:-a b} is a single value.
141
+ _WORD = (
142
+ r"""(?:`(?:[^`\\]|\\.)*`|\$\((?:[^()'"\\]|\\.|'[^']*'|"(?:[^"\\]|\\.)*"|\([^()]*\))*\)"""
143
+ r"""|\$\{[^}]*\}|"(?:[^"\\]|\\.)*"|'[^']*'|\\.|[^\s'"\\])"""
144
+ )
145
+ _ASSIGN_RE = re.compile(r"(?<![\w$])([A-Za-z_][A-Za-z0-9_]*)=(" + _WORD + r"+)?")
146
+ _PURE_REF_RE = re.compile(r"\$(?:[A-Za-z_]\w*|\{[A-Za-z_]\w*\})")
147
+ # $(manager word word ...): arguments must be plain path-like words, so no "=", quotes,
148
+ # redirects or comments can smuggle a literal secret in next to a trusted command name.
149
+ _ARG = r"-{0,2}[A-Za-z0-9_./:@+\-]+"
150
+ _CMD_SUB_RE = re.compile(r"\$\(\s*([A-Za-z0-9_.\-]+)((?:\s+" + _ARG + r")*)\s*\)")
151
+ # Commands whose output is a secret fetched at runtime, so the line holds no secret itself.
152
+ _SECRET_MANAGERS = {"pass", "gopass", "op", "security", "bw", "lpass", "vault", "aws", "gcloud",
153
+ "secret-tool", "doppler", "keyring"}
154
+
155
+
156
+ def _is_reference(value):
157
+ """True only for values that point at a variable or a secret manager, never a literal."""
158
+ if value.startswith("'"):
159
+ return False
160
+ inner = value[1:-1] if value.startswith('"') and value.endswith('"') and len(value) >= 2 else value
161
+ if _PURE_REF_RE.fullmatch(inner):
162
+ return True
163
+ m = _CMD_SUB_RE.fullmatch(inner)
164
+ return bool(m) and m.group(1) in _SECRET_MANAGERS and not find_secret_shapes(m.group(2))
165
+
166
+
167
+ def _balanced(value):
168
+ stack, quote, i = [], None, 0
169
+ pairs = {")": "(", "}": "{"}
170
+ while i < len(value):
171
+ c = value[i]
172
+ if c == "\\" and quote != "'":
173
+ i += 2
174
+ continue
175
+ if quote:
176
+ if c == quote:
177
+ quote = None
178
+ elif c in "'\"`":
179
+ quote = c
180
+ elif c in "({":
181
+ stack.append(c)
182
+ elif c in pairs:
183
+ if not stack or stack.pop() != pairs[c]:
184
+ return False
185
+ i += 1
186
+ return not stack and quote is None
187
+
188
+
189
+ def _has_secret_signal(text):
190
+ if find_secret_shapes(text):
191
+ return True
192
+ return any(is_secret_name(m.group(1)) for m in _ASSIGN_RE.finditer(text))
193
+
194
+
195
+ def _mask_assignments(text, notes, label):
196
+ out, pos = [], 0
197
+ for m in _ASSIGN_RE.finditer(text):
198
+ if m.start() < pos:
199
+ continue
200
+ name, value = m.group(1), m.group(2) or ""
201
+ out.append(text[pos:m.start()])
202
+ pos = m.end()
203
+ if re.search(r"\balias\s+$", text[:m.start()]):
204
+ # alias name: keep it, but scrub assignments inside its quoted body
205
+ simple = len(value) >= 2 and value[0] in "'\"" and value[-1] == value[0] and _balanced(value)
206
+ if simple:
207
+ out.append("%s=%s%s%s" % (name, value[0], _mask_assignments(value[1:-1], notes, label), value[-1]))
208
+ elif _has_secret_signal(value):
209
+ notes.append("%s (alias %s body)" % (label, name))
210
+ out.append("%s=%s" % (name, MASK))
211
+ else:
212
+ out.append(m.group(0))
213
+ continue
214
+ if not is_secret_name(name):
215
+ out.append(m.group(0))
216
+ continue
217
+ nxt = text[m.end():m.end() + 1]
218
+ unparsed = (bool(nxt) and not nxt.isspace() and nxt not in ";|&)") or not _balanced(value)
219
+ if value and _is_reference(value) and not unparsed:
220
+ out.append(m.group(0))
221
+ continue
222
+ if not value and not unparsed:
223
+ out.append(m.group(0))
224
+ continue
225
+ notes.append("%s (variable %s)" % (label, name))
226
+ out.append("%s=%s" % (name, MASK))
227
+ if unparsed: # quoting/grouping we cannot follow: drop the rest of the line rather than leak it
228
+ pos = len(text)
229
+ break
230
+ out.append(text[pos:])
231
+ return "".join(out)
232
+
233
+
234
+ def _mask_line(line, notes, label):
235
+ out = _mask_assignments(line, notes, label)
236
+ out, changed = _scrub_string(out)
237
+ if changed:
238
+ notes.append("%s (credential-like value)" % label)
239
+ # Fail closed: a clean result is a fixed point. If masking again still changes
240
+ # the line, something slipped past the parser, so omit the line entirely.
241
+ again = _scrub_string(_mask_assignments(out, [], label))[0]
242
+ if again != out:
243
+ notes.append("%s (could not be parsed safely; omitted)" % label)
244
+ return "# [claude-migrate] %s omitted: could not be masked safely" % label
245
+ return out
246
+
247
+
248
+ def extract_zsh_claude_lines(text):
249
+ """Claude-related, non-comment lines from a zshrc, with secrets masked.
250
+
251
+ Notes use "zshrc line N" labels only, never text from the line itself."""
252
+ lines, notes = [], []
253
+ for n, line in enumerate(text.splitlines(), 1):
254
+ if line.lstrip().startswith("#") or not _CLAUDE_LINE_RE.search(line):
255
+ continue
256
+ lines.append(_mask_line(line, notes, "zshrc line %d" % n))
257
+ return lines, notes
package/cm_sqlite.py ADDED
@@ -0,0 +1,185 @@
1
+ """Copy a tool's SQLite database without its secrets."""
2
+ import os
3
+ import re
4
+ import sqlite3
5
+ import tempfile
6
+
7
+ from cm_secrets import _STRONG_RE, MASK, is_secret_name, mask_secrets
8
+
9
+ _MAGIC = b"SQLite format 3\x00"
10
+ # api_key_id, token_hash, key_prefix: they point at or describe a credential without being one.
11
+ _REFERENCE_ENDINGS = ("id", "ids", "hash", "prefix", "name", "type", "count", "at")
12
+
13
+
14
+ def is_sqlite(path):
15
+ try:
16
+ with open(path, "rb") as f:
17
+ return f.read(16) == _MAGIC
18
+ except OSError:
19
+ return False
20
+
21
+
22
+ def _q(name):
23
+ return '"%s"' % name.replace('"', '""')
24
+
25
+
26
+ def _secret_columns(cols):
27
+ """Names of columns that hold a credential. `key` alone is a name in a key/value store."""
28
+ names = {c[1].lower() for c in cols}
29
+ out = []
30
+ for _cid, name, *_ in cols:
31
+ if name.lower() == "key" and "value" in names:
32
+ continue
33
+ if re.split(r"[^A-Za-z0-9]+", name.lower())[-1] in _REFERENCE_ENDINGS:
34
+ continue
35
+ if is_secret_name(name):
36
+ out.append(name)
37
+ return out
38
+
39
+
40
+ def _is_credential(col):
41
+ """Names that are unambiguously credentials. window_key or account_key are identifiers that are blanked
42
+ to be safe but legitimately appear in other tables, so they are not used for the leak check."""
43
+ squeezed = re.sub(r"[^A-Za-z0-9]", "", col).lower()
44
+ return squeezed == "key" or bool(_STRONG_RE.search(squeezed))
45
+
46
+
47
+ def _values(db, table, col):
48
+ """Distinct text and blob values of a column, as bytes-comparable originals."""
49
+ rows = db.execute("SELECT %s FROM %s WHERE typeof(%s) IN ('text','blob')" % (_q(col), _q(table), _q(col))).fetchall()
50
+ return {r[0] for r in rows if len(r[0]) >= 6 and r[0] not in (MASK, MASK.encode())}
51
+
52
+
53
+ def _file_contains(blob, value, encodings):
54
+ needles = [value] if isinstance(value, bytes) else [value.encode(e, "replace") for e in encodings]
55
+ return any(n in blob for n in needles)
56
+
57
+
58
+ def _leaks_logically(db, tables, values):
59
+ """True if any original value is still stored in a cell of any table (text or blob), whatever the file layout."""
60
+ for t in tables:
61
+ try:
62
+ cols = [c[1] for c in db.execute("PRAGMA table_info(%s)" % _q(t))]
63
+ except sqlite3.DatabaseError:
64
+ continue
65
+ for c in cols:
66
+ for v in values:
67
+ # text is compared as text, so SQLite converts the needle to the database encoding itself;
68
+ # bytes are compared as bytes. Both are tried on both kinds of cell.
69
+ probes = (("TEXT", v), ("BLOB", v.encode("utf-8"))) if isinstance(v, str) else (("BLOB", v),)
70
+ for kind, needle in probes:
71
+ q = "SELECT 1 FROM %s WHERE instr(CAST(%s AS %s), ?) > 0 LIMIT 1" % (_q(t), _q(c), kind)
72
+ if db.execute(q, (needle,)).fetchone():
73
+ return True
74
+ return False
75
+
76
+
77
+ def _virtual_tables(db):
78
+ try:
79
+ return [r[0] for r in db.execute("SELECT name FROM pragma_table_list WHERE type='virtual'")]
80
+ except sqlite3.DatabaseError: # SQLite older than 3.37: read the schema text, ignoring case and spacing
81
+ out = []
82
+ for name, sql in db.execute("SELECT name, sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL"):
83
+ if re.match(r"\s*create\s+virtual\s+table", sql, re.I):
84
+ out.append(name)
85
+ return out
86
+
87
+
88
+ def _has_populated_search_index(db):
89
+ """True if any virtual table, or any of its shadow tables (docs_data, docs_content, ...), holds rows."""
90
+ all_tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")]
91
+ for v in _virtual_tables(db):
92
+ for name in [v] + [t for t in all_tables if t.startswith(v + "_")]:
93
+ if name == v + "_config":
94
+ continue # settings only
95
+ # an empty FTS5 index still has two structure rows in <name>_data; content adds more
96
+ floor = 2 if name == v + "_data" else 0
97
+ try:
98
+ if db.execute("SELECT count(*) FROM (SELECT 1 FROM %s LIMIT %d)" % (_q(name), floor + 1)).fetchone()[0] > floor:
99
+ return True
100
+ except sqlite3.DatabaseError:
101
+ return True # cannot be read, so cannot be vouched for
102
+ return False
103
+
104
+
105
+ def _blank(db, table, col):
106
+ """Replace every value in a column. A UNIQUE constraint gets a per-row mask so the copy stays valid."""
107
+ # SQLite stores any type in any column: judge by what is there, so counts and timestamps survive
108
+ where = " WHERE typeof(%s) IN ('text','blob') AND %s != ''" % (_q(col), _q(col))
109
+ try:
110
+ return db.execute("UPDATE %s SET %s = ?%s" % (_q(table), _q(col), where), (MASK,)).rowcount
111
+ except sqlite3.IntegrityError:
112
+ return db.execute("UPDATE %s SET %s = ? || rowid%s" % (_q(table), _q(col), where), (MASK,)).rowcount
113
+
114
+
115
+ def sanitized_copy(path):
116
+ """Return (bytes of a consistent copy with secrets blanked, notes naming what was masked)."""
117
+ notes = []
118
+ src = sqlite3.connect("file:%s?mode=ro" % path, uri=True)
119
+ fd, tmp = tempfile.mkstemp(suffix=".sqlite")
120
+ os.close(fd)
121
+ try:
122
+ dst = sqlite3.connect(tmp)
123
+ src.backup(dst) # includes anything still sitting in the write-ahead log
124
+ src.close()
125
+ triggers = dst.execute("SELECT name, sql FROM sqlite_master WHERE type='trigger'").fetchall()
126
+ for name, _sql in triggers: # a trigger would fire on the blanking updates and copy secrets elsewhere
127
+ dst.execute("DROP TRIGGER %s" % _q(name))
128
+ originals = set()
129
+ tables = [r[0] for r in dst.execute(
130
+ "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")]
131
+ for t in tables:
132
+ try:
133
+ cols = dst.execute("PRAGMA table_info(%s)" % _q(t)).fetchall()
134
+ except sqlite3.DatabaseError:
135
+ continue # virtual tables whose module is missing here
136
+ for c in _secret_columns(cols):
137
+ if _is_credential(c):
138
+ originals |= _values(dst, t, c)
139
+ n = _blank(dst, t, c)
140
+ if n:
141
+ notes.append("%s.%s (%d rows)" % (t, c, n))
142
+ if {"key", "value"} <= {c[1].lower() for c in cols}:
143
+ _mask_key_value(dst, t, notes)
144
+ for _name, sql in triggers:
145
+ if sql:
146
+ dst.execute(sql)
147
+ dst.commit()
148
+ if originals:
149
+ # Fail closed. Full-text indexes keep their own tokenised copy of the content, which cannot
150
+ # be inspected or rebuilt generically, so a populated one is refused outright.
151
+ if _has_populated_search_index(dst):
152
+ raise ValueError("database has a search index that may still hold masked values")
153
+ if _leaks_logically(dst, tables, originals):
154
+ raise ValueError("a masked value is still present elsewhere in the database")
155
+ dst.execute("VACUUM")
156
+ encoding = dst.execute("PRAGMA encoding").fetchone()[0]
157
+ dst.close()
158
+ with open(tmp, "rb") as f:
159
+ blob = f.read()
160
+ codecs = {"UTF-8": ["utf-8"], "UTF-16le": ["utf-16-le"], "UTF-16be": ["utf-16-be"]}.get(encoding, ["utf-8", "utf-16-le", "utf-16-be"])
161
+ # second layer: the raw file, in the database's own text encoding
162
+ if any(_file_contains(blob, v, codecs) for v in originals):
163
+ raise ValueError("a masked value is still present in the database file")
164
+ return blob, notes
165
+ finally:
166
+ os.unlink(tmp)
167
+
168
+
169
+ def _mask_key_value(db, table, notes):
170
+ import json
171
+ rows = db.execute("SELECT rowid, key, value FROM %s" % _q(table)).fetchall()
172
+ hit = 0
173
+ for rowid, key, value in rows:
174
+ if not isinstance(value, str):
175
+ continue
176
+ try:
177
+ obj = json.loads(value)
178
+ except ValueError:
179
+ obj = value
180
+ new, masked = mask_secrets({str(key): obj})
181
+ if masked:
182
+ db.execute("UPDATE %s SET value = ? WHERE rowid = ?" % _q(table), (json.dumps(new[str(key)]), rowid))
183
+ hit += 1
184
+ if hit:
185
+ notes.append("%s.value (%d rows)" % (table, hit))