codiedev 0.10.0-beta.11 → 0.10.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +39 -0
- package/dist/commands/scrub.js +91 -0
- package/dist/hook.js +13 -0
- package/dist/localScrub.js +689 -0
- package/dist/localScrubReport.js +249 -0
- package/dist/scrub/rules.js +389 -0
- package/package.json +3 -2
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.keysIn = keysIn;
|
|
37
|
+
exports.mergeKeys = mergeKeys;
|
|
38
|
+
exports.describeText = describeText;
|
|
39
|
+
exports.sessionKeyOf = sessionKeyOf;
|
|
40
|
+
exports.redactExcluded = redactExcluded;
|
|
41
|
+
exports.machineId = machineId;
|
|
42
|
+
exports.buildReport = buildReport;
|
|
43
|
+
exports.sendReport = sendReport;
|
|
44
|
+
const crypto = __importStar(require("crypto"));
|
|
45
|
+
const fs = __importStar(require("fs"));
|
|
46
|
+
const os = __importStar(require("os"));
|
|
47
|
+
const path = __importStar(require("path"));
|
|
48
|
+
const rules_1 = require("./scrub/rules");
|
|
49
|
+
const captureExclusions_1 = require("./captureExclusions");
|
|
50
|
+
const MAX_KEYS_PER_SESSION = 20;
|
|
51
|
+
const TITLE_LEN = 80;
|
|
52
|
+
function keysIn(text) {
|
|
53
|
+
const { redactions } = (0, rules_1.scrubSecrets)(text);
|
|
54
|
+
if (redactions.length === 0)
|
|
55
|
+
return [];
|
|
56
|
+
const hints = new Map();
|
|
57
|
+
for (const r of redactions) {
|
|
58
|
+
const [f] = (0, rules_1.secretFindings)(text, [r]);
|
|
59
|
+
if (!f)
|
|
60
|
+
continue;
|
|
61
|
+
const k = `${f.rule}\u0000${f.hint}`;
|
|
62
|
+
const hit = hints.get(k) ?? { rule: f.rule, hint: f.hint, count: 0 };
|
|
63
|
+
hit.count += 1;
|
|
64
|
+
hints.set(k, hit);
|
|
65
|
+
}
|
|
66
|
+
return [...hints.values()].slice(0, MAX_KEYS_PER_SESSION);
|
|
67
|
+
}
|
|
68
|
+
function mergeKeys(a, b) {
|
|
69
|
+
const out = new Map(a.map((k) => [`${k.rule}\u0000${k.hint}`, { ...k }]));
|
|
70
|
+
for (const k of b) {
|
|
71
|
+
const id = `${k.rule}\u0000${k.hint}`;
|
|
72
|
+
const cur = out.get(id);
|
|
73
|
+
if (cur)
|
|
74
|
+
cur.count += k.count;
|
|
75
|
+
else
|
|
76
|
+
out.set(id, { ...k });
|
|
77
|
+
}
|
|
78
|
+
return [...out.values()].slice(0, MAX_KEYS_PER_SESSION);
|
|
79
|
+
}
|
|
80
|
+
function tidy(s) {
|
|
81
|
+
const t = s.replace(/\s+/g, " ").trim();
|
|
82
|
+
if (!t)
|
|
83
|
+
return undefined;
|
|
84
|
+
return t.length > TITLE_LEN ? `${t.slice(0, TITLE_LEN - 1)}…` : t;
|
|
85
|
+
}
|
|
86
|
+
const NOT_A_PROMPT = /^(?:<|# AGENTS\.md|Caveat:|\[Request interrupted)/;
|
|
87
|
+
function textOf(content) {
|
|
88
|
+
if (typeof content === "string")
|
|
89
|
+
return content;
|
|
90
|
+
if (Array.isArray(content)) {
|
|
91
|
+
for (const part of content) {
|
|
92
|
+
const p = part;
|
|
93
|
+
if (p && typeof p.text === "string" && (p.type === undefined || /text/.test(String(p.type))))
|
|
94
|
+
return p.text;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return undefined;
|
|
98
|
+
}
|
|
99
|
+
function firstPrompt(obj) {
|
|
100
|
+
const msg = obj.message;
|
|
101
|
+
const payload = obj.payload;
|
|
102
|
+
const data = obj.data;
|
|
103
|
+
const candidates = [];
|
|
104
|
+
if (msg?.role === "user")
|
|
105
|
+
candidates.push(msg.content);
|
|
106
|
+
if (obj.role === "user")
|
|
107
|
+
candidates.push(obj.message?.content ?? obj.content);
|
|
108
|
+
if (payload?.type === "message" && payload.role === "user")
|
|
109
|
+
candidates.push(payload.content);
|
|
110
|
+
if (obj.type === "user.message")
|
|
111
|
+
candidates.push(data?.content);
|
|
112
|
+
for (const c of candidates) {
|
|
113
|
+
const t = textOf(c);
|
|
114
|
+
if (t && !NOT_A_PROMPT.test(t.trimStart()))
|
|
115
|
+
return t;
|
|
116
|
+
}
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
function describeText(kind, cleaned) {
|
|
120
|
+
let title;
|
|
121
|
+
let named;
|
|
122
|
+
let cwd;
|
|
123
|
+
if (kind === "yaml") {
|
|
124
|
+
for (const line of cleaned.split(/\r?\n/)) {
|
|
125
|
+
const m = line.match(/^(summary|name|cwd):\s*(.*)$/);
|
|
126
|
+
if (!m)
|
|
127
|
+
continue;
|
|
128
|
+
const v = m[2].trim().replace(/^(["'])(.*)\1$/, "$2");
|
|
129
|
+
if (m[1] === "cwd")
|
|
130
|
+
cwd ??= v;
|
|
131
|
+
else
|
|
132
|
+
named ??= v;
|
|
133
|
+
}
|
|
134
|
+
return { title: named ? tidy(named) : undefined, cwd };
|
|
135
|
+
}
|
|
136
|
+
const docs = [];
|
|
137
|
+
if (kind === "jsonl") {
|
|
138
|
+
const lines = cleaned.split("\n");
|
|
139
|
+
for (let i = 0; i < lines.length && i < 2000; i++) {
|
|
140
|
+
if (!lines[i].trim())
|
|
141
|
+
continue;
|
|
142
|
+
try {
|
|
143
|
+
docs.push(JSON.parse(lines[i]));
|
|
144
|
+
}
|
|
145
|
+
catch { }
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
else if (kind === "json") {
|
|
149
|
+
try {
|
|
150
|
+
const d = JSON.parse(cleaned);
|
|
151
|
+
if (Array.isArray(d))
|
|
152
|
+
docs.push(...d);
|
|
153
|
+
else
|
|
154
|
+
docs.push(d);
|
|
155
|
+
}
|
|
156
|
+
catch { }
|
|
157
|
+
}
|
|
158
|
+
for (const d of docs) {
|
|
159
|
+
const o = d;
|
|
160
|
+
if (!o || typeof o !== "object")
|
|
161
|
+
continue;
|
|
162
|
+
if (!named && o.type === "summary" && typeof o.summary === "string")
|
|
163
|
+
named = o.summary;
|
|
164
|
+
if (!named && typeof o.thread_name === "string")
|
|
165
|
+
named = o.thread_name;
|
|
166
|
+
if (!cwd) {
|
|
167
|
+
const c = o.cwd ?? o.payload?.cwd;
|
|
168
|
+
if (typeof c === "string")
|
|
169
|
+
cwd = c;
|
|
170
|
+
}
|
|
171
|
+
if (!title) {
|
|
172
|
+
const info = o.info;
|
|
173
|
+
const t = info?.role === "user" ? textOf(o.parts) : firstPrompt(o);
|
|
174
|
+
if (t)
|
|
175
|
+
title = t;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return { title: tidy(named ?? title ?? ""), cwd };
|
|
179
|
+
}
|
|
180
|
+
function sessionKeyOf(harness, file) {
|
|
181
|
+
const base = path.basename(file);
|
|
182
|
+
if (base === "history.jsonl")
|
|
183
|
+
return "history";
|
|
184
|
+
if (harness === "copilot")
|
|
185
|
+
return path.basename(path.dirname(file));
|
|
186
|
+
const uuid = base.match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i);
|
|
187
|
+
if (uuid)
|
|
188
|
+
return uuid[1];
|
|
189
|
+
return base.replace(/\.(jsonl|json|ya?ml|txt)$/, "");
|
|
190
|
+
}
|
|
191
|
+
function redactExcluded(s, ignored) {
|
|
192
|
+
if (!ignored?.length)
|
|
193
|
+
return s;
|
|
194
|
+
if (s.cwd && !(0, captureExclusions_1.isCaptureIgnored)(s.cwd, ignored))
|
|
195
|
+
return s;
|
|
196
|
+
const { title: _t, cwd: _c, ...rest } = s;
|
|
197
|
+
return rest;
|
|
198
|
+
}
|
|
199
|
+
function machineId(dir) {
|
|
200
|
+
const file = path.join(dir, "machine-id");
|
|
201
|
+
try {
|
|
202
|
+
const id = fs.readFileSync(file, "utf8").trim();
|
|
203
|
+
if (/^[0-9a-f-]{36}$/.test(id))
|
|
204
|
+
return id;
|
|
205
|
+
}
|
|
206
|
+
catch { }
|
|
207
|
+
const id = crypto.randomUUID();
|
|
208
|
+
try {
|
|
209
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
210
|
+
fs.writeFileSync(file, id, { mode: 0o600 });
|
|
211
|
+
}
|
|
212
|
+
catch { }
|
|
213
|
+
return id;
|
|
214
|
+
}
|
|
215
|
+
const MAX_SESSIONS = 200;
|
|
216
|
+
function buildReport(input) {
|
|
217
|
+
const s = input.summary;
|
|
218
|
+
return {
|
|
219
|
+
machine: { id: input.machineId, name: os.hostname().slice(0, 80) },
|
|
220
|
+
at: input.at,
|
|
221
|
+
complete: input.complete,
|
|
222
|
+
scanned: s.scanned,
|
|
223
|
+
cleaned: s.cleaned,
|
|
224
|
+
redactions: s.redactions,
|
|
225
|
+
errors: s.errors,
|
|
226
|
+
skippedRecent: s.skippedRecent,
|
|
227
|
+
skippedLarge: s.skippedLarge,
|
|
228
|
+
byHarness: Object.entries(s.byHarness).map(([harness, t]) => ({ harness, files: t.files, redactions: t.redactions })),
|
|
229
|
+
leftInCursor: s.leftInCursor,
|
|
230
|
+
sessions: input.sessions
|
|
231
|
+
.slice(0, MAX_SESSIONS)
|
|
232
|
+
.map((d) => redactExcluded(d, input.ignoredPaths))
|
|
233
|
+
.map(({ cwd: _cwd, ...rest }) => rest),
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
async function sendReport(backendUrl, token, payload, cliVersion, fetchImpl = fetch) {
|
|
237
|
+
try {
|
|
238
|
+
const res = await fetchImpl(`${backendUrl}/api/cli/localScrub`, {
|
|
239
|
+
method: "POST",
|
|
240
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}`, "X-Codiedev-Cli-Version": cliVersion },
|
|
241
|
+
body: JSON.stringify(payload),
|
|
242
|
+
signal: AbortSignal.timeout(15_000),
|
|
243
|
+
});
|
|
244
|
+
return res.ok;
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return false;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.scrubSecrets = scrubSecrets;
|
|
4
|
+
exports.secretFindings = secretFindings;
|
|
5
|
+
exports.possibleSecrets = possibleSecrets;
|
|
6
|
+
exports.redactPossible = redactPossible;
|
|
7
|
+
const SECRET_REF_KEY = /_(?:NAME|ID|ARN|FILE|PATH|URL|URI|HOST|REGION|BUCKET|PREFIX|ENABLED|TYPE|VERSION|COUNT)$/i;
|
|
8
|
+
function looksSecret(v) {
|
|
9
|
+
if (v.includes("<") || v.includes(">"))
|
|
10
|
+
return false;
|
|
11
|
+
if (/^(your[-_]|changeme|change-me|placeholder|example|none|null|undefined|redacted|true|false)/i.test(v))
|
|
12
|
+
return false;
|
|
13
|
+
if (/[${}]/.test(v) || v.includes("$("))
|
|
14
|
+
return false;
|
|
15
|
+
if (/[()[\]{};,]/.test(v))
|
|
16
|
+
return false;
|
|
17
|
+
if (/:\/\//.test(v) || /^https?:/i.test(v))
|
|
18
|
+
return false;
|
|
19
|
+
if (/^[~./]|^[A-Za-z]:\\/.test(v))
|
|
20
|
+
return false;
|
|
21
|
+
if (/\.(pem|key|json|crt|cer|txt|log|env|conf|config|ya?ml|pub|p12|pfx|sh|js|ts|py)$/i.test(v))
|
|
22
|
+
return false;
|
|
23
|
+
if (/^[A-Za-z_$][\w$]*(\.[A-Za-z_$][\w$]*)+$/.test(v))
|
|
24
|
+
return false;
|
|
25
|
+
if (/^[A-Za-z-]+$/.test(v))
|
|
26
|
+
return false;
|
|
27
|
+
if (/^[0-9a-f]{16,}$/i.test(v))
|
|
28
|
+
return true;
|
|
29
|
+
if (v.length >= 12 && /[0-9]/.test(v) && /[A-Za-z]/.test(v))
|
|
30
|
+
return true;
|
|
31
|
+
if (v.length >= 20 && /[+/=]/.test(v))
|
|
32
|
+
return true;
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
const PLACEHOLDER = /x{6,}|\.\.\.|your|example|placeholder|changeme|redacted|dummy|fake|<|\$\{|\*{4,}/i;
|
|
36
|
+
const notPlaceholder = (v) => !PLACEHOLDER.test(v);
|
|
37
|
+
const URL_PASSWORD_PLACEHOLDER = /^(?:pass(?:word)?|secret|pwd|test|admin|root|guest)$/i;
|
|
38
|
+
const RULES = [
|
|
39
|
+
{
|
|
40
|
+
name: "pem-private-key",
|
|
41
|
+
re: /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----/g,
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
name: "jwt",
|
|
45
|
+
re: /eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/g,
|
|
46
|
+
},
|
|
47
|
+
{ name: "openai-key", re: /\bsk-proj-[A-Za-z0-9_-]{20,}/g },
|
|
48
|
+
{ name: "openai-key", re: /\bsk-[A-Za-z0-9]{20,}\b/g },
|
|
49
|
+
{ name: "stripe-key", re: /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}/g },
|
|
50
|
+
{ name: "github-token", re: /\bgh[posru]_[A-Za-z0-9]{36,}/g },
|
|
51
|
+
{ name: "github-token", re: /\bgithub_pat_[A-Za-z0-9_]{40,}/g },
|
|
52
|
+
{ name: "aws-access-key", re: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g },
|
|
53
|
+
{ name: "google-api-key", re: /\bAIza[0-9A-Za-z_-]{35}\b/g },
|
|
54
|
+
{ name: "slack-token", re: /\bxox[baprs]-[A-Za-z0-9-]{10,}/g },
|
|
55
|
+
{ name: "gitlab-token", re: /\bgl(?:pat|dt|rt|ptt|cbt|oas|soat|ffct|imt)-[A-Za-z0-9_-]{20,}/g, validate: notPlaceholder },
|
|
56
|
+
{ name: "anthropic-key", re: /\bsk-ant-(?:api|admin)\d{2}-[A-Za-z0-9_-]{40,}/g, validate: notPlaceholder },
|
|
57
|
+
{ name: "convex-deploy-key", re: /\b(?:prod|dev|preview):[a-z0-9-]+\|[A-Za-z0-9+/=]{40,}/g, validate: notPlaceholder },
|
|
58
|
+
{ name: "linear-key", re: /\blin_(?:api|oauth)_[A-Za-z0-9]{30,}/g, validate: notPlaceholder },
|
|
59
|
+
{ name: "codiedev-key", re: /\bcdv(?:m|ci|i|d|x|h|r|a|s)?_[A-Za-z0-9]{32,}\b/g, validate: notPlaceholder },
|
|
60
|
+
{ name: "codiedev-key", re: /\bcdv_(?:at|rt|dc)_[A-Za-z0-9_-]{40,}/g, validate: notPlaceholder },
|
|
61
|
+
{ name: "fireworks-key", re: /\bfw_[A-Za-z0-9]{20,}/g, validate: notPlaceholder },
|
|
62
|
+
{ name: "slack-webhook", re: /hooks\.slack\.com\/services\/T[A-Z0-9]+\/B[A-Z0-9]+\/[A-Za-z0-9]{20,}/g, validate: notPlaceholder },
|
|
63
|
+
{ name: "npm-token", re: /\bnpm_[A-Za-z0-9]{36}\b/g, validate: notPlaceholder },
|
|
64
|
+
{ name: "sendgrid-key", re: /\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/g, validate: notPlaceholder },
|
|
65
|
+
{ name: "twilio-key", re: /\bSK[0-9a-f]{32}\b/g, validate: notPlaceholder },
|
|
66
|
+
{ name: "azure-storage-key", re: /AccountKey=([A-Za-z0-9+/=]{40,})/g, group: 1, validate: notPlaceholder },
|
|
67
|
+
{ name: "azure-sas", re: /[?&]sig=([A-Za-z0-9%+/=]{30,})/g, group: 1, validate: notPlaceholder },
|
|
68
|
+
{
|
|
69
|
+
name: "url-password",
|
|
70
|
+
re: /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|rediss|mssql|sqlserver|amqps?|https?):\/\/[^:\s/"'@\\]+:([^@\s"'/\\]{6,})@[A-Za-z0-9.-]+/g,
|
|
71
|
+
group: 1,
|
|
72
|
+
validate: (v) => notPlaceholder(v) && !URL_PASSWORD_PLACEHOLDER.test(v),
|
|
73
|
+
},
|
|
74
|
+
{ name: "bearer-token", re: /\bBearer\s+([A-Za-z0-9._-]{20,})/g, group: 1 },
|
|
75
|
+
{
|
|
76
|
+
name: "secret-assignment",
|
|
77
|
+
re: /\b([A-Za-z0-9_-]*?(?:SECRET[_-]?KEY|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET|ACCESS[_-]?TOKEN|AUTH[_-]?TOKEN|REFRESH[_-]?TOKEN|API[_-]?KEY|PASSWORD|PASSWD|SECRET|TOKEN))\\?['"]?\s*[:=]\s*(?:\\?['"])?([^\s'"\\]{8,})(?:\\?['"])?/gi,
|
|
78
|
+
group: 2,
|
|
79
|
+
validate: (v, m) => !SECRET_REF_KEY.test(m[1]) &&
|
|
80
|
+
!/public/i.test(m[1]) &&
|
|
81
|
+
!/(?:page|continuation)[_-]?token$/i.test(m[1]) &&
|
|
82
|
+
m.input[m.index - 1] !== ":" &&
|
|
83
|
+
looksSecret(v),
|
|
84
|
+
},
|
|
85
|
+
];
|
|
86
|
+
function scrubSecrets(input) {
|
|
87
|
+
if (!input)
|
|
88
|
+
return { text: input, redactions: [] };
|
|
89
|
+
const candidates = [];
|
|
90
|
+
for (const rule of RULES) {
|
|
91
|
+
rule.re.lastIndex = 0;
|
|
92
|
+
let m;
|
|
93
|
+
while ((m = rule.re.exec(input)) !== null) {
|
|
94
|
+
let start = m.index;
|
|
95
|
+
let end = m.index + m[0].length;
|
|
96
|
+
if (rule.group !== undefined) {
|
|
97
|
+
const gv = m[rule.group];
|
|
98
|
+
if (gv === undefined || gv === "")
|
|
99
|
+
continue;
|
|
100
|
+
if (rule.validate && !rule.validate(gv, m))
|
|
101
|
+
continue;
|
|
102
|
+
const rel = m[0].lastIndexOf(gv);
|
|
103
|
+
if (rel >= 0) {
|
|
104
|
+
start = m.index + rel;
|
|
105
|
+
end = start + gv.length;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
else if (rule.validate && !rule.validate(m[0], m)) {
|
|
109
|
+
if (m[0].length === 0)
|
|
110
|
+
rule.re.lastIndex++;
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
if (end > start)
|
|
114
|
+
candidates.push({ rule: rule.name, start, end });
|
|
115
|
+
if (m[0].length === 0)
|
|
116
|
+
rule.re.lastIndex++;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (candidates.length === 0)
|
|
120
|
+
return { text: input, redactions: [] };
|
|
121
|
+
candidates.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
122
|
+
const chosen = [];
|
|
123
|
+
let cursor = -1;
|
|
124
|
+
for (const c of candidates) {
|
|
125
|
+
if (c.start >= cursor) {
|
|
126
|
+
chosen.push(c);
|
|
127
|
+
cursor = c.end;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
const parts = [];
|
|
131
|
+
let last = 0;
|
|
132
|
+
for (const c of chosen) {
|
|
133
|
+
parts.push(input.slice(last, c.start), `[REDACTED:${c.rule}]`);
|
|
134
|
+
last = c.end;
|
|
135
|
+
}
|
|
136
|
+
parts.push(input.slice(last));
|
|
137
|
+
return { text: parts.join(""), redactions: chosen };
|
|
138
|
+
}
|
|
139
|
+
const TOOL_CALL_TYPE = /^(?:tool_use|function_call|custom_tool_call|tool_call|toolCall|tool_request|tool\.execution_start|server_tool_use|mcp_tool_use)$/;
|
|
140
|
+
const TOOL_OUTPUT_TYPE = /^(?:tool_result|function_call_output|custom_tool_call_output|tool_output|toolResult|tool\.execution_complete|mcp_tool_result)$/;
|
|
141
|
+
const SHELL_TOOL = /bash|shell|exec|terminal|run_command|run_terminal/i;
|
|
142
|
+
const GIT_COMMIT = /\bgit\s+(?:[^\n]*\s)?(?:commit|push)\b/;
|
|
143
|
+
function locate(input, r, cache) {
|
|
144
|
+
const lineStart = input.lastIndexOf("\n", r.start - 1) + 1;
|
|
145
|
+
let parsed = cache.get(lineStart);
|
|
146
|
+
if (parsed === undefined) {
|
|
147
|
+
const lineEnd = input.indexOf("\n", r.end);
|
|
148
|
+
try {
|
|
149
|
+
parsed = JSON.parse(input.slice(lineStart, lineEnd === -1 ? input.length : lineEnd));
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
parsed = null;
|
|
153
|
+
}
|
|
154
|
+
cache.set(lineStart, parsed);
|
|
155
|
+
}
|
|
156
|
+
if (parsed === null || typeof parsed !== "object")
|
|
157
|
+
return { where: "unknown" };
|
|
158
|
+
const runs = input.slice(r.start, r.end).match(/[A-Za-z0-9]{8,}/g);
|
|
159
|
+
const needle = runs ? runs.reduce((a, b) => (b.length > a.length ? b : a)) : null;
|
|
160
|
+
if (!needle)
|
|
161
|
+
return { where: "unknown" };
|
|
162
|
+
let path = null;
|
|
163
|
+
const walk = (node, trail) => {
|
|
164
|
+
if (path)
|
|
165
|
+
return;
|
|
166
|
+
if (typeof node === "string") {
|
|
167
|
+
if (node.includes(needle))
|
|
168
|
+
path = trail;
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
if (!node || typeof node !== "object")
|
|
172
|
+
return;
|
|
173
|
+
if (Array.isArray(node)) {
|
|
174
|
+
node.forEach((c) => walk(c, [...trail, { node: c }]));
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
for (const [k, c] of Object.entries(node))
|
|
178
|
+
walk(c, [...trail, { node: c, key: k }]);
|
|
179
|
+
};
|
|
180
|
+
walk(parsed, [{ node: parsed }]);
|
|
181
|
+
if (!path)
|
|
182
|
+
return { where: "unknown" };
|
|
183
|
+
const frames = path;
|
|
184
|
+
for (let i = frames.length - 1; i >= 0; i--) {
|
|
185
|
+
const n = frames[i].node;
|
|
186
|
+
if (!n || typeof n !== "object" || Array.isArray(n))
|
|
187
|
+
continue;
|
|
188
|
+
const type = typeof n.type === "string" ? n.type : "";
|
|
189
|
+
const tool = typeof n.name === "string" ? n.name : typeof n.toolName === "string" ? n.toolName : undefined;
|
|
190
|
+
if (TOOL_CALL_TYPE.test(type)) {
|
|
191
|
+
const args = JSON.stringify(n.input ?? n.arguments ?? n.args ?? "");
|
|
192
|
+
if (tool && SHELL_TOOL.test(tool) && GIT_COMMIT.test(args))
|
|
193
|
+
return { where: "commit", tool };
|
|
194
|
+
return tool ? { where: "tool_call", tool } : { where: "tool_call" };
|
|
195
|
+
}
|
|
196
|
+
if (TOOL_OUTPUT_TYPE.test(type))
|
|
197
|
+
return tool ? { where: "tool_output", tool } : { where: "tool_output" };
|
|
198
|
+
if (n.role === "user" || (type === "user" && i === 0))
|
|
199
|
+
return { where: "prompt" };
|
|
200
|
+
if (n.role === "assistant" || (type === "assistant" && i === 0))
|
|
201
|
+
return { where: "assistant" };
|
|
202
|
+
}
|
|
203
|
+
return { where: "unknown" };
|
|
204
|
+
}
|
|
205
|
+
const MAX_LOCATIONS = 8;
|
|
206
|
+
function secretFindings(input, redactions) {
|
|
207
|
+
const seen = new Map();
|
|
208
|
+
const cache = new Map();
|
|
209
|
+
for (const r of redactions) {
|
|
210
|
+
let value = input.slice(r.start, r.end);
|
|
211
|
+
if (r.rule === "pem-private-key")
|
|
212
|
+
value = value.split("-----END")[0];
|
|
213
|
+
const hint = value.replace(/[^A-Za-z0-9]/g, "").slice(-4);
|
|
214
|
+
if (!hint)
|
|
215
|
+
continue;
|
|
216
|
+
const key = `${r.rule}\u0000${hint}`;
|
|
217
|
+
const finding = seen.get(key) ?? { rule: r.rule, hint, locations: [] };
|
|
218
|
+
seen.set(key, finding);
|
|
219
|
+
const loc = locate(input, r, cache);
|
|
220
|
+
if (finding.locations.length < MAX_LOCATIONS && !finding.locations.some((l) => l.where === loc.where && l.tool === loc.tool)) {
|
|
221
|
+
finding.locations.push(loc);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return [...seen.values()];
|
|
225
|
+
}
|
|
226
|
+
const KEYISH_ASSIGNMENT = /\b([A-Za-z][A-Za-z0-9_-]*?(?:KEY|CREDENTIALS?|CREDS?|AUTH|SIGNATURE|SIGNING[_-]?KEY|SALT|DSN|PASS|PWD|PASSPHRASE|CONN(?:ECTION)?[_-]?STRING|COOKIE))\\?['"]?\s*[:=]\s*(?:\\?['"])?([A-Za-z0-9+/=_.-]{20,200})(?:\\?['"])?/gi;
|
|
227
|
+
const NOT_SECRET_NAME = /(?:cache|idempoten|partition|sort|primary|foreign|composite|unique|hash|etag|checksum|digest|storage|object|row|item|react|list|map|lookup|index|public|publishable|pub|page|continuation|cursor|nonce|trace|span|request|correlation|commit|sha|fingerprint|kid|key[_-]?id|keyof|monkey|turkey|hotkey|shortcut)/i;
|
|
228
|
+
const PREFIXED_TOKEN = /\b((?!pk_)[a-z][a-z0-9]{1,9}_(?:live|prod|secret|sk|key|token|api)_)([A-Za-z0-9]{24,})\b/g;
|
|
229
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
230
|
+
function shannon(v) {
|
|
231
|
+
const counts = new Map();
|
|
232
|
+
for (const c of v)
|
|
233
|
+
counts.set(c, (counts.get(c) ?? 0) + 1);
|
|
234
|
+
let h = 0;
|
|
235
|
+
for (const n of counts.values()) {
|
|
236
|
+
const p = n / v.length;
|
|
237
|
+
h -= p * Math.log2(p);
|
|
238
|
+
}
|
|
239
|
+
return h;
|
|
240
|
+
}
|
|
241
|
+
function randomLooking(v) {
|
|
242
|
+
if (v.length < 20 || UUID.test(v))
|
|
243
|
+
return false;
|
|
244
|
+
if (/^\d+$/.test(v) || /^[A-Za-z]+$/.test(v))
|
|
245
|
+
return false;
|
|
246
|
+
if (/[._-]{2}|^[._-]|[._-]$/.test(v))
|
|
247
|
+
return false;
|
|
248
|
+
if (/\.[a-z]{2,4}$/i.test(v) || /^[a-z]+(?:[._-][a-z]+)+$/i.test(v))
|
|
249
|
+
return false;
|
|
250
|
+
if (PLACEHOLDER.test(v))
|
|
251
|
+
return false;
|
|
252
|
+
if (v.includes("/") && v.includes("-"))
|
|
253
|
+
return false;
|
|
254
|
+
if (/^[a-z0-9]+(?:[_-][a-z0-9]+){2,}$/.test(v))
|
|
255
|
+
return false;
|
|
256
|
+
if (!/[0-9]/.test(v) || !/[A-Za-z]/.test(v))
|
|
257
|
+
return false;
|
|
258
|
+
return shannon(v) >= 3.5;
|
|
259
|
+
}
|
|
260
|
+
function shapeOf(v) {
|
|
261
|
+
if (/^[0-9a-f]+$/i.test(v))
|
|
262
|
+
return "hex";
|
|
263
|
+
if (/[+/=]/.test(v))
|
|
264
|
+
return "base64";
|
|
265
|
+
return "alnum";
|
|
266
|
+
}
|
|
267
|
+
const LONG_RUN = /[A-Za-z0-9+/=_-]{16,}/g;
|
|
268
|
+
const maskRuns = (s) => s.replace(LONG_RUN, (r) => (/[0-9]/.test(r) && /[A-Za-z]/.test(r) ? "••••" : r));
|
|
269
|
+
function containingText(text, start, end, cache) {
|
|
270
|
+
const lineStart = text.lastIndexOf("\n", start - 1) + 1;
|
|
271
|
+
const lineEndRaw = text.indexOf("\n", end);
|
|
272
|
+
const raw = text.slice(lineStart, lineEndRaw === -1 ? text.length : lineEndRaw);
|
|
273
|
+
let parsed = cache.get(lineStart);
|
|
274
|
+
if (parsed === undefined) {
|
|
275
|
+
try {
|
|
276
|
+
parsed = JSON.parse(raw);
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
parsed = null;
|
|
280
|
+
}
|
|
281
|
+
cache.set(lineStart, parsed);
|
|
282
|
+
}
|
|
283
|
+
const value = text.slice(start, end);
|
|
284
|
+
let hit = null;
|
|
285
|
+
const walk = (node) => {
|
|
286
|
+
if (hit !== null)
|
|
287
|
+
return;
|
|
288
|
+
if (typeof node === "string") {
|
|
289
|
+
if (!node.includes(value))
|
|
290
|
+
return;
|
|
291
|
+
if (/^[[{]/.test(node.trim())) {
|
|
292
|
+
try {
|
|
293
|
+
walk(JSON.parse(node));
|
|
294
|
+
}
|
|
295
|
+
catch { }
|
|
296
|
+
}
|
|
297
|
+
if (hit === null)
|
|
298
|
+
hit = node;
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
if (node && typeof node === "object")
|
|
302
|
+
for (const c of Object.values(node))
|
|
303
|
+
walk(c);
|
|
304
|
+
};
|
|
305
|
+
if (parsed !== null)
|
|
306
|
+
walk(parsed);
|
|
307
|
+
return hit ?? raw;
|
|
308
|
+
}
|
|
309
|
+
function snippetAround(text, start, end, hint, cache) {
|
|
310
|
+
const value = text.slice(start, end);
|
|
311
|
+
const host = containingText(text, start, end, cache);
|
|
312
|
+
const at = host.indexOf(value);
|
|
313
|
+
const from = Math.max(0, at - 40);
|
|
314
|
+
const to = Math.min(host.length, at + value.length + 40);
|
|
315
|
+
const before = maskRuns(host.slice(from, at));
|
|
316
|
+
const after = maskRuns(host.slice(at + value.length, to));
|
|
317
|
+
return `${from > 0 ? "…" : ""}${before}••••${hint}${after}${to < host.length ? "…" : ""}`.replace(/\s+/g, " ").trim();
|
|
318
|
+
}
|
|
319
|
+
const MAX_POSSIBLE = 20;
|
|
320
|
+
function possibleSecrets(text, limit = MAX_POSSIBLE) {
|
|
321
|
+
if (!text)
|
|
322
|
+
return [];
|
|
323
|
+
const found = new Map();
|
|
324
|
+
const cache = new Map();
|
|
325
|
+
const add = (label, prefix, value, start) => {
|
|
326
|
+
const end = start + value.length;
|
|
327
|
+
const hint = value.replace(/[^A-Za-z0-9]/g, "").slice(-4);
|
|
328
|
+
const key = `${label.toLowerCase()}\u0000${hint}\u0000${value.length}`;
|
|
329
|
+
let item = found.get(key);
|
|
330
|
+
if (!item) {
|
|
331
|
+
if (found.size >= limit)
|
|
332
|
+
return;
|
|
333
|
+
item = { label, prefix, hint, length: value.length, shape: shapeOf(value), snippet: snippetAround(text, start, end, hint, cache), locations: [], spans: [] };
|
|
334
|
+
found.set(key, item);
|
|
335
|
+
}
|
|
336
|
+
item.spans.push({ start, end });
|
|
337
|
+
const loc = locate(text, { rule: "possible", start, end }, cache);
|
|
338
|
+
if (item.locations.length < MAX_LOCATIONS && !item.locations.some((l) => l.where === loc.where && l.tool === loc.tool))
|
|
339
|
+
item.locations.push(loc);
|
|
340
|
+
};
|
|
341
|
+
KEYISH_ASSIGNMENT.lastIndex = 0;
|
|
342
|
+
let m;
|
|
343
|
+
while ((m = KEYISH_ASSIGNMENT.exec(text)) !== null) {
|
|
344
|
+
const [whole, rawName, value] = m;
|
|
345
|
+
const name = text[m.index - 1] === "\\" ? rawName.replace(/^(?:u00[0-9a-fA-F]{2}|[ntr](?=[A-Z]))/, "") : rawName;
|
|
346
|
+
const before = text.slice(Math.max(0, m.index - 7), m.index);
|
|
347
|
+
if (/sig(?:nature)?$/i.test(name) && /(?:[?&]|\\u0026|\\u003f)$/i.test(before + rawName.slice(0, rawName.length - name.length)))
|
|
348
|
+
continue;
|
|
349
|
+
if (NOT_SECRET_NAME.test(name) || SECRET_REF_KEY.test(name))
|
|
350
|
+
continue;
|
|
351
|
+
if (text[m.index - 1] === ":" || text[m.index - 1] === ".")
|
|
352
|
+
continue;
|
|
353
|
+
if (!randomLooking(value))
|
|
354
|
+
continue;
|
|
355
|
+
add(name, undefined, value, m.index + whole.lastIndexOf(value));
|
|
356
|
+
}
|
|
357
|
+
PREFIXED_TOKEN.lastIndex = 0;
|
|
358
|
+
while ((m = PREFIXED_TOKEN.exec(text)) !== null) {
|
|
359
|
+
const [whole, prefix, body] = m;
|
|
360
|
+
if (!randomLooking(body))
|
|
361
|
+
continue;
|
|
362
|
+
const start = m.index;
|
|
363
|
+
const named = [...found.values()].find((f) => f.spans.some((sp) => sp.start < start + whole.length && start < sp.end));
|
|
364
|
+
if (named) {
|
|
365
|
+
named.prefix ??= prefix;
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
add(prefix.replace(/_$/, ""), prefix, whole, start);
|
|
369
|
+
}
|
|
370
|
+
return [...found.values()];
|
|
371
|
+
}
|
|
372
|
+
function redactPossible(text, match, rule = "reviewed-secret") {
|
|
373
|
+
const hits = possibleSecrets(text, Infinity).filter((p) => p.label.toLowerCase() === match.label.toLowerCase() && p.hint === match.hint && p.length === match.length);
|
|
374
|
+
const spans = hits.flatMap((h) => h.spans).sort((a, b) => a.start - b.start);
|
|
375
|
+
if (!spans.length)
|
|
376
|
+
return { text, redactions: [] };
|
|
377
|
+
const parts = [];
|
|
378
|
+
let last = 0;
|
|
379
|
+
const redactions = [];
|
|
380
|
+
for (const s of spans) {
|
|
381
|
+
if (s.start < last)
|
|
382
|
+
continue;
|
|
383
|
+
parts.push(text.slice(last, s.start), `[REDACTED:${rule}]`);
|
|
384
|
+
redactions.push({ rule, start: s.start, end: s.end });
|
|
385
|
+
last = s.end;
|
|
386
|
+
}
|
|
387
|
+
parts.push(text.slice(last));
|
|
388
|
+
return { text: parts.join(""), redactions };
|
|
389
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "codiedev",
|
|
3
|
-
"version": "0.10.0-beta.
|
|
3
|
+
"version": "0.10.0-beta.13",
|
|
4
4
|
"description": "Connect Claude Code, Codex, Cursor, OpenCode, or VS Code Copilot to Codiedev for team session capture and artifact collaboration — and run the governed Codie harness",
|
|
5
5
|
"bin": {
|
|
6
6
|
"codiedev": "dist/cli.js",
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
"test:harness-binaries": "node scripts/checkHarnessBinaries.mjs",
|
|
18
18
|
"test:opencode-attribution": "vitest run src/__tests__/repoResolver.test.ts src/__tests__/openCodeCapture.test.ts src/__tests__/openCodePluginGitRefs.test.ts src/__tests__/capturePayloadFinal.test.ts src/__tests__/captureDirectUpload.test.ts src/__tests__/resolveCaptureFinal.test.ts && tsc",
|
|
19
19
|
"test:watch": "vitest",
|
|
20
|
-
"test:metadata-parity": "vitest run src/__tests__/sourceMetadataMatrix.test.ts src/__tests__/copilotCapture.functional.test.ts src/__tests__/copilotCapture.test.ts src/__tests__/piCapture.functional.test.ts src/__tests__/piHooks.test.ts src/__tests__/cursorParity.functional.test.ts src/__tests__/nativeSourceParity.test.ts src/__tests__/harnessParity.functional.test.ts src/__tests__/subagents.test.ts src/__tests__/transcriptModels.test.ts src/__tests__/captureRetry.test.ts src/__tests__/limitEvent.functional.test.ts src/__tests__/captureExclusions.test.ts"
|
|
20
|
+
"test:metadata-parity": "vitest run src/__tests__/sourceMetadataMatrix.test.ts src/__tests__/copilotCapture.functional.test.ts src/__tests__/copilotCapture.test.ts src/__tests__/piCapture.functional.test.ts src/__tests__/piHooks.test.ts src/__tests__/cursorParity.functional.test.ts src/__tests__/nativeSourceParity.test.ts src/__tests__/harnessParity.functional.test.ts src/__tests__/subagents.test.ts src/__tests__/transcriptModels.test.ts src/__tests__/captureRetry.test.ts src/__tests__/limitEvent.functional.test.ts src/__tests__/captureExclusions.test.ts",
|
|
21
|
+
"test:local-scrub-live": "npm run build && node scripts/localScrubLive.mjs"
|
|
21
22
|
},
|
|
22
23
|
"files": [
|
|
23
24
|
"dist",
|