@nightloom/isy 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/agents/claude.js +18 -7
- package/dist/agents/claude.js.map +1 -1
- package/dist/agents/codex-records.js +16 -2
- package/dist/agents/codex-records.js.map +1 -1
- package/dist/agents/index.d.ts +16 -5
- package/dist/agents/index.js +21 -6
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/kimi-cli-wire.d.ts +17 -0
- package/dist/agents/kimi-cli-wire.js +312 -0
- package/dist/agents/kimi-cli-wire.js.map +1 -0
- package/dist/agents/kimi-config.d.ts +29 -0
- package/dist/agents/kimi-config.js +193 -0
- package/dist/agents/kimi-config.js.map +1 -0
- package/dist/agents/kimi-records.d.ts +0 -6
- package/dist/agents/kimi-records.js +51 -31
- package/dist/agents/kimi-records.js.map +1 -1
- package/dist/agents/kimi-tools.d.ts +30 -0
- package/dist/agents/kimi-tools.js +100 -0
- package/dist/agents/kimi-tools.js.map +1 -0
- package/dist/agents/kimi-wire.d.ts +8 -0
- package/dist/agents/kimi-wire.js +245 -80
- package/dist/agents/kimi-wire.js.map +1 -1
- package/dist/agents/kimi.d.ts +13 -2
- package/dist/agents/kimi.js +633 -114
- package/dist/agents/kimi.js.map +1 -1
- package/dist/agents/timeline.d.ts +36 -0
- package/dist/agents/timeline.js +61 -0
- package/dist/agents/timeline.js.map +1 -0
- package/dist/agents/types.d.ts +12 -0
- package/dist/api.d.ts +3 -0
- package/dist/api.js +1 -1
- package/dist/api.js.map +1 -1
- package/dist/commands/analyze.d.ts +2 -0
- package/dist/commands/analyze.js +63 -6
- package/dist/commands/analyze.js.map +1 -1
- package/dist/commands/check.js +11 -0
- package/dist/commands/check.js.map +1 -1
- package/dist/commands/health.js +2 -2
- package/dist/commands/health.js.map +1 -1
- package/dist/commands/init.js +25 -2
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/upload.d.ts +2 -0
- package/dist/commands/upload.js +38 -7
- package/dist/commands/upload.js.map +1 -1
- package/dist/detectors.js +40 -5
- package/dist/detectors.js.map +1 -1
- package/dist/hook.d.ts +12 -5
- package/dist/hook.js +73 -17
- package/dist/hook.js.map +1 -1
- package/dist/lines.d.ts +8 -0
- package/dist/lines.js +32 -0
- package/dist/lines.js.map +1 -0
- package/dist/mask-paths.js +31 -15
- package/dist/mask-paths.js.map +1 -1
- package/dist/parser.d.ts +41 -0
- package/dist/parser.js +345 -36
- package/dist/parser.js.map +1 -1
- package/dist/paths.d.ts +4 -0
- package/dist/paths.js +39 -39
- package/dist/paths.js.map +1 -1
- package/dist/plan.d.ts +3 -1
- package/dist/plan.js +5 -1
- package/dist/plan.js.map +1 -1
- package/dist/queue.d.ts +14 -2
- package/dist/queue.js +25 -3
- package/dist/queue.js.map +1 -1
- package/dist/redact.d.ts +17 -0
- package/dist/redact.js +202 -12
- package/dist/redact.js.map +1 -1
- package/dist/state.d.ts +15 -6
- package/dist/state.js +25 -9
- package/dist/state.js.map +1 -1
- package/dist/subagents.d.ts +53 -0
- package/dist/subagents.js +243 -0
- package/dist/subagents.js.map +1 -0
- package/dist/test/check.test.js +25 -3
- package/dist/test/check.test.js.map +1 -1
- package/dist/test/codex.test.js +21 -0
- package/dist/test/codex.test.js.map +1 -1
- package/dist/test/commit.test.js +9 -1
- package/dist/test/commit.test.js.map +1 -1
- package/dist/test/hook.test.js +66 -2
- package/dist/test/hook.test.js.map +1 -1
- package/dist/test/init.test.d.ts +1 -0
- package/dist/test/init.test.js +102 -0
- package/dist/test/init.test.js.map +1 -0
- package/dist/test/kimi-cli.test.d.ts +1 -0
- package/dist/test/kimi-cli.test.js +231 -0
- package/dist/test/kimi-cli.test.js.map +1 -0
- package/dist/test/kimi-code.test.d.ts +1 -0
- package/dist/test/kimi-code.test.js +302 -0
- package/dist/test/kimi-code.test.js.map +1 -0
- package/dist/test/kimi-config.test.d.ts +1 -0
- package/dist/test/kimi-config.test.js +224 -0
- package/dist/test/kimi-config.test.js.map +1 -0
- package/dist/test/kimi-hook.test.d.ts +1 -0
- package/dist/test/kimi-hook.test.js +163 -0
- package/dist/test/kimi-hook.test.js.map +1 -0
- package/dist/test/kimi-listing.test.d.ts +1 -0
- package/dist/test/kimi-listing.test.js +193 -0
- package/dist/test/kimi-listing.test.js.map +1 -0
- package/dist/test/kimi.test.js +52 -7
- package/dist/test/kimi.test.js.map +1 -1
- package/dist/test/mask-paths.test.js +31 -0
- package/dist/test/mask-paths.test.js.map +1 -1
- package/dist/test/parser.test.js +208 -5
- package/dist/test/parser.test.js.map +1 -1
- package/dist/test/plan.test.js +7 -6
- package/dist/test/plan.test.js.map +1 -1
- package/dist/test/redact.test.js +245 -8
- package/dist/test/redact.test.js.map +1 -1
- package/dist/test/stage0.test.js +49 -0
- package/dist/test/stage0.test.js.map +1 -1
- package/dist/test/subagents.test.d.ts +1 -0
- package/dist/test/subagents.test.js +290 -0
- package/dist/test/subagents.test.js.map +1 -0
- package/dist/test/sweep.test.js +25 -0
- package/dist/test/sweep.test.js.map +1 -1
- package/dist/test/upload.test.js +68 -6
- package/dist/test/upload.test.js.map +1 -1
- package/dist/types.d.ts +16 -0
- package/package.json +5 -3
- package/src/agents/claude.ts +17 -7
- package/src/agents/codex-records.ts +17 -2
- package/src/agents/index.ts +21 -6
- package/src/agents/kimi-cli-wire.ts +350 -0
- package/src/agents/kimi-config.ts +210 -0
- package/src/agents/kimi-records.ts +52 -29
- package/src/agents/kimi-tools.ts +128 -0
- package/src/agents/kimi-wire.ts +270 -82
- package/src/agents/kimi.ts +653 -113
- package/src/agents/timeline.ts +68 -0
- package/src/agents/types.ts +12 -0
- package/src/api.ts +5 -2
- package/src/commands/analyze.ts +62 -6
- package/src/commands/check.ts +11 -0
- package/src/commands/health.ts +2 -2
- package/src/commands/init.ts +26 -2
- package/src/commands/upload.ts +46 -7
- package/src/detectors.ts +39 -5
- package/src/hook.ts +81 -24
- package/src/lines.ts +31 -0
- package/src/mask-paths.ts +31 -14
- package/src/parser.ts +356 -35
- package/src/paths.ts +36 -33
- package/src/plan.ts +4 -1
- package/src/queue.ts +30 -4
- package/src/redact.ts +218 -15
- package/src/state.ts +34 -9
- package/src/subagents.ts +292 -0
- package/src/types.ts +16 -0
package/src/paths.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { createReadStream } from "node:fs";
|
|
2
1
|
import { readFile, readdir, stat } from "node:fs/promises";
|
|
3
|
-
import { createInterface } from "node:readline";
|
|
4
2
|
import { homedir } from "node:os";
|
|
5
3
|
import { basename, join } from "node:path";
|
|
4
|
+
import { fileLines } from "./lines.js";
|
|
5
|
+
import { sessionFingerprint } from "./subagents.js";
|
|
6
6
|
import type { SessionFile } from "./types.js";
|
|
7
7
|
|
|
8
8
|
export async function packageVersion(): Promise<string> {
|
|
@@ -80,11 +80,15 @@ async function sessionsInDir(dir: string): Promise<SessionFile[]> {
|
|
|
80
80
|
try {
|
|
81
81
|
const info = await stat(path);
|
|
82
82
|
if (!info.isFile()) continue;
|
|
83
|
+
// Size and mtime of the session as a whole: its subagents write files of
|
|
84
|
+
// their own, and a sweep compares against these to decide what grew.
|
|
85
|
+
const whole = await sessionFingerprint(path);
|
|
83
86
|
sessions.push({
|
|
84
87
|
sessionId: basename(entry, ".jsonl"),
|
|
85
88
|
path,
|
|
86
|
-
sizeBytes:
|
|
87
|
-
modifiedAt:
|
|
89
|
+
sizeBytes: whole.sizeBytes,
|
|
90
|
+
modifiedAt: whole.modifiedAt,
|
|
91
|
+
activeAt: info.mtime,
|
|
88
92
|
});
|
|
89
93
|
} catch {
|
|
90
94
|
continue;
|
|
@@ -94,8 +98,13 @@ async function sessionsInDir(dir: string): Promise<SessionFile[]> {
|
|
|
94
98
|
return sessions;
|
|
95
99
|
}
|
|
96
100
|
|
|
101
|
+
/** When a session itself last wrote (`SessionFile.activeAt`), for ordering sessions by recency. */
|
|
102
|
+
export function lastActive(file: SessionFile): number {
|
|
103
|
+
return (file.activeAt ?? file.modifiedAt).getTime();
|
|
104
|
+
}
|
|
105
|
+
|
|
97
106
|
function newestFirst(sessions: SessionFile[]): SessionFile[] {
|
|
98
|
-
return sessions.sort((a, b) => b
|
|
107
|
+
return sessions.sort((a, b) => lastActive(b) - lastActive(a));
|
|
99
108
|
}
|
|
100
109
|
|
|
101
110
|
export async function findSessions(cwd: string): Promise<SessionFile[]> {
|
|
@@ -144,41 +153,35 @@ const CWD_SCAN_LINES = 40;
|
|
|
144
153
|
* same field `mask-paths` roots its rewriting on.
|
|
145
154
|
*/
|
|
146
155
|
export async function firstCwd(path: string, limit = CWD_SCAN_LINES): Promise<string | undefined> {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if (
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
const cwd = (parsed as { cwd?: unknown }).cwd;
|
|
159
|
-
if (typeof cwd === "string" && cwd.length > 0) return cwd;
|
|
160
|
-
} catch {
|
|
161
|
-
continue;
|
|
162
|
-
}
|
|
156
|
+
let seen = 0;
|
|
157
|
+
for await (const line of fileLines(path)) {
|
|
158
|
+
if (seen++ >= limit) return undefined;
|
|
159
|
+
if (line.length === 0) continue;
|
|
160
|
+
try {
|
|
161
|
+
const parsed: unknown = JSON.parse(line);
|
|
162
|
+
if (typeof parsed !== "object" || parsed === null) continue;
|
|
163
|
+
const cwd = (parsed as { cwd?: unknown }).cwd;
|
|
164
|
+
if (typeof cwd === "string" && cwd.length > 0) return cwd;
|
|
165
|
+
} catch {
|
|
166
|
+
continue;
|
|
163
167
|
}
|
|
164
|
-
return undefined;
|
|
165
|
-
} finally {
|
|
166
|
-
reader.close();
|
|
167
|
-
input.destroy();
|
|
168
168
|
}
|
|
169
|
+
return undefined;
|
|
169
170
|
}
|
|
170
171
|
|
|
171
172
|
/** The first line of a file, without reading the rest: transcripts get large. */
|
|
172
173
|
export async function firstLine(path: string): Promise<string> {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
174
|
+
return (await firstLines(path, 1))[0] ?? "";
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The first `count` lines of a file, without reading the rest. */
|
|
178
|
+
export async function firstLines(path: string, count: number): Promise<string[]> {
|
|
179
|
+
const lines: string[] = [];
|
|
180
|
+
for await (const line of fileLines(path)) {
|
|
181
|
+
lines.push(line);
|
|
182
|
+
if (lines.length >= count) break;
|
|
181
183
|
}
|
|
184
|
+
return lines;
|
|
182
185
|
}
|
|
183
186
|
|
|
184
187
|
/**
|
package/src/plan.ts
CHANGED
|
@@ -46,9 +46,12 @@ function resetDay(iso: string | undefined): string | undefined {
|
|
|
46
46
|
/**
|
|
47
47
|
* The line, or nothing while there is still room. Credits count as room: they
|
|
48
48
|
* are what the gate spends once the allowance is gone (`billing.ts:checkQuota`).
|
|
49
|
+
* So does a cycle that has rolled over since: the snapshot is only as new as
|
|
50
|
+
* the last upload, which can be days old.
|
|
49
51
|
*/
|
|
50
|
-
export function planNotice(plan: PlanStatus | undefined): string | undefined {
|
|
52
|
+
export function planNotice(plan: PlanStatus | undefined, now = Date.now()): string | undefined {
|
|
51
53
|
if (!plan || plan.analysesLeft > 0 || plan.credits > 0) return undefined;
|
|
54
|
+
if (plan.resetsAt && Date.parse(plan.resetsAt) <= now) return undefined;
|
|
52
55
|
const day = resetDay(plan.resetsAt);
|
|
53
56
|
return `plan limit reached on ${plan.tier} — pull requests are not analysed${
|
|
54
57
|
day ? ` until ${day}` : ""
|
package/src/queue.ts
CHANGED
|
@@ -3,7 +3,14 @@ import { join } from "node:path";
|
|
|
3
3
|
import { queueDir } from "./paths.js";
|
|
4
4
|
import type { UploadPayload } from "./api.js";
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
/**
|
|
7
|
+
* Attempts are spaced rather than made once per run: the wait before the next
|
|
8
|
+
* one doubles from ten minutes, so the last comes nearly two days after the
|
|
9
|
+
* session was queued. Runs come minutes apart — a session's start and end,
|
|
10
|
+
* every commit — so counted per run, one short outage would use them all up.
|
|
11
|
+
*/
|
|
12
|
+
export const MAX_ATTEMPTS = 8;
|
|
13
|
+
const RETRY_AFTER_MS = 10 * 60_000;
|
|
7
14
|
const FAILED_SUFFIX = ".failed";
|
|
8
15
|
|
|
9
16
|
export interface QueueEntry {
|
|
@@ -11,6 +18,8 @@ export interface QueueEntry {
|
|
|
11
18
|
contentHash: string;
|
|
12
19
|
attempts: number;
|
|
13
20
|
queuedAt: string;
|
|
21
|
+
/** When the last attempt failed. isy 1.0 did not record it; `queuedAt` stands in. */
|
|
22
|
+
lastAttemptAt?: string;
|
|
14
23
|
lastError?: string;
|
|
15
24
|
payload: UploadPayload;
|
|
16
25
|
}
|
|
@@ -42,12 +51,17 @@ export async function enqueue(payload: UploadPayload, lastError?: string): Promi
|
|
|
42
51
|
|
|
43
52
|
const path = join(dir, entryName(payload.sessionId, payload.contentHash));
|
|
44
53
|
const existing = await readEntry(path);
|
|
54
|
+
const now = new Date().toISOString();
|
|
45
55
|
|
|
46
56
|
const entry: QueueEntry = {
|
|
47
57
|
sessionId: payload.sessionId,
|
|
48
58
|
contentHash: payload.contentHash,
|
|
49
59
|
attempts: existing ? existing.attempts : 0,
|
|
50
|
-
queuedAt: existing ? existing.queuedAt :
|
|
60
|
+
queuedAt: existing ? existing.queuedAt : now,
|
|
61
|
+
// The send that failed starts the wait, or the drain closing the same run
|
|
62
|
+
// would try again a second later. Queued again by a later failed send, an
|
|
63
|
+
// entry keeps both this and `attempts`: the queue keeps its own schedule.
|
|
64
|
+
lastAttemptAt: existing ? existing.lastAttemptAt : now,
|
|
51
65
|
lastError,
|
|
52
66
|
payload,
|
|
53
67
|
};
|
|
@@ -93,6 +107,12 @@ export async function listPending(): Promise<QueueItem[]> {
|
|
|
93
107
|
return listBySuffix(".json");
|
|
94
108
|
}
|
|
95
109
|
|
|
110
|
+
/** Whether a queued upload has waited out the delay since it last failed. */
|
|
111
|
+
export function isDue(entry: QueueEntry, now = Date.now()): boolean {
|
|
112
|
+
const last = Date.parse(entry.lastAttemptAt ?? entry.queuedAt);
|
|
113
|
+
return Number.isNaN(last) || now - last >= RETRY_AFTER_MS * 2 ** entry.attempts;
|
|
114
|
+
}
|
|
115
|
+
|
|
96
116
|
/** Uploads that ran out of attempts. Their `lastError` is the only record of why. */
|
|
97
117
|
export async function listFailed(): Promise<QueueItem[]> {
|
|
98
118
|
return listBySuffix(FAILED_SUFFIX);
|
|
@@ -103,7 +123,8 @@ export async function recordFailure(
|
|
|
103
123
|
error: string,
|
|
104
124
|
): Promise<"retained" | "abandoned"> {
|
|
105
125
|
const attempts = item.entry.attempts + 1;
|
|
106
|
-
const
|
|
126
|
+
const lastAttemptAt = new Date().toISOString();
|
|
127
|
+
const updated: QueueEntry = { ...item.entry, attempts, lastAttemptAt, lastError: error };
|
|
107
128
|
await writeFile(item.path, JSON.stringify(updated), { mode: 0o600 });
|
|
108
129
|
|
|
109
130
|
if (attempts >= MAX_ATTEMPTS) {
|
|
@@ -113,7 +134,7 @@ export async function recordFailure(
|
|
|
113
134
|
return "retained";
|
|
114
135
|
}
|
|
115
136
|
|
|
116
|
-
export async function drop(item: QueueItem): Promise<void> {
|
|
137
|
+
export async function drop(item: Pick<QueueItem, "path">): Promise<void> {
|
|
117
138
|
try {
|
|
118
139
|
await unlink(item.path);
|
|
119
140
|
} catch {
|
|
@@ -121,6 +142,11 @@ export async function drop(item: QueueItem): Promise<void> {
|
|
|
121
142
|
}
|
|
122
143
|
}
|
|
123
144
|
|
|
145
|
+
/** Bytes that went up some other way: a copy waiting out its delay here is done. */
|
|
146
|
+
export async function forget(sessionId: string, contentHash: string): Promise<void> {
|
|
147
|
+
await drop({ path: join(queueDir(), entryName(sessionId, contentHash)) });
|
|
148
|
+
}
|
|
149
|
+
|
|
124
150
|
export async function abandon(item: QueueItem, error: string): Promise<void> {
|
|
125
151
|
const updated: QueueEntry = { ...item.entry, attempts: MAX_ATTEMPTS, lastError: error };
|
|
126
152
|
await writeFile(item.path, JSON.stringify(updated), { mode: 0o600 });
|
package/src/redact.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { createReadStream } from "node:fs";
|
|
2
1
|
import { homedir } from "node:os";
|
|
3
2
|
import { basename } from "node:path";
|
|
4
|
-
import {
|
|
3
|
+
import { fileLines } from "./lines.js";
|
|
4
|
+
import { commandSegments, withoutHeredocs } from "./parser.js";
|
|
5
5
|
import {
|
|
6
6
|
PATH_KEYS,
|
|
7
7
|
REDACTION_MARKER,
|
|
@@ -23,10 +23,42 @@ export interface RedactionRule {
|
|
|
23
23
|
replacement: string;
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
+
/** `[pP][aA]…`: case by hand, so a rule can still tell a camelCase hump from a glued prefix. */
|
|
27
|
+
function anyCase(source: string): string {
|
|
28
|
+
return source.replace(/[a-z]/g, (letter) => `[${letter}${letter.toUpperCase()}]`);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const SECRET_WORD = anyCase("(?:password|passwd|secret|token|api[-_]?key|private[-_]?key|access[-_]?key)");
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* A name that holds a secret: one of the words, wherever it sits in an
|
|
35
|
+
* identifier — `PGPASSWORD`, `STRIPE_SECRET_KEY`, `aws_secret_access_key`,
|
|
36
|
+
* `x-api-key` — and followed by nothing but `KEY` and `BASE` (`SECRET_KEY_BASE`).
|
|
37
|
+
* That tail is the line between the secret and what is said about it:
|
|
38
|
+
* `tokenHash`, `max_tokens`, `TOKENS_PER_ANALYSIS` and `PASSWORD_FILE` turn up
|
|
39
|
+
* in a real corpus as often as the secrets do, and none of them is one.
|
|
40
|
+
*/
|
|
41
|
+
function secretName(atWord = ""): string {
|
|
42
|
+
return String.raw`[A-Za-z0-9_.-]*?${atWord}${SECRET_WORD}${anyCase("(?:[-_]?key)?(?:[-_]?base)?")}(?![A-Za-z0-9_.-])`;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The word glued to what came before, but not as a camelCase hump. */
|
|
46
|
+
const PLAIN_NAME = secretName("(?!(?<=[a-z0-9])[A-Z])");
|
|
47
|
+
/** `adminPassword`, `_authToken`: as often a variable in code as a key in a file. */
|
|
48
|
+
const CAMEL_NAME = secretName("(?<=[a-z0-9])(?=[A-Z])");
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* What an assigned secret looks like: long enough to be one, and not code. A
|
|
52
|
+
* number never is — usage counts sit under names like these in every record,
|
|
53
|
+
* and stdout quotes them as JSON text.
|
|
54
|
+
*/
|
|
55
|
+
const SECRET_VALUE = String.raw`(?!\[ISY_REDACTED)(?!-?\d[\d.,eE+-]*(?![^\s"'{}()[\]<>$]))[^\s"'{}()[\]<>$]{8,}`;
|
|
56
|
+
|
|
26
57
|
export const BUILTIN_RULES: RedactionRule[] = [
|
|
27
58
|
{
|
|
28
59
|
type: "private_key",
|
|
29
|
-
|
|
60
|
+
// `BLOCK` is how PGP armors one.
|
|
61
|
+
pattern: /-----BEGIN [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----|$)/g,
|
|
30
62
|
replacement: marker("private_key"),
|
|
31
63
|
},
|
|
32
64
|
{
|
|
@@ -65,19 +97,67 @@ export const BUILTIN_RULES: RedactionRule[] = [
|
|
|
65
97
|
replacement: marker("google_key"),
|
|
66
98
|
},
|
|
67
99
|
{
|
|
100
|
+
// Secret and restricted keys; `pk_` is publishable by design.
|
|
101
|
+
type: "stripe_key",
|
|
102
|
+
pattern: /\b[rs]k_(?:live|test)_[A-Za-z0-9]{16,}/g,
|
|
103
|
+
replacement: marker("stripe_key"),
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
type: "slack_token",
|
|
107
|
+
pattern: /\bxox[abposr]-[A-Za-z0-9-]{10,}/g,
|
|
108
|
+
replacement: marker("slack_token"),
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
type: "npm_token",
|
|
112
|
+
pattern: /\bnpm_[A-Za-z0-9]{36}\b/g,
|
|
113
|
+
replacement: marker("npm_token"),
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
// After every shape a bearer token can have, so a JWT or a GitHub token
|
|
117
|
+
// keeps its own label and its marker is not marked again.
|
|
118
|
+
type: "bearer_token",
|
|
119
|
+
pattern: /\b(Bearer\s+)(?!\[ISY_REDACTED)[A-Za-z0-9._~+/-]{20,}=*/gi,
|
|
120
|
+
replacement: `$1${marker("bearer_token")}`,
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
// Any scheme: a password in a URL is one whatever speaks it — rediss,
|
|
124
|
+
// amqps, https to a bare IP. The scheme is capped because `\b` lets a match
|
|
125
|
+
// start inside a dotted or dashed run: unbounded, every start rescans the
|
|
126
|
+
// rest of the run, and a long one costs minutes.
|
|
68
127
|
type: "connection_string",
|
|
69
|
-
pattern: /\b(
|
|
128
|
+
pattern: /\b([A-Za-z][A-Za-z0-9+.-]{0,31}):\/\/([^\s:/@]+):([^\s/@]+)@/g,
|
|
70
129
|
replacement: `$1://$2:${marker("connection_string")}@`,
|
|
71
130
|
},
|
|
72
131
|
{
|
|
132
|
+
// After the connection string: its password sits right before an `@` too.
|
|
133
|
+
// Starts only where a run of local-part characters starts: with `\b`, every
|
|
134
|
+
// dot, dash and `%` in a run began a new scan of the rest of it, and a
|
|
135
|
+
// version list or a URL-encoded query took minutes to get through.
|
|
136
|
+
type: "email",
|
|
137
|
+
pattern: /(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]+@(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,}\b/g,
|
|
138
|
+
replacement: marker("email"),
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
// The name may close a quote of its own: `"password": "…"`.
|
|
73
142
|
type: "assignment",
|
|
74
|
-
pattern:
|
|
75
|
-
/(?<![A-Za-z0-9])(password|passwd|secret|token|api[-_]?key|private[-_]?key)(\s*[=:]\s*)(["']?)(?!\[ISY_REDACTED)[^\s"'{}()[\]<>$]{8,}/gi,
|
|
143
|
+
pattern: new RegExp(String.raw`(?<![A-Za-z0-9_.-])(${PLAIN_NAME})(["']?\s*[=:]\s*)(["']?)${SECRET_VALUE}`, "g"),
|
|
76
144
|
replacement: `$1$2$3${marker("assignment")}`,
|
|
77
145
|
},
|
|
146
|
+
{
|
|
147
|
+
// A camelCase name counts where it is plainly data — a quoted key with a
|
|
148
|
+
// quoted value (`{"accessToken":"…"}`) or `name=value` with no space, as
|
|
149
|
+
// `.npmrc` writes `_authToken=` — and not in `adminPassword = someVariable`.
|
|
150
|
+
type: "assignment",
|
|
151
|
+
pattern: new RegExp(
|
|
152
|
+
String.raw`(?<=["'])(${CAMEL_NAME})(["']\s*:\s*["'])${SECRET_VALUE}|(?<![A-Za-z0-9_.-])(${CAMEL_NAME})(=)${SECRET_VALUE}`,
|
|
153
|
+
"g",
|
|
154
|
+
),
|
|
155
|
+
replacement: `$1$2$3$4${marker("assignment")}`,
|
|
156
|
+
},
|
|
78
157
|
];
|
|
79
158
|
|
|
80
|
-
|
|
159
|
+
/** A JSON key is data, never code, so any spelling of a secret's name counts. */
|
|
160
|
+
const SENSITIVE_KEY = new RegExp(`^${secretName()}$`);
|
|
81
161
|
const MIN_SENSITIVE_VALUE = 8;
|
|
82
162
|
const EDIT_INPUT_FIELDS = ["content", "old_string", "new_string", "new_source"];
|
|
83
163
|
|
|
@@ -146,6 +226,25 @@ function isDotenvPath(filePath: unknown): boolean {
|
|
|
146
226
|
return typeof filePath === "string" && basename(filePath).startsWith(".env");
|
|
147
227
|
}
|
|
148
228
|
|
|
229
|
+
/** Commands that print a file they are given, or load it into the shell. */
|
|
230
|
+
const SHELL_READER =
|
|
231
|
+
/^\s*(?:sudo\s+)?(?:cat|head|tail|less|more|bat|nl|grep|egrep|fgrep|rg|sed|awk|jq|xxd|od|diff|source|\.|git\s+(?:show|diff|log|blame)|get-content|gc|type)(?:\s|$)/i;
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* `cat .env`, `grep KEY config/.env.local`, `source .env`: a shell call that
|
|
235
|
+
* reads a dotenv file is as much a read of it as `Read {file_path}` is, and
|
|
236
|
+
* its output is the file. The command itself stays — it names the file, not
|
|
237
|
+
* what is in it.
|
|
238
|
+
*/
|
|
239
|
+
function readsDotenv(command: unknown): boolean {
|
|
240
|
+
if (typeof command !== "string" || !command.includes(".env")) return false;
|
|
241
|
+
return commandSegments(withoutHeredocs(command)).some(
|
|
242
|
+
(segment) =>
|
|
243
|
+
SHELL_READER.test(segment) &&
|
|
244
|
+
segment.trim().split(/\s+/).slice(1).some((word) => isDotenvPath(word.replace(/["'`]/g, "").replace(/^\w*:/, ""))),
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
|
|
149
248
|
function cutDotenvInput(input: Record<string, unknown>, context: RedactContext): void {
|
|
150
249
|
for (const field of EDIT_INPUT_FIELDS) {
|
|
151
250
|
if (typeof input[field] === "string") {
|
|
@@ -166,6 +265,10 @@ function scanDotenv(record: Record<string, unknown>, context: RedactContext): vo
|
|
|
166
265
|
if (block.type === "tool_use") {
|
|
167
266
|
const input = isObject(block.input) ? block.input : undefined;
|
|
168
267
|
if (!input) continue;
|
|
268
|
+
if ((block.name === "Bash" || block.name === "PowerShell") && readsDotenv(input.command)) {
|
|
269
|
+
if (typeof block.id === "string") context.envToolUseIds.add(block.id);
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
169
272
|
if (!isDotenvPath(input.file_path) && !isDotenvPath(input.notebook_path)) continue;
|
|
170
273
|
if (typeof block.id === "string") context.envToolUseIds.add(block.id);
|
|
171
274
|
cutDotenvInput(input, context);
|
|
@@ -225,12 +328,97 @@ function learnRoot(record: Record<string, unknown>, context: RedactContext): voi
|
|
|
225
328
|
context.roots = { ...context.roots, ...resolveRoots({ cwd }) };
|
|
226
329
|
}
|
|
227
330
|
|
|
331
|
+
/**
|
|
332
|
+
* Claude Code's `attachment` records are the harness's context for the model:
|
|
333
|
+
* the account's email (`session_context`), its organisation (`credential_org`),
|
|
334
|
+
* the whole system prompt with memory and CLAUDE.md (`prompt_snapshot`), the
|
|
335
|
+
* connectors' instructions. Nothing downstream reads past their `type`, so that
|
|
336
|
+
* is all that goes. The record itself stays: the next record names it as its
|
|
337
|
+
* `parentUuid`, and the parser walks that chain.
|
|
338
|
+
*
|
|
339
|
+
* Claude Code 2.1.30x also records the request it sends: `api-request-shape`
|
|
340
|
+
* carries the system prompt and the tool definitions, `api-request-blob` the
|
|
341
|
+
* context blocks sent beside the conversation — memory, CLAUDE.md, the email
|
|
342
|
+
* again — and `api-request` the parameters, whose `safeguards` hand a
|
|
343
|
+
* classifier the account's user name, home directory and permission rules.
|
|
344
|
+
* `bridge-session` and `artifact-autoreact-ledger` are cut the same way: one
|
|
345
|
+
* names the account and organisation that own the session's bridge, the other
|
|
346
|
+
* the account. The parser reads none of them past their type, so only the
|
|
347
|
+
* fields that name them go.
|
|
348
|
+
*/
|
|
349
|
+
const REQUEST_RECORD_KEYS = new Set(["type", "uuid", "parentUuid", "sessionId", "timestamp", "version", "shapeHash", "hash"]);
|
|
350
|
+
const STRIPPED_RECORD_TYPES = new Set([
|
|
351
|
+
"api-request",
|
|
352
|
+
"api-request-shape",
|
|
353
|
+
"api-request-blob",
|
|
354
|
+
"bridge-session",
|
|
355
|
+
"artifact-autoreact-ledger",
|
|
356
|
+
]);
|
|
357
|
+
|
|
358
|
+
function stripAttachment(record: Record<string, unknown>, context: RedactContext): void {
|
|
359
|
+
if (typeof record.type === "string" && STRIPPED_RECORD_TYPES.has(record.type)) {
|
|
360
|
+
for (const key of Object.keys(record)) if (!REQUEST_RECORD_KEYS.has(key)) delete record[key];
|
|
361
|
+
count(context, "attachment", 1);
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
if (record.type !== "attachment") return;
|
|
365
|
+
const kind = isObject(record.attachment) ? record.attachment.type : undefined;
|
|
366
|
+
record.attachment = typeof kind === "string" ? { type: kind } : {};
|
|
367
|
+
delete record.rendered;
|
|
368
|
+
delete record.renderedRole;
|
|
369
|
+
count(context, "attachment", 1);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* Harness bookkeeping Claude Code writes beside a turn, read by nothing here or
|
|
374
|
+
* on the server. `wireToolInputs` repeats every call's input, out of reach of
|
|
375
|
+
* the dotenv cut, which edits the copy in `message` (`scanDotenv`);
|
|
376
|
+
* `serverClassifierRequest` and `serverClassifierContext` are what a permission
|
|
377
|
+
* classifier was shown, the working tree's git state included;
|
|
378
|
+
* `wireIngestContext` and `permissionDecision` say where a call ran and what
|
|
379
|
+
* allowed it.
|
|
380
|
+
*/
|
|
381
|
+
const HARNESS_FIELDS = [
|
|
382
|
+
"wireToolInputs",
|
|
383
|
+
"serverClassifierRequest",
|
|
384
|
+
"serverClassifierContext",
|
|
385
|
+
"wireIngestContext",
|
|
386
|
+
"permissionDecision",
|
|
387
|
+
];
|
|
388
|
+
|
|
389
|
+
function stripHarnessFields(record: Record<string, unknown>, context: RedactContext): void {
|
|
390
|
+
let stripped = false;
|
|
391
|
+
for (const field of HARNESS_FIELDS) {
|
|
392
|
+
if (!(field in record)) continue;
|
|
393
|
+
delete record[field];
|
|
394
|
+
stripped = true;
|
|
395
|
+
}
|
|
396
|
+
if (stripped) count(context, "attachment", 1);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* A user record marked `isMeta` is the harness talking in the user's place: a
|
|
401
|
+
* skill's body, a `<system-reminder>`, a command's caveat, an extension's
|
|
402
|
+
* instructions, a peer agent's message. Read as text it is a turn the
|
|
403
|
+
* developer never typed, holding files the developer never sent, so only a
|
|
404
|
+
* marker goes. None carries a tool result, and the record stays where it is:
|
|
405
|
+
* the next one names it as its `parentUuid`.
|
|
406
|
+
*/
|
|
407
|
+
function stripMeta(record: Record<string, unknown>, context: RedactContext): void {
|
|
408
|
+
if (record.type !== "user" || record.isMeta !== true || !isObject(record.message)) return;
|
|
409
|
+
record.message = { ...record.message, content: marker("meta") };
|
|
410
|
+
count(context, "attachment", 1);
|
|
411
|
+
}
|
|
412
|
+
|
|
228
413
|
function redactRecord(
|
|
229
414
|
record: Record<string, unknown>,
|
|
230
415
|
context: RedactContext,
|
|
231
416
|
): Record<string, unknown> {
|
|
232
417
|
learnRoot(record, context);
|
|
233
418
|
scanDotenv(record, context);
|
|
419
|
+
stripAttachment(record, context);
|
|
420
|
+
stripHarnessFields(record, context);
|
|
421
|
+
stripMeta(record, context);
|
|
234
422
|
return redactValue(record, context) as Record<string, unknown>;
|
|
235
423
|
}
|
|
236
424
|
|
|
@@ -287,6 +475,13 @@ function redactOneLine(line: string, context: RedactContext): { line: string; un
|
|
|
287
475
|
return { line: JSON.stringify(redactRecord(parsed, context)), unparsed: false };
|
|
288
476
|
}
|
|
289
477
|
|
|
478
|
+
/**
|
|
479
|
+
* A transcript's lines, redacted. With no `cwd`, paths are rooted at the first
|
|
480
|
+
* record that names one; with no `home`, at the home that directory implies,
|
|
481
|
+
* else this machine's. Exported as `isy/redact` so the server can re-run the
|
|
482
|
+
* rules over a transcript an older client uploaded: there the machine is the
|
|
483
|
+
* server, so it passes `home: ""` and roots paths in the transcript alone.
|
|
484
|
+
*/
|
|
290
485
|
export function redactLines(
|
|
291
486
|
lines: Iterable<string>,
|
|
292
487
|
options: RedactOptions = {},
|
|
@@ -308,22 +503,23 @@ export function redactLines(
|
|
|
308
503
|
return { lines: output, summary: summarize(context, total, unparsed, invalid) };
|
|
309
504
|
}
|
|
310
505
|
|
|
311
|
-
|
|
312
|
-
|
|
506
|
+
/**
|
|
507
|
+
* `redactLines` for lines that arrive one at a time — a file read as it streams,
|
|
508
|
+
* or a Claude session with its subagents spliced in (`subagents.ts`). One
|
|
509
|
+
* context for the whole stream, so the working directory the first record names
|
|
510
|
+
* roots every path after it, the subagents' included.
|
|
511
|
+
*/
|
|
512
|
+
export async function redactLineStream(
|
|
513
|
+
lines: AsyncIterable<string>,
|
|
313
514
|
options: RedactOptions = {},
|
|
314
515
|
): Promise<{ lines: string[]; summary: RedactionSummary }> {
|
|
315
516
|
const invalid: string[] = [];
|
|
316
517
|
const context = createContext(options, invalid);
|
|
317
|
-
const reader = createInterface({
|
|
318
|
-
input: createReadStream(filePath, { encoding: "utf8" }),
|
|
319
|
-
crlfDelay: Infinity,
|
|
320
|
-
});
|
|
321
|
-
|
|
322
518
|
const output: string[] = [];
|
|
323
519
|
let total = 0;
|
|
324
520
|
let unparsed = 0;
|
|
325
521
|
|
|
326
|
-
for await (const line of
|
|
522
|
+
for await (const line of lines) {
|
|
327
523
|
total += 1;
|
|
328
524
|
const result = redactOneLine(line, context);
|
|
329
525
|
if (result.line.length === 0) continue;
|
|
@@ -333,3 +529,10 @@ export async function redactTranscriptFile(
|
|
|
333
529
|
|
|
334
530
|
return { lines: output, summary: summarize(context, total, unparsed, invalid) };
|
|
335
531
|
}
|
|
532
|
+
|
|
533
|
+
export function redactTranscriptFile(
|
|
534
|
+
filePath: string,
|
|
535
|
+
options: RedactOptions = {},
|
|
536
|
+
): Promise<{ lines: string[]; summary: RedactionSummary }> {
|
|
537
|
+
return redactLineStream(fileLines(filePath), options);
|
|
538
|
+
}
|
package/src/state.ts
CHANGED
|
@@ -25,6 +25,12 @@ export interface CommitMark {
|
|
|
25
25
|
export interface SessionState {
|
|
26
26
|
/** contentHash of the last upload the server accepted. */
|
|
27
27
|
uploadedHash?: string;
|
|
28
|
+
/**
|
|
29
|
+
* contentHash of an upload refused for what it holds — too large, or a body
|
|
30
|
+
* the server will not take. The same bytes would be refused again, so the
|
|
31
|
+
* fingerprint below is taken at the refusal too and a scan passes them over.
|
|
32
|
+
*/
|
|
33
|
+
rejectedHash?: string;
|
|
28
34
|
/**
|
|
29
35
|
* Transcript size and mtime at that upload. Both unchanged means the bytes
|
|
30
36
|
* are unchanged, which lets a scan skip reading a transcript entirely.
|
|
@@ -159,25 +165,44 @@ export async function updateSession(
|
|
|
159
165
|
}
|
|
160
166
|
|
|
161
167
|
/**
|
|
162
|
-
* Identity of one candidate across runs
|
|
163
|
-
* record
|
|
164
|
-
*
|
|
168
|
+
* Identity of one candidate across runs: the call it points at, or else the
|
|
169
|
+
* record, both of which keep their ids as the session grows — the same finding
|
|
170
|
+
* produces the same key at every later commit. Not the record's index: a
|
|
171
|
+
* subagent is merged into its session by time (`subagents.ts`), so a background
|
|
172
|
+
* one that goes on writing puts records ahead of later ones and moves every
|
|
173
|
+
* index after them. A candidate that points at neither has only the index.
|
|
165
174
|
*/
|
|
166
175
|
export function candidateKey(candidate: Candidate): string {
|
|
176
|
+
const anchor = candidate.toolUseId ?? candidate.uuid;
|
|
177
|
+
return [
|
|
178
|
+
candidate.category,
|
|
179
|
+
candidate.filePath ?? "",
|
|
180
|
+
anchor ?? "",
|
|
181
|
+
anchor === undefined ? candidate.recordIndex : "",
|
|
182
|
+
// Escaped, never a raw byte: a NUL in the source makes git treat the file as binary.
|
|
183
|
+
].join("\x00");
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* The key isy 1.0 wrote, index and all. Still matched, so the first commit
|
|
188
|
+
* after an upgrade does not print again what the one before it printed.
|
|
189
|
+
*/
|
|
190
|
+
function keyWithIndex(candidate: Candidate): string {
|
|
167
191
|
return [
|
|
168
192
|
candidate.category,
|
|
169
193
|
candidate.filePath ?? "",
|
|
170
194
|
candidate.uuid ?? "",
|
|
171
195
|
candidate.toolUseId ?? "",
|
|
172
196
|
candidate.recordIndex,
|
|
173
|
-
// Escaped, never a raw byte: a NUL in the source makes git treat the file as binary.
|
|
174
197
|
].join("\x00");
|
|
175
198
|
}
|
|
176
199
|
|
|
177
200
|
/** Candidates this session has not printed before. */
|
|
178
201
|
export function unseen(state: SessionState, candidates: readonly Candidate[]): Candidate[] {
|
|
179
202
|
const shown = new Set(state.shown ?? []);
|
|
180
|
-
return candidates.filter(
|
|
203
|
+
return candidates.filter(
|
|
204
|
+
(candidate) => !shown.has(candidateKey(candidate)) && !shown.has(keyWithIndex(candidate)),
|
|
205
|
+
);
|
|
181
206
|
}
|
|
182
207
|
|
|
183
208
|
/**
|
|
@@ -194,15 +219,15 @@ export function withCommit(
|
|
|
194
219
|
}
|
|
195
220
|
|
|
196
221
|
/**
|
|
197
|
-
* Whether the transcript or its commit marks changed since the last
|
|
198
|
-
*
|
|
199
|
-
* not be read at all.
|
|
222
|
+
* Whether the transcript or its commit marks changed since the last upload the
|
|
223
|
+
* server accepted or refused for good. Size and mtime both matching means the
|
|
224
|
+
* bytes match, so the file need not be read at all.
|
|
200
225
|
*/
|
|
201
226
|
export function looksUnchanged(
|
|
202
227
|
state: SessionState,
|
|
203
228
|
file: { sizeBytes: number; modifiedAt: Date },
|
|
204
229
|
): boolean {
|
|
205
|
-
if (state.uploadedHash === undefined) return false;
|
|
230
|
+
if (state.uploadedHash === undefined && state.rejectedHash === undefined) return false;
|
|
206
231
|
if (state.commits?.at(-1)?.sha !== state.uploadedCommit) return false;
|
|
207
232
|
return state.sizeBytes === file.sizeBytes && state.modifiedMs === file.modifiedAt.getTime();
|
|
208
233
|
}
|