@blogic-cz/agent-tools 1.10.1 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -44
- package/dist/credential-guard/bash.d.ts +3 -0
- package/dist/credential-guard/bash.d.ts.map +1 -0
- package/dist/credential-guard/claude-hook.d.ts.map +1 -1
- package/dist/credential-guard/engine.test.d.ts +5 -0
- package/dist/credential-guard/engine.test.d.ts.map +1 -0
- package/dist/credential-guard/index.d.ts +8 -35
- package/dist/credential-guard/index.d.ts.map +1 -1
- package/dist/credential-guard/ir.d.ts +150 -0
- package/dist/credential-guard/ir.d.ts.map +1 -0
- package/dist/credential-guard/literal-output.d.ts +2 -0
- package/dist/credential-guard/literal-output.d.ts.map +1 -0
- package/dist/credential-guard/passive-awk.d.ts +2 -0
- package/dist/credential-guard/passive-awk.d.ts.map +1 -0
- package/dist/credential-guard/passive-jq.d.ts.map +1 -1
- package/dist/credential-guard/passive-sed.d.ts +2 -0
- package/dist/credential-guard/passive-sed.d.ts.map +1 -0
- package/dist/credential-guard/passive.test.d.ts +2 -0
- package/dist/credential-guard/passive.test.d.ts.map +1 -0
- package/dist/credential-guard/paths.d.ts +14 -0
- package/dist/credential-guard/paths.d.ts.map +1 -0
- package/dist/credential-guard/policy.d.ts +47 -0
- package/dist/credential-guard/policy.d.ts.map +1 -0
- package/dist/credential-guard/powershell.d.ts +3 -0
- package/dist/credential-guard/powershell.d.ts.map +1 -0
- package/dist/credential-guard/registry.d.ts +89 -0
- package/dist/credential-guard/registry.d.ts.map +1 -0
- package/dist/credential-guard/review4.test.d.ts +2 -0
- package/dist/credential-guard/review4.test.d.ts.map +1 -0
- package/dist/credential-guard/review7.test.d.ts +2 -0
- package/dist/credential-guard/review7.test.d.ts.map +1 -0
- package/dist/credential-guard/shell-oracle.d.ts +2 -0
- package/dist/credential-guard/shell-oracle.d.ts.map +1 -0
- package/dist/credential-guard/xargs.d.ts +7 -0
- package/dist/credential-guard/xargs.d.ts.map +1 -0
- package/dist/db-tool/service.d.ts +2 -1
- package/dist/db-tool/service.d.ts.map +1 -1
- package/dist/db-tool/sql-client.d.ts +1 -0
- package/dist/db-tool/sql-client.d.ts.map +1 -1
- package/dist/gh-tool/pr/commands.d.ts.map +1 -1
- package/dist/gh-tool/read/api.d.ts +8 -0
- package/dist/gh-tool/read/api.d.ts.map +1 -1
- package/dist/gh-tool/read/cache.d.ts.map +1 -1
- package/dist/gh-tool/read/commands.d.ts +5 -0
- package/dist/gh-tool/read/commands.d.ts.map +1 -1
- package/dist/gh-tool/read/reads.d.ts +38 -0
- package/dist/gh-tool/read/reads.d.ts.map +1 -1
- package/dist/gh-tool/read/telemetry-command.d.ts +7 -0
- package/dist/gh-tool/read/telemetry-command.d.ts.map +1 -0
- package/dist/gh-tool/read/telemetry-store.d.ts +3 -0
- package/dist/gh-tool/read/telemetry-store.d.ts.map +1 -0
- package/dist/gh-tool/read/telemetry.d.ts +59 -0
- package/dist/gh-tool/read/telemetry.d.ts.map +1 -0
- package/dist/gh-tool/read/transport.d.ts.map +1 -1
- package/dist/observability-tool/logs.d.ts +1 -1
- package/dist/observability-tool/logs.d.ts.map +1 -1
- package/dist/observability-tool/metrics.d.ts +1 -1
- package/dist/observability-tool/metrics.d.ts.map +1 -1
- package/dist/observability-tool/shared.d.ts +2 -1
- package/dist/observability-tool/shared.d.ts.map +1 -1
- package/dist/observability-tool/trace.d.ts +1 -1
- package/dist/observability-tool/trace.d.ts.map +1 -1
- package/dist/shared/content-security.d.ts +3 -1
- package/dist/shared/content-security.d.ts.map +1 -1
- package/dist/shared/secret-env.d.ts +22 -0
- package/dist/shared/secret-env.d.ts.map +1 -0
- package/package.json +2 -1
- package/schemas/agent-tools.schema.json +1 -1
- package/src/credential-guard/bash.ts +769 -0
- package/src/credential-guard/claude-hook.ts +14 -14
- package/src/credential-guard/engine.test.ts +2050 -0
- package/src/credential-guard/index.ts +285 -4268
- package/src/credential-guard/ir.ts +139 -0
- package/src/credential-guard/literal-output.ts +8 -0
- package/src/credential-guard/passive-awk.ts +239 -0
- package/src/credential-guard/passive-jq.ts +183 -24
- package/src/credential-guard/passive-sed.ts +78 -0
- package/src/credential-guard/passive.test.ts +141 -0
- package/src/credential-guard/paths.ts +108 -0
- package/src/credential-guard/policy.ts +1576 -0
- package/src/credential-guard/powershell.ts +220 -0
- package/src/credential-guard/registry.ts +4720 -0
- package/src/credential-guard/review4.test.ts +83 -0
- package/src/credential-guard/review7.test.ts +51 -0
- package/src/credential-guard/rg-types.json +389 -0
- package/src/credential-guard/shell-oracle.ts +187 -0
- package/src/credential-guard/xargs.ts +65 -0
- package/src/db-tool/main.ts +2 -0
- package/src/db-tool/service.ts +122 -140
- package/src/db-tool/sql-client.ts +7 -4
- package/src/gh-tool/main.ts +30 -6
- package/src/gh-tool/pr/commands.ts +9 -2
- package/src/gh-tool/pr/core.ts +2 -2
- package/src/gh-tool/read/api.ts +73 -6
- package/src/gh-tool/read/cache.ts +13 -4
- package/src/gh-tool/read/commands.ts +17 -1
- package/src/gh-tool/read/reads.ts +84 -1
- package/src/gh-tool/read/telemetry-command.ts +45 -0
- package/src/gh-tool/read/telemetry-store.ts +85 -0
- package/src/gh-tool/read/telemetry.ts +191 -0
- package/src/gh-tool/read/transport.ts +10 -5
- package/src/gh-tool/workflow.ts +2 -2
- package/src/observability-tool/main.ts +7 -1
- package/src/observability-tool/shared.ts +20 -14
- package/src/shared/content-security.ts +7 -2
- package/src/shared/secret-env.ts +230 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { Context, Effect } from "effect";
|
|
2
|
+
import { redactSensitiveText } from "#shared/content-security";
|
|
3
|
+
|
|
4
|
+
export type RequestEvent = {
|
|
5
|
+
ts: string;
|
|
6
|
+
method: string;
|
|
7
|
+
endpoint: string;
|
|
8
|
+
status: number | null;
|
|
9
|
+
outcome: "cache" | "304" | "200" | "bypass" | "error" | "subprocess";
|
|
10
|
+
resource: string | null;
|
|
11
|
+
used: number | null;
|
|
12
|
+
remaining: number | null;
|
|
13
|
+
reset: number | null;
|
|
14
|
+
};
|
|
15
|
+
export type RequestRow = RequestEvent & { id: number; invocation: string; command: string };
|
|
16
|
+
export type TelemetryStore = {
|
|
17
|
+
record: (event: RequestEvent) => void;
|
|
18
|
+
read: (since: string) => readonly RequestRow[];
|
|
19
|
+
close: () => void;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export const Telemetry = Context.Reference<TelemetryStore | null>("gh/Telemetry", {
|
|
23
|
+
defaultValue: () => null,
|
|
24
|
+
});
|
|
25
|
+
export const CacheRequest = Context.Reference<boolean>("gh/CacheRequest", {
|
|
26
|
+
defaultValue: () => false,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
export const templateEndpoint = (path: string): string => {
|
|
30
|
+
const pathname = path.replace(/^https?:\/\/[^/]+/, "").split(/[?#]/, 1)[0] ?? "";
|
|
31
|
+
return redactSensitiveText(
|
|
32
|
+
`/${pathname.replace(/^\//, "")}`
|
|
33
|
+
.replace(/\b[0-9a-f]{7,64}\b/gi, (part) => (/^\d+$/.test(part) ? ":id" : ":sha"))
|
|
34
|
+
.replace(/\/\d+(?=\/|$)/g, "/:id"),
|
|
35
|
+
);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export const recordRequest = (event: RequestEvent): Effect.Effect<void> =>
|
|
39
|
+
Effect.gen(function* () {
|
|
40
|
+
const store = yield* Telemetry;
|
|
41
|
+
try {
|
|
42
|
+
store?.record({
|
|
43
|
+
...event,
|
|
44
|
+
endpoint:
|
|
45
|
+
event.outcome === "subprocess" ? event.endpoint : templateEndpoint(event.endpoint),
|
|
46
|
+
});
|
|
47
|
+
} catch {
|
|
48
|
+
/* Best effort. */
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
export const cacheEvent = (operation: string): RequestEvent => ({
|
|
53
|
+
ts: new Date().toISOString(),
|
|
54
|
+
method: "CACHE",
|
|
55
|
+
endpoint: `cache/${operation}`,
|
|
56
|
+
status: null,
|
|
57
|
+
outcome: "cache",
|
|
58
|
+
resource: null,
|
|
59
|
+
used: null,
|
|
60
|
+
remaining: null,
|
|
61
|
+
reset: null,
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const API_VALUE_FLAGS = new Set([
|
|
65
|
+
"-X",
|
|
66
|
+
"--method",
|
|
67
|
+
"-H",
|
|
68
|
+
"--header",
|
|
69
|
+
"-f",
|
|
70
|
+
"-F",
|
|
71
|
+
"--field",
|
|
72
|
+
"--raw-field",
|
|
73
|
+
"--jq",
|
|
74
|
+
"-q",
|
|
75
|
+
"--template",
|
|
76
|
+
"-t",
|
|
77
|
+
"--input",
|
|
78
|
+
"--cache",
|
|
79
|
+
"-p",
|
|
80
|
+
"--preview",
|
|
81
|
+
"--hostname",
|
|
82
|
+
]);
|
|
83
|
+
|
|
84
|
+
export const ghSubprocessEvent = (args: readonly string[], exitCode: number): RequestEvent => {
|
|
85
|
+
const api = args[0] === "api";
|
|
86
|
+
let endpoint: string | undefined;
|
|
87
|
+
let method: string | undefined;
|
|
88
|
+
let hasFields = false;
|
|
89
|
+
if (api) {
|
|
90
|
+
for (let index = 1; index < args.length; index++) {
|
|
91
|
+
const arg = args[index];
|
|
92
|
+
if (arg === undefined) continue;
|
|
93
|
+
if (arg === "--") {
|
|
94
|
+
endpoint ??= args[index + 1];
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
if (!arg.startsWith("-")) {
|
|
98
|
+
endpoint ??= arg;
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
const equals = arg.indexOf("=");
|
|
102
|
+
const flag = equals < 0 ? arg : arg.slice(0, equals);
|
|
103
|
+
if (!API_VALUE_FLAGS.has(flag)) continue;
|
|
104
|
+
const value = equals < 0 ? args[++index] : arg.slice(equals + 1);
|
|
105
|
+
if (flag === "-X" || flag === "--method") method = value;
|
|
106
|
+
if (flag === "-f" || flag === "-F" || flag === "--field" || flag === "--raw-field")
|
|
107
|
+
hasFields = true;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
...cacheEvent(""),
|
|
112
|
+
method: api ? (method ?? (hasFields ? "POST" : "GET")) : "GH",
|
|
113
|
+
endpoint: api ? templateEndpoint(endpoint ?? "") : args.slice(0, 2).join(" "),
|
|
114
|
+
status: exitCode >= 100 && exitCode <= 599 ? exitCode : null,
|
|
115
|
+
outcome: "subprocess",
|
|
116
|
+
};
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
const counts = (values: readonly string[]) => {
|
|
120
|
+
const groups = new Map<string, number>();
|
|
121
|
+
for (const value of values) groups.set(value, (groups.get(value) ?? 0) + 1);
|
|
122
|
+
return [...groups]
|
|
123
|
+
.map(([name, requests]) => ({ name, requests }))
|
|
124
|
+
.toSorted((a, b) => b.requests - a.requests || a.name.localeCompare(b.name));
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
const subprocessSpendsResource = (row: RequestRow, resource: string): boolean => {
|
|
128
|
+
if (row.outcome !== "subprocess") return false;
|
|
129
|
+
if (row.method === "GH") return true;
|
|
130
|
+
if (row.method === "GET" && row.endpoint === "/rate_limit") return false;
|
|
131
|
+
return row.endpoint === "/graphql" ? resource === "graphql" : resource !== "graphql";
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
export const summarizeRequests = (rows: readonly RequestRow[]) => {
|
|
135
|
+
const requests = rows.filter((row) => row.outcome !== "cache" && row.outcome !== "subprocess");
|
|
136
|
+
const subprocesses = rows.filter((row) => row.outcome === "subprocess");
|
|
137
|
+
const previous = new Map<string, { used: number; index: number }>();
|
|
138
|
+
let externalUsageEstimate = 0;
|
|
139
|
+
let intervals = 0;
|
|
140
|
+
for (const [index, row] of rows.entries()) {
|
|
141
|
+
if (row.outcome === "cache" || row.outcome === "subprocess") continue;
|
|
142
|
+
if (row.resource === null || row.used === null || row.reset === null) continue;
|
|
143
|
+
const resource = row.resource;
|
|
144
|
+
const key = `${resource}:${row.reset}`;
|
|
145
|
+
const before = previous.get(key);
|
|
146
|
+
if (before !== undefined && row.used >= before.used) {
|
|
147
|
+
const interval = rows.slice(before.index + 1, index + 1);
|
|
148
|
+
const own = interval.filter(
|
|
149
|
+
(candidate) =>
|
|
150
|
+
candidate.resource === resource &&
|
|
151
|
+
(candidate.reset === null || candidate.reset === row.reset) &&
|
|
152
|
+
candidate.status !== 304 &&
|
|
153
|
+
candidate.endpoint !== "/rate_limit",
|
|
154
|
+
).length;
|
|
155
|
+
const subprocessCalls = interval.filter((candidate) =>
|
|
156
|
+
subprocessSpendsResource(candidate, resource),
|
|
157
|
+
).length;
|
|
158
|
+
externalUsageEstimate += Math.max(0, row.used - before.used - own - subprocessCalls);
|
|
159
|
+
intervals++;
|
|
160
|
+
}
|
|
161
|
+
if (before === undefined || row.used >= before.used)
|
|
162
|
+
previous.set(key, { used: row.used, index });
|
|
163
|
+
}
|
|
164
|
+
const notModified = requests.filter((row) => row.status === 304).length;
|
|
165
|
+
return {
|
|
166
|
+
directHttpRequests: requests.length,
|
|
167
|
+
ghSubprocessCalls: subprocesses.length,
|
|
168
|
+
cacheHits: rows.filter((row) => row.outcome === "cache").length,
|
|
169
|
+
subprocessTopCommands: counts(subprocesses.map((row) => redactSensitiveText(row.command)))
|
|
170
|
+
.slice(0, 20)
|
|
171
|
+
.map(({ name, requests: calls }) => ({ name, calls })),
|
|
172
|
+
subprocessTopEndpoints: counts(subprocesses.map((row) => `${row.method} ${row.endpoint}`))
|
|
173
|
+
.slice(0, 20)
|
|
174
|
+
.map(({ name, requests: calls }) => ({ name, calls })),
|
|
175
|
+
directHttpByResource: counts(requests.map((row) => row.resource ?? "unknown")),
|
|
176
|
+
directHttpTopCommands: counts(requests.map((row) => redactSensitiveText(row.command))).slice(
|
|
177
|
+
0,
|
|
178
|
+
20,
|
|
179
|
+
),
|
|
180
|
+
directHttpTopEndpoints: counts(requests.map((row) => `${row.method} ${row.endpoint}`)).slice(
|
|
181
|
+
0,
|
|
182
|
+
20,
|
|
183
|
+
),
|
|
184
|
+
notModified,
|
|
185
|
+
ratio304: requests.length === 0 ? null : notModified / requests.length,
|
|
186
|
+
externalUsageEstimate: intervals === 0 ? null : externalUsageEstimate,
|
|
187
|
+
observedIntervals: intervals,
|
|
188
|
+
caveat:
|
|
189
|
+
"Estimate within observed reset windows only. Missing headers, concurrent requests, different tokens and dropped telemetry reduce accuracy. Direct HTTP counts are exact. Each gh subprocess call may make zero or more HTTP requests; its HTTP status and quota headers are unavailable. For this estimate, subprocess calls count as one request each. Cache hits count ReadCache only.",
|
|
190
|
+
};
|
|
191
|
+
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ghSubprocessEvent, recordRequest } from "./telemetry";
|
|
1
2
|
import { ChildProcess, ChildProcessSpawner } from "effect/process";
|
|
2
3
|
import { Context, Effect, Layer, Stream } from "effect";
|
|
3
4
|
|
|
@@ -182,13 +183,17 @@ export class GitHubService extends Context.Service<
|
|
|
182
183
|
|
|
183
184
|
const proc = yield* executor.spawn(command);
|
|
184
185
|
|
|
185
|
-
const stdoutChunk = yield*
|
|
186
|
+
const [stdoutChunk, stderrChunk] = yield* Effect.all(
|
|
187
|
+
[
|
|
188
|
+
proc.stdout.pipe(Stream.decodeText(), Stream.runCollect),
|
|
189
|
+
proc.stderr.pipe(Stream.decodeText(), Stream.runCollect),
|
|
190
|
+
],
|
|
191
|
+
{ concurrency: "unbounded" },
|
|
192
|
+
);
|
|
186
193
|
const stdout = stdoutChunk.join("");
|
|
187
|
-
|
|
188
|
-
const stderrChunk = yield* proc.stderr.pipe(Stream.decodeText(), Stream.runCollect);
|
|
189
194
|
const stderr = stderrChunk.join("");
|
|
190
|
-
|
|
191
195
|
const exitCode = yield* proc.exitCode;
|
|
196
|
+
yield* recordRequest(ghSubprocessEvent(args, Number(exitCode)));
|
|
192
197
|
|
|
193
198
|
return {
|
|
194
199
|
stdout,
|
|
@@ -246,7 +251,7 @@ export class GitHubService extends Context.Service<
|
|
|
246
251
|
const quotaFilter =
|
|
247
252
|
resource === undefined
|
|
248
253
|
? '.resources | to_entries[] | "\\(.key): reset time: \\(.value.reset | todateiso8601); remaining quota: \\(.value.remaining)"'
|
|
249
|
-
: `.resources.${resource} | "reset time: \\(.reset | todateiso8601); remaining quota: \\(.remaining)"`;
|
|
254
|
+
: `.resources.${resource} | "resource: ${resource}; reset time: \\(.reset | todateiso8601); remaining quota: \\(.remaining)"`;
|
|
250
255
|
const quota = yield* executeGh(["api", "rate_limit", "--jq", quotaFilter]).pipe(
|
|
251
256
|
Effect.orElseSucceed(() => null),
|
|
252
257
|
);
|
package/src/gh-tool/workflow.ts
CHANGED
|
@@ -298,7 +298,7 @@ const fetchAnnotations = Effect.fn("workflow.fetchAnnotations")(function* (opts:
|
|
|
298
298
|
const parts = opts.repo.split("/");
|
|
299
299
|
if (parts.length !== 2 || !parts[0] || !parts[1]) {
|
|
300
300
|
return yield* new GitHubCommandError({
|
|
301
|
-
message: `Invalid --repo format: "${opts.repo}". Expected "owner/name" (e.g. "
|
|
301
|
+
message: `Invalid --repo format: "${opts.repo}". Expected "owner/name" (e.g. "example-org/example-repo").`,
|
|
302
302
|
command: "workflow annotations",
|
|
303
303
|
exitCode: 1,
|
|
304
304
|
stderr: "",
|
|
@@ -574,7 +574,7 @@ export const fetchJobLogs = Effect.fn("workflow.fetchJobLogs")(function* (
|
|
|
574
574
|
const parts = opts.repo.split("/");
|
|
575
575
|
if (parts.length !== 2 || !parts[0] || !parts[1]) {
|
|
576
576
|
return yield* new GitHubCommandError({
|
|
577
|
-
message: `Invalid --repo format: "${opts.repo}". Expected "owner/name" (e.g. "
|
|
577
|
+
message: `Invalid --repo format: "${opts.repo}". Expected "owner/name" (e.g. "example-org/example-repo").`,
|
|
578
578
|
command: "workflow job-logs",
|
|
579
579
|
exitCode: 1,
|
|
580
580
|
stderr: "",
|
|
@@ -7,6 +7,7 @@ import { Command } from "effect/cli";
|
|
|
7
7
|
import { ConfigServiceLayer } from "#config";
|
|
8
8
|
import { AuditServiceLayer, withAudit } from "#shared/audit";
|
|
9
9
|
import { withRedactedOutput } from "#shared/output-boundary";
|
|
10
|
+
import { SecretEnvResolver } from "#shared/secret-env";
|
|
10
11
|
import { makeSchemaCommand, renderCauseToStderr, VERSION } from "#shared";
|
|
11
12
|
|
|
12
13
|
import { metricsCommand } from "./metrics";
|
|
@@ -24,7 +25,12 @@ const mainCommand = Command.make("observability-tool", {}).pipe(
|
|
|
24
25
|
|
|
25
26
|
const cli = Command.run(mainCommand, { version: VERSION, renderErrors: false });
|
|
26
27
|
|
|
27
|
-
const MainLayer = Layer.mergeAll(
|
|
28
|
+
const MainLayer = Layer.mergeAll(
|
|
29
|
+
BunServices.layer,
|
|
30
|
+
ConfigServiceLayer,
|
|
31
|
+
AuditServiceLayer,
|
|
32
|
+
SecretEnvResolver.liveLayer,
|
|
33
|
+
);
|
|
28
34
|
|
|
29
35
|
const program = withRedactedOutput(withAudit("observability", cli)).pipe(
|
|
30
36
|
Effect.provide(MainLayer),
|
|
@@ -5,7 +5,8 @@ import { ConfigService, getToolConfig } from "#config";
|
|
|
5
5
|
import type { ObservabilityConfig } from "#config";
|
|
6
6
|
|
|
7
7
|
import { ObservabilityToolError } from "./errors";
|
|
8
|
-
import {
|
|
8
|
+
import { redactSensitiveText, unsafeOutboundTextReason } from "#shared/content-security";
|
|
9
|
+
import { SecretEnvResolver } from "#shared/secret-env";
|
|
9
10
|
import type {
|
|
10
11
|
DsQueryOpts,
|
|
11
12
|
DsQueryResponse,
|
|
@@ -48,18 +49,15 @@ export const profileOption = Flag.optional(
|
|
|
48
49
|
),
|
|
49
50
|
);
|
|
50
51
|
|
|
51
|
-
function resolveToken(
|
|
52
|
+
function resolveToken(
|
|
53
|
+
resolver: SecretEnvResolver["Service"],
|
|
54
|
+
tokenEnvVar?: string,
|
|
55
|
+
): Promise<string | undefined> {
|
|
52
56
|
if (!tokenEnvVar) {
|
|
53
|
-
return undefined;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
const token = process.env[tokenEnvVar];
|
|
57
|
-
if (!token) {
|
|
58
|
-
throw new Error(`${tokenEnvVar} environment variable is not set`);
|
|
57
|
+
return Promise.resolve(undefined);
|
|
59
58
|
}
|
|
60
59
|
|
|
61
|
-
|
|
62
|
-
return token;
|
|
60
|
+
return resolver.resolveSecretEnv(tokenEnvVar);
|
|
63
61
|
}
|
|
64
62
|
|
|
65
63
|
async function discoverDatasources(url: string, token?: string): Promise<GrafanaDatasource[]> {
|
|
@@ -68,7 +66,9 @@ async function discoverDatasources(url: string, token?: string): Promise<Grafana
|
|
|
68
66
|
|
|
69
67
|
if (!response.ok) {
|
|
70
68
|
const body = await response.text().catch(() => "");
|
|
71
|
-
throw new Error(
|
|
69
|
+
throw new Error(
|
|
70
|
+
redactSensitiveText(`Grafana API ${response.status}: /api/datasources — ${body}`),
|
|
71
|
+
);
|
|
72
72
|
}
|
|
73
73
|
|
|
74
74
|
return (await response.json()) as GrafanaDatasource[];
|
|
@@ -102,13 +102,14 @@ export function requireTempoUid(
|
|
|
102
102
|
async function resolveFromProfile(
|
|
103
103
|
profile: ObservabilityConfig | undefined,
|
|
104
104
|
env: string,
|
|
105
|
+
resolver: SecretEnvResolver["Service"],
|
|
105
106
|
): Promise<ObservabilityEnvConfig | undefined> {
|
|
106
107
|
const environment = profile?.environments[env];
|
|
107
108
|
if (!environment) {
|
|
108
109
|
return undefined;
|
|
109
110
|
}
|
|
110
111
|
|
|
111
|
-
const token = resolveToken(environment.tokenEnvVar);
|
|
112
|
+
const token = await resolveToken(resolver, environment.tokenEnvVar);
|
|
112
113
|
const datasources = await discoverDatasources(environment.url, token);
|
|
113
114
|
|
|
114
115
|
// Tempo is optional: metrics/logs work on Grafana instances without it (e.g. Percona PMM).
|
|
@@ -157,6 +158,7 @@ function resolveFromEnv(
|
|
|
157
158
|
export const resolveConfig = (env: string, profile: Option.Option<string>) =>
|
|
158
159
|
Effect.gen(function* () {
|
|
159
160
|
const config = yield* ConfigService;
|
|
161
|
+
const secretEnvResolver = yield* SecretEnvResolver;
|
|
160
162
|
const profileName = Option.getOrUndefined(profile);
|
|
161
163
|
const observabilityConfig = getToolConfig<ObservabilityConfig>(
|
|
162
164
|
config,
|
|
@@ -166,7 +168,11 @@ export const resolveConfig = (env: string, profile: Option.Option<string>) =>
|
|
|
166
168
|
|
|
167
169
|
return yield* Effect.tryPromise({
|
|
168
170
|
try: async () => {
|
|
169
|
-
const profileResolved = await resolveFromProfile(
|
|
171
|
+
const profileResolved = await resolveFromProfile(
|
|
172
|
+
observabilityConfig,
|
|
173
|
+
env,
|
|
174
|
+
secretEnvResolver,
|
|
175
|
+
);
|
|
170
176
|
if (profileResolved) {
|
|
171
177
|
return profileResolved;
|
|
172
178
|
}
|
|
@@ -258,7 +264,7 @@ export function observabilityFetch<T>(
|
|
|
258
264
|
|
|
259
265
|
if (!response.ok) {
|
|
260
266
|
const body = await response.text().catch(() => "");
|
|
261
|
-
throw new Error(`Grafana API ${response.status}: ${path} — ${body}`);
|
|
267
|
+
throw new Error(redactSensitiveText(`Grafana API ${response.status}: ${path} — ${body}`));
|
|
262
268
|
}
|
|
263
269
|
|
|
264
270
|
return (await response.json()) as T;
|
|
@@ -12,8 +12,13 @@ const MIN_SECRET_LENGTH = 8;
|
|
|
12
12
|
const resolvedSecrets = new Set<string>();
|
|
13
13
|
|
|
14
14
|
/** Redact a credential a tool resolved outside a sensitively named environment variable. */
|
|
15
|
-
export const registerResolvedSecret = (
|
|
16
|
-
|
|
15
|
+
export const registerResolvedSecret = (
|
|
16
|
+
value: string,
|
|
17
|
+
options: { readonly allowShort?: boolean } = {},
|
|
18
|
+
) => {
|
|
19
|
+
if (value.length > 0 && (options.allowShort === true || value.length >= MIN_SECRET_LENGTH)) {
|
|
20
|
+
resolvedSecrets.add(value);
|
|
21
|
+
}
|
|
17
22
|
};
|
|
18
23
|
|
|
19
24
|
const sensitiveEnvironmentValues = (environment: Record<string, string | undefined>) =>
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { relative, resolve as resolvePath } from "node:path";
|
|
4
|
+
import { Context, Layer } from "effect";
|
|
5
|
+
|
|
6
|
+
import { registerResolvedSecret } from "./content-security";
|
|
7
|
+
|
|
8
|
+
const SECRET_NAME_PATTERN = /^[A-Z_][A-Z0-9_]*$/;
|
|
9
|
+
const CONFIG_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
10
|
+
const DEFAULT_TIMEOUT_MS = 10_000;
|
|
11
|
+
const STARTUP_ENV_NAMES = [
|
|
12
|
+
"PATH",
|
|
13
|
+
"HOME",
|
|
14
|
+
"USER",
|
|
15
|
+
"LOGNAME",
|
|
16
|
+
"SHELL",
|
|
17
|
+
"TERM",
|
|
18
|
+
"LANG",
|
|
19
|
+
"LC_ALL",
|
|
20
|
+
"TMPDIR",
|
|
21
|
+
"XDG_CONFIG_HOME",
|
|
22
|
+
"ZDOTDIR",
|
|
23
|
+
] as const;
|
|
24
|
+
|
|
25
|
+
export type LookupResult =
|
|
26
|
+
| { readonly _tag: "found"; readonly value: string }
|
|
27
|
+
| { readonly _tag: "absent" }
|
|
28
|
+
| { readonly _tag: "failed"; readonly reason: string };
|
|
29
|
+
|
|
30
|
+
export type ZshRunner = (name: string, timeoutMs: number) => Promise<LookupResult>;
|
|
31
|
+
|
|
32
|
+
const failed = (name: string): LookupResult => ({ _tag: "failed", reason: name });
|
|
33
|
+
|
|
34
|
+
type FrameInspection =
|
|
35
|
+
| { readonly _tag: "incomplete" }
|
|
36
|
+
| { readonly _tag: "malformed" }
|
|
37
|
+
| { readonly _tag: "complete"; readonly result: LookupResult };
|
|
38
|
+
|
|
39
|
+
const inspectFrame = (frame: Buffer): FrameInspection => {
|
|
40
|
+
const firstNewline = frame.indexOf(10);
|
|
41
|
+
if (firstNewline < 0) {
|
|
42
|
+
const prefix = frame.toString("ascii");
|
|
43
|
+
return "set".startsWith(prefix) || "unset".startsWith(prefix)
|
|
44
|
+
? { _tag: "incomplete" }
|
|
45
|
+
: { _tag: "malformed" };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const state = frame.subarray(0, firstNewline).toString("ascii");
|
|
49
|
+
if (state === "unset") {
|
|
50
|
+
return firstNewline === frame.length - 1
|
|
51
|
+
? { _tag: "complete", result: { _tag: "absent" } }
|
|
52
|
+
: { _tag: "malformed" };
|
|
53
|
+
}
|
|
54
|
+
if (state !== "set") return { _tag: "malformed" };
|
|
55
|
+
|
|
56
|
+
const lengthEnd = frame.indexOf(10, firstNewline + 1);
|
|
57
|
+
if (lengthEnd < 0) {
|
|
58
|
+
const lengthPrefix = frame.subarray(firstNewline + 1).toString("ascii");
|
|
59
|
+
if (/^(0|[1-9][0-9]*)$/.test(lengthPrefix) || lengthPrefix === "") {
|
|
60
|
+
return { _tag: "incomplete" };
|
|
61
|
+
}
|
|
62
|
+
return { _tag: "malformed" };
|
|
63
|
+
}
|
|
64
|
+
const lengthText = frame.subarray(firstNewline + 1, lengthEnd).toString("ascii");
|
|
65
|
+
if (!/^(0|[1-9][0-9]*)$/.test(lengthText)) return { _tag: "malformed" };
|
|
66
|
+
|
|
67
|
+
const length = Number(lengthText);
|
|
68
|
+
const start = lengthEnd + 1;
|
|
69
|
+
if (!Number.isSafeInteger(length)) return { _tag: "malformed" };
|
|
70
|
+
if (start + length < frame.length) return { _tag: "malformed" };
|
|
71
|
+
if (start + length > frame.length) return { _tag: "incomplete" };
|
|
72
|
+
return {
|
|
73
|
+
_tag: "complete",
|
|
74
|
+
result: { _tag: "found", value: frame.subarray(start).toString("utf8") },
|
|
75
|
+
};
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const runInteractiveZsh: ZshRunner = (name, timeoutMs) =>
|
|
79
|
+
new Promise((resolve) => {
|
|
80
|
+
if (process.env.VITEST !== undefined) {
|
|
81
|
+
const home = process.env.HOME;
|
|
82
|
+
const zdotdir = process.env.ZDOTDIR;
|
|
83
|
+
const relativeZdotdir = zdotdir ? relative(tmpdir(), resolvePath(zdotdir)) : "..";
|
|
84
|
+
if (
|
|
85
|
+
!home ||
|
|
86
|
+
!zdotdir ||
|
|
87
|
+
resolvePath(home) !== resolvePath(zdotdir) ||
|
|
88
|
+
relativeZdotdir === ".." ||
|
|
89
|
+
relativeZdotdir.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)
|
|
90
|
+
) {
|
|
91
|
+
resolve(failed(name));
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const childEnv = Object.fromEntries(
|
|
97
|
+
STARTUP_ENV_NAMES.flatMap((key) => {
|
|
98
|
+
const value = process.env[key];
|
|
99
|
+
return value === undefined ? [] : [[key, value]];
|
|
100
|
+
}),
|
|
101
|
+
);
|
|
102
|
+
const script = `export LC_ALL=C; if (( \${+parameters[$1]} )); then value=\${(P)1}; print -ru3 -- "set"; print -ru3 -- "\${#value}"; print -rn -- "$value" >&3; else print -ru3 -- "unset"; fi`;
|
|
103
|
+
|
|
104
|
+
let child: ReturnType<typeof spawn>;
|
|
105
|
+
try {
|
|
106
|
+
child = spawn("zsh", ["-i", "-c", script, "agent-tools-secret-env", name], {
|
|
107
|
+
detached: true,
|
|
108
|
+
env: childEnv,
|
|
109
|
+
stdio: ["ignore", "ignore", "ignore", "pipe"],
|
|
110
|
+
});
|
|
111
|
+
} catch {
|
|
112
|
+
resolve(failed(name));
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const stream = child.stdio[3];
|
|
117
|
+
if (!stream || typeof stream === "number") {
|
|
118
|
+
child.kill("SIGKILL");
|
|
119
|
+
resolve(failed(name));
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const chunks: Buffer[] = [];
|
|
124
|
+
let exited = false;
|
|
125
|
+
let ended = false;
|
|
126
|
+
let exitCode: number | null = null;
|
|
127
|
+
let settled = false;
|
|
128
|
+
const killProcessGroup = () => {
|
|
129
|
+
if (child.pid === undefined) return;
|
|
130
|
+
try {
|
|
131
|
+
process.kill(-child.pid, "SIGKILL");
|
|
132
|
+
} catch {
|
|
133
|
+
// The process group may already have exited.
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
const finish = (result: LookupResult) => {
|
|
137
|
+
if (settled) return;
|
|
138
|
+
settled = true;
|
|
139
|
+
clearTimeout(timer);
|
|
140
|
+
stream.destroy();
|
|
141
|
+
killProcessGroup();
|
|
142
|
+
resolve(result);
|
|
143
|
+
};
|
|
144
|
+
const checkFrame = () => {
|
|
145
|
+
if (settled) return;
|
|
146
|
+
const inspection = inspectFrame(Buffer.concat(chunks));
|
|
147
|
+
if (inspection._tag === "malformed") return finish(failed(name));
|
|
148
|
+
if (!exited) return;
|
|
149
|
+
if (exitCode !== 0) return finish(failed(name));
|
|
150
|
+
if (inspection._tag === "complete") finish(inspection.result);
|
|
151
|
+
else if (ended) finish(failed(name));
|
|
152
|
+
};
|
|
153
|
+
const timer = setTimeout(() => {
|
|
154
|
+
finish(failed(name));
|
|
155
|
+
}, timeoutMs);
|
|
156
|
+
|
|
157
|
+
stream.on("data", (chunk: Buffer | string) => {
|
|
158
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
159
|
+
checkFrame();
|
|
160
|
+
});
|
|
161
|
+
stream.on("end", () => {
|
|
162
|
+
ended = true;
|
|
163
|
+
checkFrame();
|
|
164
|
+
});
|
|
165
|
+
stream.on("error", () => finish(failed(name)));
|
|
166
|
+
child.on("error", () => finish(failed(name)));
|
|
167
|
+
child.on("exit", (code) => {
|
|
168
|
+
exited = true;
|
|
169
|
+
exitCode = code;
|
|
170
|
+
checkFrame();
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
export class SecretEnvResolver extends Context.Service<
|
|
175
|
+
SecretEnvResolver,
|
|
176
|
+
{
|
|
177
|
+
readonly resolveSecretEnv: (name: string) => Promise<string>;
|
|
178
|
+
readonly lookupConfigEnv: (name: string) => Promise<LookupResult>;
|
|
179
|
+
}
|
|
180
|
+
>()("@agent-tools/SecretEnvResolver") {
|
|
181
|
+
static get liveLayer() {
|
|
182
|
+
return makeSecretEnvResolverLayer();
|
|
183
|
+
}
|
|
184
|
+
static readonly testLayer = (runner: ZshRunner = () => Promise.resolve({ _tag: "absent" })) =>
|
|
185
|
+
makeSecretEnvResolverLayer(runner);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export function makeSecretEnvResolverLayer(
|
|
189
|
+
runner: ZshRunner = runInteractiveZsh,
|
|
190
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
191
|
+
) {
|
|
192
|
+
return Layer.sync(SecretEnvResolver, () => {
|
|
193
|
+
const cache = new Map<string, Promise<LookupResult>>();
|
|
194
|
+
const lookup = (name: string, emptyProcessValueWins: boolean): Promise<LookupResult> => {
|
|
195
|
+
const fromProcess = process.env[name];
|
|
196
|
+
if (fromProcess !== undefined && (emptyProcessValueWins || fromProcess !== "")) {
|
|
197
|
+
return Promise.resolve({ _tag: "found", value: fromProcess });
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
let result = cache.get(name);
|
|
201
|
+
if (!result) {
|
|
202
|
+
result = Promise.resolve()
|
|
203
|
+
.then(() => runner(name, timeoutMs))
|
|
204
|
+
.catch(() => failed(name));
|
|
205
|
+
cache.set(name, result);
|
|
206
|
+
}
|
|
207
|
+
return result;
|
|
208
|
+
};
|
|
209
|
+
const lookupConfigEnv = (name: string): Promise<LookupResult> => {
|
|
210
|
+
if (!CONFIG_NAME_PATTERN.test(name)) return Promise.resolve(failed(name));
|
|
211
|
+
return lookup(name, true);
|
|
212
|
+
};
|
|
213
|
+
const resolveSecretEnv = async (name: string): Promise<string> => {
|
|
214
|
+
if (!SECRET_NAME_PATTERN.test(name)) {
|
|
215
|
+
throw new Error(`Invalid environment variable name: ${name}`);
|
|
216
|
+
}
|
|
217
|
+
const result = await lookup(name, false);
|
|
218
|
+
if (result._tag !== "found") {
|
|
219
|
+
throw new Error(`${name} is not in the environment or in the interactive zsh`);
|
|
220
|
+
}
|
|
221
|
+
if (result.value.length < 4) {
|
|
222
|
+
throw new Error(`${name} is too short to redact safely`);
|
|
223
|
+
}
|
|
224
|
+
registerResolvedSecret(result.value, { allowShort: true });
|
|
225
|
+
return result.value;
|
|
226
|
+
};
|
|
227
|
+
|
|
228
|
+
return SecretEnvResolver.of({ resolveSecretEnv, lookupConfigEnv });
|
|
229
|
+
});
|
|
230
|
+
}
|