agentwrangler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +191 -0
- package/README.md +116 -0
- package/dist/apply/jobs.js +429 -0
- package/dist/apply/open-terminal-child.mjs +98 -0
- package/dist/apply/open-terminal.js +221 -0
- package/dist/apply/settings-gen.js +35 -0
- package/dist/cli/agentwrangler.js +18 -0
- package/dist/daemon/config.js +51 -0
- package/dist/daemon/http.js +258 -0
- package/dist/daemon/index.js +372 -0
- package/dist/daemon/outcomes-pass.js +82 -0
- package/dist/daemon/readiness.js +15 -0
- package/dist/daemon/router.js +756 -0
- package/dist/daemon/static.js +146 -0
- package/dist/db/migrate.js +72 -0
- package/dist/db/migrations/001_observe.sql +196 -0
- package/dist/db/migrations/002_indexes.sql +6 -0
- package/dist/db/migrations/003_context_inventory_history.sql +20 -0
- package/dist/db/migrations/004_apply_jobs.sql +17 -0
- package/dist/db/migrations/005_tool_event_metadata.sql +17 -0
- package/dist/db/migrations/006_d7_query_indexes.sql +9 -0
- package/dist/db/migrations/007_work_item_branch_keys.sql +11 -0
- package/dist/db/migrations/008_thinking_tokens.sql +1 -0
- package/dist/db/migrations/009_user_turn_count.sql +1 -0
- package/dist/db/migrations/010_workspace_cwd.sql +1 -0
- package/dist/db/migrations/011_reports.sql +1 -0
- package/dist/db/migrations/012_reconcile_indexes.sql +2 -0
- package/dist/db/migrations/013_friction_fields.sql +5 -0
- package/dist/db/migrations/014_session_churn.sql +11 -0
- package/dist/db/migrations/015_gap_aggregates.sql +6 -0
- package/dist/db/open.js +30 -0
- package/dist/detector/benchmark-anchors.js +36 -0
- package/dist/detector/calibration.js +302 -0
- package/dist/detector/context-history-retention.js +312 -0
- package/dist/detector/context-probe.js +574 -0
- package/dist/detector/d1-source-identity.js +25 -0
- package/dist/detector/detectors/d10_catalog_footprint.js +146 -0
- package/dist/detector/detectors/d1_ctx_always_loaded.js +203 -0
- package/dist/detector/detectors/d2_session_long_full_context.js +119 -0
- package/dist/detector/detectors/d4_model_mismatch.js +258 -0
- package/dist/detector/detectors/d5_limit_burn_forecast.js +138 -0
- package/dist/detector/detectors/d6_tool_result_bloat.js +301 -0
- package/dist/detector/detectors/d7_loop_retry_waste.js +345 -0
- package/dist/detector/detectors/d8_cache_write_churn.js +201 -0
- package/dist/detector/detectors/d9_idle_background_session.js +101 -0
- package/dist/detector/engine.js +88 -0
- package/dist/detector/index.js +17 -0
- package/dist/detector/measurement.js +426 -0
- package/dist/detector/practice-registry.js +259 -0
- package/dist/detector/registry.js +32 -0
- package/dist/detector/savings.js +249 -0
- package/dist/detector/types.js +14 -0
- package/dist/evidence/common/approved-input.js +632 -0
- package/dist/evidence/common/boundary.js +84 -0
- package/dist/evidence/common/canonical.js +55 -0
- package/dist/evidence/common/redaction.js +321 -0
- package/dist/evidence/common/sqlite.js +25 -0
- package/dist/evidence/common/state.js +29 -0
- package/dist/evidence/cond1/cli.js +289 -0
- package/dist/evidence/cond1/packet.js +407 -0
- package/dist/evidence/cond1/prepare.js +295 -0
- package/dist/evidence/cond1/score.js +349 -0
- package/dist/evidence/cond1/types.js +1 -0
- package/dist/evidence/create-approval.js +365 -0
- package/dist/evidence/create-scratch.js +542 -0
- package/dist/evidence/d7/cli.js +113 -0
- package/dist/evidence/d7/measure.js +193 -0
- package/dist/evidence/d7/types.js +1 -0
- package/dist/evidence/discover-approval.js +492 -0
- package/dist/evidence/g2/adjudicate.js +20 -0
- package/dist/evidence/g2/cli.js +207 -0
- package/dist/evidence/g2/kappa.js +39 -0
- package/dist/evidence/g2/pipeline.js +92 -0
- package/dist/evidence/g2/store.js +14 -0
- package/dist/evidence/github/client.js +1 -0
- package/dist/evidence/github/gh-cli-client.js +301 -0
- package/dist/evidence/r3/cli.js +209 -0
- package/dist/evidence/r3/evaluate.js +417 -0
- package/dist/evidence/r3/packet.js +162 -0
- package/dist/evidence/r3/prepare.js +405 -0
- package/dist/evidence/r3/score.js +341 -0
- package/dist/evidence/r3/transcript.js +155 -0
- package/dist/evidence/r3/types.js +4 -0
- package/dist/hook/context-budget-hook.mjs +138 -0
- package/dist/hook/danger-guard-denylist.json +27 -0
- package/dist/hook/danger-guard-hook.mjs +167 -0
- package/dist/hook/install.js +0 -0
- package/dist/hook/limit-burn-hook.mjs +127 -0
- package/dist/hook/loop-guard-hook.mjs +104 -0
- package/dist/hook/precompact-checkpoint-hook.mjs +123 -0
- package/dist/ingest/churn-collector.js +122 -0
- package/dist/ingest/detector-hook.js +52 -0
- package/dist/ingest/discovery.js +207 -0
- package/dist/ingest/health.js +43 -0
- package/dist/ingest/index.js +28 -0
- package/dist/ingest/ingestor.js +509 -0
- package/dist/ingest/parser.js +344 -0
- package/dist/ingest/pricing.js +153 -0
- package/dist/ingest/reconcile.js +52 -0
- package/dist/ingest/tail.js +152 -0
- package/dist/ingest/types.js +24 -0
- package/dist/ingest/workspace-mapping.js +114 -0
- package/dist/oauth/anthropic-api-key.js +88 -0
- package/dist/oauth/count-tokens.js +86 -0
- package/dist/oauth/credentials.js +171 -0
- package/dist/oauth/judge-g2-client.js +154 -0
- package/dist/oauth/usage.js +167 -0
- package/dist/outcomes/branch-key.js +49 -0
- package/dist/outcomes/conclusions.js +45 -0
- package/dist/outcomes/derive.js +94 -0
- package/dist/outcomes/finding-extractors.js +131 -0
- package/dist/outcomes/findings.js +237 -0
- package/dist/outcomes/github/client.js +367 -0
- package/dist/outcomes/github/credential.js +195 -0
- package/dist/outcomes/github/gh-cli-client.js +340 -0
- package/dist/outcomes/linker.js +486 -0
- package/dist/outcomes/pool.js +24 -0
- package/dist/outcomes/sync.js +276 -0
- package/dist/query/api/agents-liveness.js +182 -0
- package/dist/query/api/burn-status.js +50 -0
- package/dist/query/api/context-budget.js +114 -0
- package/dist/query/api/context-composition.js +67 -0
- package/dist/query/api/cost-per-success.js +104 -0
- package/dist/query/api/delivery.js +92 -0
- package/dist/query/api/effectiveness.js +254 -0
- package/dist/query/api/efficiency-headroom.js +74 -0
- package/dist/query/api/headroom-trend.js +105 -0
- package/dist/query/api/hook-config.js +75 -0
- package/dist/query/api/hook-install.js +8 -0
- package/dist/query/api/hot-sessions.js +17 -0
- package/dist/query/api/idle-sessions.js +52 -0
- package/dist/query/api/index.js +40 -0
- package/dist/query/api/loop-guard.js +90 -0
- package/dist/query/api/offload-share.js +41 -0
- package/dist/query/api/outcomes.js +218 -0
- package/dist/query/api/overview.js +535 -0
- package/dist/query/api/rec-prompt.js +138 -0
- package/dist/query/api/recommendations-ledger.js +111 -0
- package/dist/query/api/recommendations.js +514 -0
- package/dist/query/api/reports.js +78 -0
- package/dist/query/api/self-churn.js +77 -0
- package/dist/query/api/self-percentiles.js +109 -0
- package/dist/query/api/session-drivers.js +153 -0
- package/dist/query/api/settings.js +85 -0
- package/dist/query/api/spend-flavor.js +234 -0
- package/dist/query/api/trends.js +155 -0
- package/dist/query/cap-weighted.js +119 -0
- package/dist/query/db-context.js +42 -0
- package/dist/query/envelope.js +71 -0
- package/dist/query/forecast.js +191 -0
- package/dist/query/settings-store.js +441 -0
- package/dist/query/spend.js +171 -0
- package/dist/query/trends.js +194 -0
- package/dist/ui/assets/index-DnRKgc21.css +1 -0
- package/dist/ui/assets/index-h1Q1wWq5.js +168 -0
- package/dist/ui/index.html +39 -0
- package/package.json +59 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
export class EvidenceBoundaryError extends Error {
|
|
4
|
+
code;
|
|
5
|
+
constructor(code) {
|
|
6
|
+
super(code);
|
|
7
|
+
this.code = code;
|
|
8
|
+
this.name = "EvidenceBoundaryError";
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
function comparisonPath(value) {
|
|
12
|
+
const normalized = path.normalize(value).replace(/[\\/]+$/u, "");
|
|
13
|
+
return process.platform === "win32" ? normalized.toLowerCase() : normalized;
|
|
14
|
+
}
|
|
15
|
+
function isWithin(parent, child) {
|
|
16
|
+
const relative = path.relative(parent, child);
|
|
17
|
+
return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== "..");
|
|
18
|
+
}
|
|
19
|
+
/** Resolve symlinks/junctions through the nearest existing ancestor of a future path. */
|
|
20
|
+
export function resolveProspectivePath(inputPath) {
|
|
21
|
+
let existing = path.resolve(inputPath);
|
|
22
|
+
const suffix = [];
|
|
23
|
+
while (!fs.existsSync(existing)) {
|
|
24
|
+
const parent = path.dirname(existing);
|
|
25
|
+
if (parent === existing)
|
|
26
|
+
throw new EvidenceBoundaryError("path_has_no_existing_ancestor");
|
|
27
|
+
suffix.unshift(path.basename(existing));
|
|
28
|
+
existing = parent;
|
|
29
|
+
}
|
|
30
|
+
return path.join(fs.realpathSync.native(existing), ...suffix);
|
|
31
|
+
}
|
|
32
|
+
export function readFileIdentity(filePath) {
|
|
33
|
+
const stat = fs.statSync(filePath, { bigint: true });
|
|
34
|
+
if (!stat.isFile())
|
|
35
|
+
throw new EvidenceBoundaryError("database_path_not_file");
|
|
36
|
+
return {
|
|
37
|
+
device: stat.dev.toString(),
|
|
38
|
+
inode: stat.ino.toString(),
|
|
39
|
+
size: stat.size.toString(),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
export function sameFileIdentity(left, right) {
|
|
43
|
+
return left.inode !== "0" && left.device === right.device && left.inode === right.inode;
|
|
44
|
+
}
|
|
45
|
+
export function assertScratchBoundary(input) {
|
|
46
|
+
if (input.sourceDbPath.length === 0 ||
|
|
47
|
+
input.scratchDbPath.length === 0 ||
|
|
48
|
+
input.scratchStatePath.length === 0) {
|
|
49
|
+
throw new EvidenceBoundaryError("empty_path");
|
|
50
|
+
}
|
|
51
|
+
if (!fs.existsSync(input.sourceDbPath))
|
|
52
|
+
throw new EvidenceBoundaryError("source_missing");
|
|
53
|
+
const sourceDbPath = fs.realpathSync.native(input.sourceDbPath);
|
|
54
|
+
const sourceIdentity = readFileIdentity(sourceDbPath);
|
|
55
|
+
const repositoryRoot = fs.realpathSync.native(input.repositoryRoot);
|
|
56
|
+
const scratchDbPath = resolveProspectivePath(input.scratchDbPath);
|
|
57
|
+
const scratchStatePath = resolveProspectivePath(input.scratchStatePath);
|
|
58
|
+
if (isWithin(repositoryRoot, scratchDbPath) || isWithin(repositoryRoot, scratchStatePath)) {
|
|
59
|
+
throw new EvidenceBoundaryError("output_inside_repository");
|
|
60
|
+
}
|
|
61
|
+
if (comparisonPath(sourceDbPath) === comparisonPath(scratchDbPath)) {
|
|
62
|
+
throw new EvidenceBoundaryError("scratch_aliases_source");
|
|
63
|
+
}
|
|
64
|
+
if (fs.existsSync(scratchDbPath)) {
|
|
65
|
+
const targetIdentity = readFileIdentity(scratchDbPath);
|
|
66
|
+
if (sameFileIdentity(sourceIdentity, targetIdentity)) {
|
|
67
|
+
throw new EvidenceBoundaryError("scratch_aliases_source");
|
|
68
|
+
}
|
|
69
|
+
throw new EvidenceBoundaryError("scratch_already_exists");
|
|
70
|
+
}
|
|
71
|
+
if (fs.existsSync(scratchStatePath)) {
|
|
72
|
+
throw new EvidenceBoundaryError("scratch_state_already_exists");
|
|
73
|
+
}
|
|
74
|
+
if (isWithin(scratchStatePath, scratchDbPath) || isWithin(scratchDbPath, scratchStatePath)) {
|
|
75
|
+
throw new EvidenceBoundaryError("overlapping_output_paths");
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
sourceDbPath,
|
|
79
|
+
scratchDbPath,
|
|
80
|
+
scratchStatePath,
|
|
81
|
+
repositoryRoot,
|
|
82
|
+
sourceIdentity,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
function serializeCanonical(value, seen) {
|
|
4
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") {
|
|
5
|
+
return JSON.stringify(value);
|
|
6
|
+
}
|
|
7
|
+
if (typeof value === "number") {
|
|
8
|
+
if (!Number.isFinite(value))
|
|
9
|
+
throw new Error("canonical_non_finite_number");
|
|
10
|
+
return JSON.stringify(value);
|
|
11
|
+
}
|
|
12
|
+
if (typeof value !== "object")
|
|
13
|
+
throw new Error("canonical_unsupported_value");
|
|
14
|
+
if (seen.has(value))
|
|
15
|
+
throw new Error("canonical_cycle");
|
|
16
|
+
seen.add(value);
|
|
17
|
+
try {
|
|
18
|
+
if (Array.isArray(value)) {
|
|
19
|
+
return `[${value.map((item) => serializeCanonical(item, seen)).join(",")}]`;
|
|
20
|
+
}
|
|
21
|
+
const prototype = Object.getPrototypeOf(value);
|
|
22
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
23
|
+
throw new Error("canonical_non_plain_object");
|
|
24
|
+
}
|
|
25
|
+
const record = value;
|
|
26
|
+
return `{${Object.keys(record)
|
|
27
|
+
.sort()
|
|
28
|
+
.map((key) => `${JSON.stringify(key)}:${serializeCanonical(record[key], seen)}`)
|
|
29
|
+
.join(",")}}`;
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
seen.delete(value);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** Deterministic JSON with recursively sorted object keys and preserved array order. */
|
|
36
|
+
export function canonicalJson(value) {
|
|
37
|
+
return serializeCanonical(value, new Set());
|
|
38
|
+
}
|
|
39
|
+
export function sha256Bytes(value) {
|
|
40
|
+
return createHash("sha256").update(value).digest("hex");
|
|
41
|
+
}
|
|
42
|
+
export function sha256Canonical(value) {
|
|
43
|
+
return sha256Bytes(canonicalJson(value));
|
|
44
|
+
}
|
|
45
|
+
/** Stream a file into SHA-256 so database size is not duplicated in memory. */
|
|
46
|
+
export async function sha256File(filePath) {
|
|
47
|
+
const hash = createHash("sha256");
|
|
48
|
+
await new Promise((resolve, reject) => {
|
|
49
|
+
const input = fs.createReadStream(filePath);
|
|
50
|
+
input.on("data", (chunk) => hash.update(chunk));
|
|
51
|
+
input.once("error", reject);
|
|
52
|
+
input.once("end", resolve);
|
|
53
|
+
});
|
|
54
|
+
return hash.digest("hex");
|
|
55
|
+
}
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
export const EVIDENCE_EXCERPT_MAX_BYTES = 512;
|
|
2
|
+
const REPLACEMENTS = [
|
|
3
|
+
[/\b(?:https?|ftp):\/\/[^\s<>"'`]+/giu, "[REDACTED:URL]"],
|
|
4
|
+
[/\bwww\.[^\s<>"'`]+/giu, "[REDACTED:URL]"],
|
|
5
|
+
[
|
|
6
|
+
/"(?:[A-Za-z]:[\\/]|\\\\|\/|~[\\/]|\.{1,2}[\\/]|[A-Za-z0-9._~-]+[\\/])[^"\r\n]+"/gu,
|
|
7
|
+
"[REDACTED:PATH]",
|
|
8
|
+
],
|
|
9
|
+
[
|
|
10
|
+
/'(?:[A-Za-z]:[\\/]|\\\\|\/|~[\\/]|\.{1,2}[\\/]|[A-Za-z0-9._~-]+[\\/])[^'\r\n]+'/gu,
|
|
11
|
+
"[REDACTED:PATH]",
|
|
12
|
+
],
|
|
13
|
+
[
|
|
14
|
+
/`(?:[A-Za-z]:[\\/]|\\\\|\/|~[\\/]|\.{1,2}[\\/]|[A-Za-z0-9._~-]+[\\/])[^`\r\n]+`/gu,
|
|
15
|
+
"[REDACTED:PATH]",
|
|
16
|
+
],
|
|
17
|
+
[
|
|
18
|
+
/(^|[\s"'`()\[\]{}<>=:,;])(?:[A-Za-z]:[\\/]|\\\\)[^\s<>"'`()\[\]{}=,;]*/gmu,
|
|
19
|
+
"$1[REDACTED:PATH]",
|
|
20
|
+
],
|
|
21
|
+
[
|
|
22
|
+
/(^|[\s"'`()\[\]{}<>=:,;])(?:~[\\/]|\.{1,2}[\\/]|\/)[^\s<>"'`()\[\]{}=,;]*/gmu,
|
|
23
|
+
"$1[REDACTED:PATH]",
|
|
24
|
+
],
|
|
25
|
+
[/\brefs\/(?:heads|remotes|tags)\/[A-Za-z0-9._~/-]+/gu, "[REDACTED:REF]"],
|
|
26
|
+
[/\b(?:origin|upstream)\/[A-Za-z0-9._~/-]+/gu, "[REDACTED:REF]"],
|
|
27
|
+
[
|
|
28
|
+
/\b(?:feat(?:ure)?|fix|bugfix|hotfix|release|chore|docs|test|refactor|dependabot|renovate)\/[A-Za-z0-9._~/-]+/giu,
|
|
29
|
+
"[REDACTED:REF]",
|
|
30
|
+
],
|
|
31
|
+
[
|
|
32
|
+
/\b(?:branch|ref|head\.ref)\s*(?::|=|\bis\b)?\s*["'`]?[A-Za-z0-9._~/-]+["'`]?/giu,
|
|
33
|
+
"[REDACTED:REF]",
|
|
34
|
+
],
|
|
35
|
+
[/\b[A-Za-z0-9._~-]+(?:[\\/][A-Za-z0-9._~/-]+)+/gu, "[REDACTED:PATH]"],
|
|
36
|
+
[/\bBearer\s+[^\s,;]+/giu, "[REDACTED:CREDENTIAL]"],
|
|
37
|
+
[
|
|
38
|
+
/\b(?:token|password|passwd|secret|api[_-]?key|authorization)\s*(?::|=|\bis\b)?\s*"[^"\r\n]+"/giu,
|
|
39
|
+
"[REDACTED:CREDENTIAL]",
|
|
40
|
+
],
|
|
41
|
+
[
|
|
42
|
+
/\b(?:token|password|passwd|secret|api[_-]?key|authorization)\s*(?::|=|\bis\b)?\s*'[^'\r\n]+'/giu,
|
|
43
|
+
"[REDACTED:CREDENTIAL]",
|
|
44
|
+
],
|
|
45
|
+
[
|
|
46
|
+
/\b(?:token|password|passwd|secret|api[_-]?key|authorization)\s*(?::|=|\bis\b)?\s*`[^`\r\n]+`/giu,
|
|
47
|
+
"[REDACTED:CREDENTIAL]",
|
|
48
|
+
],
|
|
49
|
+
[
|
|
50
|
+
/\b(?:token|password|passwd|secret|api[_-]?key|authorization)\s*[:=]\s*[^\s,;]+/giu,
|
|
51
|
+
"[REDACTED:CREDENTIAL]",
|
|
52
|
+
],
|
|
53
|
+
[/\bgh[pousr]_[A-Za-z0-9_]{20,}\b/gu, "[REDACTED:CREDENTIAL]"],
|
|
54
|
+
[
|
|
55
|
+
/\b(?:glpat-|npm_|sk_(?:test|live)_|xox[baprs]-)[A-Za-z0-9_-]{16,}\b/gu,
|
|
56
|
+
"[REDACTED:CREDENTIAL]",
|
|
57
|
+
],
|
|
58
|
+
[/\bAKIA[0-9A-Z]{16}\b/gu, "[REDACTED:CREDENTIAL]"],
|
|
59
|
+
[/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/gu, "[REDACTED:CREDENTIAL]"],
|
|
60
|
+
[/\b[A-Za-z0-9._%+-]+:[^\s/@]+@/gu, "[REDACTED:CREDENTIAL]@"],
|
|
61
|
+
[/\b(?:password|passwd|pwd)\s+(?:is\s+)?["'`]?[^\s,;"'`]+["'`]?/giu, "[REDACTED:CREDENTIAL]"],
|
|
62
|
+
[/\b(?:[0-9a-f]{64}|[0-9a-f]{40})\b/giu, "[REDACTED:COMMIT]"],
|
|
63
|
+
[/\b(?:commit|sha|revision)\s*(?::|=|\bis\b)?\s*[0-9a-f]{7,39}\b/giu, "[REDACTED:COMMIT]"],
|
|
64
|
+
[/\b[0-9a-f]{7,39}\b/giu, "[REDACTED:COMMIT]"],
|
|
65
|
+
];
|
|
66
|
+
const UNSAFE_PATTERNS = REPLACEMENTS.map(([pattern]) => pattern);
|
|
67
|
+
/**
|
|
68
|
+
* Separator homoglyphs that render as ASCII "/" or "\". Normalized to their ASCII
|
|
69
|
+
* form BEFORE any path/gate matching so a no-space path written with confusable
|
|
70
|
+
* slashes cannot slip past the ASCII-only path detectors. Over-normalization is
|
|
71
|
+
* acceptable (fail-closed): worst case a benign glyph becomes a redactable token.
|
|
72
|
+
*/
|
|
73
|
+
const SEPARATOR_HOMOGLYPHS = [
|
|
74
|
+
// U+2044 fraction, U+2215 division, U+29F8 big solidus, U+FF0F fullwidth solidus
|
|
75
|
+
[/[⁄∕⧸/]/gu, "/"],
|
|
76
|
+
// U+2216 set minus, U+29F9 big reverse solidus, U+FE68 small reverse solidus, U+FF3C fullwidth reverse solidus
|
|
77
|
+
[/[∖⧹﹨\]/gu, "\\"],
|
|
78
|
+
];
|
|
79
|
+
function normalizeSeparators(value) {
|
|
80
|
+
let output = value;
|
|
81
|
+
for (const [pattern, replacement] of SEPARATOR_HOMOGLYPHS) {
|
|
82
|
+
output = output.replace(pattern, replacement);
|
|
83
|
+
}
|
|
84
|
+
return output;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Fail-closed backstop for separator confusables the enumerated normalization does
|
|
88
|
+
* not cover. A non-ASCII codepoint wedged directly between two ASCII path-word
|
|
89
|
+
* characters — with no space — is a separator/segment joiner the ASCII-only path
|
|
90
|
+
* detectors cannot see: slash look-alikes (box-drawing `╱`, math diagonals, …),
|
|
91
|
+
* ideographic strokes (`丿`, `ノ`), or mixed-script segment joiners. Latin letters
|
|
92
|
+
* and combining marks are exempted so accented prose (café, naïve, café) is
|
|
93
|
+
* preserved; everything else in that position drops the excerpt. This makes safety
|
|
94
|
+
* class-based rather than dependent on an ever-drifting codepoint allowlist.
|
|
95
|
+
*/
|
|
96
|
+
function hasNonAsciiSeparatorWedge(value) {
|
|
97
|
+
return /[A-Za-z0-9._~-](?![\p{Script=Latin}\p{M}])[^\p{ASCII}][A-Za-z0-9._~-]/u.test(value);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* A run of one or more CASED letters from a non-Latin alphabet (Cyrillic, Greek,
|
|
101
|
+
* Armenian, Cherokee, …). These render as ASCII look-alikes but are not Latin
|
|
102
|
+
* script, so the ASCII-anchored path/ref rules never match them. Caseless scripts
|
|
103
|
+
* (`\p{Lo}`: CJK, most emoji) and Latin script (accented prose, and fullwidth Latin
|
|
104
|
+
* once NFKC-folded to ASCII) are excluded so bilingual prose survives.
|
|
105
|
+
*/
|
|
106
|
+
const CONFUSABLE_CASED_RUN = "(?:(?![\\p{Script=Latin}\\p{ASCII}])[\\p{Lu}\\p{Ll}\\p{Lt}\\p{M}])+";
|
|
107
|
+
/**
|
|
108
|
+
* Fail-closed backstop for the ADV-3 residual: a run of confusable cased letters
|
|
109
|
+
* ("оре" reads as "ope"). When such a run sits at a segment boundary it is NOT wedged
|
|
110
|
+
* between two ASCII word chars, so `hasNonAsciiSeparatorWedge` misses it — the ASCII
|
|
111
|
+
* portion of the segment gets absorbed into a `[REDACTED:…]` marker while the
|
|
112
|
+
* confusable prefix/suffix survives (`оре[REDACTED:PATH]`). Detecting the run directly
|
|
113
|
+
* against a marker (optionally across one residual path slash, with no separating
|
|
114
|
+
* space) proves it is the non-ASCII remnant of a redacted sensitive segment, so drop
|
|
115
|
+
* the excerpt.
|
|
116
|
+
*/
|
|
117
|
+
function hasConfusableRedactionAdjacency(value) {
|
|
118
|
+
const marker = "\\[REDACTED:[A-Z]+\\]";
|
|
119
|
+
const beforeMarker = new RegExp(`${CONFUSABLE_CASED_RUN}[\\\\/]?${marker}`, "u");
|
|
120
|
+
const afterMarker = new RegExp(`${marker}@?[\\\\/]?${CONFUSABLE_CASED_RUN}`, "u");
|
|
121
|
+
return beforeMarker.test(value) || afterMarker.test(value);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Marker-independent companion to `hasConfusableRedactionAdjacency` (ADV-3, class A).
|
|
125
|
+
* A path built entirely from confusable letters (`оре/секрет`) — or an ASCII segment
|
|
126
|
+
* joined to a confusable one (`home/секрет`, `секрет/home`) — produces NO
|
|
127
|
+
* `[REDACTED:…]` marker at all, because every path/ref rule is ASCII-anchored, so the
|
|
128
|
+
* marker-adjacency check never fires and `hasNonAsciiSeparatorWedge` misses (a separator
|
|
129
|
+
* flanked by two non-ASCII letters is outside its ASCII word-char triple). Keying off
|
|
130
|
+
* the raw join instead: a confusable cased run sitting FLUSH (no separating space)
|
|
131
|
+
* against a path separator is a path shape whose confusable segment a human reads as
|
|
132
|
+
* ASCII — drop it. Two separator kinds are treated as path joins:
|
|
133
|
+
* 1. an ASCII `/` or `\`;
|
|
134
|
+
* 2. a single NON-ASCII, non-whitespace char (a slash/separator look-alike that is
|
|
135
|
+
* neither NFKC-folded nor in SEPARATOR_HOMOGLYPHS: box-drawing `╱`, ideographic
|
|
136
|
+
* stroke `丿`, katakana `ノ`, modifier `ː`, …) joining a confusable run to another
|
|
137
|
+
* confusable run or to an ASCII path-word char.
|
|
138
|
+
* ASCII punctuation other than `/`/`\` is deliberately NOT a join here: only `/` and
|
|
139
|
+
* `\` are path separators in this module (ASCII `foo.bar`/`foo-bar` are left
|
|
140
|
+
* unredacted), so a confusable run around an ASCII `.` `-` `:` stays prose — Russian
|
|
141
|
+
* `из-за`, `т.е.`, `т.к.` are preserved. Fail-closed: like the ASCII path rule that
|
|
142
|
+
* redacts `and/or`, this drops confusable `word/word` flush to a separator (Cyrillic
|
|
143
|
+
* `и/или`). Space-separated prose (`проект uses /home`, `修复 …`) is not flush — and
|
|
144
|
+
* caseless CJK/emoji are outside the run class — so both survive.
|
|
145
|
+
*
|
|
146
|
+
* The joiner is a single char that is a SEPARATOR look-alike: non-ASCII, non-whitespace,
|
|
147
|
+
* not run material (a cased letter or combining mark), and not ordinary in-word
|
|
148
|
+
* punctuation (dashes `\p{Pd}`, quotes/apostrophes `\p{Pf}`/`\p{Pi}`/U+02BC, middle dots
|
|
149
|
+
* U+00B7/U+0387). What remains matches the caseless slash look-alikes (`╱` symbol,
|
|
150
|
+
* `丿`/`ノ` letter-other, `ː` modifier letter) that sit BETWEEN confusable segments,
|
|
151
|
+
* while a confusable run's own letters can never serve as the joiner (that would split
|
|
152
|
+
* any long Cyrillic word `прое·к·т` and nuke all prose). Excluding in-word punctuation
|
|
153
|
+
* keeps common non-Latin prose alive — Ukrainian apostrophe words (`п’ять`, `об’єкт`,
|
|
154
|
+
* U+2019 and U+02BC), typographic-dash compounds (`кто‐то`, `да—нет`), and middle-dot
|
|
155
|
+
* text — mirroring how the module leaves ASCII `foo-bar`/`foo.bar` unredacted (only
|
|
156
|
+
* `/` and `\` are separators). Requiring a confusable run on at least one side, joined
|
|
157
|
+
* to another run OR to an ASCII path-word char, keeps ordinary ASCII+Cyrillic adjacency
|
|
158
|
+
* (`homeоре`) and ASCII-hyphenated Cyrillic (`из-за`) alive. Fail-closed: like the ASCII
|
|
159
|
+
* path rule that redacts `and/or`, this drops confusable `word/word` flush to a
|
|
160
|
+
* separator (`и/или`); space-separated prose (`проект uses /home`) is preserved.
|
|
161
|
+
*/
|
|
162
|
+
function hasConfusablePathSeparatorAdjacency(value) {
|
|
163
|
+
const run = CONFUSABLE_CASED_RUN;
|
|
164
|
+
// A separator look-alike: non-ASCII, non-whitespace, not run material (cased letter /
|
|
165
|
+
// mark), and not ordinary in-word punctuation (dashes, quotes/apostrophes, middle dots).
|
|
166
|
+
const joiner = "[^\\p{ASCII}\\s\\p{Lu}\\p{Ll}\\p{Lt}\\p{M}\\p{Pd}\\p{Pf}\\p{Pi}\\u{02BC}\\u{00B7}\\u{0387}]";
|
|
167
|
+
const word = "[A-Za-z0-9._~-]";
|
|
168
|
+
return (new RegExp(`${run}[\\\\/]`, "u").test(value) ||
|
|
169
|
+
new RegExp(`[\\\\/]${run}`, "u").test(value) ||
|
|
170
|
+
new RegExp(`${run}${joiner}+${run}`, "u").test(value) ||
|
|
171
|
+
new RegExp(`${word}${joiner}+${run}`, "u").test(value) ||
|
|
172
|
+
new RegExp(`${run}${joiner}+${word}`, "u").test(value));
|
|
173
|
+
}
|
|
174
|
+
const REDACTION_MARKER_FRAME = /\[REDACTED:[^\]\r\n]+\]/gu;
|
|
175
|
+
/**
|
|
176
|
+
* Marker-spoof guard (SEC-3). A genuine marker is an ASCII `[REDACTED:…]` token this
|
|
177
|
+
* module emits. SEC-2's NFKC fold opened a spoof surface: compatibility characters —
|
|
178
|
+
* fullwidth brackets/colon (`[REDACTED:PATH]`, U+FF3B/U+FF3D/U+FF1A) or a compat char
|
|
179
|
+
* wedged inside an ASCII frame (`[REDACTED:㍱]` U+3371 → `[REDACTED:hPa]`) — fold to a
|
|
180
|
+
* literal `[REDACTED:…]` token the module never produced. Detect that whole class by
|
|
181
|
+
* frame identity: a genuine marker is pure ASCII and NFKC-stable, so it appears
|
|
182
|
+
* byte-identically in the raw input. Any `[REDACTED:…]`-framed token in the folded form
|
|
183
|
+
* whose exact text is NOT a substring of the raw input was created or altered by the
|
|
184
|
+
* fold → forged. Fail-closed: drop the excerpt.
|
|
185
|
+
*
|
|
186
|
+
* Scope note: this deliberately does NOT drop an ASCII `[REDACTED:…]` already present
|
|
187
|
+
* verbatim in the raw input (it is a substring of raw → kept). That token is
|
|
188
|
+
* indistinguishable from the output of a prior redaction pass, and dropping it would
|
|
189
|
+
* break idempotency — the redactor's own output must re-redact to itself (see
|
|
190
|
+
* `parseCond1Prepared`, which re-runs this on stored excerpts and requires identity).
|
|
191
|
+
* The "raw input already contains a literal ASCII marker" case predates SEC-2 and can
|
|
192
|
+
* only be closed by a private-use sentinel scheme (rejected as over-engineering for a
|
|
193
|
+
* low-severity, no-data-leak integrity nit); it is left open.
|
|
194
|
+
*/
|
|
195
|
+
function hasFoldSynthesizedRedactionMarker(raw, folded) {
|
|
196
|
+
const frames = folded.match(REDACTION_MARKER_FRAME);
|
|
197
|
+
if (frames === null)
|
|
198
|
+
return false;
|
|
199
|
+
return frames.some((frame) => !raw.includes(frame));
|
|
200
|
+
}
|
|
201
|
+
function validUnicode(value) {
|
|
202
|
+
for (let index = 0; index < value.length; index++) {
|
|
203
|
+
const code = value.charCodeAt(index);
|
|
204
|
+
if (code >= 0xd800 && code <= 0xdbff) {
|
|
205
|
+
const next = value.charCodeAt(index + 1);
|
|
206
|
+
if (next < 0xdc00 || next > 0xdfff)
|
|
207
|
+
return false;
|
|
208
|
+
index++;
|
|
209
|
+
}
|
|
210
|
+
else if (code >= 0xdc00 && code <= 0xdfff) {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return true;
|
|
215
|
+
}
|
|
216
|
+
function hasDisallowedControl(value) {
|
|
217
|
+
for (const character of value) {
|
|
218
|
+
const code = character.codePointAt(0);
|
|
219
|
+
if (code !== undefined &&
|
|
220
|
+
(code <= 0x08 ||
|
|
221
|
+
code === 0x0b ||
|
|
222
|
+
code === 0x0c ||
|
|
223
|
+
(code >= 0x0e && code <= 0x1f) ||
|
|
224
|
+
code === 0x7f)) {
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return false;
|
|
229
|
+
}
|
|
230
|
+
function containsUnsafe(value) {
|
|
231
|
+
for (const pattern of UNSAFE_PATTERNS) {
|
|
232
|
+
pattern.lastIndex = 0;
|
|
233
|
+
if (pattern.test(value))
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
return false;
|
|
237
|
+
}
|
|
238
|
+
function hasWindowsRelativePathShape(value) {
|
|
239
|
+
if (/(?:^|[^\p{L}\p{N}._~-])[A-Za-z]:(?![\\/])(?=\S)/u.test(value)) {
|
|
240
|
+
return true;
|
|
241
|
+
}
|
|
242
|
+
for (let index = 0; index < value.length; index++) {
|
|
243
|
+
if (value[index] !== "\\" || value[index + 1] === "\\")
|
|
244
|
+
continue;
|
|
245
|
+
const previous = value[index - 1];
|
|
246
|
+
if (previous !== undefined && /[\p{L}\p{N}._~-]/u.test(previous))
|
|
247
|
+
continue;
|
|
248
|
+
const drive = value[index - 2];
|
|
249
|
+
const driveBoundary = value[index - 3];
|
|
250
|
+
const isDriveAbsolute = previous === ":" &&
|
|
251
|
+
drive !== undefined &&
|
|
252
|
+
/[A-Za-z]/u.test(drive) &&
|
|
253
|
+
(driveBoundary === undefined || !/[\p{L}\p{N}._~-]/u.test(driveBoundary));
|
|
254
|
+
if (isDriveAbsolute)
|
|
255
|
+
continue;
|
|
256
|
+
const next = value[index + 1];
|
|
257
|
+
if (next !== undefined && !/\s/u.test(next))
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
function hasResidualSensitivePrefix(value) {
|
|
263
|
+
return (/(?:^|[^\p{L}\p{N}._~-])(?:[A-Za-z]:[\\/]|\\\\|~[\\/]|\.{1,2}[\\/]|\/)/u.test(value) ||
|
|
264
|
+
hasWindowsRelativePathShape(value));
|
|
265
|
+
}
|
|
266
|
+
function hasAmbiguousUnquotedSpacedPath(value) {
|
|
267
|
+
const probe = value
|
|
268
|
+
.replace(/\b(?:https?|ftp):\/\/[^\s<>"'`]+/giu, "[MASKED]")
|
|
269
|
+
.replace(/["'`][^"'`\r\n]*[\\/][^"'`\r\n]*["'`]/gu, "[MASKED]")
|
|
270
|
+
.replace(/\brefs\/(?:heads|remotes|tags)\/[A-Za-z0-9._~/-]+/gu, "[MASKED]")
|
|
271
|
+
.replace(/\b(?:origin|upstream)\/[A-Za-z0-9._~/-]+/gu, "[MASKED]")
|
|
272
|
+
.replace(/\b(?:feat(?:ure)?|fix|bugfix|hotfix|release|chore|docs|test|refactor|dependabot|renovate)\/[A-Za-z0-9._~/-]+/giu, "[MASKED]");
|
|
273
|
+
return (/(^|[\s"'`()\[\]{}<>=:,;])(?:[A-Za-z]:[\\/]|\\\\|\/|~[\\/]|\.{1,2}[\\/])[^\s"'`()\[\]{}<>]*\s+[^\s\r\n]/mu.test(probe) ||
|
|
274
|
+
/(^|[\s"'`()\[\]{}<>=:,;])[A-Za-z._~-][A-Za-z0-9._~-]*[\\/][^\s"'`()\[\]{}<>]*\s+[^\s\r\n]/mu.test(probe));
|
|
275
|
+
}
|
|
276
|
+
function truncateUtf8(value) {
|
|
277
|
+
let bytes = 0;
|
|
278
|
+
let output = "";
|
|
279
|
+
for (const codePoint of value) {
|
|
280
|
+
const width = Buffer.byteLength(codePoint, "utf8");
|
|
281
|
+
if (bytes + width > EVIDENCE_EXCERPT_MAX_BYTES)
|
|
282
|
+
break;
|
|
283
|
+
output += codePoint;
|
|
284
|
+
bytes += width;
|
|
285
|
+
}
|
|
286
|
+
return output;
|
|
287
|
+
}
|
|
288
|
+
/** Deterministically redact one bounded excerpt, or drop it when it cannot be proven safe. */
|
|
289
|
+
export function redactEvidenceExcerpt(value) {
|
|
290
|
+
if (typeof value !== "string" ||
|
|
291
|
+
!validUnicode(value) ||
|
|
292
|
+
hasDisallowedControl(value) ||
|
|
293
|
+
/\p{Cf}/u.test(value)) {
|
|
294
|
+
return null;
|
|
295
|
+
}
|
|
296
|
+
// NFKC-fold first so compatibility homographs (fullwidth Latin home → home,
|
|
297
|
+
// fullwidth solidus / → /, ligatures, …) collapse to their ASCII skeleton and the
|
|
298
|
+
// ASCII-anchored detectors below can see them. NFKC leaves Cyrillic/Greek and
|
|
299
|
+
// canonical accented Latin unchanged, so it does not affect those paths.
|
|
300
|
+
const folded = value.normalize("NFKC");
|
|
301
|
+
// Then normalize separator homoglyphs to ASCII before any path/gate matching so
|
|
302
|
+
// confusable slashes cannot bypass the ASCII-only path detectors below.
|
|
303
|
+
const normalized = normalizeSeparators(folded);
|
|
304
|
+
if (hasFoldSynthesizedRedactionMarker(value, normalized) ||
|
|
305
|
+
hasWindowsRelativePathShape(normalized) ||
|
|
306
|
+
hasAmbiguousUnquotedSpacedPath(normalized) ||
|
|
307
|
+
hasNonAsciiSeparatorWedge(normalized) ||
|
|
308
|
+
hasConfusablePathSeparatorAdjacency(normalized)) {
|
|
309
|
+
return null;
|
|
310
|
+
}
|
|
311
|
+
let redacted = normalized;
|
|
312
|
+
for (const [pattern, replacement] of REPLACEMENTS)
|
|
313
|
+
redacted = redacted.replace(pattern, replacement);
|
|
314
|
+
if (hasResidualSensitivePrefix(redacted) || hasConfusableRedactionAdjacency(redacted))
|
|
315
|
+
return null;
|
|
316
|
+
const bounded = truncateUtf8(redacted);
|
|
317
|
+
if (!validUnicode(bounded) || containsUnsafe(bounded) || hasResidualSensitivePrefix(bounded)) {
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
return bounded;
|
|
321
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
/** Open an existing SQLite file with both OS-level readonly and SQLite query-only guards. */
|
|
3
|
+
export function openQueryOnlyDb(dbPath) {
|
|
4
|
+
const db = new Database(dbPath, { readonly: true, fileMustExist: true });
|
|
5
|
+
try {
|
|
6
|
+
db.pragma("query_only = ON");
|
|
7
|
+
const queryOnly = db.pragma("query_only", { simple: true });
|
|
8
|
+
if (queryOnly !== 1)
|
|
9
|
+
throw new Error("query_only_not_enabled");
|
|
10
|
+
return db;
|
|
11
|
+
}
|
|
12
|
+
catch (error) {
|
|
13
|
+
db.close();
|
|
14
|
+
throw error;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export function assertDatabaseIntegrity(db) {
|
|
18
|
+
const integrity = db.pragma("integrity_check");
|
|
19
|
+
if (integrity.length !== 1 || integrity[0]?.integrity_check !== "ok") {
|
|
20
|
+
throw new Error("sqlite_integrity_check_failed");
|
|
21
|
+
}
|
|
22
|
+
const foreignKeys = db.pragma("foreign_key_check");
|
|
23
|
+
if (foreignKeys.length !== 0)
|
|
24
|
+
throw new Error("sqlite_foreign_key_check_failed");
|
|
25
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { EvidenceBoundaryError } from "./boundary.js";
|
|
5
|
+
import { canonicalJson, sha256Bytes } from "./canonical.js";
|
|
6
|
+
/** Create one private canonical JSON artifact through write-fsync-rename in its approved state dir. */
|
|
7
|
+
export function writeApprovedStateJson(stateDirectory, fileName, value) {
|
|
8
|
+
if (path.basename(fileName) !== fileName || fileName.length === 0) {
|
|
9
|
+
throw new EvidenceBoundaryError("invalid_state_file_name");
|
|
10
|
+
}
|
|
11
|
+
const stateStat = fs.statSync(stateDirectory);
|
|
12
|
+
if (!stateStat.isDirectory())
|
|
13
|
+
throw new EvidenceBoundaryError("scratch_state_not_directory");
|
|
14
|
+
const destination = path.join(stateDirectory, fileName);
|
|
15
|
+
if (fs.existsSync(destination))
|
|
16
|
+
throw new EvidenceBoundaryError("state_file_already_exists");
|
|
17
|
+
const bytes = `${canonicalJson(value)}\n`;
|
|
18
|
+
const temporary = path.join(stateDirectory, `.${fileName}.${process.pid}.${randomUUID()}.tmp`);
|
|
19
|
+
const descriptor = fs.openSync(temporary, "wx", 0o600);
|
|
20
|
+
try {
|
|
21
|
+
fs.writeFileSync(descriptor, bytes, { encoding: "utf8" });
|
|
22
|
+
fs.fsyncSync(descriptor);
|
|
23
|
+
}
|
|
24
|
+
finally {
|
|
25
|
+
fs.closeSync(descriptor);
|
|
26
|
+
}
|
|
27
|
+
fs.renameSync(temporary, destination);
|
|
28
|
+
return { path: destination, sha256: sha256Bytes(bytes) };
|
|
29
|
+
}
|