@link-assistant/hive-mind 2.12.3 → 2.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/package.json +1 -1
- package/src/agent.prompts.lib.mjs +12 -0
- package/src/claude.prompts.lib.mjs +9 -0
- package/src/codex.prompts.lib.mjs +9 -0
- package/src/credential-sanitization-core.lib.mjs +104 -9
- package/src/encoded-credential-detection.lib.mjs +1009 -0
- package/src/formal-ai-prompt.lib.mjs +29 -0
- package/src/formal-ai.lib.mjs +30 -0
- package/src/gemini.prompts.lib.mjs +9 -0
- package/src/github-url-parser.lib.mjs +8 -0
- package/src/github.lib.mjs +3 -2
- package/src/opencode.prompts.lib.mjs +9 -0
- package/src/qwen.prompts.lib.mjs +9 -0
- package/src/session-kill-recovery.lib.mjs +9 -1
- package/src/solve.mjs +10 -1
- package/src/solve.restart-shared.lib.mjs +6 -0
- package/src/token-sanitization.lib.mjs +182 -5
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the small repository objective sent through native CLIs to Formal AI.
|
|
3
|
+
*
|
|
4
|
+
* Formal AI owns the agent policy for its model. Repeating Hive Mind's native
|
|
5
|
+
* provider policy in the request both wastes context and exposes incidental
|
|
6
|
+
* shell-language cues to Formal AI's deterministic intent router (#2158).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { isFormalAiModel } from './formal-ai-model.lib.mjs';
|
|
10
|
+
|
|
11
|
+
export const buildFormalAiRepositoryPrompt = params => {
|
|
12
|
+
if (!isFormalAiModel(params?.argv?.model)) return null;
|
|
13
|
+
|
|
14
|
+
const { issueUrl, issueNumber, prNumber, prUrl, branchName, isContinueMode, feedbackLines, owner, repo } = params;
|
|
15
|
+
const issueReference = isContinueMode && issueNumber && owner && repo ? `https://github.com/${owner}/${repo}/issues/${issueNumber}` : issueUrl || `the issue linked to pull request ${prNumber}`;
|
|
16
|
+
const lines = [`Resolve the GitHub issue at ${issueReference} in this repository.`];
|
|
17
|
+
|
|
18
|
+
if (branchName) lines.push(`Keep the solution on branch ${branchName}.`);
|
|
19
|
+
if (prUrl) lines.push(`Update the pull request at ${prUrl}.`);
|
|
20
|
+
// The review text is caller-controlled and can contain command snippets.
|
|
21
|
+
// Point Formal AI to the canonical PR instead of copying those snippets into
|
|
22
|
+
// the intent-classification request.
|
|
23
|
+
if (feedbackLines?.length && prUrl) lines.push('Review and address all feedback recorded on that pull request.');
|
|
24
|
+
|
|
25
|
+
lines.push('', 'Implement and verify the solution before reporting completion.', isContinueMode ? 'Continue.' : 'Proceed.');
|
|
26
|
+
return `${lines.join('\n')}\n`;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export default { buildFormalAiRepositoryPrompt };
|
package/src/formal-ai.lib.mjs
CHANGED
|
@@ -217,6 +217,36 @@ export const createPreparedToolResult = preparedCommand => ({
|
|
|
217
217
|
errorDuringExecution: false,
|
|
218
218
|
});
|
|
219
219
|
|
|
220
|
+
const FORMAL_AI_NON_EXECUTION_PATTERNS = [/^\s*planned,\s*not executed\b/im, /^\s*planned_not_executed\s*$/im, /^\s*terminal_state\s+["']?planned_not_executed\b/im];
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Turn Formal AI's explicit non-execution terminal state into a failed tool
|
|
224
|
+
* result (issue #2158).
|
|
225
|
+
*
|
|
226
|
+
* Formal AI 0.339.1 truthfully reports repository work as
|
|
227
|
+
* `planned_not_executed`, but each native CLI exits zero. Treating that process
|
|
228
|
+
* exit as a successful solve made `--auto-restart-until-mergeable` repeat the
|
|
229
|
+
* same deterministic plan five times. Keep the model's summary for evidence,
|
|
230
|
+
* while giving Hive Mind an actionable terminal failure.
|
|
231
|
+
*/
|
|
232
|
+
export const classifyFormalAiToolResult = ({ model, toolResult } = {}) => {
|
|
233
|
+
if (!toolResult || !isFormalAiModel(model) || toolResult.success === false) return toolResult;
|
|
234
|
+
|
|
235
|
+
const evidence = [toolResult.resultSummary, toolResult.result, toolResult.lastMessage, toolResult.output].filter(value => typeof value === 'string').join('\n');
|
|
236
|
+
if (!FORMAL_AI_NON_EXECUTION_PATTERNS.some(pattern => pattern.test(evidence))) return toolResult;
|
|
237
|
+
|
|
238
|
+
return {
|
|
239
|
+
...toolResult,
|
|
240
|
+
success: false,
|
|
241
|
+
errorDuringExecution: true,
|
|
242
|
+
formalAiNonExecution: true,
|
|
243
|
+
errorInfo: {
|
|
244
|
+
code: 'FORMAL_AI_PLANNED_NOT_EXECUTED',
|
|
245
|
+
message: "Formal AI did not execute repository work; it returned the terminal state planned_not_executed. Fix or upgrade Formal AI's repository-work executor before retrying.",
|
|
246
|
+
},
|
|
247
|
+
};
|
|
248
|
+
};
|
|
249
|
+
|
|
220
250
|
export const logPreparedToolCommand = async ({ argv, fullCommand, log, formatAligned }) => {
|
|
221
251
|
await log(`\n${formatAligned('📝', 'Raw command:', '')}`);
|
|
222
252
|
await log(fullCommand);
|
|
@@ -9,6 +9,8 @@ import { getThinkingPromptInstruction } from './thinking-prompt.lib.mjs';
|
|
|
9
9
|
import { buildWorkLanguageDirective } from './work-language.prompts.lib.mjs';
|
|
10
10
|
import { buildRequestedBaseBranchDirective } from './solve-option-contract.prompts.lib.mjs';
|
|
11
11
|
import { buildIssueResearchPrompt } from './deep-analysis.lib.mjs';
|
|
12
|
+
import { buildFormalAiRepositoryPrompt } from './formal-ai-prompt.lib.mjs';
|
|
13
|
+
import { isFormalAiModel } from './formal-ai-model.lib.mjs';
|
|
12
14
|
|
|
13
15
|
/**
|
|
14
16
|
* Build the user prompt for Gemini
|
|
@@ -16,6 +18,9 @@ import { buildIssueResearchPrompt } from './deep-analysis.lib.mjs';
|
|
|
16
18
|
* @returns {string} The formatted user prompt
|
|
17
19
|
*/
|
|
18
20
|
export const buildUserPrompt = params => {
|
|
21
|
+
const formalAiPrompt = buildFormalAiRepositoryPrompt(params);
|
|
22
|
+
if (formalAiPrompt !== null) return formalAiPrompt;
|
|
23
|
+
|
|
19
24
|
const { issueUrl, issueNumber, prNumber, prUrl, branchName, tempDir, workspaceTmpDir, isContinueMode, forkedRepo, feedbackLines, forkActionsUrl, owner, repo, argv } = params;
|
|
20
25
|
|
|
21
26
|
const promptLines = [];
|
|
@@ -81,6 +86,10 @@ export const buildUserPrompt = params => {
|
|
|
81
86
|
export const buildSystemPrompt = params => {
|
|
82
87
|
const { owner, repo, issueNumber, prNumber, branchName, workspaceTmpDir, argv, modelSupportsVision, forkedRepo } = params;
|
|
83
88
|
|
|
89
|
+
// Issue #2158: keep caller workflow instructions out of Formal AI's task
|
|
90
|
+
// classifier. Formal AI provides its own execution policy.
|
|
91
|
+
if (isFormalAiModel(argv?.model)) return '';
|
|
92
|
+
|
|
84
93
|
const screenshotRepoPath = argv?.fork && forkedRepo ? forkedRepo : `${owner}/${repo}`;
|
|
85
94
|
|
|
86
95
|
let workspaceInstructions = '';
|
|
@@ -241,6 +241,14 @@ export function normalizeGitHubUrl(url) {
|
|
|
241
241
|
const parsed = parseGitHubUrl(url);
|
|
242
242
|
return parsed.valid ? parsed.normalized : null;
|
|
243
243
|
}
|
|
244
|
+
|
|
245
|
+
/** Build the canonical web URL for a pull request already identified by GitHub. */
|
|
246
|
+
export function buildGitHubPullRequestUrl({ owner, repo, number } = {}) {
|
|
247
|
+
if (!owner || !repo || !Number.isInteger(Number(number)) || Number(number) <= 0) {
|
|
248
|
+
throw new TypeError('A GitHub pull request URL requires owner, repo, and a positive integer number');
|
|
249
|
+
}
|
|
250
|
+
return `https://github.com/${owner}/${repo}/pull/${Number(number)}`;
|
|
251
|
+
}
|
|
244
252
|
/**
|
|
245
253
|
* Check if a URL is a valid GitHub URL of a specific type
|
|
246
254
|
* @param {string} url - The URL to check
|
package/src/github.lib.mjs
CHANGED
|
@@ -20,8 +20,8 @@ export { buildCostInfoString };
|
|
|
20
20
|
// #1756: route gh exec calls through transient + rate-limit retry wrapper
|
|
21
21
|
import { execGhWithRetry } from './github-rate-limit.lib.mjs';
|
|
22
22
|
import { QUIET_PROBE } from './quiet-probe.lib.mjs'; // issues #2130, #2135: keep read-only probe payloads out of the attached log
|
|
23
|
-
import { isGitHubUrlType, normalizeGitHubUrl, parseGitHubUrl } from './github-url-parser.lib.mjs';
|
|
24
|
-
export { isGitHubUrlType, normalizeGitHubUrl, parseGitHubUrl };
|
|
23
|
+
import { buildGitHubPullRequestUrl, isGitHubUrlType, normalizeGitHubUrl, parseGitHubUrl } from './github-url-parser.lib.mjs';
|
|
24
|
+
export { buildGitHubPullRequestUrl, isGitHubUrlType, normalizeGitHubUrl, parseGitHubUrl };
|
|
25
25
|
// Issue #1625: Named marker constants (single source of truth) + in-memory tracking for tool-posted comments. See tool-comments.lib.mjs for design.
|
|
26
26
|
import { SOLUTION_DRAFT_LOG_MARKER, SOLUTION_DRAFT_FAILED_MARKER, SOLUTION_DRAFT_FINISHED_WITH_ERRORS_MARKER, USAGE_LIMIT_REACHED_MARKER, NOW_WORKING_SESSION_IS_ENDED_MARKER, postTrackedComment, postTrackedCommentFromFile } from './tool-comments.lib.mjs';
|
|
27
27
|
export const maskGitHubToken = maskToken; // Alias for backward compatibility
|
|
@@ -1168,6 +1168,7 @@ export default {
|
|
|
1168
1168
|
fetchProjectIssues,
|
|
1169
1169
|
isRateLimitError,
|
|
1170
1170
|
batchCheckPullRequestsForIssues,
|
|
1171
|
+
buildGitHubPullRequestUrl,
|
|
1171
1172
|
parseGitHubUrl,
|
|
1172
1173
|
normalizeGitHubUrl,
|
|
1173
1174
|
isGitHubUrlType,
|
|
@@ -9,6 +9,8 @@ import { getThinkingPromptInstruction } from './thinking-prompt.lib.mjs';
|
|
|
9
9
|
import { buildWorkLanguageDirective } from './work-language.prompts.lib.mjs';
|
|
10
10
|
import { buildRequestedBaseBranchDirective } from './solve-option-contract.prompts.lib.mjs';
|
|
11
11
|
import { buildIssueResearchPrompt } from './deep-analysis.lib.mjs';
|
|
12
|
+
import { buildFormalAiRepositoryPrompt } from './formal-ai-prompt.lib.mjs';
|
|
13
|
+
import { isFormalAiModel } from './formal-ai-model.lib.mjs';
|
|
12
14
|
|
|
13
15
|
/**
|
|
14
16
|
* Build the user prompt for OpenCode
|
|
@@ -16,6 +18,9 @@ import { buildIssueResearchPrompt } from './deep-analysis.lib.mjs';
|
|
|
16
18
|
* @returns {string} The formatted user prompt
|
|
17
19
|
*/
|
|
18
20
|
export const buildUserPrompt = params => {
|
|
21
|
+
const formalAiPrompt = buildFormalAiRepositoryPrompt(params);
|
|
22
|
+
if (formalAiPrompt !== null) return formalAiPrompt;
|
|
23
|
+
|
|
19
24
|
const { issueUrl, issueNumber, prNumber, prUrl, branchName, tempDir, workspaceTmpDir, isContinueMode, forkedRepo, feedbackLines, forkActionsUrl, owner, repo, argv } = params;
|
|
20
25
|
|
|
21
26
|
const promptLines = [];
|
|
@@ -92,6 +97,10 @@ export const buildUserPrompt = params => {
|
|
|
92
97
|
export const buildSystemPrompt = params => {
|
|
93
98
|
const { owner, repo, issueNumber, prNumber, branchName, workspaceTmpDir, argv, modelSupportsVision, forkedRepo } = params;
|
|
94
99
|
|
|
100
|
+
// Issue #2158: keep caller workflow instructions out of Formal AI's task
|
|
101
|
+
// classifier. Formal AI provides its own execution policy.
|
|
102
|
+
if (isFormalAiModel(argv?.model)) return '';
|
|
103
|
+
|
|
95
104
|
// When in fork mode, screenshots are pushed to the fork, not the original repo
|
|
96
105
|
const screenshotRepoPath = argv?.fork && forkedRepo ? forkedRepo : `${owner}/${repo}`;
|
|
97
106
|
|
package/src/qwen.prompts.lib.mjs
CHANGED
|
@@ -9,6 +9,8 @@ import { getThinkingPromptInstruction } from './thinking-prompt.lib.mjs';
|
|
|
9
9
|
import { buildWorkLanguageDirective } from './work-language.prompts.lib.mjs';
|
|
10
10
|
import { buildRequestedBaseBranchDirective } from './solve-option-contract.prompts.lib.mjs';
|
|
11
11
|
import { buildIssueResearchPrompt } from './deep-analysis.lib.mjs';
|
|
12
|
+
import { buildFormalAiRepositoryPrompt } from './formal-ai-prompt.lib.mjs';
|
|
13
|
+
import { isFormalAiModel } from './formal-ai-model.lib.mjs';
|
|
12
14
|
|
|
13
15
|
/**
|
|
14
16
|
* Build the user prompt for Qwen Code
|
|
@@ -16,6 +18,9 @@ import { buildIssueResearchPrompt } from './deep-analysis.lib.mjs';
|
|
|
16
18
|
* @returns {string} The formatted user prompt
|
|
17
19
|
*/
|
|
18
20
|
export const buildUserPrompt = params => {
|
|
21
|
+
const formalAiPrompt = buildFormalAiRepositoryPrompt(params);
|
|
22
|
+
if (formalAiPrompt !== null) return formalAiPrompt;
|
|
23
|
+
|
|
19
24
|
const { issueUrl, issueNumber, prNumber, prUrl, branchName, tempDir, workspaceTmpDir, isContinueMode, forkedRepo, feedbackLines, forkActionsUrl, owner, repo, argv, tool = 'qwen' } = params;
|
|
20
25
|
|
|
21
26
|
const promptLines = [];
|
|
@@ -81,6 +86,10 @@ export const buildUserPrompt = params => {
|
|
|
81
86
|
export const buildSystemPrompt = params => {
|
|
82
87
|
const { owner, repo, issueNumber, prNumber, branchName, workspaceTmpDir, argv, modelSupportsVision, forkedRepo } = params;
|
|
83
88
|
|
|
89
|
+
// Issue #2158: keep caller workflow instructions out of Formal AI's task
|
|
90
|
+
// classifier. Formal AI provides its own execution policy.
|
|
91
|
+
if (isFormalAiModel(argv?.model)) return '';
|
|
92
|
+
|
|
84
93
|
const screenshotRepoPath = argv?.fork && forkedRepo ? forkedRepo : `${owner}/${repo}`;
|
|
85
94
|
|
|
86
95
|
let workspaceInstructions = '';
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
|
|
25
25
|
import { spawn } from 'child_process';
|
|
26
26
|
import { describeChildExit } from './child-exit.lib.mjs';
|
|
27
|
+
import { sanitizeForPublication } from './token-sanitization.lib.mjs'; // issue #2156: this body is published to a pull request
|
|
27
28
|
import { KILL_CAUSE_DISK_FULL, KILL_CAUSE_FORCED_KILL, KILL_CAUSE_OUT_OF_MEMORY } from './session-kill-diagnostics.lib.mjs';
|
|
28
29
|
import { ON_SESSION_KILL_RESUME } from './session-kill-policy.lib.mjs';
|
|
29
30
|
|
|
@@ -161,6 +162,13 @@ const defaultUnlink = async filePath => {
|
|
|
161
162
|
* `--body-file` (not `--body`) is used deliberately: the notice contains
|
|
162
163
|
* backticks and newlines that would otherwise have to survive shell quoting.
|
|
163
164
|
*
|
|
165
|
+
* Issue #2156: the body is sanitized here rather than by the caller. It carries
|
|
166
|
+
* kill diagnostics and a resume command, both assembled from process and log
|
|
167
|
+
* data, so it is a publication boundary like any other and must fail closed.
|
|
168
|
+
* The array-argument `gh` invocation below is invisible to the
|
|
169
|
+
* `require-sanitized-output` ESLint rule, which is exactly how this path stayed
|
|
170
|
+
* unsanitized; the rule now understands this shape too.
|
|
171
|
+
*
|
|
164
172
|
* @param {Object} options
|
|
165
173
|
* @param {string} options.pullRequestUrl
|
|
166
174
|
* @param {string} options.body
|
|
@@ -178,7 +186,7 @@ export async function postKillRecoveryNotice({ pullRequestUrl, body, runCommand
|
|
|
178
186
|
|
|
179
187
|
const bodyFile = `${tempDir.replace(/\/$/, '')}/hive-mind-kill-notice-${fileSuffix}.md`;
|
|
180
188
|
try {
|
|
181
|
-
await writeFile(bodyFile, body);
|
|
189
|
+
await writeFile(bodyFile, await sanitizeForPublication(body));
|
|
182
190
|
const result = await runCommand('gh', ['pr', 'comment', pullRequestUrl, '--body-file', bodyFile]);
|
|
183
191
|
if (result?.code === 0) {
|
|
184
192
|
const url = String(result.stdout || '').trim() || null;
|
package/src/solve.mjs
CHANGED
|
@@ -69,6 +69,7 @@ const { validateAndExitOnInvalidClaudeSubAgentModel, validateAndExitOnInvalidMod
|
|
|
69
69
|
const { autoAcceptInviteForRepo } = await import('./solve.accept-invite.lib.mjs');
|
|
70
70
|
const { handleAutoForkOption, handleMaintainerForkAccess } = await import('./solve.fork-detection.lib.mjs');
|
|
71
71
|
const { resolveUncommittedChangesTool } = await import('./solve.tool-uncommitted.lib.mjs');
|
|
72
|
+
const { classifyFormalAiToolResult } = await import('./formal-ai.lib.mjs');
|
|
72
73
|
const logFile = await initializeLogFile(null);
|
|
73
74
|
const versionInfo = await getVersionInfo();
|
|
74
75
|
const rawCommand = await logSolveStartup(versionInfo);
|
|
@@ -535,7 +536,10 @@ try {
|
|
|
535
536
|
let prUrl = null;
|
|
536
537
|
// In continue mode, we already have the PR details
|
|
537
538
|
if (isContinueMode) {
|
|
538
|
-
|
|
539
|
+
// Issue #2158: auto-continue can discover a PR while the input remains an
|
|
540
|
+
// issue URL. Passing that issue URL as "Your prepared Pull Request" sent
|
|
541
|
+
// the first Formal AI attempt to the wrong GitHub entity.
|
|
542
|
+
prUrl = githubLib.buildGitHubPullRequestUrl({ owner, repo, number: prNumber });
|
|
539
543
|
// prNumber is already set from earlier when we parsed the PR
|
|
540
544
|
}
|
|
541
545
|
// Handle auto PR creation using the new module
|
|
@@ -757,6 +761,11 @@ try {
|
|
|
757
761
|
});
|
|
758
762
|
toolResult = claudeResult;
|
|
759
763
|
}
|
|
764
|
+
toolResult = classifyFormalAiToolResult({ model: argv.model, toolResult });
|
|
765
|
+
if (toolResult?.formalAiNonExecution) {
|
|
766
|
+
await log(`❌ ${toolResult.errorInfo.message}`, { level: 'error' });
|
|
767
|
+
await log(' The deterministic terminal response will not be retried as a mergeability problem.', { level: 'error' });
|
|
768
|
+
}
|
|
760
769
|
try {
|
|
761
770
|
await recordAfterAgentSize({ tempDir, beforeBytes: cleanupContext.diskDiagnostics?.beforeBytes ?? null, log });
|
|
762
771
|
} catch (diskError) {
|
|
@@ -34,6 +34,7 @@ const { log, formatAligned, extractToolErrorCore } = lib;
|
|
|
34
34
|
const { ensurePullRequestBaseBranch } = await import('./solve.pr-base-guard.lib.mjs');
|
|
35
35
|
const { ensureAiToolScratchIgnored, filterAiToolScratchFromStatus } = await import('./ai-tool-scratch.lib.mjs');
|
|
36
36
|
const { RESOURCE_PHASE_RESTART_AFTER, RESOURCE_PHASE_RESTART_BEFORE, recordResourceSnapshot } = await import('./solve.resource-diagnostics.lib.mjs');
|
|
37
|
+
const { classifyFormalAiToolResult } = await import('./formal-ai.lib.mjs');
|
|
37
38
|
// Issue #2123: shared draft/ready transitions for working sessions.
|
|
38
39
|
const { ensurePullRequestIsDraft } = await import('./pr-draft-state.lib.mjs');
|
|
39
40
|
|
|
@@ -492,6 +493,11 @@ export const executeToolIteration = async params => {
|
|
|
492
493
|
});
|
|
493
494
|
}
|
|
494
495
|
|
|
496
|
+
toolResult = classifyFormalAiToolResult({ model: argv.model, toolResult });
|
|
497
|
+
if (toolResult?.formalAiNonExecution) {
|
|
498
|
+
await log(`❌ ${toolResult.errorInfo.message}`, { level: 'error' });
|
|
499
|
+
}
|
|
500
|
+
|
|
495
501
|
await ensurePullRequestBaseBranch({ owner, repo, prNumber, argv, log, formatAligned, $ });
|
|
496
502
|
await recordResourceSnapshot({
|
|
497
503
|
phase: RESOURCE_PHASE_RESTART_AFTER,
|
|
@@ -19,6 +19,7 @@ import { ensureUseM } from './use-m-bootstrap.lib.mjs';
|
|
|
19
19
|
// lib.mjs, so it must not depend on this asynchronous Secretlint layer.
|
|
20
20
|
import { log, isENOSPC } from './lib.mjs';
|
|
21
21
|
import { CREDENTIAL_SANITIZATION_ERROR_CODE, CREDENTIAL_SANITIZATION_FAILURE_MESSAGE, createCredentialStreamSanitizer, findCredentialResiduals, maskToken, sanitizeCredentialText } from './credential-sanitization-core.lib.mjs';
|
|
22
|
+
import { findDecodableRuns, findEncodedKnownTokenRuns, sanitizeEncodedCredentials } from './encoded-credential-detection.lib.mjs'; // issue #2156: credentials that only appear re-encoded
|
|
22
23
|
import { reportError } from './sentry.lib.mjs';
|
|
23
24
|
|
|
24
25
|
export { createCredentialStreamSanitizer };
|
|
@@ -518,6 +519,130 @@ const sanitizeCredentialTextPreservingExclusions = (input, excludedSet) => {
|
|
|
518
519
|
return output;
|
|
519
520
|
};
|
|
520
521
|
|
|
522
|
+
// ---------------------------------------------------------------------------
|
|
523
|
+
// Issue #2156 — known-local tokens that only appear in an encoded form
|
|
524
|
+
// ---------------------------------------------------------------------------
|
|
525
|
+
// The leak in this issue was a `gho_` token that the GHCR token endpoint echoed
|
|
526
|
+
// back base64-encoded inside a JSON body. Every masking layer we had compared
|
|
527
|
+
// bytes literally, so the encoded copy walked straight through. These helpers
|
|
528
|
+
// mask the *encoded* occurrences of tokens we already hold locally.
|
|
529
|
+
// ---------------------------------------------------------------------------
|
|
530
|
+
|
|
531
|
+
/** Encoded-scan recursion limit: base64-of-base64-of-base64 and no deeper. */
|
|
532
|
+
const MAX_ENCODED_KNOWN_TOKEN_DEPTH = 2;
|
|
533
|
+
|
|
534
|
+
/**
|
|
535
|
+
* Replace every verbatim occurrence of the supplied token values.
|
|
536
|
+
*
|
|
537
|
+
* @param {string} text
|
|
538
|
+
* @param {Array<string>} values already filtered and de-duplicated
|
|
539
|
+
* @returns {string}
|
|
540
|
+
*/
|
|
541
|
+
const maskKnownTokenValues = (text, values) => {
|
|
542
|
+
let output = text;
|
|
543
|
+
for (const value of values) {
|
|
544
|
+
if (output.includes(value)) output = output.split(value).join(maskToken(value));
|
|
545
|
+
}
|
|
546
|
+
return output;
|
|
547
|
+
};
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Narrow a raw token list to the values worth searching for.
|
|
551
|
+
*
|
|
552
|
+
* @param {Array<string|{value: string}>} tokens
|
|
553
|
+
* @param {Set<string>} [excludedSet] issue #1745 user-content carve-out
|
|
554
|
+
* @returns {Array<string>}
|
|
555
|
+
*/
|
|
556
|
+
const usableTokenValues = (tokens, excludedSet) => [...new Set((tokens || []).map(t => (typeof t === 'string' ? t : t?.value)).filter(value => typeof value === 'string' && value.length >= 12))].filter(value => !excludedSet?.has(value));
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Mask encoded occurrences of known-local tokens.
|
|
560
|
+
*
|
|
561
|
+
* Decoded payloads are rebuilt rather than dropped: a base64 blob that merely
|
|
562
|
+
* *contains* the token keeps its other fields and stays parseable, and the
|
|
563
|
+
* masked token retains its first/last characters for debugging — the same
|
|
564
|
+
* contract plaintext masking has always offered.
|
|
565
|
+
*
|
|
566
|
+
* @param {string} text
|
|
567
|
+
* @param {Array<string>} values from {@link usableTokenValues}
|
|
568
|
+
* @param {number} [depth] internal recursion counter
|
|
569
|
+
* @returns {string}
|
|
570
|
+
*/
|
|
571
|
+
const maskEncodedKnownTokens = (text, values, depth = 0) => {
|
|
572
|
+
if (values.length === 0) return text;
|
|
573
|
+
return sanitizeEncodedCredentials(text, {
|
|
574
|
+
knownTokens: values,
|
|
575
|
+
sanitizePlaintext: decoded => {
|
|
576
|
+
const masked = maskKnownTokenValues(decoded, values);
|
|
577
|
+
// Peel nested encodings so base64-of-base64 is covered too.
|
|
578
|
+
return depth >= MAX_ENCODED_KNOWN_TOKEN_DEPTH ? masked : maskEncodedKnownTokens(masked, values, depth + 1);
|
|
579
|
+
},
|
|
580
|
+
});
|
|
581
|
+
};
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* Mask encoded runs whose *decoded* payload Secretlint recognises.
|
|
585
|
+
*
|
|
586
|
+
* This is the redundancy the issue asks for, aimed at where it actually helps.
|
|
587
|
+
* Secretlint is blind to encoding: its GitHub rule flags a bare `gho_…` but
|
|
588
|
+
* reports nothing for the same token base64-encoded, and neither does any other
|
|
589
|
+
* pattern scanner, because a pattern scanner matches the bytes it is given.
|
|
590
|
+
* Adding a third scanner alongside the first two would therefore have changed
|
|
591
|
+
* nothing about this incident. Decoding first and *then* asking both detectors
|
|
592
|
+
* is what closes the gap, so the external rule set is applied to the decoded
|
|
593
|
+
* payload exactly as the maintained core already is.
|
|
594
|
+
*
|
|
595
|
+
* The two detectors stay independent: this runs whether or not the core found
|
|
596
|
+
* anything, so a credential format Secretlint knows and we do not is still
|
|
597
|
+
* caught once it is decoded.
|
|
598
|
+
*
|
|
599
|
+
* @param {string} text
|
|
600
|
+
* @param {Set<string>} [excludedSet] issue #1745 user-content carve-out
|
|
601
|
+
* @returns {Promise<{text: string, masked: number, ruleIds: Array<string>}>}
|
|
602
|
+
*/
|
|
603
|
+
const maskEncodedSecretsWithSecretlint = async (text, excludedSet) => {
|
|
604
|
+
const runs = findDecodableRuns(text);
|
|
605
|
+
if (runs.length === 0) return { text, masked: 0, ruleIds: [] };
|
|
606
|
+
|
|
607
|
+
// Each payload is scanned on its own rather than as one joined document: a
|
|
608
|
+
// rule that matched across a join boundary would blame a run that is
|
|
609
|
+
// innocent, and masking an innocent run destroys log content.
|
|
610
|
+
const verdicts = await Promise.all(runs.map(run => detectSecretsWithSecretlint(run.decoded)));
|
|
611
|
+
|
|
612
|
+
// Keyed by decoded content, because that is what the sync layer hands back
|
|
613
|
+
// when it re-walks the same runs below. Two runs that decode identically are
|
|
614
|
+
// masked identically, which is what we want.
|
|
615
|
+
const maskedPayloads = new Map();
|
|
616
|
+
const ruleIds = new Set();
|
|
617
|
+
for (const [index, findings] of verdicts.entries()) {
|
|
618
|
+
const usable = findings.filter(finding => !excludedSet?.has(finding.token));
|
|
619
|
+
if (usable.length === 0) continue;
|
|
620
|
+
const { decoded } = runs[index];
|
|
621
|
+
|
|
622
|
+
// Mask inside the decoded payload so the surrounding structure survives.
|
|
623
|
+
// Ranges are spliced from the end so earlier offsets stay valid.
|
|
624
|
+
let payload = decoded;
|
|
625
|
+
for (const finding of [...usable].sort((a, b) => b.start - a.start)) {
|
|
626
|
+
if (payload.substring(finding.start, finding.end) !== finding.token) continue;
|
|
627
|
+
payload = payload.substring(0, finding.start) + maskToken(finding.token) + payload.substring(finding.end);
|
|
628
|
+
ruleIds.add(finding.ruleId);
|
|
629
|
+
}
|
|
630
|
+
if (payload === decoded) continue;
|
|
631
|
+
maskedPayloads.set(decoded, payload);
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
if (maskedPayloads.size === 0) return { text, masked: 0, ruleIds: [] };
|
|
635
|
+
|
|
636
|
+
// Re-encoding, round-trip verification and overlap merging are the sync
|
|
637
|
+
// layer's job. Driving it with a lookup of payloads we have already masked
|
|
638
|
+
// means the two paths cannot disagree about what a masked run looks like.
|
|
639
|
+
const output = sanitizeEncodedCredentials(text, {
|
|
640
|
+
sanitizePlaintext: decoded => maskedPayloads.get(decoded) ?? decoded,
|
|
641
|
+
});
|
|
642
|
+
|
|
643
|
+
return { text: output, masked: maskedPayloads.size, ruleIds: [...ruleIds] };
|
|
644
|
+
};
|
|
645
|
+
|
|
521
646
|
/**
|
|
522
647
|
* Sanitize arbitrary outbound output by masking sensitive tokens while avoiding false positives
|
|
523
648
|
* Uses DUAL APPROACH: Both secretlint AND custom patterns run independently
|
|
@@ -543,6 +668,8 @@ export const sanitizeOutput = async (output, options = {}) => {
|
|
|
543
668
|
const stats = {
|
|
544
669
|
knownTokens: 0,
|
|
545
670
|
secretlintDetections: 0,
|
|
671
|
+
encodedSecretlintDetections: 0,
|
|
672
|
+
encodedSecretlintRuleIds: [],
|
|
546
673
|
customDetections: 0,
|
|
547
674
|
secretlintOnlyWarnings: [],
|
|
548
675
|
customOnlyDetections: [],
|
|
@@ -571,6 +698,17 @@ export const sanitizeOutput = async (output, options = {}) => {
|
|
|
571
698
|
}
|
|
572
699
|
}
|
|
573
700
|
}
|
|
701
|
+
|
|
702
|
+
// Issue #2156: the same tokens, base64/hex/percent-encoded. Byte-for-byte
|
|
703
|
+
// comparison above cannot see those copies.
|
|
704
|
+
const encodableTokens = usableTokenValues(allKnownTokens, excludedSet);
|
|
705
|
+
const beforeEncoded = sanitized;
|
|
706
|
+
sanitized = maskEncodedKnownTokens(sanitized, encodableTokens);
|
|
707
|
+
if (sanitized !== beforeEncoded) {
|
|
708
|
+
stats.knownTokens++;
|
|
709
|
+
sanitizationStats.knownTokenMasks++;
|
|
710
|
+
sanitizationStats.totalMasked++;
|
|
711
|
+
}
|
|
574
712
|
}
|
|
575
713
|
|
|
576
714
|
if (skipOutputSanitization) {
|
|
@@ -663,6 +801,21 @@ export const sanitizeOutput = async (output, options = {}) => {
|
|
|
663
801
|
}
|
|
664
802
|
}
|
|
665
803
|
|
|
804
|
+
// Step 3b (issue #2156): everything above compares against the *surface*
|
|
805
|
+
// text, so a credential that only ever appears encoded is invisible to it —
|
|
806
|
+
// that is exactly how the leaked token survived. The maintained core
|
|
807
|
+
// already reads decoded payloads; run the external rule set over them too,
|
|
808
|
+
// so the two layers cover the same ground and either one can be the catch.
|
|
809
|
+
const beforeEncodedScan = sanitized;
|
|
810
|
+
const encodedScan = await maskEncodedSecretsWithSecretlint(sanitized, excludedSet);
|
|
811
|
+
if (encodedScan.text !== beforeEncodedScan) {
|
|
812
|
+
sanitized = encodedScan.text;
|
|
813
|
+
stats.encodedSecretlintDetections += encodedScan.masked;
|
|
814
|
+
stats.encodedSecretlintRuleIds = encodedScan.ruleIds;
|
|
815
|
+
sanitizationStats.patternMasks += encodedScan.masked;
|
|
816
|
+
sanitizationStats.totalMasked += encodedScan.masked;
|
|
817
|
+
}
|
|
818
|
+
|
|
666
819
|
// Step 4: Handle 40-char hex tokens specially - only mask if NOT in safe context
|
|
667
820
|
// These could be GitHub tokens OR git commit hashes/gist IDs
|
|
668
821
|
const hexPattern = /(?:^|[\s:=])([a-f0-9]{40})(?=[\s\n]|$)/gm;
|
|
@@ -701,11 +854,14 @@ export const sanitizeOutput = async (output, options = {}) => {
|
|
|
701
854
|
}
|
|
702
855
|
|
|
703
856
|
// Summary logging
|
|
704
|
-
const totalMasked = allSecrets.size + hexReplacements.length + stats.knownTokens;
|
|
857
|
+
const totalMasked = allSecrets.size + hexReplacements.length + stats.knownTokens + stats.encodedSecretlintDetections;
|
|
705
858
|
if (global.verboseMode && totalMasked > 0) {
|
|
706
859
|
await log(` 🔒 Sanitized ${totalMasked} secrets using dual approach:`, { verbose: true });
|
|
707
860
|
await log(` • Known tokens: ${stats.knownTokens}`, { verbose: true });
|
|
708
861
|
await log(` • Secretlint: ${stats.secretlintDetections} detections`, { verbose: true });
|
|
862
|
+
if (stats.encodedSecretlintDetections > 0) {
|
|
863
|
+
await log(` • Secretlint (encoded payloads): ${stats.encodedSecretlintDetections} run(s) [${stats.encodedSecretlintRuleIds.join(', ')}]`, { verbose: true });
|
|
864
|
+
}
|
|
709
865
|
await log(` • Custom patterns: ${stats.customDetections} detections`, { verbose: true });
|
|
710
866
|
await log(` • Hex tokens: ${hexReplacements.length}`, { verbose: true });
|
|
711
867
|
if (stats.secretlintOnlyWarnings.length > 0) {
|
|
@@ -929,16 +1085,29 @@ export const getAllKnownLocalTokens = async () => {
|
|
|
929
1085
|
* @param {Array<{value: string, name?: string, source?: string}>} [tokens]
|
|
930
1086
|
* Pre-fetched token list (if you already called getAllKnownLocalTokens).
|
|
931
1087
|
* Pass an explicit list to avoid re-running `gh auth status` per check.
|
|
932
|
-
*
|
|
933
|
-
*
|
|
1088
|
+
* Issue #2156: a token that appears only base64/hex/percent-encoded is a leak
|
|
1089
|
+
* just the same — GitHub's own secret scanning decodes before matching, which
|
|
1090
|
+
* is exactly how the revocation in that issue was triggered. Encoded hits are
|
|
1091
|
+
* reported with the encoding that matched so operators can tell the two cases
|
|
1092
|
+
* apart in the fail-closed publication error path.
|
|
1093
|
+
*
|
|
1094
|
+
* @returns {Promise<Array<{name: string, source: string, encoding: string}>>}
|
|
1095
|
+
* list of token identifiers that were found in the text (NOT the values
|
|
1096
|
+
* themselves).
|
|
934
1097
|
*/
|
|
935
1098
|
export const containsKnownToken = async (text, tokens) => {
|
|
936
1099
|
if (typeof text !== 'string' || text.length === 0) return [];
|
|
937
1100
|
const list = tokens || (await getAllKnownLocalTokens());
|
|
938
1101
|
const hits = [];
|
|
939
1102
|
for (const t of list) {
|
|
940
|
-
if (t.value
|
|
941
|
-
|
|
1103
|
+
if (!t.value) continue;
|
|
1104
|
+
if (text.includes(t.value)) {
|
|
1105
|
+
hits.push({ name: t.name, source: t.source, encoding: 'plaintext' });
|
|
1106
|
+
continue;
|
|
1107
|
+
}
|
|
1108
|
+
const encodedRuns = findEncodedKnownTokenRuns(text, [t.value]);
|
|
1109
|
+
if (encodedRuns.length > 0) {
|
|
1110
|
+
hits.push({ name: t.name, source: t.source, encoding: encodedRuns[0].encoding });
|
|
942
1111
|
}
|
|
943
1112
|
}
|
|
944
1113
|
return hits;
|
|
@@ -979,6 +1148,14 @@ export const sanitizeCommentBody = async (body, options = {}) => {
|
|
|
979
1148
|
sanitizationStats.totalMasked++;
|
|
980
1149
|
}
|
|
981
1150
|
}
|
|
1151
|
+
|
|
1152
|
+
// Issue #2156: the same tokens, re-encoded (base64/hex/percent/escapes).
|
|
1153
|
+
const beforeEncoded = sanitized;
|
|
1154
|
+
sanitized = maskEncodedKnownTokens(sanitized, usableTokenValues(knownTokens, excludedSet));
|
|
1155
|
+
if (sanitized !== beforeEncoded) {
|
|
1156
|
+
sanitizationStats.knownTokenMasks++;
|
|
1157
|
+
sanitizationStats.totalMasked++;
|
|
1158
|
+
}
|
|
982
1159
|
}
|
|
983
1160
|
|
|
984
1161
|
// Pass 2: regex + secretlint sweep for anything else.
|