@runeditrun/rer 0.2.74 → 0.2.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/cli/export.mjs +1 -1
- package/cli/listing.mjs +20 -0
- package/cli/main.mjs +1 -0
- package/cli/parse.mjs +2 -1
- package/cli/portability-migrations.mjs +46 -0
- package/cli/portability-worker.mjs +7 -20
- package/cli/portability.mjs +5 -4
- package/cli/server-registry.mjs +43 -18
- package/cli/server-resources.mjs +38 -9
- package/cli/site-check.mjs +4 -2
- package/cli/site-prepare.mjs +6 -0
- package/cli/site-render.mjs +7 -1
- package/cli/site-runtime.mjs +55 -0
- package/cli/site.mjs +45 -17
- package/global/CHANGELOG.md +13 -0
- package/global/README.md +7 -2
- package/global/launch/state.mjs +5 -4
- package/global/launch/stop.mjs +30 -8
- package/global/main.mjs +2 -1
- package/modules/deploy-kit/README.md +47 -2
- package/modules/deploy-kit/owner-cli.mjs +2 -1
- package/modules/deploy-kit/site-readback.mjs +12 -2
- package/modules/deploy-kit/site-runtime.mjs +33 -0
- package/modules/deploy-kit/site.mjs +27 -8
- package/modules/job-queue/command.mjs +8 -3
- package/modules/portability/README.md +40 -11
- package/modules/portability/backfills.mjs +29 -16
- package/modules/portability/bridge.mjs +24 -29
- package/modules/portability/import.mjs +1 -2
- package/modules/portability/index.d.mts +1 -1
- package/modules/portability/ledgers.mjs +22 -0
- package/modules/portability/migration-history.mjs +2 -2
- package/modules/release/README.md +135 -3
- package/modules/release/catalogue.mjs +6 -2
- package/modules/release/listing.mjs +64 -0
- package/modules/release/look.mjs +26 -0
- package/modules/release/previous-listing.mjs +44 -0
- package/modules/release/produce.mjs +3 -0
- package/modules/release/screenshots.mjs +51 -0
- package/modules/release/signature.mjs +3 -2
- package/modules/release/source.mjs +19 -6
- package/modules/release/statement.mjs +8 -4
- package/modules/site/docs-search/SearchField.tsx +18 -0
- package/modules/site/docs-search/artifacts.mjs +54 -0
- package/modules/site/docs-search/client.ts +194 -0
- package/modules/site/docs-search/index-build.mjs +63 -0
- package/modules/site/docs-search/retrieve.ts +16 -0
- package/modules/site/docs-search/search.css +19 -0
- package/modules/site/docs-search/sections.ts +24 -0
- package/modules/site/docs-search/types.ts +14 -0
- package/modules/site/docs-search/worker.ts +39 -0
- package/onboarding/process-cleanup.mjs +5 -5
- package/onboarding/process-scope.mjs +1 -1
- package/onboarding/process-table.mjs +6 -4
- package/onboarding/process-tree.mjs +4 -4
- package/onboarding/schema/json-schema.mjs +4 -1
- package/package.json +1 -1
- package/scripts/publish-check/README.md +14 -0
- package/scripts/publish-check/assignment.py +48 -0
- package/scripts/publish-check/credential_assignments.mjs +75 -0
- package/scripts/publish-check/credential_assignments.py +70 -0
- package/scripts/publish-check/credential_literals.mjs +66 -0
- package/scripts/publish-check/credential_syntax.mjs +29 -0
- package/scripts/publish-check/env_values.py +14 -1
- package/scripts/publish-check/history_scan.py +9 -2
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { SearchSection } from './types';
|
|
2
|
+
|
|
3
|
+
// Run on the renderer's tree after it assigns ids. Search can only link to
|
|
4
|
+
// headings the published page actually has, including duplicate heading ids.
|
|
5
|
+
type Node = { type: string; value?: string; tagName?: string; properties?: Record<string, unknown>; children?: Node[] };
|
|
6
|
+
const textOf = (node: Node): string => node.value ?? (node.children ?? []).map(textOf).join('');
|
|
7
|
+
const collapse = (text: string) => text.replace(/\s+/g, ' ').trim();
|
|
8
|
+
|
|
9
|
+
export function headingSections(nodes: Node[], title: string, titleId: string): SearchSection[] {
|
|
10
|
+
const sections: SearchSection[] = [];
|
|
11
|
+
let parent = '';
|
|
12
|
+
let current: SearchSection = { anchor: titleId, titlePath: [title], text: '' };
|
|
13
|
+
const finish = () => { if (current.text.trim()) sections.push({ ...current, text: current.text.trim() }); };
|
|
14
|
+
for (const node of nodes) {
|
|
15
|
+
if (node.tagName === 'h2' || node.tagName === 'h3') {
|
|
16
|
+
finish();
|
|
17
|
+
const heading = collapse(textOf(node));
|
|
18
|
+
if (node.tagName === 'h2') parent = heading;
|
|
19
|
+
current = { anchor: String(node.properties?.id), titlePath: [title, ...(node.tagName === 'h3' && parent ? [parent] : []), heading], text: '' };
|
|
20
|
+
} else current.text += `${collapse(textOf(node))}\n\n`;
|
|
21
|
+
}
|
|
22
|
+
finish();
|
|
23
|
+
return sections;
|
|
24
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type SearchSection = { anchor: string; titlePath: string[]; text: string };
|
|
2
|
+
export type SearchPage = { href: string; searchSections: SearchSection[] };
|
|
3
|
+
export type SearchChunk = SearchSection & { href: string; hash: string };
|
|
4
|
+
export type SearchIndex = {
|
|
5
|
+
version: 1;
|
|
6
|
+
model: string;
|
|
7
|
+
pooling: 'cls';
|
|
8
|
+
dimensions: number;
|
|
9
|
+
corpusHash: string;
|
|
10
|
+
chunks: (SearchChunk & { vector: number[] })[];
|
|
11
|
+
};
|
|
12
|
+
export type Embeddings = { shape: number[]; data: number[][] };
|
|
13
|
+
export type Embed = (model: string, input: { text: string[]; pooling: 'cls' }) => Promise<Embeddings>;
|
|
14
|
+
export type SearchResult = { href: string; titlePath: string[]; snippet: string; score: number };
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { search } from './retrieve';
|
|
2
|
+
import type { Embed, SearchIndex } from './types';
|
|
3
|
+
|
|
4
|
+
type Environment = { ASSETS: { fetch(request: Request): Promise<Response> }; AI: { run: Embed } };
|
|
5
|
+
const json = (data: unknown, status = 200) => Response.json(data, { status, headers: { 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff' } });
|
|
6
|
+
|
|
7
|
+
/** No app state, credentials or database. Install on the site's Worker only. */
|
|
8
|
+
export function createSearchWorker({ corpusHash, endpoint = '/docs/search', minimumScore = 0.55, indexAsset = '/docs-search-vectors.json', model = '@cf/baai/bge-base-en-v1.5', pooling = 'cls' }: { corpusHash: string; endpoint?: string; minimumScore?: number; indexAsset?: string; model?: string; pooling?: string }) {
|
|
9
|
+
let loaded: SearchIndex | undefined;
|
|
10
|
+
async function load(request: Request, env: Environment): Promise<SearchIndex> {
|
|
11
|
+
if (loaded) return loaded;
|
|
12
|
+
const response = await env.ASSETS.fetch(new Request(new URL(indexAsset, request.url)));
|
|
13
|
+
if (!response.ok) throw Error('Search index unavailable');
|
|
14
|
+
const index = await response.json() as SearchIndex;
|
|
15
|
+
if (index.corpusHash !== corpusHash || index.version !== 1 || index.pooling !== pooling || index.model !== model || !index.dimensions || !Array.isArray(index.chunks)) throw Error('Invalid search index');
|
|
16
|
+
loaded = index;
|
|
17
|
+
return index;
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
async fetch(request: Request, env: Environment): Promise<Response> {
|
|
21
|
+
const url = new URL(request.url);
|
|
22
|
+
if (url.hostname.startsWith('www.')) return Response.redirect(`https://${url.hostname.slice(4)}${url.pathname}${url.search}`, 301);
|
|
23
|
+
if (url.pathname !== endpoint) return env.ASSETS.fetch(request);
|
|
24
|
+
if (request.method !== 'GET') return json({ error: 'Method not allowed' }, 405);
|
|
25
|
+
const query = (url.searchParams.get('q') ?? '').trim();
|
|
26
|
+
if (query.length < 2) return json({ results: [] });
|
|
27
|
+
if (query.length > 240) return json({ error: 'Query too long' }, 400);
|
|
28
|
+
try {
|
|
29
|
+
// The generated entry contains the corpus digest, so docs changes also
|
|
30
|
+
// change Worker code. Binding-only isolate reuse cannot retain old vectors.
|
|
31
|
+
const index = await load(request, env);
|
|
32
|
+
const vectors = await env.AI.run(index.model, { text: [`Represent this sentence for searching relevant passages: ${query}`], pooling: index.pooling });
|
|
33
|
+
return json({ results: search(index, vectors.data[0], minimumScore) });
|
|
34
|
+
} catch {
|
|
35
|
+
return json({ error: 'Search unavailable' }, 503);
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { setTimeout as delay } from 'node:timers/promises'
|
|
2
|
-
import { inspectionMs, readProcessIdentity, readProcessTable, sameBirth,
|
|
2
|
+
import { inspectionMs, readProcessIdentity, readProcessTable, sameBirth, validPid } from './process-table.mjs'
|
|
3
3
|
import { readOwnedTree } from './process-tree.mjs'
|
|
4
4
|
|
|
5
5
|
const graceMs = 1000
|
|
@@ -8,7 +8,7 @@ const cleanupMs = 30000
|
|
|
8
8
|
const pollMs = process.platform === 'darwin' ? 5 : 25
|
|
9
9
|
const isZombie = (member) => member.state.startsWith('Z')
|
|
10
10
|
const isStopped = (member) => /^[Tt]/.test(member.state)
|
|
11
|
-
const key = (member) => `${member.pid}:${member.birth}
|
|
11
|
+
const key = (member) => `${member.pid}:${member.birth}`
|
|
12
12
|
const isLeaf = (member, group) => !group.some((other) => other.ppid === member.pid)
|
|
13
13
|
|
|
14
14
|
// Ask every owned process to shut down gracefully, including supervisors that
|
|
@@ -24,7 +24,7 @@ export async function stopOwnedGroup(child, signal, ownership) {
|
|
|
24
24
|
const inspect = async (timeout) => {
|
|
25
25
|
const current = await readOwnedTree(child, ownership, timeout)
|
|
26
26
|
// Retain departed group leaders as provenance, not as signal targets.
|
|
27
|
-
for (const member of current) if (!ownership.some(saved =>
|
|
27
|
+
for (const member of current) if (!ownership.some(saved => sameBirth(saved, member))) ownership.push(member)
|
|
28
28
|
return current
|
|
29
29
|
}
|
|
30
30
|
let group
|
|
@@ -45,7 +45,7 @@ export async function stopOwnedGroup(child, signal, ownership) {
|
|
|
45
45
|
// signal. Never send a negative/group signal that can kill parents first.
|
|
46
46
|
const current = requireLeaf ? await readProcessTable(inspectionMs) : []
|
|
47
47
|
const target = await readProcessIdentity(member.pid, inspectionMs)
|
|
48
|
-
if (!
|
|
48
|
+
if (!sameBirth(member, target) || isZombie(target)) return false
|
|
49
49
|
if (requireLeaf && !isLeaf(target, current)) return false
|
|
50
50
|
if (name === 'SIGKILL' && !isStopped(target)) return false
|
|
51
51
|
if (requireLeaf && target.pid === pgid && current.some(other => other.pid !== pgid
|
|
@@ -66,7 +66,7 @@ export async function stopOwnedGroup(child, signal, ownership) {
|
|
|
66
66
|
const leader = ownership.find(member => member.pid === pgid)
|
|
67
67
|
if (leader && child.exitCode == null && child.signalCode == null) {
|
|
68
68
|
const live = await readProcessIdentity(pgid, inspectionMs)
|
|
69
|
-
if (
|
|
69
|
+
if (sameBirth(leader, live) && live.ppid === process.pid && !isZombie(live)) {
|
|
70
70
|
try { process.kill(pgid, signal); signalled.set(key(live), Date.now()) }
|
|
71
71
|
catch (error) { if (error.code !== 'ESRCH') throw new Error(`Could not send ${signal} to owned process ${pgid}.`) }
|
|
72
72
|
}
|
|
@@ -90,7 +90,7 @@ export async function scopedProcesses(scope, table, timeout = inspectionMs, prev
|
|
|
90
90
|
}
|
|
91
91
|
// A recycled PID must not transfer a token reading to its successor. PGID may
|
|
92
92
|
// legitimately change, so follow the same birth identity across that change.
|
|
93
|
-
// The caller rechecks each match's
|
|
93
|
+
// The caller rechecks each match's PID/start identity, including across exec.
|
|
94
94
|
return matches
|
|
95
95
|
}
|
|
96
96
|
|
|
@@ -119,10 +119,13 @@ export async function readProcessIdentity(pid, timeout = inspectionMs) {
|
|
|
119
119
|
return withinBound(timeout, (remaining) => psIdentity(pid, remaining))
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
+
// macOS ps comm is read from the process's own argv, which process.title
|
|
123
|
+
// rewrites (the job supervisor retitles itself after launch). ucomm is the
|
|
124
|
+
// kernel's executable name: set only by exec, so it survives a retitle.
|
|
122
125
|
async function psIdentity(pid, timeout) {
|
|
123
126
|
let stdout
|
|
124
127
|
try {
|
|
125
|
-
({ stdout } = await execute('/bin/ps', ['-p', String(pid), '-o', 'pid=', '-o', 'ppid=', '-o', 'pgid=', '-o', 'uid=', '-o', 'stat=', '-o', 'lstart=', '-o', '
|
|
128
|
+
({ stdout } = await execute('/bin/ps', ['-p', String(pid), '-o', 'pid=', '-o', 'ppid=', '-o', 'pgid=', '-o', 'uid=', '-o', 'stat=', '-o', 'lstart=', '-o', 'ucomm='],
|
|
126
129
|
{ env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' }, timeout, killSignal: 'SIGKILL', maxBuffer: 1024 * 1024 }))
|
|
127
130
|
} catch (error) { if (error.code === 1 && !error.stdout?.trim()) return; throw transient('Could not inspect process identity.') }
|
|
128
131
|
const match = stdout.match(/^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.{24})\s+(.+?)\s*$/)
|
|
@@ -150,7 +153,6 @@ export async function readProcessTable(timeout = inspectionMs, pgid, read = proc
|
|
|
150
153
|
}, extension)
|
|
151
154
|
}
|
|
152
155
|
|
|
156
|
+
// Custody follows a process lifetime: exec changes its executable name without
|
|
157
|
+
// changing its PID or start time. Names are metadata, never lifetime identity.
|
|
153
158
|
export const sameBirth = (left, right) => validPid(left?.pid) && left.pid === right?.pid && left.birth === right?.birth
|
|
154
|
-
|
|
155
|
-
export const sameProcess = (left, right) => validPid(left?.pid) && left?.pid === right?.pid
|
|
156
|
-
&& left?.birth === right?.birth && left?.command === right?.command
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { scopedProcesses } from './process-scope.mjs'
|
|
2
|
-
import { inspectionMs, readProcessIdentity, readProcessTable, sameBirth,
|
|
2
|
+
import { inspectionMs, readProcessIdentity, readProcessTable, sameBirth, validPid } from './process-table.mjs'
|
|
3
3
|
|
|
4
|
-
// Retain PID/start
|
|
4
|
+
// Retain PID/start identities across exec and after close. An inherited marker
|
|
5
5
|
// recovers descendants after reparenting; group membership and cwd never grant
|
|
6
6
|
// ownership. Inspect executable identity only for actual custody candidates.
|
|
7
7
|
export async function readOwnedTree(child, previous = [], timeout = inspectionMs, scope = previous.scope) {
|
|
@@ -9,12 +9,12 @@ export async function readOwnedTree(child, previous = [], timeout = inspectionMs
|
|
|
9
9
|
const table = await readProcessTable(timeout)
|
|
10
10
|
const candidates = table.filter(member => previous.some(saved => sameBirth(member, saved)))
|
|
11
11
|
let owned = (await Promise.all(candidates.map(member => readProcessIdentity(member.pid, timeout))))
|
|
12
|
-
.filter(member => member && previous.some(saved =>
|
|
12
|
+
.filter(member => member && previous.some(saved => sameBirth(member, saved)))
|
|
13
13
|
if (scope) {
|
|
14
14
|
for (const member of await scopedProcesses(scope, table, timeout, previous)) {
|
|
15
15
|
if (owned.some(saved => sameBirth(saved, member))) continue
|
|
16
16
|
const live = await readProcessIdentity(member.pid, timeout)
|
|
17
|
-
if (sameBirth(member, live) && !owned.some(saved =>
|
|
17
|
+
if (sameBirth(member, live) && !owned.some(saved => sameBirth(saved, live))) owned.push(live)
|
|
18
18
|
}
|
|
19
19
|
} else if (!previous.length && child.exitCode == null && child.signalCode == null) {
|
|
20
20
|
const leader = await readProcessIdentity(child.pid, timeout)
|
|
@@ -23,7 +23,7 @@ function isDeepStrictEqual(left, right) {
|
|
|
23
23
|
const SUPPORTED = new Set([
|
|
24
24
|
'$schema', '$id', '$ref', '$defs', 'title', 'description',
|
|
25
25
|
'type', 'enum', 'const', 'required', 'properties', 'additionalProperties',
|
|
26
|
-
'propertyNames', 'items', 'minItems', 'minLength', 'pattern', 'format',
|
|
26
|
+
'propertyNames', 'items', 'minItems', 'maxItems', 'minLength', 'pattern', 'format',
|
|
27
27
|
'allOf', 'if', 'then', 'uniqueItems'
|
|
28
28
|
]);
|
|
29
29
|
|
|
@@ -181,6 +181,9 @@ export function validate(schema, data, { root = schema, path = '$', label = '' }
|
|
|
181
181
|
if (schema.minItems !== undefined && data.length < schema.minItems) {
|
|
182
182
|
fail(`needs at least ${schema.minItems} item(s)`);
|
|
183
183
|
}
|
|
184
|
+
if (schema.maxItems !== undefined && data.length > schema.maxItems) {
|
|
185
|
+
fail(`needs at most ${schema.maxItems} item(s)`);
|
|
186
|
+
}
|
|
184
187
|
if (schema.items) {
|
|
185
188
|
data.forEach((item, index) => {
|
|
186
189
|
errors.push(...validate(schema.items, item, { root, path: `${path}[${index}]` }));
|
package/package.json
CHANGED
|
@@ -27,6 +27,20 @@ Hexadecimal credential assignments also use a Gitleaks rule without generic
|
|
|
27
27
|
entropy or English stopword filters. Both assignment rules go through the same
|
|
28
28
|
location-bound Source/module identity classification; provider and environment findings
|
|
29
29
|
remain independent.
|
|
30
|
+
`assignment.py` narrows only comma-separated pairs: the complete left token
|
|
31
|
+
must contain only letters, digits, underscores or hyphens. Gitleaks still
|
|
32
|
+
requires a secret-shaped right value, using its inherited entropy/length policy
|
|
33
|
+
or the hex rule. Native locations bind the check to the full key in files and
|
|
34
|
+
messages, so a filename or path's key-like suffix cannot become a key. Other
|
|
35
|
+
assignment operators retain their existing grammar.
|
|
36
|
+
|
|
37
|
+
`env_values.py` only hashes decoded dotenv values that have at least 16
|
|
38
|
+
characters, have Shannon entropy greater than 3.5 bits per character (the pinned
|
|
39
|
+
Gitleaks generic-api-key threshold), or start with a known token prefix:
|
|
40
|
+
`sk_`, `sk-`, `pk_live`, `ghp_`, `github_pat_`, `xox`, `AKIA`, `SG.`, or `eyJ`.
|
|
41
|
+
Prefix matching is case-sensitive. Ordinary short words, statuses and short
|
|
42
|
+
numeric IDs therefore do not become environment matches. Eligible values are still
|
|
43
|
+
compared only through in-memory digests and cannot be waived.
|
|
30
44
|
`tree_scan.py` applies the kept text and licence policy. `history_scan.py`
|
|
31
45
|
assigns release, working-tree and historical scopes. Inspection returns each
|
|
32
46
|
text's repository references instead of resolving them; `history_scan.py`
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Require a complete identifier key for comma-separated credential pairs."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
from hit_location import match_spans
|
|
5
|
+
|
|
6
|
+
IDENTIFIER = re.compile(br'[A-Za-z0-9_-]+\Z')
|
|
7
|
+
COMMA = re.compile(br''',[\x60'"\s=]{0,5}\Z''')
|
|
8
|
+
BOUNDARIES = b" \t\r\n\v\f\"'`[]{}(),;:=<>|?"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def identifier_before(data, end):
|
|
12
|
+
left = data[:end].rstrip()
|
|
13
|
+
if not left:
|
|
14
|
+
return False
|
|
15
|
+
end = len(left)
|
|
16
|
+
if left[-1] in b"\"'":
|
|
17
|
+
quote = left[-1]
|
|
18
|
+
start = end - 2
|
|
19
|
+
while start >= 0:
|
|
20
|
+
if left[start] == quote:
|
|
21
|
+
previous = start - 1
|
|
22
|
+
while previous >= 0 and left[previous] == ord('\\'):
|
|
23
|
+
previous -= 1
|
|
24
|
+
if (start - previous - 1) % 2 == 0:
|
|
25
|
+
return bool(IDENTIFIER.fullmatch(left[start + 1:end - 1]))
|
|
26
|
+
start -= 1
|
|
27
|
+
return False
|
|
28
|
+
start = end
|
|
29
|
+
while start and left[start - 1] not in BOUNDARIES:
|
|
30
|
+
start -= 1
|
|
31
|
+
return bool(IDENTIFIER.fullmatch(left[start:end]))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def is_assignment(data, hit):
|
|
35
|
+
# Gitleaks alone decides whether the right value meets its entropy/length
|
|
36
|
+
# policy (or the hex rule). Every other assignment operator is unchanged.
|
|
37
|
+
if hit.get('RuleID') not in ('generic-api-key', 'hex-api-key'):
|
|
38
|
+
return True
|
|
39
|
+
match = hit.get('Match', '').encode('utf-8')
|
|
40
|
+
secret = hit.get('Secret', '').encode('utf-8')
|
|
41
|
+
value = match.rfind(secret) if secret else -1
|
|
42
|
+
comma = COMMA.search(match[:value]) if value >= 0 else None
|
|
43
|
+
if comma is None:
|
|
44
|
+
return True
|
|
45
|
+
spans = match_spans(data, hit)
|
|
46
|
+
# Keep findings whose exact native location cannot be resolved. Reading the
|
|
47
|
+
# whole left token also rejects a key-like suffix inside a path or string.
|
|
48
|
+
return not spans or any(identifier_before(data, start + comma.start()) for start, _ in spans)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/** SEC-3 credential assignment: literal syntax, entropy and fixture/reference semantics. */
|
|
2
|
+
import {codeSource, typeDeclarations} from './credential_syntax.mjs';
|
|
3
|
+
import {carriesCredential, isPlaceholder, cssLiteral} from './credential_literals.mjs';
|
|
4
|
+
|
|
5
|
+
function shannonEntropy(value) {
|
|
6
|
+
const counts = new Map();
|
|
7
|
+
for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1);
|
|
8
|
+
const length = [...value].length;
|
|
9
|
+
let entropy = 0;
|
|
10
|
+
for (const count of counts.values()) { const p = count / length; entropy -= p * Math.log2(p); }
|
|
11
|
+
return entropy;
|
|
12
|
+
}
|
|
13
|
+
// Bounded name parts keep matching linear on long lines (no unbounded backtracking before the keyword).
|
|
14
|
+
const assignment=/(?<![A-Za-z0-9_.$-])([A-Za-z0-9_.$-]{0,48}(?:secret|token|passw(?:or)?d|api[_-]?key|private[_-]?key)[A-Za-z0-9_]{0,48})["'\]]?[ \t]{0,8}(?::=|:|=)[ \t\r\n]{0,8}(["'`])([^"'`\s]{16,512})\2/gi
|
|
15
|
+
// Unquoted source identifiers and expressions are references, never literal credential values.
|
|
16
|
+
const numberAssignment=/(?<![A-Za-z0-9_.$-])([A-Za-z0-9_.$-]{0,48}(?:secret|token|passw(?:or)?d|api[_-]?key|private[_-]?key)[A-Za-z0-9_]{0,48})[ \t]{0,8}(?::=|:|=)[ \t]{0,8}([+-]?(?:0x[\da-f]{16,512}|\d{16,512}))(?![\w.$])/gi
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
function parameterType(text,match){
|
|
20
|
+
// A colon directly after a parameter name describes its type. Defaults use '=' and still scan.
|
|
21
|
+
const afterName=match[0].slice(match[1].length)
|
|
22
|
+
return /^[ \t]*:/.test(afterName)&&/(?:^|[,(])[ \t\r\n]*$/.test(text.slice(Math.max(0,match.index -256),match.index))
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const envAssignment=/^[ \t]{0,8}(?:export[ \t]+)?([A-Za-z0-9_]{0,48}(?:SECRET|TOKEN|PASSW(?:OR)?D|API_?KEY|PRIVATE_?KEY)[A-Za-z0-9_]{0,48})[ \t]*=[ \t]*([^\s"'`#]{16,512})[ \t]*$/gim
|
|
26
|
+
|
|
27
|
+
/** SEC-3: names that look like credentials assigned a literal of at least 16 characters with entropy ≥ 3.5 bits, other than CSS assigned to a token name (cssLiteral). */
|
|
28
|
+
function findCredentialAssignments(text,options={}) {
|
|
29
|
+
const found=[],types=options.code===false?[]:typeDeclarations(text)
|
|
30
|
+
const consider=(name,value,index)=>{
|
|
31
|
+
// Theme and design tokens are named token; CSS is never a secret's, password's or key's value.
|
|
32
|
+
if(types.some(({start,end})=>index>=start&&index<end)||shannonEntropy(value)<3.5||!/secret|passw|key/i.test(name)&&cssLiteral(value))return
|
|
33
|
+
if(/^(process\.env|import\.meta\.env|env\.|c\.env)/.test(value))return
|
|
34
|
+
found.push({name,value,index,placeholder:isPlaceholder(value)})
|
|
35
|
+
}
|
|
36
|
+
for(const match of text.matchAll(assignment))if(!parameterType(text,match))consider(match[1],match[3],match.index +match[0].lastIndexOf(match[3]))
|
|
37
|
+
for(const pattern of [numberAssignment,...(options.code===false?[envAssignment]:[])])for(const match of text.matchAll(pattern))if(!parameterType(text,match))consider(match[1],match[2],match.index +match[0].lastIndexOf(match[2]))
|
|
38
|
+
return found
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// A URL or module/route path is a reference, not the credential value itself.
|
|
42
|
+
// Known credential formats are still scanned throughout those literals.
|
|
43
|
+
function referenceLiteral(value){
|
|
44
|
+
if(/^(?:\.{1,2}\/|\/)[A-Za-z0-9_./{}:-]+$/.test(value))return true
|
|
45
|
+
try{const url=new URL(value);return ['https:','http:'].includes(url.protocol)&&!url.username&&!url.password&&!url.search&&!url.hash}catch{return false}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// In call('/route?token='+encodeURIComponent(value)), the quote closes the URL.
|
|
49
|
+
// The assignment heuristic otherwise consumes the following JS expression as a quoted value.
|
|
50
|
+
function queryConcatenation(text,match){
|
|
51
|
+
return match.value.startsWith('+')&&/[?&][A-Za-z0-9_.-]+=["'`]$/.test(text.slice(Math.max(0,match.index-128),match.index))
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
/** The hosted rule scans every file; its extension determines identifier/type syntax only. */
|
|
56
|
+
export function sourceCredentialFindings(text, path) {
|
|
57
|
+
return findCredentialAssignments(text, {code: codeSource(path)})
|
|
58
|
+
.filter(match => !match.placeholder && !queryConcatenation(text, match) &&
|
|
59
|
+
(carriesCredential(match.value) || !/^synthetic[-_]/i.test(match.value) && !referenceLiteral(match.value)))
|
|
60
|
+
.map(match => ({index: match.index, rule: 'SEC-3 credential assignment'}))
|
|
61
|
+
.sort((a, b) => a.index - b.index);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// A single worker serves the Python scanner's files without a process per blob.
|
|
65
|
+
// The protocol returns locations only, including on malformed input.
|
|
66
|
+
if (process.argv[1] === import.meta.filename) {
|
|
67
|
+
const {createInterface} = await import('node:readline');
|
|
68
|
+
for await (const input of createInterface({input: process.stdin, crlfDelay: Infinity})) {
|
|
69
|
+
try {
|
|
70
|
+
const {text, path} = JSON.parse(input);
|
|
71
|
+
const findings = sourceCredentialFindings(text, path).map(({index}) => ({line: text.slice(0, index).split('\n').length}));
|
|
72
|
+
process.stdout.write(JSON.stringify(findings) + '\n');
|
|
73
|
+
} catch { process.stdout.write('null\n'); }
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Register SEC-3 with text inspection and value-free, line-aware reports."""
|
|
2
|
+
import atexit
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import subprocess
|
|
7
|
+
from git_history import InspectionError
|
|
8
|
+
from report import Findings as PublicationFindings
|
|
9
|
+
from tree_scan import text_rules as publication_text_rules
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class AssignmentFinding:
|
|
14
|
+
line: int
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AssignmentScanner:
|
|
18
|
+
def __init__(self):
|
|
19
|
+
self.worker = None
|
|
20
|
+
atexit.register(self.close)
|
|
21
|
+
|
|
22
|
+
def findings(self, data, path):
|
|
23
|
+
try:
|
|
24
|
+
if self.worker is None:
|
|
25
|
+
self.worker = subprocess.Popen(
|
|
26
|
+
['node', str(Path(__file__).with_suffix('.mjs'))],
|
|
27
|
+
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
|
28
|
+
text=True, encoding='utf-8')
|
|
29
|
+
self.worker.stdin.write(json.dumps(dict(text=data.decode('utf-8', 'replace'), path=path)) + '\n')
|
|
30
|
+
self.worker.stdin.flush()
|
|
31
|
+
result = json.loads(self.worker.stdout.readline())
|
|
32
|
+
if not isinstance(result, list) or any(not isinstance(hit.get('line'), int) or hit['line'] < 1 for hit in result):
|
|
33
|
+
raise ValueError('invalid SEC-3 response')
|
|
34
|
+
return {AssignmentFinding(hit['line']) for hit in result}
|
|
35
|
+
except (OSError, ValueError, AttributeError, TypeError):
|
|
36
|
+
raise InspectionError('scan.credential-assignment-incomplete') from None
|
|
37
|
+
|
|
38
|
+
def close(self):
|
|
39
|
+
if self.worker is not None:
|
|
40
|
+
self.worker.stdin.close()
|
|
41
|
+
self.worker.wait()
|
|
42
|
+
self.worker.stdout.close()
|
|
43
|
+
self.worker = None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
scanner = AssignmentScanner()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def text_rules(data, path, config, env_hashes, **message_options):
|
|
50
|
+
rules, provenance, references = publication_text_rules(data, path, config, env_hashes,
|
|
51
|
+
**message_options)
|
|
52
|
+
# History explicitly supplies True for commits and False for annotated tags.
|
|
53
|
+
# File calls omit that flag, even when the file has a metadata-looking name.
|
|
54
|
+
if 'commit_message' not in message_options:
|
|
55
|
+
rules.update(scanner.findings(data, path))
|
|
56
|
+
return rules, provenance, references
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Findings(PublicationFindings):
|
|
60
|
+
def add(self, rule, path, commit=None, oid=None, scope='newly-public', bucket=None, scanner_rule=None):
|
|
61
|
+
if not isinstance(rule, AssignmentFinding):
|
|
62
|
+
return super().add(rule, path, commit, oid, scope, bucket, scanner_rule)
|
|
63
|
+
bucket = bucket or ('preExisting' if scope == 'pre-existing' else 'findings')
|
|
64
|
+
# The internal discriminator keeps separate lines separate while using
|
|
65
|
+
# the existing history/scope deduplication and path redaction.
|
|
66
|
+
discriminator = 'SEC-3:' + str(rule.line)
|
|
67
|
+
super().add('SEC-3', path, commit, oid, scope, bucket, discriminator)
|
|
68
|
+
item = self.seen[bucket, 'SEC-3', path, oid, discriminator]
|
|
69
|
+
item['line'] = rule.line
|
|
70
|
+
item.pop('scannerRule', None)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/** Exact credential formats, placeholders and strictly shaped CSS literals for SEC-3. */
|
|
2
|
+
const SECRET_PATTERNS=[
|
|
3
|
+
{kind:'PEM private key',regex:/-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----/g},
|
|
4
|
+
{kind:'AWS access key',regex:/\bAKIA[0-9A-Z]{16}\b/g},
|
|
5
|
+
{kind:'GitHub token',regex:/\bgh[pousr]_[A-Za-z0-9]{36,}\b/g},
|
|
6
|
+
{kind:'GitHub fine-grained token',regex:/\bgithub_pat_[A-Za-z0-9_]{22,}/g},
|
|
7
|
+
{kind:'Anthropic key',regex:/\bsk-ant-[A-Za-z0-9_-]{20,}/g},
|
|
8
|
+
{kind:'OpenAI key',regex:/\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}/g},
|
|
9
|
+
{kind:'Stripe live key',regex:/\b(?:sk|rk)_live_[0-9A-Za-z]{20,}/g},
|
|
10
|
+
{kind:'Slack token',regex:/\bxox[abprs]-[A-Za-z0-9-]{10,}/g},
|
|
11
|
+
{kind:'Google API key',regex:/\bAIza[0-9A-Za-z_-]{35}/g},
|
|
12
|
+
{kind:'SendGrid key',regex:/\bSG\.[\w-]{22}\.[\w-]{43}/g},
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
const placeholderMarkers=['example','placeholder','xxxx','test-only','<','${']
|
|
16
|
+
// A reader's instruction in place of the value: "your" or "replace with" followed by lowercase words, no digit (your-scoped-deploy-token).
|
|
17
|
+
const instruction=/^(?:your|replace[-_]with)(?:[-_][a-z]+)+$/
|
|
18
|
+
// A credential variable's name standing for its value: UPPER_SNAKE letters, no digit, ending in what it holds (MAIL_PROVIDER_API_KEY).
|
|
19
|
+
const variableName=/^[A-Z]+(?:_[A-Z]+)*_(?:SECRET|TOKEN|PASSWORD|PASSWD|KEY)$/
|
|
20
|
+
// Exact credential formats: SEC-1's, plus Stripe test and publishable keys and JWTs, which SEC-3 judges by name.
|
|
21
|
+
const exactFormats=[...SECRET_PATTERNS.map(({regex})=>new RegExp(regex.source)),/\b[spr]k_(?:live|test)_[0-9A-Za-z]{8,}/,/\beyJ[\w-]{8,}\.eyJ[\w-]{8,}\./]
|
|
22
|
+
/** A value carrying an exact credential format is a placeholder only by an explicit marker, and never a fixture, a reference or CSS. */
|
|
23
|
+
export function carriesCredential(value) {return exactFormats.some(format=>format.test(value))}
|
|
24
|
+
/** Values that are documentation, templates, instructions or a credential variable's name rather than credentials. */
|
|
25
|
+
export function isPlaceholder(value) {
|
|
26
|
+
const lower=value.toLowerCase()
|
|
27
|
+
return placeholderMarkers.some(marker=>lower.includes(marker))||!carriesCredential(value)&&(instruction.test(value)||variableName.test(value))
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// CSS that SEC-3 admits is made only of recognisable CSS shapes; anything else in a literal is judged by the normal rules.
|
|
31
|
+
// Identifiers are lowercase letters and single dashes with no digit and numbers have at most four digits either side of the
|
|
32
|
+
// point, so no hex, base64, digit or mixed-case run hides in a name, keyword or number.
|
|
33
|
+
const cssIdent='-{0,2}[a-z]+(?:-[a-z]+)*'
|
|
34
|
+
const cssAtom=`(?:#(?:[0-9a-fA-F]{3,4}|[0-9a-fA-F]{6}|[0-9a-fA-F]{8})|[+-]?(?:\\d{1,4}(?:\\.\\d{1,4})?|\\.\\d{1,4})(?:%|px|r?em|ex|ch|[dls]?v[hw]|vmin|vmax|cq[ihw]|fr|deg|rad|turn|m?s|pt|dppx)?|${cssIdent}|"[A-Za-z ]+"|'[A-Za-z ]+'|\x01)`
|
|
35
|
+
const cssValue=new RegExp(`^${cssAtom}(?:[\\s,/]+${cssAtom})*(?:!important)?$`)
|
|
36
|
+
const cssArguments=new RegExp(`^${cssAtom}(?:(?:[\\s,/*]|\\s[+-]\\s)+${cssAtom})*$`)
|
|
37
|
+
const cssFunction=/\b(?:var|calc|min|max|clamp|rgba?|hsla?)\(([^()]*)\)/g
|
|
38
|
+
// A compound selector: an optional element, then classes, ids, pseudo-classes and attributes, each with its own sigil.
|
|
39
|
+
const cssSubSelector=`(?:[.#]${cssIdent}|::?${cssIdent}|\\[${cssIdent}(?:[~|^$*]?=${cssIdent})?\\]|\x02)`
|
|
40
|
+
const cssCompound=`(?:(?:h[1-6]|${cssIdent}|\\*|&)${cssSubSelector}*|${cssSubSelector}+)`
|
|
41
|
+
const cssSelector=new RegExp(`^${cssCompound}(?:[\\s,>+~]+${cssCompound})*$`)
|
|
42
|
+
const cssPseudoFunction=/:(?:where|is|not|has)\(([^()]*)\)/g
|
|
43
|
+
const cssProperty=new RegExp(`^${cssIdent}$`),cssCustomProperty=/^--[a-z]+(?:-[a-z]+)*$/
|
|
44
|
+
const cssDeclarations=(text,custom)=>text.split(';').every(declaration=>{
|
|
45
|
+
if(!declaration)return true
|
|
46
|
+
const colon=declaration.indexOf(':')
|
|
47
|
+
return colon>0&&(custom?cssCustomProperty:cssProperty).test(declaration.slice(0,colon))&&cssValue.test(declaration.slice(colon+1))
|
|
48
|
+
})
|
|
49
|
+
/** A literal written in CSS: custom property declarations (--name:value;…) or selector{declarations} rules, whose every
|
|
50
|
+
* value is a hex colour, a number with unit or %, an identifier, a quoted font name, or var/calc/min/max/clamp/rgb/hsl of those,
|
|
51
|
+
* with at least one value that is not a bare identifier. */
|
|
52
|
+
export function cssLiteral(value) {
|
|
53
|
+
if(!/[:{]/.test(value)||/[\x01\x02]/.test(value)||carriesCredential(value)||!/#[0-9a-fA-F]{3}|\d|\b(?:var|calc|min|max|clamp|rgba?|hsla?)\(/.test(value))return false
|
|
54
|
+
let text=value,previous
|
|
55
|
+
do{
|
|
56
|
+
previous=text
|
|
57
|
+
text=text.replace(cssFunction,(whole,args)=>cssArguments.test(args)?'\x01':whole).replace(cssPseudoFunction,(whole,args)=>cssSelector.test(args)?'\x02':whole)
|
|
58
|
+
}while(text!==previous)
|
|
59
|
+
const parts=text.split('}')
|
|
60
|
+
return parts.every((part,index)=>{
|
|
61
|
+
const open=part.indexOf('{')
|
|
62
|
+
// Before any rule: custom properties only (a continued block or a design-token list). Between rules: nothing.
|
|
63
|
+
if(open<0)return index===0?cssDeclarations(part,true):!part
|
|
64
|
+
return part.indexOf('{',open+1)<0&&cssSelector.test(part.slice(0,open))&&cssDeclarations(part.slice(open+1),false)
|
|
65
|
+
})
|
|
66
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** Lexical context for the SEC-3 source rule. No source values leave this module. */
|
|
2
|
+
/** These source languages use bare names as expressions; configuration files can have unquoted string literals. */
|
|
3
|
+
export const codeSource=(path)=>path===undefined||/\.(?:[cm]?[jt]sx?|py|rs|go|java|kt|cs|cpp|cc|c|h|swift|rb)$/i.test(path)
|
|
4
|
+
|
|
5
|
+
export const sourceTokens=/\/\/[^\n]*|\/\*[\s\S]*?(?:\*\/|$)|\/(?![*/])(?:\\[\s\S]?|\[(?:\\[\s\S]?|[^\]\\])*(?:\]|$)|[^/\\[\n])+(?:\/[a-z]*|$)|"(?:\\[\s\S]?|[^"\\])*(?:"|$)|'(?:\\[\s\S]?|[^'\\])*(?:'|$)|`(?:\\[\s\S]?|[^`\\])*(?:`|$)|[A-Za-z_$][\w$]*|[^\s]/g
|
|
6
|
+
|
|
7
|
+
/** Type declarations are recognized outside comments and strings, so text cannot hide later assignments. */
|
|
8
|
+
export function typeDeclarations(text) {
|
|
9
|
+
if(!/\b(?:interface|type)\s+[A-Za-z_$]/.test(text))return []
|
|
10
|
+
const tokens=[...text.matchAll(sourceTokens)].filter(token=>!token[0].startsWith('//')&&!token[0].startsWith('/*'))
|
|
11
|
+
const ranges=[]
|
|
12
|
+
for(let i=0;i<tokens.length;i++){
|
|
13
|
+
const start=tokens[i],kind=start[0]
|
|
14
|
+
if(!['interface','type'].includes(kind)||!/^\w+$/.test(tokens[i+1]?.[0]??''))continue
|
|
15
|
+
let j=i+2
|
|
16
|
+
while(j<tokens.length&&!['{','=',';',':'].includes(tokens[j][0]))j++
|
|
17
|
+
if(tokens[j]?.[0]!== (kind==='interface'?'{':'='))continue
|
|
18
|
+
let depth=kind==='interface'?1:0,end=tokens[j].index +1
|
|
19
|
+
for(j++;j<tokens.length;j++){
|
|
20
|
+
const token=tokens[j],value=token[0]
|
|
21
|
+
if(depth===0&&(value===';'||text.slice(end,token.index).includes('\n')&&!['|','&','{'].includes(value)))break
|
|
22
|
+
if(['{','(','['].includes(value))depth++
|
|
23
|
+
if(['}',')',']'].includes(value)&&--depth===0){end=token.index +1;if(kind==='interface')break;continue}
|
|
24
|
+
end=token.index +value.length
|
|
25
|
+
}
|
|
26
|
+
ranges.push({start:start.index,end,continuation:j===tokens.length?(kind==='interface'?'interface ScanContext ':'type ScanContext = ')+ '{'.repeat(Math.max(0,depth)):''})
|
|
27
|
+
}
|
|
28
|
+
return ranges
|
|
29
|
+
}
|
|
@@ -1,9 +1,22 @@
|
|
|
1
1
|
"""Read dotenv values into in-memory digests; never retain or report plaintext."""
|
|
2
|
+
from collections import Counter
|
|
2
3
|
import hashlib
|
|
4
|
+
from math import log2
|
|
3
5
|
import re
|
|
4
6
|
from pathlib import Path
|
|
5
7
|
|
|
6
8
|
|
|
9
|
+
TOKEN_PREFIXES = ('sk_', 'sk-', 'pk_live', 'ghp_', 'github_pat_', 'xox', 'AKIA', 'SG.', 'eyJ')
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def secret_shaped(value: str) -> bool:
|
|
13
|
+
if len(value) >= 16 or value.startswith(TOKEN_PREFIXES):
|
|
14
|
+
return True
|
|
15
|
+
# Gitleaks 8.30.1's inherited generic-api-key rule uses 3.5 bits/character.
|
|
16
|
+
frequencies = (count / len(value) for count in Counter(value).values())
|
|
17
|
+
return -sum(p * log2(p) for p in frequencies) > 3.5
|
|
18
|
+
|
|
19
|
+
|
|
7
20
|
def digest(value):
|
|
8
21
|
return hashlib.sha256(value).digest()
|
|
9
22
|
|
|
@@ -69,6 +82,6 @@ def value_hashes(text):
|
|
|
69
82
|
end = len(text) if end < 0 else end
|
|
70
83
|
value = re.split(r'\s+#', text[cursor:end], maxsplit=1)[0].strip()
|
|
71
84
|
cursor = end + 1
|
|
72
|
-
if value:
|
|
85
|
+
if value and secret_shaped(value):
|
|
73
86
|
hashes.add(digest(value.encode()))
|
|
74
87
|
return hashes
|
|
@@ -16,6 +16,8 @@ from tree_scan import context, text_rules, working_files, working_regular
|
|
|
16
16
|
from public_hygiene import reference_rules
|
|
17
17
|
from content_identities import ContentIdentities, candidate as identity_candidate
|
|
18
18
|
from checksum_maps import ChecksumMaps
|
|
19
|
+
from credential_assignments import text_rules, Findings
|
|
20
|
+
from assignment import is_assignment
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300, annotation_file=None):
|
|
@@ -137,6 +139,9 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
137
139
|
def read_regular(path, scoped_entries=scoped_entries):
|
|
138
140
|
entry = scoped_entries.get(path)
|
|
139
141
|
return (history.blob(entry['oid']), entry['mode']) if entry and entry['kind'] == 'blob' and entry['mode'] in ('100644', '100755') else None
|
|
142
|
+
held = working_regular(Path(history.repo), path) if mode == 'dir' else read_regular(path)
|
|
143
|
+
if held and not is_assignment(held[0], hit):
|
|
144
|
+
continue
|
|
140
145
|
if identity_scope not in checksums:
|
|
141
146
|
checksums[identity_scope] = timings.call('checksums.prepare', ChecksumMaps,
|
|
142
147
|
working_paths if mode == 'dir' else scoped_entries,
|
|
@@ -150,7 +155,6 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
150
155
|
continue
|
|
151
156
|
else:
|
|
152
157
|
# Identity documents use their own exact structural locations.
|
|
153
|
-
held = working_regular(Path(history.repo), path) if mode == 'dir' else read_regular(path)
|
|
154
158
|
if held and checksums[identity_scope].matches(held[0], hit):
|
|
155
159
|
continue
|
|
156
160
|
add(secret_rule(hit['RuleID']), path, commit, oid, scope, scanner_rule=hit['RuleID'])
|
|
@@ -181,7 +185,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
181
185
|
length = (data + b'\n').count(b'\n')
|
|
182
186
|
lines.append((count, count + length, path, commit, scope))
|
|
183
187
|
count += length
|
|
184
|
-
|
|
188
|
+
message_data = b''.join(messages)
|
|
189
|
+
for hit in timings.call('gitleaks.messages', secrets, binary, 'stdin', data=message_data, timeout=timeout, redact=False):
|
|
190
|
+
if not is_assignment(message_data, hit):
|
|
191
|
+
continue
|
|
185
192
|
line = hit.get('StartLine', 0)
|
|
186
193
|
location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
|
|
187
194
|
add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
|