@runeditrun/rer 0.2.75 → 0.2.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/cli/listing.mjs +20 -0
- package/cli/main.mjs +1 -0
- package/cli/parse.mjs +2 -1
- package/cli/portability-migrations.mjs +46 -0
- package/cli/portability-worker.mjs +7 -20
- package/cli/portability.mjs +5 -4
- package/cli/server-registry.mjs +43 -18
- package/cli/server-resources.mjs +38 -9
- package/global/CHANGELOG.md +8 -0
- package/global/README.md +7 -2
- package/global/launch/state.mjs +5 -4
- package/global/launch/stop.mjs +30 -8
- package/global/main.mjs +2 -1
- package/modules/job-queue/command.mjs +8 -3
- package/modules/portability/README.md +40 -11
- package/modules/portability/backfills.mjs +29 -16
- package/modules/portability/bridge.mjs +24 -29
- package/modules/portability/import.mjs +1 -2
- package/modules/portability/index.d.mts +1 -1
- package/modules/portability/ledgers.mjs +22 -0
- package/modules/portability/migration-history.mjs +2 -2
- package/modules/release/README.md +135 -3
- package/modules/release/catalogue.mjs +6 -2
- package/modules/release/listing.mjs +64 -0
- package/modules/release/look.mjs +26 -0
- package/modules/release/previous-listing.mjs +44 -0
- package/modules/release/produce.mjs +3 -0
- package/modules/release/screenshots.mjs +51 -0
- package/modules/release/signature.mjs +3 -2
- package/modules/release/source.mjs +19 -6
- package/modules/release/statement.mjs +8 -4
- package/onboarding/process-cleanup.mjs +5 -5
- package/onboarding/process-scope.mjs +1 -1
- package/onboarding/process-table.mjs +6 -4
- package/onboarding/process-tree.mjs +4 -4
- package/onboarding/schema/json-schema.mjs +4 -1
- package/package.json +1 -1
- package/scripts/publish-check/README.md +14 -0
- package/scripts/publish-check/assignment.py +48 -0
- package/scripts/publish-check/credential_assignments.mjs +75 -0
- package/scripts/publish-check/credential_assignments.py +70 -0
- package/scripts/publish-check/credential_literals.mjs +66 -0
- package/scripts/publish-check/credential_syntax.mjs +29 -0
- package/scripts/publish-check/env_values.py +14 -1
- package/scripts/publish-check/history_scan.py +9 -2
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/** Exact credential formats, placeholders and strictly shaped CSS literals for SEC-3. */
|
|
2
|
+
const SECRET_PATTERNS=[
|
|
3
|
+
{kind:'PEM private key',regex:/-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----/g},
|
|
4
|
+
{kind:'AWS access key',regex:/\bAKIA[0-9A-Z]{16}\b/g},
|
|
5
|
+
{kind:'GitHub token',regex:/\bgh[pousr]_[A-Za-z0-9]{36,}\b/g},
|
|
6
|
+
{kind:'GitHub fine-grained token',regex:/\bgithub_pat_[A-Za-z0-9_]{22,}/g},
|
|
7
|
+
{kind:'Anthropic key',regex:/\bsk-ant-[A-Za-z0-9_-]{20,}/g},
|
|
8
|
+
{kind:'OpenAI key',regex:/\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}/g},
|
|
9
|
+
{kind:'Stripe live key',regex:/\b(?:sk|rk)_live_[0-9A-Za-z]{20,}/g},
|
|
10
|
+
{kind:'Slack token',regex:/\bxox[abprs]-[A-Za-z0-9-]{10,}/g},
|
|
11
|
+
{kind:'Google API key',regex:/\bAIza[0-9A-Za-z_-]{35}/g},
|
|
12
|
+
{kind:'SendGrid key',regex:/\bSG\.[\w-]{22}\.[\w-]{43}/g},
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
const placeholderMarkers=['example','placeholder','xxxx','test-only','<','${']
|
|
16
|
+
// A reader's instruction in place of the value: "your" or "replace with" followed by lowercase words, no digit (your-scoped-deploy-token).
|
|
17
|
+
const instruction=/^(?:your|replace[-_]with)(?:[-_][a-z]+)+$/
|
|
18
|
+
// A credential variable's name standing for its value: UPPER_SNAKE letters, no digit, ending in what it holds (MAIL_PROVIDER_API_KEY).
|
|
19
|
+
const variableName=/^[A-Z]+(?:_[A-Z]+)*_(?:SECRET|TOKEN|PASSWORD|PASSWD|KEY)$/
|
|
20
|
+
// Exact credential formats: SEC-1's, plus Stripe test and publishable keys and JWTs, which SEC-3 judges by name.
|
|
21
|
+
const exactFormats=[...SECRET_PATTERNS.map(({regex})=>new RegExp(regex.source)),/\b[spr]k_(?:live|test)_[0-9A-Za-z]{8,}/,/\beyJ[\w-]{8,}\.eyJ[\w-]{8,}\./]
|
|
22
|
+
/** A value carrying an exact credential format is a placeholder only by an explicit marker, and never a fixture, a reference or CSS. */
|
|
23
|
+
export function carriesCredential(value) {return exactFormats.some(format=>format.test(value))}
|
|
24
|
+
/** Values that are documentation, templates, instructions or a credential variable's name rather than credentials. */
|
|
25
|
+
export function isPlaceholder(value) {
|
|
26
|
+
const lower=value.toLowerCase()
|
|
27
|
+
return placeholderMarkers.some(marker=>lower.includes(marker))||!carriesCredential(value)&&(instruction.test(value)||variableName.test(value))
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// CSS that SEC-3 admits is made only of recognisable CSS shapes; anything else in a literal is judged by the normal rules.
|
|
31
|
+
// Identifiers are lowercase letters and single dashes with no digit and numbers have at most four digits either side of the
|
|
32
|
+
// point, so no hex, base64, digit or mixed-case run hides in a name, keyword or number.
|
|
33
|
+
const cssIdent='-{0,2}[a-z]+(?:-[a-z]+)*'
|
|
34
|
+
const cssAtom=`(?:#(?:[0-9a-fA-F]{3,4}|[0-9a-fA-F]{6}|[0-9a-fA-F]{8})|[+-]?(?:\\d{1,4}(?:\\.\\d{1,4})?|\\.\\d{1,4})(?:%|px|r?em|ex|ch|[dls]?v[hw]|vmin|vmax|cq[ihw]|fr|deg|rad|turn|m?s|pt|dppx)?|${cssIdent}|"[A-Za-z ]+"|'[A-Za-z ]+'|\x01)`
|
|
35
|
+
const cssValue=new RegExp(`^${cssAtom}(?:[\\s,/]+${cssAtom})*(?:!important)?$`)
|
|
36
|
+
const cssArguments=new RegExp(`^${cssAtom}(?:(?:[\\s,/*]|\\s[+-]\\s)+${cssAtom})*$`)
|
|
37
|
+
const cssFunction=/\b(?:var|calc|min|max|clamp|rgba?|hsla?)\(([^()]*)\)/g
|
|
38
|
+
// A compound selector: an optional element, then classes, ids, pseudo-classes and attributes, each with its own sigil.
|
|
39
|
+
const cssSubSelector=`(?:[.#]${cssIdent}|::?${cssIdent}|\\[${cssIdent}(?:[~|^$*]?=${cssIdent})?\\]|\x02)`
|
|
40
|
+
const cssCompound=`(?:(?:h[1-6]|${cssIdent}|\\*|&)${cssSubSelector}*|${cssSubSelector}+)`
|
|
41
|
+
const cssSelector=new RegExp(`^${cssCompound}(?:[\\s,>+~]+${cssCompound})*$`)
|
|
42
|
+
const cssPseudoFunction=/:(?:where|is|not|has)\(([^()]*)\)/g
|
|
43
|
+
const cssProperty=new RegExp(`^${cssIdent}$`),cssCustomProperty=/^--[a-z]+(?:-[a-z]+)*$/
|
|
44
|
+
const cssDeclarations=(text,custom)=>text.split(';').every(declaration=>{
|
|
45
|
+
if(!declaration)return true
|
|
46
|
+
const colon=declaration.indexOf(':')
|
|
47
|
+
return colon>0&&(custom?cssCustomProperty:cssProperty).test(declaration.slice(0,colon))&&cssValue.test(declaration.slice(colon+1))
|
|
48
|
+
})
|
|
49
|
+
/** A literal written in CSS: custom property declarations (--name:value;…) or selector{declarations} rules, whose every
|
|
50
|
+
* value is a hex colour, a number with unit or %, an identifier, a quoted font name, or var/calc/min/max/clamp/rgb/hsl of those,
|
|
51
|
+
* with at least one value that is not a bare identifier. */
|
|
52
|
+
export function cssLiteral(value) {
|
|
53
|
+
if(!/[:{]/.test(value)||/[\x01\x02]/.test(value)||carriesCredential(value)||!/#[0-9a-fA-F]{3}|\d|\b(?:var|calc|min|max|clamp|rgba?|hsla?)\(/.test(value))return false
|
|
54
|
+
let text=value,previous
|
|
55
|
+
do{
|
|
56
|
+
previous=text
|
|
57
|
+
text=text.replace(cssFunction,(whole,args)=>cssArguments.test(args)?'\x01':whole).replace(cssPseudoFunction,(whole,args)=>cssSelector.test(args)?'\x02':whole)
|
|
58
|
+
}while(text!==previous)
|
|
59
|
+
const parts=text.split('}')
|
|
60
|
+
return parts.every((part,index)=>{
|
|
61
|
+
const open=part.indexOf('{')
|
|
62
|
+
// Before any rule: custom properties only (a continued block or a design-token list). Between rules: nothing.
|
|
63
|
+
if(open<0)return index===0?cssDeclarations(part,true):!part
|
|
64
|
+
return part.indexOf('{',open+1)<0&&cssSelector.test(part.slice(0,open))&&cssDeclarations(part.slice(open+1),false)
|
|
65
|
+
})
|
|
66
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** Lexical context for the SEC-3 source rule. No source values leave this module. */
|
|
2
|
+
/** These source languages use bare names as expressions; configuration files can have unquoted string literals. */
|
|
3
|
+
export const codeSource=(path)=>path===undefined||/\.(?:[cm]?[jt]sx?|py|rs|go|java|kt|cs|cpp|cc|c|h|swift|rb)$/i.test(path)
|
|
4
|
+
|
|
5
|
+
export const sourceTokens=/\/\/[^\n]*|\/\*[\s\S]*?(?:\*\/|$)|\/(?![*/])(?:\\[\s\S]?|\[(?:\\[\s\S]?|[^\]\\])*(?:\]|$)|[^/\\[\n])+(?:\/[a-z]*|$)|"(?:\\[\s\S]?|[^"\\])*(?:"|$)|'(?:\\[\s\S]?|[^'\\])*(?:'|$)|`(?:\\[\s\S]?|[^`\\])*(?:`|$)|[A-Za-z_$][\w$]*|[^\s]/g
|
|
6
|
+
|
|
7
|
+
/** Type declarations are recognized outside comments and strings, so text cannot hide later assignments. */
|
|
8
|
+
export function typeDeclarations(text) {
|
|
9
|
+
if(!/\b(?:interface|type)\s+[A-Za-z_$]/.test(text))return []
|
|
10
|
+
const tokens=[...text.matchAll(sourceTokens)].filter(token=>!token[0].startsWith('//')&&!token[0].startsWith('/*'))
|
|
11
|
+
const ranges=[]
|
|
12
|
+
for(let i=0;i<tokens.length;i++){
|
|
13
|
+
const start=tokens[i],kind=start[0]
|
|
14
|
+
if(!['interface','type'].includes(kind)||!/^\w+$/.test(tokens[i+1]?.[0]??''))continue
|
|
15
|
+
let j=i+2
|
|
16
|
+
while(j<tokens.length&&!['{','=',';',':'].includes(tokens[j][0]))j++
|
|
17
|
+
if(tokens[j]?.[0]!== (kind==='interface'?'{':'='))continue
|
|
18
|
+
let depth=kind==='interface'?1:0,end=tokens[j].index +1
|
|
19
|
+
for(j++;j<tokens.length;j++){
|
|
20
|
+
const token=tokens[j],value=token[0]
|
|
21
|
+
if(depth===0&&(value===';'||text.slice(end,token.index).includes('\n')&&!['|','&','{'].includes(value)))break
|
|
22
|
+
if(['{','(','['].includes(value))depth++
|
|
23
|
+
if(['}',')',']'].includes(value)&&--depth===0){end=token.index +1;if(kind==='interface')break;continue}
|
|
24
|
+
end=token.index +value.length
|
|
25
|
+
}
|
|
26
|
+
ranges.push({start:start.index,end,continuation:j===tokens.length?(kind==='interface'?'interface ScanContext ':'type ScanContext = ')+ '{'.repeat(Math.max(0,depth)):''})
|
|
27
|
+
}
|
|
28
|
+
return ranges
|
|
29
|
+
}
|
|
@@ -1,9 +1,22 @@
|
|
|
1
1
|
"""Read dotenv values into in-memory digests; never retain or report plaintext."""
|
|
2
|
+
from collections import Counter
|
|
2
3
|
import hashlib
|
|
4
|
+
from math import log2
|
|
3
5
|
import re
|
|
4
6
|
from pathlib import Path
|
|
5
7
|
|
|
6
8
|
|
|
9
|
+
TOKEN_PREFIXES = ('sk_', 'sk-', 'pk_live', 'ghp_', 'github_pat_', 'xox', 'AKIA', 'SG.', 'eyJ')
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def secret_shaped(value: str) -> bool:
|
|
13
|
+
if len(value) >= 16 or value.startswith(TOKEN_PREFIXES):
|
|
14
|
+
return True
|
|
15
|
+
# Gitleaks 8.30.1's inherited generic-api-key rule uses 3.5 bits/character.
|
|
16
|
+
frequencies = (count / len(value) for count in Counter(value).values())
|
|
17
|
+
return -sum(p * log2(p) for p in frequencies) > 3.5
|
|
18
|
+
|
|
19
|
+
|
|
7
20
|
def digest(value):
|
|
8
21
|
return hashlib.sha256(value).digest()
|
|
9
22
|
|
|
@@ -69,6 +82,6 @@ def value_hashes(text):
|
|
|
69
82
|
end = len(text) if end < 0 else end
|
|
70
83
|
value = re.split(r'\s+#', text[cursor:end], maxsplit=1)[0].strip()
|
|
71
84
|
cursor = end + 1
|
|
72
|
-
if value:
|
|
85
|
+
if value and secret_shaped(value):
|
|
73
86
|
hashes.add(digest(value.encode()))
|
|
74
87
|
return hashes
|
|
@@ -16,6 +16,8 @@ from tree_scan import context, text_rules, working_files, working_regular
|
|
|
16
16
|
from public_hygiene import reference_rules
|
|
17
17
|
from content_identities import ContentIdentities, candidate as identity_candidate
|
|
18
18
|
from checksum_maps import ChecksumMaps
|
|
19
|
+
from credential_assignments import text_rules, Findings
|
|
20
|
+
from assignment import is_assignment
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300, annotation_file=None):
|
|
@@ -137,6 +139,9 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
137
139
|
def read_regular(path, scoped_entries=scoped_entries):
|
|
138
140
|
entry = scoped_entries.get(path)
|
|
139
141
|
return (history.blob(entry['oid']), entry['mode']) if entry and entry['kind'] == 'blob' and entry['mode'] in ('100644', '100755') else None
|
|
142
|
+
held = working_regular(Path(history.repo), path) if mode == 'dir' else read_regular(path)
|
|
143
|
+
if held and not is_assignment(held[0], hit):
|
|
144
|
+
continue
|
|
140
145
|
if identity_scope not in checksums:
|
|
141
146
|
checksums[identity_scope] = timings.call('checksums.prepare', ChecksumMaps,
|
|
142
147
|
working_paths if mode == 'dir' else scoped_entries,
|
|
@@ -150,7 +155,6 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
150
155
|
continue
|
|
151
156
|
else:
|
|
152
157
|
# Identity documents use their own exact structural locations.
|
|
153
|
-
held = working_regular(Path(history.repo), path) if mode == 'dir' else read_regular(path)
|
|
154
158
|
if held and checksums[identity_scope].matches(held[0], hit):
|
|
155
159
|
continue
|
|
156
160
|
add(secret_rule(hit['RuleID']), path, commit, oid, scope, scanner_rule=hit['RuleID'])
|
|
@@ -181,7 +185,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
181
185
|
length = (data + b'\n').count(b'\n')
|
|
182
186
|
lines.append((count, count + length, path, commit, scope))
|
|
183
187
|
count += length
|
|
184
|
-
|
|
188
|
+
message_data = b''.join(messages)
|
|
189
|
+
for hit in timings.call('gitleaks.messages', secrets, binary, 'stdin', data=message_data, timeout=timeout, redact=False):
|
|
190
|
+
if not is_assignment(message_data, hit):
|
|
191
|
+
continue
|
|
185
192
|
line = hit.get('StartLine', 0)
|
|
186
193
|
location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
|
|
187
194
|
add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
|