@runeditrun/rer 0.2.75 → 0.2.76

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +7 -2
  2. package/cli/listing.mjs +20 -0
  3. package/cli/main.mjs +1 -0
  4. package/cli/parse.mjs +2 -1
  5. package/cli/portability-migrations.mjs +46 -0
  6. package/cli/portability-worker.mjs +7 -20
  7. package/cli/portability.mjs +5 -4
  8. package/cli/server-registry.mjs +43 -18
  9. package/cli/server-resources.mjs +38 -9
  10. package/global/CHANGELOG.md +8 -0
  11. package/global/README.md +7 -2
  12. package/global/launch/state.mjs +5 -4
  13. package/global/launch/stop.mjs +30 -8
  14. package/global/main.mjs +2 -1
  15. package/modules/job-queue/command.mjs +8 -3
  16. package/modules/portability/README.md +40 -11
  17. package/modules/portability/backfills.mjs +29 -16
  18. package/modules/portability/bridge.mjs +24 -29
  19. package/modules/portability/import.mjs +1 -2
  20. package/modules/portability/index.d.mts +1 -1
  21. package/modules/portability/ledgers.mjs +22 -0
  22. package/modules/portability/migration-history.mjs +2 -2
  23. package/modules/release/README.md +135 -3
  24. package/modules/release/catalogue.mjs +6 -2
  25. package/modules/release/listing.mjs +64 -0
  26. package/modules/release/look.mjs +26 -0
  27. package/modules/release/previous-listing.mjs +44 -0
  28. package/modules/release/produce.mjs +3 -0
  29. package/modules/release/screenshots.mjs +51 -0
  30. package/modules/release/signature.mjs +3 -2
  31. package/modules/release/source.mjs +19 -6
  32. package/modules/release/statement.mjs +8 -4
  33. package/onboarding/process-cleanup.mjs +5 -5
  34. package/onboarding/process-scope.mjs +1 -1
  35. package/onboarding/process-table.mjs +6 -4
  36. package/onboarding/process-tree.mjs +4 -4
  37. package/onboarding/schema/json-schema.mjs +4 -1
  38. package/package.json +1 -1
  39. package/scripts/publish-check/README.md +14 -0
  40. package/scripts/publish-check/assignment.py +48 -0
  41. package/scripts/publish-check/credential_assignments.mjs +75 -0
  42. package/scripts/publish-check/credential_assignments.py +70 -0
  43. package/scripts/publish-check/credential_literals.mjs +66 -0
  44. package/scripts/publish-check/credential_syntax.mjs +29 -0
  45. package/scripts/publish-check/env_values.py +14 -1
  46. package/scripts/publish-check/history_scan.py +9 -2
@@ -0,0 +1,66 @@
1
+ /** Exact credential formats, placeholders and strictly shaped CSS literals for SEC-3. */
2
+ const SECRET_PATTERNS=[
3
+ {kind:'PEM private key',regex:/-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----/g},
4
+ {kind:'AWS access key',regex:/\bAKIA[0-9A-Z]{16}\b/g},
5
+ {kind:'GitHub token',regex:/\bgh[pousr]_[A-Za-z0-9]{36,}\b/g},
6
+ {kind:'GitHub fine-grained token',regex:/\bgithub_pat_[A-Za-z0-9_]{22,}/g},
7
+ {kind:'Anthropic key',regex:/\bsk-ant-[A-Za-z0-9_-]{20,}/g},
8
+ {kind:'OpenAI key',regex:/\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}/g},
9
+ {kind:'Stripe live key',regex:/\b(?:sk|rk)_live_[0-9A-Za-z]{20,}/g},
10
+ {kind:'Slack token',regex:/\bxox[abprs]-[A-Za-z0-9-]{10,}/g},
11
+ {kind:'Google API key',regex:/\bAIza[0-9A-Za-z_-]{35}/g},
12
+ {kind:'SendGrid key',regex:/\bSG\.[\w-]{22}\.[\w-]{43}/g},
13
+ ]
14
+
15
+ const placeholderMarkers=['example','placeholder','xxxx','test-only','<','${']
16
+ // A reader's instruction in place of the value: "your" or "replace with" followed by lowercase words, no digit (your-scoped-deploy-token).
17
+ const instruction=/^(?:your|replace[-_]with)(?:[-_][a-z]+)+$/
18
+ // A credential variable's name standing for its value: UPPER_SNAKE letters, no digit, ending in what it holds (MAIL_PROVIDER_API_KEY).
19
+ const variableName=/^[A-Z]+(?:_[A-Z]+)*_(?:SECRET|TOKEN|PASSWORD|PASSWD|KEY)$/
20
+ // Exact credential formats: SEC-1's, plus Stripe test and publishable keys and JWTs, which SEC-3 judges by name.
21
+ const exactFormats=[...SECRET_PATTERNS.map(({regex})=>new RegExp(regex.source)),/\b[spr]k_(?:live|test)_[0-9A-Za-z]{8,}/,/\beyJ[\w-]{8,}\.eyJ[\w-]{8,}\./]
22
+ /** A value carrying an exact credential format is a placeholder only by an explicit marker, and never a fixture, a reference or CSS. */
23
+ export function carriesCredential(value) {return exactFormats.some(format=>format.test(value))}
24
+ /** Values that are documentation, templates, instructions or a credential variable's name rather than credentials. */
25
+ export function isPlaceholder(value) {
26
+ const lower=value.toLowerCase()
27
+ return placeholderMarkers.some(marker=>lower.includes(marker))||!carriesCredential(value)&&(instruction.test(value)||variableName.test(value))
28
+ }
29
+
30
+ // CSS that SEC-3 admits is made only of recognisable CSS shapes; anything else in a literal is judged by the normal rules.
31
+ // Identifiers are lowercase letters and single dashes with no digit and numbers have at most four digits either side of the
32
+ // point, so no hex, base64, digit or mixed-case run hides in a name, keyword or number.
33
+ const cssIdent='-{0,2}[a-z]+(?:-[a-z]+)*'
34
+ const cssAtom=`(?:#(?:[0-9a-fA-F]{3,4}|[0-9a-fA-F]{6}|[0-9a-fA-F]{8})|[+-]?(?:\\d{1,4}(?:\\.\\d{1,4})?|\\.\\d{1,4})(?:%|px|r?em|ex|ch|[dls]?v[hw]|vmin|vmax|cq[ihw]|fr|deg|rad|turn|m?s|pt|dppx)?|${cssIdent}|"[A-Za-z ]+"|'[A-Za-z ]+'|\x01)`
35
+ const cssValue=new RegExp(`^${cssAtom}(?:[\\s,/]+${cssAtom})*(?:!important)?$`)
36
+ const cssArguments=new RegExp(`^${cssAtom}(?:(?:[\\s,/*]|\\s[+-]\\s)+${cssAtom})*$`)
37
+ const cssFunction=/\b(?:var|calc|min|max|clamp|rgba?|hsla?)\(([^()]*)\)/g
38
+ // A compound selector: an optional element, then classes, ids, pseudo-classes and attributes, each with its own sigil.
39
+ const cssSubSelector=`(?:[.#]${cssIdent}|::?${cssIdent}|\\[${cssIdent}(?:[~|^$*]?=${cssIdent})?\\]|\x02)`
40
+ const cssCompound=`(?:(?:h[1-6]|${cssIdent}|\\*|&)${cssSubSelector}*|${cssSubSelector}+)`
41
+ const cssSelector=new RegExp(`^${cssCompound}(?:[\\s,>+~]+${cssCompound})*$`)
42
+ const cssPseudoFunction=/:(?:where|is|not|has)\(([^()]*)\)/g
43
+ const cssProperty=new RegExp(`^${cssIdent}$`),cssCustomProperty=/^--[a-z]+(?:-[a-z]+)*$/
44
+ const cssDeclarations=(text,custom)=>text.split(';').every(declaration=>{
45
+ if(!declaration)return true
46
+ const colon=declaration.indexOf(':')
47
+ return colon>0&&(custom?cssCustomProperty:cssProperty).test(declaration.slice(0,colon))&&cssValue.test(declaration.slice(colon+1))
48
+ })
49
+ /** A literal written in CSS: custom property declarations (--name:value;…) or selector{declarations} rules, whose every
50
+ * value is a hex colour, a number with unit or %, an identifier, a quoted font name, or var/calc/min/max/clamp/rgb/hsl of those,
51
+ * with at least one value that is not a bare identifier. */
52
+ export function cssLiteral(value) {
53
+ if(!/[:{]/.test(value)||/[\x01\x02]/.test(value)||carriesCredential(value)||!/#[0-9a-fA-F]{3}|\d|\b(?:var|calc|min|max|clamp|rgba?|hsla?)\(/.test(value))return false
54
+ let text=value,previous
55
+ do{
56
+ previous=text
57
+ text=text.replace(cssFunction,(whole,args)=>cssArguments.test(args)?'\x01':whole).replace(cssPseudoFunction,(whole,args)=>cssSelector.test(args)?'\x02':whole)
58
+ }while(text!==previous)
59
+ const parts=text.split('}')
60
+ return parts.every((part,index)=>{
61
+ const open=part.indexOf('{')
62
+ // Before any rule: custom properties only (a continued block or a design-token list). Between rules: nothing.
63
+ if(open<0)return index===0?cssDeclarations(part,true):!part
64
+ return part.indexOf('{',open+1)<0&&cssSelector.test(part.slice(0,open))&&cssDeclarations(part.slice(open+1),false)
65
+ })
66
+ }
@@ -0,0 +1,29 @@
1
+ /** Lexical context for the SEC-3 source rule. No source values leave this module. */
2
+ /** These source languages use bare names as expressions; configuration files can have unquoted string literals. */
3
+ export const codeSource=(path)=>path===undefined||/\.(?:[cm]?[jt]sx?|py|rs|go|java|kt|cs|cpp|cc|c|h|swift|rb)$/i.test(path)
4
+
5
+ export const sourceTokens=/\/\/[^\n]*|\/\*[\s\S]*?(?:\*\/|$)|\/(?![*/])(?:\\[\s\S]?|\[(?:\\[\s\S]?|[^\]\\])*(?:\]|$)|[^/\\[\n])+(?:\/[a-z]*|$)|"(?:\\[\s\S]?|[^"\\])*(?:"|$)|'(?:\\[\s\S]?|[^'\\])*(?:'|$)|`(?:\\[\s\S]?|[^`\\])*(?:`|$)|[A-Za-z_$][\w$]*|[^\s]/g
6
+
7
+ /** Type declarations are recognized outside comments and strings, so text cannot hide later assignments. */
8
+ export function typeDeclarations(text) {
9
+ if(!/\b(?:interface|type)\s+[A-Za-z_$]/.test(text))return []
10
+ const tokens=[...text.matchAll(sourceTokens)].filter(token=>!token[0].startsWith('//')&&!token[0].startsWith('/*'))
11
+ const ranges=[]
12
+ for(let i=0;i<tokens.length;i++){
13
+ const start=tokens[i],kind=start[0]
14
+ if(!['interface','type'].includes(kind)||!/^\w+$/.test(tokens[i+1]?.[0]??''))continue
15
+ let j=i+2
16
+ while(j<tokens.length&&!['{','=',';',':'].includes(tokens[j][0]))j++
17
+ if(tokens[j]?.[0]!== (kind==='interface'?'{':'='))continue
18
+ let depth=kind==='interface'?1:0,end=tokens[j].index +1
19
+ for(j++;j<tokens.length;j++){
20
+ const token=tokens[j],value=token[0]
21
+ if(depth===0&&(value===';'||text.slice(end,token.index).includes('\n')&&!['|','&','{'].includes(value)))break
22
+ if(['{','(','['].includes(value))depth++
23
+ if(['}',')',']'].includes(value)&&--depth===0){end=token.index +1;if(kind==='interface')break;continue}
24
+ end=token.index +value.length
25
+ }
26
+ ranges.push({start:start.index,end,continuation:j===tokens.length?(kind==='interface'?'interface ScanContext ':'type ScanContext = ')+ '{'.repeat(Math.max(0,depth)):''})
27
+ }
28
+ return ranges
29
+ }
@@ -1,9 +1,22 @@
1
1
  """Read dotenv values into in-memory digests; never retain or report plaintext."""
2
+ from collections import Counter
2
3
  import hashlib
4
+ from math import log2
3
5
  import re
4
6
  from pathlib import Path
5
7
 
6
8
 
9
+ TOKEN_PREFIXES = ('sk_', 'sk-', 'pk_live', 'ghp_', 'github_pat_', 'xox', 'AKIA', 'SG.', 'eyJ')
10
+
11
+
12
+ def secret_shaped(value: str) -> bool:
13
+ if len(value) >= 16 or value.startswith(TOKEN_PREFIXES):
14
+ return True
15
+ # Gitleaks 8.30.1's inherited generic-api-key rule uses 3.5 bits/character.
16
+ frequencies = (count / len(value) for count in Counter(value).values())
17
+ return -sum(p * log2(p) for p in frequencies) > 3.5
18
+
19
+
7
20
  def digest(value):
8
21
  return hashlib.sha256(value).digest()
9
22
 
@@ -69,6 +82,6 @@ def value_hashes(text):
69
82
  end = len(text) if end < 0 else end
70
83
  value = re.split(r'\s+#', text[cursor:end], maxsplit=1)[0].strip()
71
84
  cursor = end + 1
72
- if value:
85
+ if value and secret_shaped(value):
73
86
  hashes.add(digest(value.encode()))
74
87
  return hashes
@@ -16,6 +16,8 @@ from tree_scan import context, text_rules, working_files, working_regular
16
16
  from public_hygiene import reference_rules
17
17
  from content_identities import ContentIdentities, candidate as identity_candidate
18
18
  from checksum_maps import ChecksumMaps
19
+ from credential_assignments import text_rules, Findings
20
+ from assignment import is_assignment
19
21
 
20
22
 
21
23
  def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, repository=None, *, cache_dir=None, timeout=300, annotation_file=None):
@@ -137,6 +139,9 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
137
139
  def read_regular(path, scoped_entries=scoped_entries):
138
140
  entry = scoped_entries.get(path)
139
141
  return (history.blob(entry['oid']), entry['mode']) if entry and entry['kind'] == 'blob' and entry['mode'] in ('100644', '100755') else None
142
+ held = working_regular(Path(history.repo), path) if mode == 'dir' else read_regular(path)
143
+ if held and not is_assignment(held[0], hit):
144
+ continue
140
145
  if identity_scope not in checksums:
141
146
  checksums[identity_scope] = timings.call('checksums.prepare', ChecksumMaps,
142
147
  working_paths if mode == 'dir' else scoped_entries,
@@ -150,7 +155,6 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
150
155
  continue
151
156
  else:
152
157
  # Identity documents use their own exact structural locations.
153
- held = working_regular(Path(history.repo), path) if mode == 'dir' else read_regular(path)
154
158
  if held and checksums[identity_scope].matches(held[0], hit):
155
159
  continue
156
160
  add(secret_rule(hit['RuleID']), path, commit, oid, scope, scanner_rule=hit['RuleID'])
@@ -181,7 +185,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
181
185
  length = (data + b'\n').count(b'\n')
182
186
  lines.append((count, count + length, path, commit, scope))
183
187
  count += length
184
- for hit in timings.call('gitleaks.messages', secrets, binary, 'stdin', data=b''.join(messages), timeout=timeout):
188
+ message_data = b''.join(messages)
189
+ for hit in timings.call('gitleaks.messages', secrets, binary, 'stdin', data=message_data, timeout=timeout, redact=False):
190
+ if not is_assignment(message_data, hit):
191
+ continue
185
192
  line = hit.get('StartLine', 0)
186
193
  location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
187
194
  add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])