@joe-broadhead/opencode-mem 0.9.5 → 0.9.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1281 -0
- package/README.md +105 -31
- package/docs/agent-protocol.md +28 -27
- package/docs/permissions.md +148 -0
- package/package.json +24 -12
- package/skills/mem.pages/SKILL.md +35 -16
- package/skills/mem.sharing/SKILL.md +39 -6
- package/skills/mem.sources/SKILL.md +41 -10
- package/skills/mem.usage/SKILL.md +37 -16
- package/src/backend/records.js +8 -0
- package/src/backend/v3.js +15 -10
- package/src/call-shapes.js +18 -25
- package/src/capture-pause.js +136 -0
- package/src/capture.js +7 -5
- package/src/collector-service.js +212 -35
- package/src/collector.js +186 -47
- package/src/commands.js +37 -0
- package/src/compaction.js +4 -2
- package/src/config.js +151 -1
- package/src/curation.js +6 -10
- package/src/events.js +144 -36
- package/src/forget-purge.js +377 -0
- package/src/forgotten.js +152 -0
- package/src/hooks.js +9 -4
- package/src/index.js +5 -3
- package/src/permissions.js +116 -38
- package/src/protocol.js +21 -11
- package/src/records.js +4 -1
- package/src/runtime.js +202 -15
- package/src/shapes.js +63 -0
- package/src/sources.js +183 -12
- package/src/spaces.js +58 -1
- package/src/spool.js +442 -64
- package/src/stated.js +334 -0
- package/src/storable.js +477 -0
- package/src/tools/aliases.js +36 -147
- package/src/tools/commitments.js +69 -12
- package/src/tools/index.js +25 -43
- package/src/tools/ingest.js +33 -128
- package/src/tools/memory.js +503 -904
- package/src/tools/page-sections.js +147 -0
- package/src/tools/page.js +314 -82
- package/src/tools/recall-browse.js +173 -0
- package/src/tools/recall.js +36 -8
- package/src/tools/share.js +494 -131
- package/src/tools/shared.js +97 -19
- package/src/tools/spaces.js +194 -101
- package/src/tools/status.js +31 -3
- package/src/tools/wiki.js +5 -13
- package/src/util.js +10 -59
- package/src/views/profile.js +2 -3
- package/src/views/records-reads.js +6 -1
- package/src/erase.js +0 -36
package/src/storable.js
ADDED
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What may be stored, decided once for every write path (owner decision D3,
|
|
3
|
+
* revised as D3' in 0.9.9).
|
|
4
|
+
*
|
|
5
|
+
* - **Secrets are refused everywhere**: API keys, tokens, private keys,
|
|
6
|
+
* passwords in assignments, connection strings with credentials.
|
|
7
|
+
* - **A person's email address** is refused on an explicit write
|
|
8
|
+
* (`mem_remember`, `mem_revise`, `mem_page`, `mem_commitments`, records
|
|
9
|
+
* writes), as 0.9.5 refused it; **a home-directory path** is refused in a
|
|
10
|
+
* memory and stored with a warning on a page or a commitment
|
|
11
|
+
* (`EXPLICIT_POLICY`). An automatic write (turn capture, the host's
|
|
12
|
+
* compaction summary, the compaction checkpoint, routines) masks both.
|
|
13
|
+
*
|
|
14
|
+
* An explicit write that holds a secret is refused whole: the person or agent
|
|
15
|
+
* can say it again without the value. An automatic write has no one to ask, so
|
|
16
|
+
* each line that holds a secret is withheld (a private key block whole) and the
|
|
17
|
+
* rest is kept, masked.
|
|
18
|
+
*
|
|
19
|
+
* The plugin (src/storable.js) and the records service (records/src/storable.js)
|
|
20
|
+
* carry this file byte for byte: `npm run check:copies` fails on any drift. It
|
|
21
|
+
* imports nothing, so either side can load it.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** The write paths and how each treats an email or a local path. */
|
|
25
|
+
export const STORABLE_PATHS = Object.freeze({
|
|
26
|
+
remember: "explicit",
|
|
27
|
+
page: "explicit",
|
|
28
|
+
commitment: "explicit",
|
|
29
|
+
import: "explicit",
|
|
30
|
+
capture: "automatic",
|
|
31
|
+
summary: "automatic",
|
|
32
|
+
checkpoint: "automatic",
|
|
33
|
+
routine: "automatic",
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* What an explicit write does with a person's email address and a
|
|
38
|
+
* home-directory path (owner decision D3', 0.9.9). Emails are refused on every
|
|
39
|
+
* explicit write a person or agent makes, as 0.9.5 refused them in memories.
|
|
40
|
+
* A role address (`support@`, `oncall@`, …) is a service, not a person: it is
|
|
41
|
+
* refused in a memory and stored with a warning on a page or a commitment.
|
|
42
|
+
* Paths are refused in memories, as in 0.9.5, and stored with a warning on a
|
|
43
|
+
* page or a commitment, which 0.9.5 never checked. A collected event (import)
|
|
44
|
+
* is the person's own source text: it keeps all of them, and a routine that
|
|
45
|
+
* writes from it masks them.
|
|
46
|
+
*/
|
|
47
|
+
export const EXPLICIT_POLICY = Object.freeze({
|
|
48
|
+
remember: Object.freeze({ email: "refuse", role: "refuse", path: "refuse" }),
|
|
49
|
+
page: Object.freeze({ email: "refuse", role: "warn", path: "warn" }),
|
|
50
|
+
commitment: Object.freeze({ email: "refuse", role: "warn", path: "warn" }),
|
|
51
|
+
import: Object.freeze({ email: "warn", role: "warn", path: "warn" }),
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
/** The refusal messages of an explicit write, by reason; the agent relays them. */
|
|
55
|
+
export const REFUSALS = Object.freeze({
|
|
56
|
+
email:
|
|
57
|
+
"Nothing was stored: Memory doesn't store email addresses. An email address is personal data, not a durable fact. Keep the decision, and link the person's page (such as [[people/ama]]) instead of the address.",
|
|
58
|
+
local_path: "not stored: an absolute local path is repo state, not a fact. It is wrong on every other machine.",
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
/** What an automatic write keeps in place of a line that held a secret. */
|
|
62
|
+
export const WITHHELD_SECRET = "[not stored: a credential]"
|
|
63
|
+
export const EMAIL_MASK = "<email>"
|
|
64
|
+
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
// Redaction for logs and ask excerpts (not storage). The local-path rules are
|
|
67
|
+
// shared with the derived core (opencode-mem-deploy build/core/memory-redact.ts)
|
|
68
|
+
// through test/fixtures/redaction-vectors.json, so keep their output stable.
|
|
69
|
+
|
|
70
|
+
const SECRET_PATTERNS = Object.freeze([
|
|
71
|
+
// Authorization-style headers consume the whole value, including a scheme.
|
|
72
|
+
[/\b(authorization|x-api-key)\b["']?(\s*[:=]\s*)[^\r\n,;]+/gi, "$1$2[redacted]"],
|
|
73
|
+
// Token assignments, passwords, and API keys. The value may be quoted.
|
|
74
|
+
[
|
|
75
|
+
/\b(api[_-]?key|access[_-]?token|refresh[_-]?token|token|password|passwd|secret)\b["']?(\s*[:=]\s*)(?:"[^"\n]*"|'[^'\n]*'|[^\s,;]+)/gi,
|
|
76
|
+
"$1$2[redacted]",
|
|
77
|
+
],
|
|
78
|
+
[/\bBearer\s+[A-Za-z0-9._~+/=-]{3,}/gi, "Bearer [redacted]"],
|
|
79
|
+
[/\bsk-[A-Za-z0-9_-]{6,}/g, "[redacted-key]"],
|
|
80
|
+
[/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{8,}\b/g, "[redacted-token]"],
|
|
81
|
+
])
|
|
82
|
+
|
|
83
|
+
/** Redact values that look like credentials. Used on every text this plugin logs or shows in an ask. */
|
|
84
|
+
export function redactSecrets(value) {
|
|
85
|
+
let text = String(value ?? "")
|
|
86
|
+
for (const [pattern, replacement] of SECRET_PATTERNS) text = text.replace(pattern, replacement)
|
|
87
|
+
return text
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Absolute local paths: they name the user and the machine, and are wrong on
|
|
91
|
+
// every other one.
|
|
92
|
+
const PATH_SEGMENT = String.raw`[^\s/\\"'\x60<>|()\[\]{},;]+`
|
|
93
|
+
const LOCAL_PATHS = Object.freeze([
|
|
94
|
+
// POSIX: home and machine-local roots, optionally as a file:// url.
|
|
95
|
+
new RegExp(
|
|
96
|
+
String.raw`(?<![\w.~/\\-])(?:file://)?/(?:Users|home|root|private|var|tmp|Volumes|mnt|media|srv)(?:/${PATH_SEGMENT})+/?`,
|
|
97
|
+
"g",
|
|
98
|
+
),
|
|
99
|
+
// The home directory as ~.
|
|
100
|
+
new RegExp(String.raw`(?<![\w.~/\\-])~(?:/${PATH_SEGMENT})+/?`, "g"),
|
|
101
|
+
// Windows: a drive or a UNC share, with \ or / (or JSON's \\) between parts.
|
|
102
|
+
new RegExp(String.raw`(?<![\w.~/\\-])(?:[A-Za-z]:|\\\\${PATH_SEGMENT})(?:[\\/]+${PATH_SEGMENT})+[\\/]*`, "g"),
|
|
103
|
+
])
|
|
104
|
+
|
|
105
|
+
/** `<path>/<last part>`, or `<path>` when the last part is a user's name. */
|
|
106
|
+
function neutralPath(match) {
|
|
107
|
+
const trail = /[.,:;!?]+$/.exec(match)?.[0] ?? ""
|
|
108
|
+
const body = match.slice(0, match.length - trail.length).replace(/^file:\/\//, "")
|
|
109
|
+
const parts = body.split(/[\\/]+/).filter(Boolean)
|
|
110
|
+
const last = parts.at(-1)
|
|
111
|
+
const before = parts.at(-2)
|
|
112
|
+
const named = parts.length >= 2 && last && !/^(?:users|home)$/i.test(before ?? "") && !/^[A-Za-z]:$/.test(last)
|
|
113
|
+
return `${named ? `<path>/${last}` : "<path>"}${trail}`
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Replace absolute local paths (macOS, Linux, Windows, `~`) with a neutral
|
|
118
|
+
* placeholder that keeps the file name: a macOS home path to `app.ts` becomes
|
|
119
|
+
* `<path>/app.ts`.
|
|
120
|
+
*/
|
|
121
|
+
export function redactLocalPaths(value) {
|
|
122
|
+
let text = String(value ?? "")
|
|
123
|
+
for (const pattern of LOCAL_PATHS) text = text.replace(pattern, neutralPath)
|
|
124
|
+
return text
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
// Detection for storage.
|
|
129
|
+
|
|
130
|
+
/** An email-shaped address: local part, then host. `emailKind` says whether it is a person's. */
|
|
131
|
+
const EMAIL = /(?<![\w.+-])([A-Za-z0-9._%+-]+)@([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,})(?![\w-])/g
|
|
132
|
+
|
|
133
|
+
/** Senders and system users: not a person (`git@`, `notifications@`, `ec2-user@…`, `deploy@…`). */
|
|
134
|
+
const NOT_PERSONAL_LOCAL =
|
|
135
|
+
/^(?:git|hg|svn|no-?reply(?:[+._-].*)?|do-?not-?reply|notifications?|root|ubuntu|ec2-user|centos|debian|fedora|azureuser|opc|vagrant|ansible|deploy|deployer|jenkins|postgres|www-data)$/i
|
|
136
|
+
/** Hosts that are machines, not mailboxes: internal names, cloud instances, GitHub's no-reply users. */
|
|
137
|
+
const NOT_PERSONAL_HOST =
|
|
138
|
+
/(?:^|\.)(?:internal|local|lan|localdomain|localhost)$|\.amazonaws\.com$|\.cloudapp\.azure\.com$|\.googleusercontent\.com$|(?:^|\.)noreply\.[a-z0-9.-]+$/i
|
|
139
|
+
/** An ssh, scp, sftp, rsync or mosh command on this line, with only options and paths between it and the address. */
|
|
140
|
+
const SSH_BEFORE = /\b(?:ssh|scp|sftp|rsync|mosh|ssh-copy-id)\s+(?:[-\w.\/~:=]+\s+)*$/i
|
|
141
|
+
/** A URL's userinfo: a user before the host, or a user and a password before it. */
|
|
142
|
+
const URL_USERINFO_BEFORE = /[a-z][a-z0-9+.-]*:\/\/(?:[^\s/@:]*:)?$/i
|
|
143
|
+
/** The scp and git remote form `user@host:path`. */
|
|
144
|
+
const REMOTE_AFTER = /^:(?:[/~.]|[\w.-]+\/)/
|
|
145
|
+
/** Role addresses: a service or a team, not a person. */
|
|
146
|
+
const ROLE_LOCAL = /^(?:support|security|oncall|ops|abuse|noc|help|info|admin|billing|privacy)$/i
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* What an email-shaped match is: `person`, `role` (a fixed list: `support@`,
|
|
150
|
+
* `oncall@`, …), or `none` (ssh and cloud user-at-host, URL userinfo, remote
|
|
151
|
+
* `user@host:path`, service and no-reply senders).
|
|
152
|
+
*/
|
|
153
|
+
function emailKind(match, text) {
|
|
154
|
+
const [whole, local, host] = match
|
|
155
|
+
const before = text.slice(0, match.index)
|
|
156
|
+
const lineBefore = before.slice(before.lastIndexOf("\n") + 1)
|
|
157
|
+
if (URL_USERINFO_BEFORE.test(before)) return "none"
|
|
158
|
+
if (SSH_BEFORE.test(lineBefore)) return "none"
|
|
159
|
+
if (REMOTE_AFTER.test(text.slice(match.index + whole.length, match.index + whole.length + 40))) return "none"
|
|
160
|
+
if (NOT_PERSONAL_LOCAL.test(local) || NOT_PERSONAL_HOST.test(host)) return "none"
|
|
161
|
+
return ROLE_LOCAL.test(local) ? "role" : "person"
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* A home directory: `/Users/<name>…`, `/home/<user>/…`, `~/…`, `C:\Users\…`.
|
|
166
|
+
* Not `/var`, `/tmp`, `/srv`, `/mnt` or an API route: they name no person.
|
|
167
|
+
*/
|
|
168
|
+
const HOME_PATHS = Object.freeze([
|
|
169
|
+
new RegExp(String.raw`(?<![\w.~/\\-])(?:file://)?/(?:Users(?:/${PATH_SEGMENT})+|home/${PATH_SEGMENT}(?:/${PATH_SEGMENT})+)/?`, "g"),
|
|
170
|
+
new RegExp(String.raw`(?<![\w.~/\\-])~(?:/${PATH_SEGMENT})+/?`, "g"),
|
|
171
|
+
new RegExp(String.raw`(?<![\w.~/\\-])[A-Za-z]:[\\/]+Users(?:[\\/]+${PATH_SEGMENT})+[\\/]*`, "gi"),
|
|
172
|
+
])
|
|
173
|
+
|
|
174
|
+
/** A private key block, from its BEGIN line to its END line (or the end of the text). */
|
|
175
|
+
const PRIVATE_KEY_BLOCK = /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----|$)/g
|
|
176
|
+
|
|
177
|
+
/** Well-known credential shapes, found anywhere. Each match is checked for documentation. */
|
|
178
|
+
const SHAPED = Object.freeze([
|
|
179
|
+
// At a word start, with a key-like tail: 20 or more key characters, letters and digits.
|
|
180
|
+
["api key", /(?<![A-Za-z0-9_-])sk-[A-Za-z0-9_-]{20,}/g, (match) => /\d/.test(match.slice(3)) && /[A-Za-z]/.test(match.slice(3))],
|
|
181
|
+
["api key", /\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}/g],
|
|
182
|
+
["api key", /\bAIza[0-9A-Za-z_-]{30,}/g],
|
|
183
|
+
["api key", /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g],
|
|
184
|
+
["api key", /\b(?:AccountKey|SharedAccessKey)=[A-Za-z0-9+/]{20,}={0,2}/g],
|
|
185
|
+
["token", /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{8,}/g],
|
|
186
|
+
["token", /\bgithub_pat_[A-Za-z0-9_]{20,}/g],
|
|
187
|
+
["token", /\bglpat-[A-Za-z0-9_-]{16,}/g],
|
|
188
|
+
["token", /\bxox[bpars]-[A-Za-z0-9-]{10,}/g],
|
|
189
|
+
["token", /\bnpm_[A-Za-z0-9]{30,}/g],
|
|
190
|
+
["token", /\bhf_[A-Za-z0-9]{30,}/g],
|
|
191
|
+
["token", /\bdop_v1_[A-Za-z0-9]{40,}/g],
|
|
192
|
+
["token", /https:\/\/hooks\.slack\.com\/(?:services|workflows|triggers)\/[A-Za-z0-9_/-]{20,}/g],
|
|
193
|
+
["token", /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g],
|
|
194
|
+
])
|
|
195
|
+
|
|
196
|
+
/** `Bearer <token>`, `Authorization: <value>`, `x-api-key: <value>`. */
|
|
197
|
+
const BEARER = /\bBearer\s+([A-Za-z0-9._~+/=-]+)/gi
|
|
198
|
+
const AUTH_HEADER = /\b(?:authorization|x-api-key)\b["']?\s*[:=]\s*(?:"([^"\n]*)"|'([^'\n]*)'|([^\s,;]+(?:[ \t]+[^\s,;]+)?))/gi
|
|
199
|
+
|
|
200
|
+
/** `user:password@host` inside a URL; the password may itself hold `@`. */
|
|
201
|
+
const CONNECTION = /\b[a-z][a-z0-9+.-]*:\/\/([^\s:@/]+):([^\s/]*)@[^\s@/]+/gi
|
|
202
|
+
|
|
203
|
+
const CREDENTIAL_KEYS =
|
|
204
|
+
"api[_-]?key|apikey|access[_-]?key|access[_-]?token|refresh[_-]?token|auth[_-]?token|client[_-]?secret|secret[_-]?key|private[_-]?key|passphrase|password|passwd|pwd|token|secret"
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* An assignment of a credential: `password = hunter2`, `DB_PASSWORD="x"`,
|
|
208
|
+
* `"api_key": "…"`, `password is: x`. The key may carry a prefix
|
|
209
|
+
* (`OPENAI_API_KEY`), and the value may sit on the next line (pretty-printed
|
|
210
|
+
* JSON, YAML). The value is checked by `credentialValue`, so prose
|
|
211
|
+
* ("password: rotated quarterly", "token: 4096") and code
|
|
212
|
+
* (`getToken(session)`) are not secrets.
|
|
213
|
+
*/
|
|
214
|
+
const ASSIGNMENT = new RegExp(
|
|
215
|
+
String.raw`(?<![A-Za-z0-9])([A-Za-z0-9_.-]*?(${CREDENTIAL_KEYS}))["']?((?:[ \t]+(?:is|was))?[ \t]*[:=][ \t]*(?:\r?\n[ \t]*)?)(?:"([^"\n]*)"|'([^'\n]*)'|` + "`([^`\\n]*)`" + String.raw`|([^\s,;&]+))`,
|
|
216
|
+
"gi",
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
const PLACEHOLDER =
|
|
220
|
+
/^(?:\[[^\]]*\]|<[^>]*>|\{\{.*\}\}|\$\{?[A-Za-z_][\w.]*\}?|%[A-Za-z_]\w*%|\*+|x{3,}|\.{2,}|…|-+|true|false|null|nil|none|undefined|string|str|number|int|bool|required|optional|env|secret|password|token|redacted|placeholder|changeme|example|your[_-]?\w*|(?:process\.env|os\.environ|env)\b.*)$/i
|
|
221
|
+
|
|
222
|
+
/** The jwt.io sample token's payload and signature: documentation, not a credential. */
|
|
223
|
+
const JWT_IO_SAMPLE = /eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ|SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c/
|
|
224
|
+
|
|
225
|
+
/** Characters of a credential: no spaces, mostly token alphabet. */
|
|
226
|
+
const TOKENISH = /^[A-Za-z0-9._~+/=!@#$%^&*()\-:]+$/
|
|
227
|
+
|
|
228
|
+
/** A value or a token that is documentation: a placeholder, an example, a template. */
|
|
229
|
+
function documentation(value) {
|
|
230
|
+
const v = String(value ?? "").trim()
|
|
231
|
+
return (
|
|
232
|
+
PLACEHOLDER.test(v) ||
|
|
233
|
+
/example/i.test(v) ||
|
|
234
|
+
/x{4,}/i.test(v) ||
|
|
235
|
+
/<[^>]*>/.test(v) ||
|
|
236
|
+
/\byour[-_][\w-]*here\b/i.test(v) ||
|
|
237
|
+
/\$\{|\{\{/.test(v) ||
|
|
238
|
+
JWT_IO_SAMPLE.test(v) ||
|
|
239
|
+
/[…]|\.{3}$/.test(v)
|
|
240
|
+
)
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** A value that is code, not a literal: a member access, a call, a path, an environment read. */
|
|
244
|
+
function codeValue(value) {
|
|
245
|
+
const v = String(value ?? "").trim()
|
|
246
|
+
return (
|
|
247
|
+
// `session.accessToken`, `config.secretKey`: identifiers, not a random token with digits in it.
|
|
248
|
+
(/^[A-Za-z_$][\w$]*(?:\??\.[A-Za-z_$][\w$]*)+;?$/.test(v) && (v.match(/\d/g) ?? []).length <= 2) ||
|
|
249
|
+
/^(?:await\s+)?[A-Za-z_$][\w$.]*\(/.test(v) ||
|
|
250
|
+
/^(?:~|\.{1,2})?\/[\w./-]*$/.test(v) ||
|
|
251
|
+
/^(?:process\.env|os\.environ|import\.meta\.env)\b/.test(v)
|
|
252
|
+
)
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** Shannon entropy of a string, in bits per character. */
|
|
256
|
+
function entropy(value) {
|
|
257
|
+
const counts = new Map()
|
|
258
|
+
for (const char of value) counts.set(char, (counts.get(char) ?? 0) + 1)
|
|
259
|
+
let bits = 0
|
|
260
|
+
for (const count of counts.values()) {
|
|
261
|
+
const p = count / value.length
|
|
262
|
+
bits -= p * Math.log2(p)
|
|
263
|
+
}
|
|
264
|
+
return bits
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** A bare token: `min` or more token characters, a letter and a digit, not a word or a placeholder. */
|
|
268
|
+
function tokenShaped(value, min = 12) {
|
|
269
|
+
const v = String(value ?? "").trim()
|
|
270
|
+
return v.length >= min && TOKENISH.test(v) && /\d/.test(v) && /[A-Za-z]/.test(v) && entropy(v) >= 3 && !documentation(v) && !codeValue(v)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** Prose that a passphrase does not hold. */
|
|
274
|
+
const PROSE_WORDS = new Set(
|
|
275
|
+
"a an the is are was were be been must should shall will can may not no never always every each at least most of to in on for and or with by from via see use uses used set stored managed rotated rotate required policy characters chars minimum maximum change changed expires expire reset same as one this that it its".split(" "),
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
/** `password: correct horse battery staple`: three to eight plain words, no prose, no full stop. */
|
|
279
|
+
function passphrase(rest) {
|
|
280
|
+
const line = String(rest ?? "").trim()
|
|
281
|
+
if (line.length < 16 || /[.!?]$/.test(line)) return false
|
|
282
|
+
const words = line.split(/\s+/)
|
|
283
|
+
return words.length >= 3 && words.length <= 8 && words.every((word) => /^[A-Za-z][A-Za-z'-]*$/.test(word) && !PROSE_WORDS.has(word.toLowerCase()))
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Whether the value of a credential assignment is one. A quoted value counts
|
|
288
|
+
* from 4 characters (8 for a token or secret); a bare one needs 6 (8) and a
|
|
289
|
+
* digit, a symbol or an inner capital, which a word of prose does not have.
|
|
290
|
+
* A value in a URL query (`?token=…&`) needs 16 characters and the entropy of
|
|
291
|
+
* a random token.
|
|
292
|
+
*/
|
|
293
|
+
function credentialValue(value, quoted, key, query) {
|
|
294
|
+
const v = String(value ?? "").trim()
|
|
295
|
+
if (!v || documentation(v)) return false
|
|
296
|
+
if (query) return tokenShaped(v, 16)
|
|
297
|
+
const loose = /^(?:token|secret)$/i.test(key)
|
|
298
|
+
if (quoted) return v.length >= (loose ? 8 : 4) && !/^\d+(?:\.\d+)?$/.test(v) && !/^[A-Za-z_$][\w$.]*\(|^\$\{/.test(v)
|
|
299
|
+
if (codeValue(v) || !TOKENISH.test(v) || /^\d+(?:\.\d+)?$/.test(v)) return false
|
|
300
|
+
if (v.length < (loose ? 8 : 6)) return false
|
|
301
|
+
return /\d/.test(v) || /[^A-Za-z0-9]/.test(v) || /.[A-Z]/.test(v)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** The kinds of secret in one piece of text: "private key", "api key", "token", "password", "connection string". */
|
|
305
|
+
export function secretKinds(text) {
|
|
306
|
+
const value = String(text ?? "")
|
|
307
|
+
const kinds = new Set()
|
|
308
|
+
if (new RegExp(PRIVATE_KEY_BLOCK.source).test(value)) kinds.add("private key")
|
|
309
|
+
for (const [kind, pattern, valid] of SHAPED) {
|
|
310
|
+
for (const match of value.matchAll(pattern)) if (!documentation(match[0]) && (!valid || valid(match[0]))) kinds.add(kind)
|
|
311
|
+
}
|
|
312
|
+
for (const match of value.matchAll(BEARER)) if (tokenShaped(match[1])) kinds.add("token")
|
|
313
|
+
for (const match of value.matchAll(AUTH_HEADER)) {
|
|
314
|
+
const v = String(match[1] ?? match[2] ?? match[3] ?? "").trim().replace(/^(?:Bearer|Basic|Token|Bot)\s+/i, "")
|
|
315
|
+
if (tokenShaped(v)) kinds.add("token")
|
|
316
|
+
}
|
|
317
|
+
for (const match of value.matchAll(CONNECTION)) if (match[2].length >= 3 && !documentation(match[2])) kinds.add("connection string")
|
|
318
|
+
for (const match of value.matchAll(ASSIGNMENT)) {
|
|
319
|
+
const key = match[2]
|
|
320
|
+
const quoted = match[4] ?? match[5] ?? match[6]
|
|
321
|
+
const bare = match[7]
|
|
322
|
+
const crossed = /\n/.test(match[3])
|
|
323
|
+
// A value on the next line that is itself a key ("DB_HOST=x", "host:") is not this key's value.
|
|
324
|
+
if (crossed && bare !== undefined && /^[A-Za-z_][\w.-]*[:=]/.test(bare)) continue
|
|
325
|
+
// In a URL query (`?token=…&page=2`) the value must look random.
|
|
326
|
+
const query = /[?&]/.test(value[match.index - 1] ?? "")
|
|
327
|
+
const password = /pass|pwd/i.test(key)
|
|
328
|
+
let found = credentialValue(quoted ?? bare, quoted !== undefined, key, query)
|
|
329
|
+
if (!found && password && quoted === undefined && !query) {
|
|
330
|
+
const start = match.index + match[0].length - (bare ?? "").length
|
|
331
|
+
const end = value.indexOf("\n", start)
|
|
332
|
+
found = passphrase(value.slice(start, end === -1 ? value.length : end))
|
|
333
|
+
}
|
|
334
|
+
if (found) kinds.add(password ? "password" : /token/i.test(key) ? "token" : "api key")
|
|
335
|
+
}
|
|
336
|
+
return [...kinds]
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** The email addresses in the text, by kind: a person's, and role addresses. */
|
|
340
|
+
function emailCounts(text) {
|
|
341
|
+
const value = String(text)
|
|
342
|
+
const counts = { person: 0, role: 0 }
|
|
343
|
+
for (const match of value.matchAll(EMAIL)) {
|
|
344
|
+
const kind = emailKind(match, value)
|
|
345
|
+
if (kind !== "none") counts[kind] += 1
|
|
346
|
+
}
|
|
347
|
+
return counts
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function countPaths(text) {
|
|
351
|
+
let n = 0
|
|
352
|
+
for (const pattern of HOME_PATHS) n += (String(text).match(pattern) ?? []).length
|
|
353
|
+
return n
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** Mask a person's emails and home-directory paths. */
|
|
357
|
+
function maskPersonal(text) {
|
|
358
|
+
const value = String(text)
|
|
359
|
+
let masked = ""
|
|
360
|
+
let last = 0
|
|
361
|
+
for (const match of value.matchAll(EMAIL)) {
|
|
362
|
+
masked += value.slice(last, match.index) + (emailKind(match, value) === "none" ? match[0] : EMAIL_MASK)
|
|
363
|
+
last = match.index + match[0].length
|
|
364
|
+
}
|
|
365
|
+
masked += value.slice(last)
|
|
366
|
+
for (const pattern of HOME_PATHS) masked = masked.replace(pattern, neutralPath)
|
|
367
|
+
return masked
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Withhold what holds a secret, in an automatic write: a private key block
|
|
372
|
+
* whole, each line that holds one, and a key line together with the value
|
|
373
|
+
* line after it ("password:" then the value). When a secret is still found
|
|
374
|
+
* (it spans lines some other way), the whole text is withheld.
|
|
375
|
+
*/
|
|
376
|
+
function withholdSecrets(text) {
|
|
377
|
+
let withheld = 0
|
|
378
|
+
const blocks = String(text).replace(PRIVATE_KEY_BLOCK, () => {
|
|
379
|
+
withheld += 1
|
|
380
|
+
return WITHHELD_SECRET
|
|
381
|
+
})
|
|
382
|
+
const lines = blocks.split("\n")
|
|
383
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
384
|
+
const line = lines[index]
|
|
385
|
+
if (line.trim() === WITHHELD_SECRET) continue
|
|
386
|
+
if (secretKinds(line).length > 0) {
|
|
387
|
+
lines[index] = WITHHELD_SECRET
|
|
388
|
+
withheld += 1
|
|
389
|
+
continue
|
|
390
|
+
}
|
|
391
|
+
// A key line and its value line: neither holds the secret alone, the two together do.
|
|
392
|
+
const next = lines[index + 1]
|
|
393
|
+
if (next !== undefined && next.trim() !== WITHHELD_SECRET && secretKinds(next).length === 0 && secretKinds(`${line}\n${next}`).length > 0) {
|
|
394
|
+
lines.splice(index, 2, WITHHELD_SECRET)
|
|
395
|
+
withheld += 1
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
const kept = lines.join("\n")
|
|
399
|
+
if (secretKinds(kept).length > 0) return { text: WITHHELD_SECRET, withheld: withheld + 1, whole: true }
|
|
400
|
+
return { text: kept, withheld, whole: false }
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Decide what of `text` may be stored on write path `path`.
|
|
405
|
+
*
|
|
406
|
+
* - `refuse`: it holds a secret, or (an explicit write, by `EXPLICIT_POLICY`)
|
|
407
|
+
* a person's email address or a home-directory path. An explicit write
|
|
408
|
+
* stores nothing (`text` is ""; `reason` is secret, email or local_path);
|
|
409
|
+
* an automatic one keeps `text` with what holds the secret withheld (a line,
|
|
410
|
+
* a key line and its value line, or the whole text) and the rest masked.
|
|
411
|
+
* - `warn`: an explicit write holding what its policy stores with a warning;
|
|
412
|
+
* `text` is unchanged and `message` is the warning to show.
|
|
413
|
+
* - `mask`: an automatic write holding an email or a home-directory path;
|
|
414
|
+
* `text` has them masked.
|
|
415
|
+
* - `store`: nothing to change.
|
|
416
|
+
*
|
|
417
|
+
* @param {unknown} text
|
|
418
|
+
* @param {keyof typeof STORABLE_PATHS} path
|
|
419
|
+
* @returns {{ outcome: "store"|"warn"|"mask"|"refuse", reason: ""|"secret"|"email"|"local_path", text: string, secrets: string[], emails: number, paths: number, withheld: number, message: string }}
|
|
420
|
+
*/
|
|
421
|
+
export function storable(text, path) {
|
|
422
|
+
const mode = STORABLE_PATHS[path]
|
|
423
|
+
if (!mode) throw new Error(`storable: unknown write path ${JSON.stringify(path)}`)
|
|
424
|
+
const value = String(text ?? "")
|
|
425
|
+
const secrets = secretKinds(value)
|
|
426
|
+
if (mode === "explicit") {
|
|
427
|
+
const counts = emailCounts(value)
|
|
428
|
+
const emails = counts.person
|
|
429
|
+
const roles = counts.role
|
|
430
|
+
const paths = countPaths(value)
|
|
431
|
+
if (secrets.length > 0) {
|
|
432
|
+
return {
|
|
433
|
+
outcome: "refuse",
|
|
434
|
+
reason: "secret",
|
|
435
|
+
text: "",
|
|
436
|
+
secrets,
|
|
437
|
+
emails,
|
|
438
|
+
paths,
|
|
439
|
+
withheld: 0,
|
|
440
|
+
message: `not stored: the text holds a secret (${secrets.join(", ")}). Secrets are never stored; say it again without the value.`,
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
const policy = EXPLICIT_POLICY[path]
|
|
444
|
+
if ((emails > 0 && policy.email === "refuse") || (roles > 0 && policy.role === "refuse")) return { outcome: "refuse", reason: "email", text: "", secrets, emails: emails + roles, paths, withheld: 0, message: REFUSALS.email }
|
|
445
|
+
if (paths > 0 && policy.path === "refuse") return { outcome: "refuse", reason: "local_path", text: "", secrets, emails, paths, withheld: 0, message: REFUSALS.local_path }
|
|
446
|
+
if (emails + roles + paths > 0) {
|
|
447
|
+
const what = [emails ? `${emails} email address${emails === 1 ? "" : "es"}` : "", roles ? `${roles} role email address${roles === 1 ? "" : "es"}` : "", paths ? `${paths} home-directory path${paths === 1 ? "" : "s"}` : ""].filter(Boolean).join(" and ")
|
|
448
|
+
return {
|
|
449
|
+
outcome: "warn",
|
|
450
|
+
reason: "",
|
|
451
|
+
text: value,
|
|
452
|
+
secrets,
|
|
453
|
+
emails: emails + roles,
|
|
454
|
+
paths,
|
|
455
|
+
withheld: 0,
|
|
456
|
+
message: `Warning: stored with ${what}. Personal addresses and one machine's paths age badly; correct it without them if they are not needed.`,
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
return { outcome: "store", reason: "", text: value, secrets, emails, paths, withheld: 0, message: "" }
|
|
460
|
+
}
|
|
461
|
+
const held = secrets.length > 0 ? withholdSecrets(value) : { text: value, withheld: 0, whole: false }
|
|
462
|
+
const counts = emailCounts(held.text)
|
|
463
|
+
const emails = counts.person + counts.role
|
|
464
|
+
const paths = countPaths(held.text)
|
|
465
|
+
const masked = emails + paths > 0 ? maskPersonal(held.text) : held.text
|
|
466
|
+
const outcome = held.withheld > 0 ? "refuse" : emails + paths > 0 ? "mask" : "store"
|
|
467
|
+
return {
|
|
468
|
+
outcome,
|
|
469
|
+
reason: held.withheld > 0 ? "secret" : "",
|
|
470
|
+
text: masked,
|
|
471
|
+
secrets,
|
|
472
|
+
emails,
|
|
473
|
+
paths,
|
|
474
|
+
withheld: held.withheld,
|
|
475
|
+
message: held.whole ? `the whole text withheld: it holds a secret (${secrets.join(", ")}) across lines` : held.withheld > 0 ? `${held.withheld} part(s) holding a secret (${secrets.join(", ")}) withheld` : "",
|
|
476
|
+
}
|
|
477
|
+
}
|