@heyquilt/seam 0.27.0 → 0.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/seam.js +117 -17
- package/lib/quilts/agent-tokens.js +148 -0
- package/lib/quilts/archiver.js +241 -0
- package/lib/quilts/attest.js +212 -0
- package/lib/quilts/bisect-run.js +3 -2
- package/lib/quilts/block.js +497 -55
- package/lib/quilts/cas.js +78 -0
- package/lib/quilts/cbor.js +2 -0
- package/lib/quilts/claims.js +409 -0
- package/lib/quilts/cli.js +611 -69
- package/lib/quilts/client.js +305 -19
- package/lib/quilts/conflict-format.js +266 -0
- package/lib/quilts/daemon.js +174 -0
- package/lib/quilts/describe.js +281 -38
- package/lib/quilts/dict.js +34 -0
- package/lib/quilts/exit-archive.js +184 -0
- package/lib/quilts/export-format.js +52 -0
- package/lib/quilts/export-git.js +27 -3
- package/lib/quilts/export-lfs.js +52 -0
- package/lib/quilts/export-sign.js +49 -0
- package/lib/quilts/fastcdc.js +186 -0
- package/lib/quilts/hash.js +17 -2
- package/lib/quilts/ignored.js +46 -0
- package/lib/quilts/import-git.js +54 -2
- package/lib/quilts/journal.js +105 -0
- package/lib/quilts/limits.js +197 -0
- package/lib/quilts/linepolicy.js +109 -0
- package/lib/quilts/pack.js +146 -18
- package/lib/quilts/persona.js +70 -0
- package/lib/quilts/records.js +7 -3
- package/lib/quilts/recover.js +37 -0
- package/lib/quilts/runner.js +79 -12
- package/lib/quilts/sealed.js +231 -0
- package/lib/quilts/secrets.js +86 -0
- package/lib/quilts/signing.js +7 -3
- package/lib/quilts/tarball.js +107 -0
- package/lib/quilts/textdiff.js +7 -5
- package/lib/quilts/tools.js +401 -83
- package/lib/quilts/tree.js +351 -0
- package/lib/quilts/want.js +51 -0
- package/lib/quilts/webhook.js +116 -0
- package/package.json +3 -2
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// The server's ignore check (05 "Ignore rules and the .env rule"). A hosted agent or a hand-written
|
|
2
|
+
// client never runs a client-side matcher, so with `ignoredPath` on (SEAM_IGNORED_PATH=1) a commit's
|
|
3
|
+
// puts are refused with 422 ignored_path when a built-in names the path (always, tracked or not:
|
|
4
|
+
// a .gitignore cannot un-ignore one), or when the path is not in the first parent's tree and a
|
|
5
|
+
// .gitignore on its way down in that tree matches it (the deepest file with a rule wins, as in git).
|
|
6
|
+
// Deletes always go through, so a tracked .env from an import can be removed.
|
|
7
|
+
import ignore from 'ignore'
|
|
8
|
+
|
|
9
|
+
/** Always ignored, everywhere. No tool's own folders: only what no project wants committed. */
|
|
10
|
+
export const BUILTIN_IGNORED = ['.git', '.seam', 'node_modules', '.DS_Store', 'Thumbs.db', '.next', '.turbo', '.nuxt', '.svelte-kit', '.parcel-cache', '.vercel', '.env', '.env.*', '!.env.example', '*.swp', '*.swo', '*~', '.#*', '.seamignore']
|
|
11
|
+
const BUILTIN = ignore().add(BUILTIN_IGNORED)
|
|
12
|
+
|
|
13
|
+
export const builtinIgnored = (path) => BUILTIN.ignores(path)
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The paths among `puts` the rules refuse, in the order given. `tracked(path)` says whether the
|
|
17
|
+
* path is in the parent tree; `readText(path)` gives a file of the parent tree as text, or null.
|
|
18
|
+
*/
|
|
19
|
+
export async function ignoredPaths (puts, { tracked, readText }) {
|
|
20
|
+
const rules = new Map() // folder → ignore instance or null
|
|
21
|
+
const rulesIn = async (dir) => {
|
|
22
|
+
if (!rules.has(dir)) {
|
|
23
|
+
const text = await readText(dir ? `${dir}/.gitignore` : '.gitignore')
|
|
24
|
+
rules.set(dir, text == null ? null : ignore().add(text))
|
|
25
|
+
}
|
|
26
|
+
return rules.get(dir)
|
|
27
|
+
}
|
|
28
|
+
const out = []
|
|
29
|
+
for (const p of puts) {
|
|
30
|
+
if (builtinIgnored(p)) { out.push(p); continue }
|
|
31
|
+
if (await tracked(p)) continue
|
|
32
|
+
const parts = p.split('/')
|
|
33
|
+
let verdict = false
|
|
34
|
+
// Deepest first: the first .gitignore with a rule for the path decides.
|
|
35
|
+
for (let i = parts.length - 1; i >= 0; i--) {
|
|
36
|
+
const dir = parts.slice(0, i).join('/')
|
|
37
|
+
const ig = await rulesIn(dir)
|
|
38
|
+
if (!ig) continue
|
|
39
|
+
const res = ig.test(parts.slice(i).join('/'))
|
|
40
|
+
if (res.ignored) { verdict = true; break }
|
|
41
|
+
if (res.unignored) break
|
|
42
|
+
}
|
|
43
|
+
if (verdict) out.push(p)
|
|
44
|
+
}
|
|
45
|
+
return out
|
|
46
|
+
}
|
package/lib/quilts/import-git.js
CHANGED
|
@@ -77,7 +77,33 @@ function diffTree (repo, parent, sha) {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
const agentEmail = /@agents\.noreply\.heyquilt\.com$/i
|
|
80
|
-
|
|
80
|
+
/**
|
|
81
|
+
* A git author's member (14, "Authors to members"): `authors` (email → member id, like
|
|
82
|
+
* .mailmap, any case) wins; anything else is the stub member of its email. The server refuses a
|
|
83
|
+
* mapped id that is not a member of the quilt.
|
|
84
|
+
*/
|
|
85
|
+
export function checkAuthors (authors) {
|
|
86
|
+
if (authors == null) return {}
|
|
87
|
+
if (typeof authors !== 'object' || Array.isArray(authors)) throw Object.assign(new Error('authors: an object of email → member id'), { status: 400, code: 'bad_request' })
|
|
88
|
+
const out = {}
|
|
89
|
+
const entries = Object.entries(authors)
|
|
90
|
+
if (entries.length > 10000) throw Object.assign(new Error('authors: at most 10000 emails'), { status: 400, code: 'bad_request' })
|
|
91
|
+
for (const [email, id] of entries) {
|
|
92
|
+
if (!email.includes('@') || email.length > 320 || typeof id !== 'string' || !id || id.length > 200 || id.startsWith('stub:')) throw Object.assign(new Error(`authors: ${email.slice(0, 80)} must map to a member id`), { status: 400, code: 'bad_request' })
|
|
93
|
+
out[email.trim().toLowerCase()] = id
|
|
94
|
+
}
|
|
95
|
+
return out
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const memberFor = (authors = {}) => {
|
|
99
|
+
const map = new Map(Object.entries(authors || {}).map(([e, id]) => [e.trim().toLowerCase(), String(id)]))
|
|
100
|
+
return (who) => {
|
|
101
|
+
const email = String(who.email || '').toLowerCase()
|
|
102
|
+
const kind = agentEmail.test(email) ? 'agent' : 'person'
|
|
103
|
+
const mapped = email && map.get(email)
|
|
104
|
+
return { member_id: mapped || `stub:${String(who.email || who.name).toLowerCase()}`, kind, name: who.name || who.email }
|
|
105
|
+
}
|
|
106
|
+
}
|
|
81
107
|
|
|
82
108
|
/**
|
|
83
109
|
* The branches to import: [{ name, sha }], HEAD's branch first as main. Local branches, then
|
|
@@ -115,7 +141,28 @@ function gitTags (repo) {
|
|
|
115
141
|
* where only the commits it does not have yet are added. Returns { quilt, slug, commits,
|
|
116
142
|
* patches, lines, tags, report, validated? }.
|
|
117
143
|
*/
|
|
118
|
-
|
|
144
|
+
/**
|
|
145
|
+
* The authors an import would bring in (14, awaiting_authors): [{ email, name, commits }] over the
|
|
146
|
+
* same branches importGit reads, the most commits first. Emails in lower case; the latest name.
|
|
147
|
+
*/
|
|
148
|
+
export function gitAuthors (repo, { follow = false, onlyHead = false } = {}) {
|
|
149
|
+
const head = git(repo, ['rev-parse', 'HEAD']).toString().trim()
|
|
150
|
+
const refs = onlyHead ? [{ sha: head }] : branches(repo, head).filter((r) => !follow || r.name === 'main' || r.name.startsWith('release/'))
|
|
151
|
+
const out = new Map()
|
|
152
|
+
for (const row of git(repo, ['log', '--format=%aE%x00%aN', ...new Set(refs.map((r) => r.sha))]).toString().split('\n')) {
|
|
153
|
+
if (!row) continue
|
|
154
|
+
const [e, name] = row.split('\0')
|
|
155
|
+
const email = e.trim().toLowerCase()
|
|
156
|
+
const a = out.get(email) || { email, name: name || email, commits: 0 }
|
|
157
|
+
a.commits++
|
|
158
|
+
out.set(email, a)
|
|
159
|
+
}
|
|
160
|
+
return [...out.values()].sort((a, b) => b.commits - a.commits || (a.email < b.email ? -1 : 1))
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export async function importGit (client, { repo, slug, batch = 100, log = () => {}, validate = false, limit = 0, onlyHead = false, fresh = false, follow = false, authors = {} }) {
|
|
164
|
+
const explicit = checkAuthors(authors)
|
|
165
|
+
let member = memberFor(explicit)
|
|
119
166
|
const head = git(repo, ['rev-parse', 'HEAD']).toString().trim()
|
|
120
167
|
// follow: only main and release/* (14: the lines an import follows).
|
|
121
168
|
const refs = onlyHead ? [{ name: 'main', sha: head, git: 'HEAD' }] : branches(repo, head).filter((r) => !follow || r.name === 'main' || r.name.startsWith('release/'))
|
|
@@ -140,6 +187,11 @@ export async function importGit (client, { repo, slug, batch = 100, log = () =>
|
|
|
140
187
|
for (const [sha, id] of Object.entries(known)) ids.set(sha, id)
|
|
141
188
|
}
|
|
142
189
|
}
|
|
190
|
+
// Authors to members, step 2 (14): the emails of new commits that are a member's account email.
|
|
191
|
+
const emails = [...new Set(all.filter((sha) => !ids.has(sha)).map((sha) => String(info.get(sha).author?.email || '').trim().toLowerCase()).filter((e) => e.includes('@') && e.length <= 320 && !explicit[e]))]
|
|
192
|
+
const found = {}
|
|
193
|
+
for (let i = 0; i < emails.length && client.importEmails; i += 5000) Object.assign(found, (await client.importEmails(quilt.id, emails.slice(i, i + 5000))).authors)
|
|
194
|
+
member = memberFor({ ...found, ...explicit })
|
|
143
195
|
const lineTips = new Map((again ? await client.lines(quilt.id) : [{ name: 'main', tip: null }]).map((l) => [l.name, l.tip]))
|
|
144
196
|
log(again ? `Adding to ${quilt.slug}: ${all.length - ids.size} new commits of ${all.length}…` : `Importing ${all.length} commits on ${refs.length} branches into ${quilt.slug}…`)
|
|
145
197
|
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// The tip journal (plans/quilt-source-control/03, "The tip journal"; 04): the other durable copy
|
|
2
|
+
// of order, between the last WAL segment and the database. After a `merge`, `move` or `square`
|
|
3
|
+
// row commits (and a `policy` row naming a line: making, freezing or unfreezing it, since replay
|
|
4
|
+
// rebuilds `lines` from those too), the authority appends it to the quilt's current journal group
|
|
5
|
+
// and acknowledges the write only when that group's PUT succeeded. Writes that land within one
|
|
6
|
+
// 200 ms window share one group, so a quilt costs at most one PUT per window. `commit` rows are
|
|
7
|
+
// not journaled: their content is in pack trailers (15's `recover-gap`).
|
|
8
|
+
//
|
|
9
|
+
// A group is the same canonical log records a segment holds, in the segment encoding (archiver.js:
|
|
10
|
+
// 'QWS1', compressed body, hashed trailer), stored at q/<quilt_id>/j/<first_seq>-<last_seq>.qtj.
|
|
11
|
+
// Groups whose rows are all in sealed segments are redundant; gc() deletes them 24 h after that.
|
|
12
|
+
// Behind SEAM_TIP_JOURNAL=1 (index.js), since it adds a PUT to the ack of every merge.
|
|
13
|
+
import { encodeSegment, decodeSegment } from './archiver.js'
|
|
14
|
+
|
|
15
|
+
export const JOURNAL_WINDOW_MS = 200
|
|
16
|
+
export const JOURNAL_KEEP_MS = 24 * 60 * 60 * 1000
|
|
17
|
+
const ZERO = '0'.repeat(64)
|
|
18
|
+
const pad = (n) => String(n).padStart(12, '0')
|
|
19
|
+
export const groupName = (from, to) => `${pad(from)}-${pad(to)}.qtj`
|
|
20
|
+
const parseName = (name) => { const m = /^(\d{12})-(\d{12})\.qtj$/.exec(name); return m ? { from: Number(m[1]), to: Number(m[2]) } : null }
|
|
21
|
+
|
|
22
|
+
/** Whether a log row goes in the journal. */
|
|
23
|
+
export const journaled = (row) => row.kind === 'merge' || row.kind === 'move' || row.kind === 'square' ||
|
|
24
|
+
(row.kind === 'policy' && !!row.line && ['line', 'freeze', 'unfreeze'].includes(row.payload?.op))
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The journal over a store and an object store. `sync(q)` resolves once every journaled row the
|
|
28
|
+
* quilt has now is in object storage (callers in one window share the PUT); it rejects when the
|
|
29
|
+
* PUT fails, so the write it guards is not acknowledged. `gc(q)` deletes redundant groups.
|
|
30
|
+
*/
|
|
31
|
+
export function createJournal ({ store, objects, now = Date.now, windowMs = JOURNAL_WINDOW_MS, keepMs = JOURNAL_KEEP_MS }) {
|
|
32
|
+
const done = new Map() // quilt -> the last seq looked at and written
|
|
33
|
+
const pending = new Map() // quilt -> the window's promise
|
|
34
|
+
|
|
35
|
+
async function startSeq (q) {
|
|
36
|
+
if (done.has(q)) return done.get(q)
|
|
37
|
+
const quilt = await store.getQuilt(q)
|
|
38
|
+
let at = quilt?.archive_seq || 0
|
|
39
|
+
for (const name of await objects.listFiles(q, 'j')) { const g = parseName(name); if (g && g.to > at) at = g.to }
|
|
40
|
+
return at
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async function flush (q) {
|
|
44
|
+
pending.delete(q)
|
|
45
|
+
const after = await startSeq(q)
|
|
46
|
+
const rows = []
|
|
47
|
+
let last = after
|
|
48
|
+
for (;;) {
|
|
49
|
+
const page = await store.logRows(q, { after: last, limit: 1000 })
|
|
50
|
+
if (!page.length) break
|
|
51
|
+
for (const row of page) { last = row.seq; if (journaled(row)) rows.push(row) }
|
|
52
|
+
if (page.length < 1000) break
|
|
53
|
+
}
|
|
54
|
+
if (rows.length) {
|
|
55
|
+
const from = rows[0].seq
|
|
56
|
+
const to = rows[rows.length - 1].seq
|
|
57
|
+
const { bytes } = encodeSegment({ quilt_id: q, from_seq: from, to_seq: to, prev: ZERO, records: rows.map((row) => ({ t: 'log', row })) })
|
|
58
|
+
await objects.putFile(q, 'j', groupName(from, to), bytes)
|
|
59
|
+
}
|
|
60
|
+
if (last > (done.get(q) || 0)) done.set(q, last)
|
|
61
|
+
return { rows: rows.length, through: last }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function sync (q) {
|
|
65
|
+
if (pending.has(q)) return pending.get(q)
|
|
66
|
+
const p = new Promise((resolve) => setTimeout(resolve, windowMs)).then(() => flush(q))
|
|
67
|
+
pending.set(q, p)
|
|
68
|
+
return p
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// When each group was first seen redundant (all its rows sealed in segments), in this process.
|
|
72
|
+
// A restart only delays a deletion; it never deletes early.
|
|
73
|
+
const redundantSince = new Map()
|
|
74
|
+
async function gc (q) {
|
|
75
|
+
const quilt = await store.getQuilt(q)
|
|
76
|
+
const sealed = quilt?.archive_seq || 0
|
|
77
|
+
let deleted = 0
|
|
78
|
+
for (const name of await objects.listFiles(q, 'j')) {
|
|
79
|
+
const g = parseName(name)
|
|
80
|
+
if (!g || g.to > sealed) continue
|
|
81
|
+
const key = `${q}/${name}`
|
|
82
|
+
if (!redundantSince.has(key)) redundantSince.set(key, now())
|
|
83
|
+
if (now() - redundantSince.get(key) < keepMs) continue
|
|
84
|
+
await objects.deleteFile(q, 'j', name)
|
|
85
|
+
redundantSince.delete(key)
|
|
86
|
+
deleted++
|
|
87
|
+
}
|
|
88
|
+
return { deleted }
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return { sync, gc }
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Every journaled row with seq past `after`, from the quilt's groups, in seq order (each group's hash checked). */
|
|
95
|
+
export async function readJournal (objects, q, after = 0) {
|
|
96
|
+
const rows = new Map()
|
|
97
|
+
for (const name of await objects.listFiles(q, 'j')) {
|
|
98
|
+
const g = parseName(name)
|
|
99
|
+
if (!g || g.to <= after) continue
|
|
100
|
+
const { trailer, records } = decodeSegment(await objects.readFile(q, 'j', name))
|
|
101
|
+
if (trailer.quilt_id !== q) throw new Error(`journal group ${name} belongs to quilt ${trailer.quilt_id}`)
|
|
102
|
+
for (const r of records) if (r.t === 'log' && r.row.seq > after) rows.set(r.row.seq, r.row)
|
|
103
|
+
}
|
|
104
|
+
return [...rows.values()].sort((a, b) => a.seq - b.seq)
|
|
105
|
+
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
// Per-token rate limits (plans/quilt-source-control/04 "Rate limiting and load shedding",
|
|
2
|
+
// decision 40): counted per token, so an agent's key and the person it works for are limited
|
|
3
|
+
// apart; a person's token gets five times an agent's numbers (11). A limit answers at once with
|
|
4
|
+
// 429 rate_limited, `limit` naming its row and `retry_after` seconds; nothing waits.
|
|
5
|
+
// Counted in this API process (sliding windows of request times), where a token can reach one
|
|
6
|
+
// allowance per machine; or, with `shared` (a store, SEAM_SHARED_LIMITS=1), in the database's
|
|
7
|
+
// fixed windows (quilt_rate_take), one allowance however many machines answer.
|
|
8
|
+
import crypto from 'node:crypto'
|
|
9
|
+
import { QuiltError } from './errors.js'
|
|
10
|
+
|
|
11
|
+
const MIN = 60 * 1000
|
|
12
|
+
const HOUR = 60 * MIN
|
|
13
|
+
const DAY = 24 * HOUR
|
|
14
|
+
|
|
15
|
+
/** 04's rows for an agent token: [name, count, window ms, which requests]. */
|
|
16
|
+
export const LIMITS = [
|
|
17
|
+
['token_rate', 600, MIN, 'rpc'],
|
|
18
|
+
['commit_rate', 60, MIN, 'commit'],
|
|
19
|
+
['commit_rate', 2000, DAY, 'commit'],
|
|
20
|
+
['merge_rate', 30, HOUR, 'merge']
|
|
21
|
+
]
|
|
22
|
+
export const PERSON_FACTOR = 5
|
|
23
|
+
|
|
24
|
+
/** 04's fetch_rate row: 50 want-lists (POST /blobs) a second per token; on with `fetchRate` (SEAM_FETCH_RATE=1). */
|
|
25
|
+
export const FETCH_RATE = ['fetch_rate', 50, 1000, 'fetch']
|
|
26
|
+
|
|
27
|
+
// Rows that answer a code of their own (12's refusal list); the others answer rate_limited naming the row.
|
|
28
|
+
const OWN_CODES = new Set(['quilt_busy', 'fetch_rate', 'fallback_rate', 'upload_rate'])
|
|
29
|
+
|
|
30
|
+
/** 04's fallback_rate: a request with `x-quilt-fallback` (a daemon with no relay) gets a tenth of every token row, want-lists included. */
|
|
31
|
+
export const fallbackRows = (limits) => limits.map(([, count, windowMs, kind]) => ['fallback_rate', Math.max(1, Math.floor(count / 10)), windowMs, kind])
|
|
32
|
+
|
|
33
|
+
/** 04's auth_lockout row: 10 failed authentications a minute lock a token for a minute after its last failure. */
|
|
34
|
+
export const AUTH_LOCKOUT = { failures: 10, windowMs: MIN, lockMs: MIN }
|
|
35
|
+
|
|
36
|
+
/** 04's rows for a whole quilt, across every token that writes into it; a person's token is not ×5. */
|
|
37
|
+
export const QUILT_LIMITS = [
|
|
38
|
+
['quilt_busy', 6000, HOUR, 'commit'],
|
|
39
|
+
['upload_rate', 2 ** 40, DAY, 'upload'], // 1 TiB of packs a day
|
|
40
|
+
['quilt_busy', 200, 1000, 'write'] // 200 sequenced writes a second (rows only ever appended: the shared store keys by index)
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
/** 04's rows for one member in one quilt (a person's ×5): bytes of packs uploaded. */
|
|
44
|
+
export const MEMBER_LIMITS = [
|
|
45
|
+
['upload_rate', 100 * 2 ** 30, HOUR, 'upload'] // 100 GiB an hour: a 500 GiB file in five hours
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
/** Which counters a request takes: object traffic (packs, blob and object reads) takes none. */
|
|
49
|
+
export function classify (method, url, body) {
|
|
50
|
+
const p = String(url).split('?')[0]
|
|
51
|
+
if (method === 'POST' && /^\/v1\/quilts\/[^/]+\/(blobs|objects)$/.test(p)) return ['fetch'] // a want-list
|
|
52
|
+
if (method === 'POST' && /^\/v1\/quilts\/[^/]+\/packs$/.test(p)) return ['upload'] // counted in bytes, per member and per quilt
|
|
53
|
+
if (/\/(packs|objects|blobs)(\/|$)/.test(p)) return []
|
|
54
|
+
// A sequenced write: any change inside one quilt but have? (an upload check); quilt_busy counts them.
|
|
55
|
+
const write = method !== 'GET' && method !== 'HEAD' && /^\/v1\/quilts\/[^/]+\/./.test(p) && !/\/have$/.test(p) ? ['write'] : []
|
|
56
|
+
if (method === 'POST' && /\/squares\/[^/]+\/commit$/.test(p)) return ['rpc', 'commit', ...write]
|
|
57
|
+
if (method === 'POST' && /\/lines\/[^/]+\/merge$/.test(p)) return body && body.dry_run ? ['rpc'] : ['rpc', 'merge', ...write] // a dry run is never merge-limited
|
|
58
|
+
return ['rpc', ...write]
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The token's key: a hash of its Authorization header (the token itself is never kept). */
|
|
62
|
+
export const tokenKey = (req, me) => {
|
|
63
|
+
const h = req && req.headers && req.headers.authorization
|
|
64
|
+
return h ? crypto.createHash('sha256').update(h).digest('hex').slice(0, 32) : `member:${me.member_id}:${me.kind}`
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const refuse = (name, max, windowMs, retry, scope) => new QuiltError(429, OWN_CODES.has(name) ? name : 'rate_limited', `rate limit ${name}: ${max} in ${windowMs >= DAY ? 'a day' : windowMs >= HOUR ? 'an hour' : windowMs >= MIN ? 'a minute' : 'a second'} for this ${scope}; try again in ${retry} s`, { limit: name, retry_after: retry })
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* A token's own `rate` (11.9: QuiltToken.rate, downward only) as a share per row name: `rate[name]`
|
|
71
|
+
* is the count for that name's first (shortest) row, and its longer rows shrink by the same share.
|
|
72
|
+
* A number above 04's is ignored (never raises).
|
|
73
|
+
*/
|
|
74
|
+
export function rateShares (limits, rate) {
|
|
75
|
+
if (!rate) return null
|
|
76
|
+
const out = {}
|
|
77
|
+
for (const [name, value] of Object.entries(rate)) {
|
|
78
|
+
const first = limits.find(([n]) => n === name)
|
|
79
|
+
if (first && Number.isInteger(value) && value > 0 && value < first[1]) out[name] = value / first[1]
|
|
80
|
+
}
|
|
81
|
+
return out
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** The names a token's `rate` may lower: 04's per-token rows, fetch_rate included. */
|
|
85
|
+
export const RATE_NAMES = [...new Set([...LIMITS, FETCH_RATE].map(([n]) => n))]
|
|
86
|
+
export const rateDefault = (name) => [...LIMITS, FETCH_RATE].find(([n]) => n === name)[1]
|
|
87
|
+
|
|
88
|
+
/** A limiter for the token rows (`take`) and the quilt rows (`takeQuilt`, keyed `quilt:<id>`). */
|
|
89
|
+
export function createLimiter ({ now = Date.now, fetchRate = false, limits = fetchRate ? [...LIMITS, FETCH_RATE] : LIMITS, quiltLimits = QUILT_LIMITS, memberLimits = MEMBER_LIMITS, personFactor = PERSON_FACTOR, authLockout = AUTH_LOCKOUT, shared = null } = {}) {
|
|
90
|
+
const make = shared ? sharedCounter : localCounter
|
|
91
|
+
const token = make({ now, limits, personFactor, shared, scope: 'token' })
|
|
92
|
+
const quilt = make({ now, limits: quiltLimits, personFactor: 1, shared, scope: 'quilt' })
|
|
93
|
+
const member = make({ now, limits: memberLimits, personFactor, shared, scope: 'member in this quilt' })
|
|
94
|
+
const fallback = make({ now, limits: fallbackRows(limits.some(([n]) => n === 'fetch_rate') ? limits : [...limits, FETCH_RATE]), personFactor, shared, scope: 'token' })
|
|
95
|
+
return {
|
|
96
|
+
/** Counts one request of `kinds` for `key`; `scale` (a reader with no token, 11.9) shrinks the allowance. */
|
|
97
|
+
// `rate`: the token's own lower numbers (11.9), { commit_rate: 10, … }.
|
|
98
|
+
take: (key, kinds, person, scale = 1, rate = null) => token(key, kinds, person, 1, scale, rateShares(limits, rate)),
|
|
99
|
+
/** Counts a fallback request (x-quilt-fallback) against a tenth of the token rows, apart from its normal count. */
|
|
100
|
+
takeFallback: (key, kinds, person) => fallback(`fallback:${key}`, kinds, person),
|
|
101
|
+
/** Counts one request of `kinds` against the quilt's own rows, whoever sends it. */
|
|
102
|
+
takeQuilt: (quiltId, kinds) => quilt(`quilt:${quiltId}`, kinds, false),
|
|
103
|
+
/** Counts `bytes` of a pack upload against the member's rows in the quilt, then the quilt's. */
|
|
104
|
+
async takeUpload (quiltId, memberId, bytes, person) {
|
|
105
|
+
await member(`upload:${quiltId}:${memberId}`, ['upload'], person, bytes)
|
|
106
|
+
await quilt(`quilt:${quiltId}`, ['upload'], false, bytes)
|
|
107
|
+
},
|
|
108
|
+
...(shared ? sharedLockout({ now, shared, ...authLockout }) : lockout({ now, ...authLockout }))
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// auth_lockout with `shared` (SEAM_SHARED_LIMITS=1): failures counted in the store's fixed
|
|
113
|
+
// one-minute windows, checked with a read that counts nothing (`peek`); locked until the window
|
|
114
|
+
// that saw the 10th failure ends.
|
|
115
|
+
function sharedLockout ({ now, shared, failures, windowMs }) {
|
|
116
|
+
const rows = [{ i: 0, window_ms: windowMs, max: failures }]
|
|
117
|
+
return {
|
|
118
|
+
async authCheck (key) {
|
|
119
|
+
const r = await shared.rateTake({ key: `authfail:${key}`, at: now(), rows, peek: true })
|
|
120
|
+
if (r && r.ok === false) throw new QuiltError(429, 'auth_lockout', `too many failed sign-ins with these credentials; try again in ${r.retry_after} s`, { limit: 'auth_lockout', retry_after: r.retry_after })
|
|
121
|
+
},
|
|
122
|
+
async authFailed (key) { await shared.rateTake({ key: `authfail:${key}`, at: now(), rows }) }
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// auth_lockout (04, 11) counted in this process.
|
|
127
|
+
function lockout ({ now, failures, windowMs, lockMs }) {
|
|
128
|
+
const fails = new Map() // token key → { times: failure times in the window, until: locked until }
|
|
129
|
+
return {
|
|
130
|
+
/** Throws 429 auth_lockout while `key` is locked, before its credentials are even checked. */
|
|
131
|
+
authCheck (key) {
|
|
132
|
+
const f = fails.get(key)
|
|
133
|
+
const t = now()
|
|
134
|
+
if (f && f.until > t) {
|
|
135
|
+
const retry = Math.max(1, Math.ceil((f.until - t) / 1000))
|
|
136
|
+
throw new QuiltError(429, 'auth_lockout', `too many failed sign-ins with these credentials; try again in ${retry} s`, { limit: 'auth_lockout', retry_after: retry })
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
/** Records a failed authentication; the 10th in a minute, and each one after, locks for a minute from it. */
|
|
140
|
+
authFailed (key) {
|
|
141
|
+
const t = now()
|
|
142
|
+
let f = fails.get(key)
|
|
143
|
+
if (!f) { f = { times: [], until: 0 }; fails.set(key, f) }
|
|
144
|
+
f.times = f.times.filter((x) => x > t - windowMs)
|
|
145
|
+
f.times.push(t)
|
|
146
|
+
if (f.times.length >= failures) f.until = Math.max(f.until, t + lockMs)
|
|
147
|
+
if (fails.size > 10000) for (const [k, v] of fails) if (v.until <= t && !v.times.some((x) => x > t - windowMs)) fails.delete(k)
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function sharedCounter ({ now, limits, personFactor, shared, scope }) {
|
|
153
|
+
/** Counts one request of `kinds` for `key` in the shared store, or rejects 429 without counting. */
|
|
154
|
+
return async function take (key, kinds, person, n = 1, scale = 1, shares = null) {
|
|
155
|
+
const rows = []
|
|
156
|
+
limits.forEach(([name, count, windowMs, kind], i) => { if (kinds.includes(kind)) rows.push({ i, window_ms: windowMs, max: Math.max(1, Math.floor((person ? count * personFactor : count) * scale * (shares?.[name] ?? 1))) }) })
|
|
157
|
+
if (!rows.length) return
|
|
158
|
+
const r = await shared.rateTake({ key, at: now(), rows, ...(n !== 1 ? { n } : {}) })
|
|
159
|
+
if (r && r.ok === false) {
|
|
160
|
+
const [name, , windowMs] = limits[r.row]
|
|
161
|
+
throw refuse(name, rows.find((x) => x.i === r.row).max, windowMs, r.retry_after, scope)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function localCounter ({ now, limits, personFactor, scope }) {
|
|
167
|
+
const seen = new Map() // key + row index → { list: [time, amount] oldest first, sum }
|
|
168
|
+
|
|
169
|
+
/** Counts `n` (1 a request, or bytes) of `kinds` for `key`, or throws 429 without counting it. */
|
|
170
|
+
return function take (key, kinds, person, n = 1, scale = 1, shares = null) {
|
|
171
|
+
const t = now()
|
|
172
|
+
const rows = []
|
|
173
|
+
for (let i = 0; i < limits.length; i++) {
|
|
174
|
+
const [name, count, windowMs, kind] = limits[i]
|
|
175
|
+
if (!kinds.includes(kind)) continue
|
|
176
|
+
// `scale`: a reader with no token (a public quilt, 11.9) gets a tenth, counted per address.
|
|
177
|
+
const max = Math.max(1, Math.floor((person ? count * personFactor : count) * scale * (shares?.[name] ?? 1)))
|
|
178
|
+
const id = `${key}\u0000${i}`
|
|
179
|
+
let w = seen.get(id)
|
|
180
|
+
if (!w) { w = { list: [], sum: 0 }; seen.set(id, w) }
|
|
181
|
+
let drop = 0
|
|
182
|
+
while (drop < w.list.length && w.list[drop][0] <= t - windowMs) w.sum -= w.list[drop++][1]
|
|
183
|
+
if (drop) w.list.splice(0, drop)
|
|
184
|
+
if (w.sum >= max) { // full: free again once enough of the oldest have slid out
|
|
185
|
+
let left = w.sum
|
|
186
|
+
let j = 0
|
|
187
|
+
while (left >= max) left -= w.list[j++][1]
|
|
188
|
+
const retry = Math.max(1, Math.ceil((w.list[j - 1][0] + windowMs - t) / 1000))
|
|
189
|
+
throw refuse(name, max, windowMs, retry, scope)
|
|
190
|
+
}
|
|
191
|
+
rows.push(w)
|
|
192
|
+
}
|
|
193
|
+
for (const w of rows) { w.list.push([t, n]); w.sum += n }
|
|
194
|
+
// Forget keys with nothing left in any window, so the map stays the size of the active ones.
|
|
195
|
+
if (seen.size > 10000) for (const [id, w] of seen) if (!w.list.length || w.list[w.list.length - 1][0] <= t - DAY) seen.delete(id)
|
|
196
|
+
}
|
|
197
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// A line's policy as versions (plans/quilt-source-control/11.4, 13 "Where a policy lives and who
|
|
2
|
+
// changes it"): every change is a new version and a policy log row, and any version can be read
|
|
3
|
+
// back. On a server with SEAM_POLICY_APPROVAL=1 a change waits: a second person who is an admin
|
|
4
|
+
// approves it, or, where the proposer is the quilt's only person with admin, the proposer
|
|
5
|
+
// confirms it with a signature from one of their own signing keys. An agent never proposes,
|
|
6
|
+
// approves or confirms a policy, and nobody approves their own.
|
|
7
|
+
import crypto from 'node:crypto'
|
|
8
|
+
import { QuiltError } from './errors.js'
|
|
9
|
+
import { canonical } from './attest.js'
|
|
10
|
+
import { verifyText, localKey } from './signing.js'
|
|
11
|
+
|
|
12
|
+
/** What a proposer signs to confirm a version alone: the quilt, line, version and the body's hash. */
|
|
13
|
+
export const confirmText = (quiltId, line, version, body) => `seam-policy-confirm-v1\n${quiltId}\n${line}\n${version}\n${crypto.createHash('sha256').update(canonical(body)).digest('hex')}\n`
|
|
14
|
+
|
|
15
|
+
const view = (v) => ({ version: v.version, state: v.state, base: v.base, body: v.body, proposed_by: v.proposed_by, at: v.at, ...(v.decided_by ? { decided_by: v.decided_by, decided_at: v.decided_at } : {}), ...(v.confirmed_key ? { confirmed_with: v.confirmed_key } : {}) })
|
|
16
|
+
|
|
17
|
+
export function linePolicyOps (svc, { approval = false } = {}) {
|
|
18
|
+
const { store, now, quiltFor, fromStore } = svc.internals
|
|
19
|
+
|
|
20
|
+
const versionsOf = async (qid, line) => store.linePolicyVersions(qid, line)
|
|
21
|
+
const activeOf = (list) => list.filter((v) => v.state === 'active').reduce((m, v) => Math.max(m, v.version), 0)
|
|
22
|
+
|
|
23
|
+
/** The quilt's members who are people with the admin role. */
|
|
24
|
+
async function personAdmins (qid) { return (await store.members(qid)).filter((m) => m.level === 'admin' && m.member_id.startsWith('person:')).map((m) => m.member_id) }
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Records `body` as the line's next version: active at once, or pending a second admin (or the
|
|
28
|
+
* proposer's signed confirmation) on a server that asks for approval.
|
|
29
|
+
*/
|
|
30
|
+
async function propose (q, me, line, body) {
|
|
31
|
+
const state = approval ? 'pending' : 'active'
|
|
32
|
+
// The store numbers the version and records its base (the active version) in one call.
|
|
33
|
+
let row
|
|
34
|
+
try { row = await store.addLinePolicyVersion({ quilt_id: q.id, line, body, state, proposed_by: me.member_id, at: now() }) } catch (err) { throw fromStore(err) }
|
|
35
|
+
const version = row.version
|
|
36
|
+
if (!approval) return { version }
|
|
37
|
+
const admins = await personAdmins(q.id)
|
|
38
|
+
const alone = admins.filter((a) => a !== me.member_id).length === 0
|
|
39
|
+
return { version, pending: { version, needs: alone ? 'confirmation' : 'approval', how: alone ? `you are the only person with admin: confirm it with your signing key (seam policy confirm ${line} ${version})` : `another person with admin approves it (seam policy approve ${line} ${version})` } }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function pendingVersion (me, ref, line, version) {
|
|
43
|
+
const q = await quiltFor(me, ref, 'admin')
|
|
44
|
+
if (me.kind !== 'person') throw new QuiltError(403, 'forbidden', 'only a person approves a line\'s policy; an agent\'s approval never counts')
|
|
45
|
+
const v = (await versionsOf(q.id, String(line))).find((x) => x.version === Number(version))
|
|
46
|
+
if (!v) throw new QuiltError(404, 'not_found', `${line} has no policy version ${version}`)
|
|
47
|
+
if (v.state !== 'pending') throw new QuiltError(409, 'policy_state', `version ${version} is ${v.state}`)
|
|
48
|
+
return { q, v }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function decide (q, line, v, decision, me, extra = {}) {
|
|
52
|
+
try { await store.decideLinePolicy({ quilt_id: q.id, line, version: v.version, decision, by: me.member_id, at: now(), ...extra }) } catch (err) {
|
|
53
|
+
if (err.code === 'policy_moved') throw new QuiltError(409, 'policy_moved', `${line}'s policy changed since version ${v.version} was proposed: propose it again`)
|
|
54
|
+
throw fromStore(err)
|
|
55
|
+
}
|
|
56
|
+
return linePolicyVersions(me, q.id, line)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** POST /lines/:line/policy/versions/:v/approve: a second person with admin makes it active. */
|
|
60
|
+
async function approveLinePolicy (me, ref, line, version) {
|
|
61
|
+
const { q, v } = await pendingVersion(me, ref, line, version)
|
|
62
|
+
if (v.proposed_by === me.member_id) throw new QuiltError(403, 'own_policy', 'nobody approves a policy they proposed: another person with admin does, or, alone, confirm it with your signing key')
|
|
63
|
+
return decide(q, String(line), v, 'approve', me)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** POST /lines/:line/policy/versions/:v/confirm { key, signature }: the only admin confirms with a signing key. */
|
|
67
|
+
async function confirmLinePolicy (me, ref, line, version, { key, signature } = {}) {
|
|
68
|
+
const { q, v } = await pendingVersion(me, ref, line, version)
|
|
69
|
+
if (v.proposed_by !== me.member_id) throw new QuiltError(403, 'forbidden', 'only the proposer confirms a version; others approve it')
|
|
70
|
+
if ((await personAdmins(q.id)).some((a) => a !== me.member_id)) throw new QuiltError(409, 'needs_approval', 'another person in this quilt has admin: they approve it (a signed confirmation is for a quilt with one admin)')
|
|
71
|
+
const row = key ? await store.signingKey(String(key)) : null
|
|
72
|
+
if (!row || row.member !== me.member_id || row.removed_at) throw new QuiltError(400, 'bad_key', `${key || 'key'} is not one of your signing keys (seam key add)`)
|
|
73
|
+
if (!verifyText(row.public_key, confirmText(q.id, String(line), v.version, v.body), signature)) throw new QuiltError(422, 'bad_signature', 'the signature is not over this version')
|
|
74
|
+
return decide(q, String(line), v, 'approve', me, { confirmed_key: row.id })
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** POST /lines/:line/policy/versions/:v/reject: any person with admin, the proposer included. */
|
|
78
|
+
async function rejectLinePolicy (me, ref, line, version) {
|
|
79
|
+
const { q, v } = await pendingVersion(me, ref, line, version)
|
|
80
|
+
return decide(q, String(line), v, 'reject', me)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** GET /lines/:line/policy/versions: every version, newest first, and which one is active. */
|
|
84
|
+
async function linePolicyVersions (me, ref, line) {
|
|
85
|
+
const q = await quiltFor(me, ref, 'read')
|
|
86
|
+
const list = await versionsOf(q.id, String(line))
|
|
87
|
+
return { line: String(line), active: activeOf(list), versions: list.slice().sort((a, b) => b.version - a.version).map(view) }
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** One version (GET /lines/:line/policy?version=). */
|
|
91
|
+
async function linePolicyVersion (me, ref, line, version) {
|
|
92
|
+
const r = await linePolicyVersions(me, ref, line)
|
|
93
|
+
const v = r.versions.find((x) => x.version === Number(version))
|
|
94
|
+
if (!v) throw new QuiltError(404, 'not_found', `${line} has no policy version ${version}`)
|
|
95
|
+
return { line: String(line), ...v.body, version: v.version, state: v.state }
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return { propose, approveLinePolicy, confirmLinePolicy, rejectLinePolicy, linePolicyVersions, linePolicyVersion }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** `seam policy confirm`: signs a pending version with this computer's signing key and sends it. */
|
|
102
|
+
export async function confirmWithLocalKey (client, quilt, line, version) {
|
|
103
|
+
const k = localKey()
|
|
104
|
+
if (!k) throw new Error('No signing key on this computer: run seam key add first.')
|
|
105
|
+
const [info, list] = [await client.quilt(quilt), await client.linePolicyVersions(quilt, line)]
|
|
106
|
+
const v = list.versions.find((x) => x.version === version)
|
|
107
|
+
if (!v) throw new Error(`${line} has no policy version ${version}`)
|
|
108
|
+
return client.confirmLinePolicy(quilt, line, version, { key: k.key, signature: k.signText(confirmText(info.id, line, version, v.body)) })
|
|
109
|
+
}
|