@nacre.work/core 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +9 -0
- package/dist/authz/filter.d.ts +54 -3
- package/dist/authz/filter.d.ts.map +1 -1
- package/dist/authz/filter.js +45 -9
- package/dist/authz/filter.js.map +1 -1
- package/dist/authz/index.d.ts +2 -2
- package/dist/authz/index.d.ts.map +1 -1
- package/dist/authz/index.js +1 -1
- package/dist/authz/index.js.map +1 -1
- package/dist/config.d.ts +33 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +137 -3
- package/dist/config.js.map +1 -1
- package/dist/db/client.d.ts +39 -0
- package/dist/db/client.d.ts.map +1 -1
- package/dist/db/client.js +82 -0
- package/dist/db/client.js.map +1 -1
- package/dist/db/migrate.d.ts.map +1 -1
- package/dist/db/migrate.js +49 -0
- package/dist/db/migrate.js.map +1 -1
- package/dist/index.d.ts +15 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -3
- package/dist/index.js.map +1 -1
- package/dist/lifecycle.d.ts +57 -0
- package/dist/lifecycle.d.ts.map +1 -0
- package/dist/lifecycle.js +97 -0
- package/dist/lifecycle.js.map +1 -0
- package/dist/metadata.d.ts +50 -0
- package/dist/metadata.d.ts.map +1 -0
- package/dist/metadata.js +127 -0
- package/dist/metadata.js.map +1 -0
- package/dist/metrics.d.ts +18 -1
- package/dist/metrics.d.ts.map +1 -1
- package/dist/metrics.js +77 -2
- package/dist/metrics.js.map +1 -1
- package/dist/migrations/0008_authenticating_lookup.sql +124 -0
- package/dist/migrations/0009_refresh_tokens.sql +73 -0
- package/dist/migrations/0010_definer_search_path.sql +22 -0
- package/dist/migrations/0011_sweep_lease.sql +36 -0
- package/dist/migrations/0012_prune.sql +118 -0
- package/dist/migrations/0013_reindex.sql +65 -0
- package/dist/oauth.d.ts +45 -0
- package/dist/oauth.d.ts.map +1 -0
- package/dist/oauth.js +53 -0
- package/dist/oauth.js.map +1 -0
- package/dist/observability.d.ts.map +1 -1
- package/dist/observability.js +81 -13
- package/dist/observability.js.map +1 -1
- package/dist/passwords.d.ts +37 -0
- package/dist/passwords.d.ts.map +1 -0
- package/dist/passwords.js +234 -0
- package/dist/passwords.js.map +1 -0
- package/dist/redis.d.ts +46 -0
- package/dist/redis.d.ts.map +1 -0
- package/dist/redis.js +234 -0
- package/dist/redis.js.map +1 -0
- package/dist/reindex.d.ts +87 -0
- package/dist/reindex.d.ts.map +1 -0
- package/dist/reindex.js +97 -0
- package/dist/reindex.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/vector/query.d.ts +49 -4
- package/dist/vector/query.d.ts.map +1 -1
- package/dist/vector/query.js +43 -10
- package/dist/vector/query.js.map +1 -1
- package/dist/vector/search.d.ts +109 -4
- package/dist/vector/search.d.ts.map +1 -1
- package/dist/vector/search.js +166 -7
- package/dist/vector/search.js.map +1 -1
- package/package.json +26 -8
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { randomBytes, scrypt as scryptCallback, timingSafeEqual } from 'node:crypto';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
/**
|
|
4
|
+
* Password hashing, with scrypt from the standard library.
|
|
5
|
+
*
|
|
6
|
+
* Argon2id is the first choice in every current guideline and it is not
|
|
7
|
+
* available here without a dependency — either a native module built by
|
|
8
|
+
* node-gyp, or one shipping prebuilt binaries for every platform an operator
|
|
9
|
+
* might self-host on. Both are things every operator of a security product
|
|
10
|
+
* inherits, and the second is a supply chain that ships machine code.
|
|
11
|
+
*
|
|
12
|
+
* scrypt is the documented alternative in the same guidelines, it is memory-
|
|
13
|
+
* hard, and it is in `node:crypto`. The parameters below are OWASP's minimum
|
|
14
|
+
* for it. This is a deliberate second choice rather than an oversight, and the
|
|
15
|
+
* encoding carries its parameters so that moving to Argon2id later is a new
|
|
16
|
+
* prefix and a rehash on next login rather than a migration that invalidates
|
|
17
|
+
* every password.
|
|
18
|
+
*
|
|
19
|
+
* ## Why the cost is a constant and not configuration
|
|
20
|
+
*
|
|
21
|
+
* A tunable would let an operator turn it down, and the only reason anyone ever
|
|
22
|
+
* does is that logins felt slow. The cost is the entire defence: it is what
|
|
23
|
+
* makes a stolen `password_hash` column expensive rather than a wordlist away
|
|
24
|
+
* from plaintext.
|
|
25
|
+
*/
|
|
26
|
+
// promisify picks the overload without options, and the options are the whole
|
|
27
|
+
// point here — the cost parameters live in them.
|
|
28
|
+
const scrypt = promisify(scryptCallback);
|
|
29
|
+
/** OWASP's minimum for scrypt: N=2^17, r=8, p=1. */
|
|
30
|
+
const N = 131_072;
|
|
31
|
+
const R = 8;
|
|
32
|
+
const P = 1;
|
|
33
|
+
const KEY_BYTES = 64;
|
|
34
|
+
const SALT_BYTES = 32;
|
|
35
|
+
/**
|
|
36
|
+
* 128 * N * r is 134 MB, and Node's default `maxmem` is 32 MB — without this
|
|
37
|
+
* every call fails with an error about memory rather than a wrong password,
|
|
38
|
+
* which is a confusing way to find out. The headroom is for the internal
|
|
39
|
+
* allocations on top of the working set.
|
|
40
|
+
*/
|
|
41
|
+
const MAX_MEM = 256 * 1024 * 1024;
|
|
42
|
+
/* ─────────────────────── the gate, and why there is one ───────────────────────
|
|
43
|
+
*
|
|
44
|
+
* `crypto.scrypt` is asynchronous in the sense that it does not block the event
|
|
45
|
+
* loop — it runs on libuv's thread pool. That pool has four threads by default,
|
|
46
|
+
* and it is not the crypto pool: `dns.lookup`, `fs`, and zlib are on it too.
|
|
47
|
+
*
|
|
48
|
+
* So four concurrent sign-in attempts occupy the whole pool for the duration of
|
|
49
|
+
* a deliberately expensive hash, and the fifth thing to want a thread waits.
|
|
50
|
+
* When that thing is `getaddrinfo` for Postgres, the request path stops on a
|
|
51
|
+
* name lookup — which looks like a database problem, on a dashboard, at the
|
|
52
|
+
* exact moment somebody is spraying the login endpoint. Each call also holds
|
|
53
|
+
* 134 MB while it runs, so the unbounded version is 134 MB times however many
|
|
54
|
+
* requests arrive at once.
|
|
55
|
+
*
|
|
56
|
+
* The rate limiter does not cover this. It counts per email address, and an
|
|
57
|
+
* attacker rotating addresses never repeats one; it counts in Redis, so it
|
|
58
|
+
* fails open by design; and it runs per request while this is about how many
|
|
59
|
+
* run *at the same time*, which is a different quantity. A limit of ten per
|
|
60
|
+
* fifteen minutes still permits every attempt to arrive in the same second.
|
|
61
|
+
*
|
|
62
|
+
* ─── the numbers ───
|
|
63
|
+
*
|
|
64
|
+
* Two concurrent hashes, so half the default pool stays free for the DNS and
|
|
65
|
+
* file I/O the rest of the process needs. Read from UV_THREADPOOL_SIZE where an
|
|
66
|
+
* operator has raised it, because the point is a fraction of the pool rather
|
|
67
|
+
* than the number two.
|
|
68
|
+
*
|
|
69
|
+
* The queue is bounded as well, and that bound is the load-shedding one: a
|
|
70
|
+
* queue is cheap in memory — a waiting request holds no scrypt buffer — but an
|
|
71
|
+
* unbounded one converts a flood into unbounded latency for the legitimate
|
|
72
|
+
* user, who ends up behind ten thousand guesses. Past the bound the call is
|
|
73
|
+
* refused immediately and the caller answers 503, which is the honest response
|
|
74
|
+
* to "this process cannot verify a password right now".
|
|
75
|
+
*
|
|
76
|
+
* Refusing is not an oracle. It depends on how loaded the process is and not at
|
|
77
|
+
* all on whether the account exists, and it is returned identically to the
|
|
78
|
+
* caller whether the address matched a user or not.
|
|
79
|
+
*/
|
|
80
|
+
const poolSize = Number(process.env.UV_THREADPOOL_SIZE ?? 4);
|
|
81
|
+
const MAX_CONCURRENT = Math.max(1, Math.floor((Number.isFinite(poolSize) ? poolSize : 4) / 2));
|
|
82
|
+
const MAX_QUEUED = 64;
|
|
83
|
+
/** Raised when the process is already verifying as many passwords as it will. */
|
|
84
|
+
export class TooBusy extends Error {
|
|
85
|
+
constructor() {
|
|
86
|
+
super('too many passwords are being verified at once');
|
|
87
|
+
this.name = 'TooBusy';
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
let active = 0;
|
|
91
|
+
const waiting = [];
|
|
92
|
+
async function gated(work) {
|
|
93
|
+
if (active >= MAX_CONCURRENT) {
|
|
94
|
+
if (waiting.length >= MAX_QUEUED)
|
|
95
|
+
throw new TooBusy();
|
|
96
|
+
// Waits to be *handed* a permit, and does not increment on waking.
|
|
97
|
+
//
|
|
98
|
+
// The obvious version releases the permit, wakes a waiter, and lets it take
|
|
99
|
+
// a slot on its own — which leaves a window between the release and the
|
|
100
|
+
// waiter's turn where an arriving call sees a free slot and takes it too.
|
|
101
|
+
// As it happens that window is not reachable from a single Node event loop:
|
|
102
|
+
// the waiter's continuation is queued as a microtask at the moment of the
|
|
103
|
+
// release, so it runs before anything that could have arrived afterwards.
|
|
104
|
+
// Tried to reproduce it and could not.
|
|
105
|
+
//
|
|
106
|
+
// Handing the permit over regardless, because "correct, given the current
|
|
107
|
+
// microtask ordering of the runtime" is a bad thing for a bound to depend
|
|
108
|
+
// on, and this version does not depend on it at all: the count never
|
|
109
|
+
// changes when a permit changes owner, so there is no window to reason
|
|
110
|
+
// about. It is also shorter.
|
|
111
|
+
await new Promise((resolve) => waiting.push(resolve));
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
active++;
|
|
115
|
+
}
|
|
116
|
+
try {
|
|
117
|
+
return await work();
|
|
118
|
+
}
|
|
119
|
+
finally {
|
|
120
|
+
// Hand the permit over rather than release it. `active` is unchanged when
|
|
121
|
+
// there is a waiter, because the slot never becomes free — it changes
|
|
122
|
+
// owner. One waiter per completion, so a queue drains at the rate work
|
|
123
|
+
// finishes and never all at once.
|
|
124
|
+
const next = waiting.shift();
|
|
125
|
+
if (next === undefined)
|
|
126
|
+
active--;
|
|
127
|
+
else
|
|
128
|
+
next();
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/** For a metric and for tests. Not a decision anything makes. */
|
|
132
|
+
export const hashingLoad = () => ({
|
|
133
|
+
active,
|
|
134
|
+
queued: waiting.length,
|
|
135
|
+
limit: MAX_CONCURRENT,
|
|
136
|
+
});
|
|
137
|
+
/**
|
|
138
|
+
* `scrypt$N$r$p$salt$hash`, base64url, parameters first.
|
|
139
|
+
*
|
|
140
|
+
* Self-describing on purpose. A bare hash cannot be verified after the cost is
|
|
141
|
+
* raised, so raising it means invalidating every password in the database;
|
|
142
|
+
* carrying the parameters means old hashes keep verifying and get rewritten as
|
|
143
|
+
* people sign in.
|
|
144
|
+
*/
|
|
145
|
+
export async function hashPassword(password) {
|
|
146
|
+
const salt = randomBytes(SALT_BYTES);
|
|
147
|
+
const key = await gated(() => scrypt(password.normalize('NFKC'), salt, KEY_BYTES, {
|
|
148
|
+
N,
|
|
149
|
+
r: R,
|
|
150
|
+
p: P,
|
|
151
|
+
maxmem: MAX_MEM,
|
|
152
|
+
}));
|
|
153
|
+
return `scrypt$${N}$${R}$${P}$${salt.toString('base64url')}$${key.toString('base64url')}`;
|
|
154
|
+
}
|
|
155
|
+
function parse(encoded) {
|
|
156
|
+
const parts = encoded.split('$');
|
|
157
|
+
if (parts.length !== 6 || parts[0] !== 'scrypt')
|
|
158
|
+
return undefined;
|
|
159
|
+
const n = Number(parts[1]);
|
|
160
|
+
const r = Number(parts[2]);
|
|
161
|
+
const p = Number(parts[3]);
|
|
162
|
+
if (!Number.isInteger(n) || !Number.isInteger(r) || !Number.isInteger(p))
|
|
163
|
+
return undefined;
|
|
164
|
+
// A stored record could otherwise ask for a work factor large enough to hang
|
|
165
|
+
// the process, which turns one poisoned row into an outage.
|
|
166
|
+
if (n < 1024 || n > 1 << 22 || r < 1 || r > 32 || p < 1 || p > 16)
|
|
167
|
+
return undefined;
|
|
168
|
+
try {
|
|
169
|
+
return {
|
|
170
|
+
n,
|
|
171
|
+
r,
|
|
172
|
+
p,
|
|
173
|
+
salt: Buffer.from(parts[4], 'base64url'),
|
|
174
|
+
key: Buffer.from(parts[5], 'base64url'),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Verify, in time that does not depend on whether the password was right.
|
|
183
|
+
*
|
|
184
|
+
* A malformed or absent hash answers false rather than throwing: the caller
|
|
185
|
+
* turns every failure into the same refusal, and an exception on one of them
|
|
186
|
+
* would be a way to tell them apart from the outside.
|
|
187
|
+
*/
|
|
188
|
+
export async function verifyPassword(password, encoded) {
|
|
189
|
+
const parsed = parse(encoded);
|
|
190
|
+
if (parsed === undefined)
|
|
191
|
+
return false;
|
|
192
|
+
const key = await gated(() => scrypt(password.normalize('NFKC'), parsed.salt, parsed.key.length, {
|
|
193
|
+
N: parsed.n,
|
|
194
|
+
r: parsed.r,
|
|
195
|
+
p: parsed.p,
|
|
196
|
+
maxmem: MAX_MEM,
|
|
197
|
+
}));
|
|
198
|
+
if (key.length !== parsed.key.length)
|
|
199
|
+
return false;
|
|
200
|
+
return timingSafeEqual(key, parsed.key);
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Whether a stored hash was made with parameters weaker than the current ones.
|
|
204
|
+
*
|
|
205
|
+
* The caller rehashes on a successful sign-in, which is the only moment the
|
|
206
|
+
* plaintext exists. Without this, raising the cost protects nobody who already
|
|
207
|
+
* has an account.
|
|
208
|
+
*/
|
|
209
|
+
export function needsRehash(encoded) {
|
|
210
|
+
const parsed = parse(encoded);
|
|
211
|
+
if (parsed === undefined)
|
|
212
|
+
return true;
|
|
213
|
+
return parsed.n < N || parsed.r < R || parsed.p < P;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Spend the same work as a real verification, and fail.
|
|
217
|
+
*
|
|
218
|
+
* For the case where no user matched. Returning early there makes the response
|
|
219
|
+
* time say whether an address has an account — the enumeration oracle that
|
|
220
|
+
* makes a careful 404 elsewhere pointless. This burns a scrypt call against a
|
|
221
|
+
* fixed hash so the two paths cost the same.
|
|
222
|
+
*
|
|
223
|
+
* It is not exact, and cannot be: a database round trip that finds nothing is
|
|
224
|
+
* shorter than one that finds a row. It removes the difference that matters,
|
|
225
|
+
* which is the one measured in hundreds of milliseconds rather than in tenths
|
|
226
|
+
* of one.
|
|
227
|
+
*/
|
|
228
|
+
let decoy;
|
|
229
|
+
export async function spendVerificationTime(password) {
|
|
230
|
+
decoy ??= await hashPassword(randomBytes(32).toString('base64url'));
|
|
231
|
+
await verifyPassword(password, decoy);
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=passwords.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"passwords.js","sourceRoot":"","sources":["../passwords.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,IAAI,cAAc,EAAE,eAAe,EAAsB,MAAM,aAAa,CAAA;AACxG,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,8EAA8E;AAC9E,iDAAiD;AACjD,MAAM,MAAM,GAAG,SAAS,CAAC,cAAc,CAKnB,CAAA;AAEpB,oDAAoD;AACpD,MAAM,CAAC,GAAG,OAAO,CAAA;AACjB,MAAM,CAAC,GAAG,CAAC,CAAA;AACX,MAAM,CAAC,GAAG,CAAC,CAAA;AACX,MAAM,SAAS,GAAG,EAAE,CAAA;AACpB,MAAM,UAAU,GAAG,EAAE,CAAA;AAErB;;;;;GAKG;AACH,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAA;AAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,CAAC,CAAA;AAC5D,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;AAC9F,MAAM,UAAU,GAAG,EAAE,CAAA;AAErB,iFAAiF;AACjF,MAAM,OAAO,OAAQ,SAAQ,KAAK;IAChC;QACE,KAAK,CAAC,+CAA+C,CAAC,CAAA;QACtD,IAAI,CAAC,IAAI,GAAG,SAAS,CAAA;IACvB,CAAC;CACF;AAED,IAAI,MAAM,GAAG,CAAC,CAAA;AACd,MAAM,OAAO,GAAmB,EAAE,CAAA;AAElC,KAAK,UAAU,KAAK,CAAI,IAAsB;IAC5C,IAAI,MAAM,IAAI,cAAc,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM,IAAI,OAAO,EAAE,CAAA;QACrD,mEAAmE;QACnE,EAAE;QACF,4EAA4E;QAC5E,wEAAwE;QACxE,0EAA0E;QAC1E,4EAA4E;QAC5E,0EAA0E;QAC1E,0EAA0E;QAC1E,uCAAuC;QACvC,EAAE;QACF,0EAA0E;QAC1E,0EAA0E;QAC1E,qEAAqE;QACrE,uEAAuE;QACvE,6BAA6B;QAC7B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAA;IAC7D,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,CAAA;IACV,CAAC;IAED,IAAI,CAAC;QACH,OAAO,MAAM,IAAI,EAAE,CAAA;IACrB,CAAC;YAAS,CAAC;QACT,0EAA0E;QAC1E,sEAAsE;QACtE,uEAAuE;QACvE,kCAAkC;QAClC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAA;QAC5B,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,EAAE,CAAA;;YAC3B,IAAI,EAAE,CAAA;IACb,CAAC;AACH,CAAC;AAED,iEAAiE;AACjE,MAAM,CAAC,MAAM,WAAW,GAAG,GAAsD,EAAE,CAAC,CAAC;IACnF,MAAM;IACN,MAAM,EAAE,OAAO,CAAC,MAAM;IACtB,KAAK,EAAE,cAAc;CACtB,CAAC,CAAA;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,IAAI,GAAG,WAAW,CAAC,UAAU,CAAC,CAAA;IACpC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,CAC3B,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;QAClD,CAAC;QACD,CAAC,EAAE,CAAC;QACJ,CAAC,EAAE,CAAC;QACJ,MAAM,EAAE,OAAO;KAChB,CAAC,CACH,CAAA;IACD,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAA;AAC3F,CAAC;AAUD,SAAS,KAAK,CAAC,OAAe;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAChC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAEjE,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;QAAE,OAAO,SAAS,CAAA;IAC1F,6EAA6E;IAC7E,4DAA4D;IAC5D,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE;QAAE,OAAO,SAAS,CAAA;IAEnF,IAAI,CAAC;QACH,OAAO;YACL,CAAC;YACD,CAAC;YACD,CAAC;YACD,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAW,EAAE,WAAW,CAAC;YAClD,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAW,EAAE,WAAW,CAAC;SAClD,CAAA;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB,EAAE,OAAe;IACpE,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAA;IAC7B,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,KAAK,CAAA;IAEtC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,CAC3B,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE;QACjE,CAAC,EAAE,MAAM,CAAC,CAAC;QACX,CAAC,EAAE,MAAM,CAAC,CAAC;QACX,CAAC,EAAE,MAAM,CAAC,CAAC;QACX,MAAM,EAAE,OAAO;KAChB,CAAC,CACH,CAAA;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM;QAAE,OAAO,KAAK,CAAA;IAClD,OAAO,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;AACzC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAA;IAC7B,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAA;IACrC,OAAO,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAA;AACrD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,IAAI,KAAyB,CAAA;AAC7B,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,QAAgB;IAC1D,KAAK,KAAK,MAAM,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAA;IACnE,MAAM,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IACrC,OAAO,KAAK,CAAA;AACd,CAAC"}
|
package/dist/redis.d.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A very small Redis client.
|
|
3
|
+
*
|
|
4
|
+
* Written rather than installed, for the same reason `metrics.ts` was: the
|
|
5
|
+
* commands needed here are five, RESP2 is a text protocol that fits on a page,
|
|
6
|
+
* and a dependency in a self-hosted security product is something every
|
|
7
|
+
* operator inherits. The alternative was pulling a client with its own
|
|
8
|
+
* transitive tree to run `INCR`.
|
|
9
|
+
*
|
|
10
|
+
* `NACRE_REDIS_URL` has been required configuration and Redis has been in every
|
|
11
|
+
* Compose profile — with the API waiting on its healthcheck — since before
|
|
12
|
+
* anything connected to it. That is what this closes: a service every
|
|
13
|
+
* deployment had to run, for nothing.
|
|
14
|
+
*
|
|
15
|
+
* Not a general-purpose client. No pub/sub, no cluster, no pipelining beyond
|
|
16
|
+
* what the callers here need, no reconnect backoff worth the name. When
|
|
17
|
+
* something needs more than this, replace it rather than growing it.
|
|
18
|
+
*/
|
|
19
|
+
export interface RedisOptions {
|
|
20
|
+
readonly url: string;
|
|
21
|
+
/** How long to wait for a reply before giving up on the connection. */
|
|
22
|
+
readonly timeoutMs?: number;
|
|
23
|
+
}
|
|
24
|
+
type Reply = string | number | null | Reply[];
|
|
25
|
+
export declare class RedisError extends Error {
|
|
26
|
+
constructor(message: string, options?: {
|
|
27
|
+
cause?: unknown;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
export declare class Redis {
|
|
31
|
+
#private;
|
|
32
|
+
constructor(options: RedisOptions);
|
|
33
|
+
/** One command. Reconnects once if the connection went away between calls. */
|
|
34
|
+
command(...parts: string[]): Promise<Reply>;
|
|
35
|
+
/**
|
|
36
|
+
* Several commands written together, answered in order.
|
|
37
|
+
*
|
|
38
|
+
* Not a transaction — no MULTI. The callers here want one round trip, not
|
|
39
|
+
* atomicity across commands, and saying so is better than implying more.
|
|
40
|
+
*/
|
|
41
|
+
pipeline(...commands: readonly (readonly string[])[]): Promise<Reply[]>;
|
|
42
|
+
ping(): Promise<boolean>;
|
|
43
|
+
close(): void;
|
|
44
|
+
}
|
|
45
|
+
export {};
|
|
46
|
+
//# sourceMappingURL=redis.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.d.ts","sourceRoot":"","sources":["../redis.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,uEAAuE;IACvE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,KAAK,KAAK,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,KAAK,EAAE,CAAA;AAW7C,qBAAa,UAAW,SAAQ,KAAK;gBACvB,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAI3D;AAED,qBAAa,KAAK;;gBAUJ,OAAO,EAAE,YAAY;IAiHjC,8EAA8E;IACxE,OAAO,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IAWjD;;;;;OAKG;IACG,QAAQ,CAAC,GAAG,QAAQ,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IA6BvE,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC;IAQ9B,KAAK,IAAI,IAAI;CAId"}
|
package/dist/redis.js
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { connect } from 'node:net';
|
|
2
|
+
import { once } from 'node:events';
|
|
3
|
+
const CRLF = '\r\n';
|
|
4
|
+
/** RESP2 is length-prefixed, so a command is its parts with their byte lengths. */
|
|
5
|
+
function encode(parts) {
|
|
6
|
+
let out = `*${parts.length}${CRLF}`;
|
|
7
|
+
for (const p of parts)
|
|
8
|
+
out += `$${Buffer.byteLength(p)}${CRLF}${p}${CRLF}`;
|
|
9
|
+
return out;
|
|
10
|
+
}
|
|
11
|
+
export class RedisError extends Error {
|
|
12
|
+
constructor(message, options) {
|
|
13
|
+
super(message, options);
|
|
14
|
+
this.name = 'RedisError';
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export class Redis {
|
|
18
|
+
#url;
|
|
19
|
+
#timeoutMs;
|
|
20
|
+
#socket;
|
|
21
|
+
#buffer = Buffer.alloc(0);
|
|
22
|
+
/** One resolver per in-flight command, in the order they were sent. */
|
|
23
|
+
#pending = [];
|
|
24
|
+
#connecting;
|
|
25
|
+
constructor(options) {
|
|
26
|
+
this.#url = new URL(options.url);
|
|
27
|
+
this.#timeoutMs = options.timeoutMs ?? 2000;
|
|
28
|
+
}
|
|
29
|
+
async #connectOnce() {
|
|
30
|
+
const socket = connect({
|
|
31
|
+
host: this.#url.hostname,
|
|
32
|
+
port: Number(this.#url.port || 6379),
|
|
33
|
+
});
|
|
34
|
+
socket.setNoDelay(true);
|
|
35
|
+
// A connect deadline, which `#timeoutMs` alone did not give: that guards
|
|
36
|
+
// replies, and a reply cannot time out on a connection that never opened.
|
|
37
|
+
//
|
|
38
|
+
// A restarted Redis refuses immediately and fails open in milliseconds,
|
|
39
|
+
// which is the case the fail-open was designed against. A *partitioned* one
|
|
40
|
+
// — a firewall dropping packets, a node that vanished — blackholes the SYN,
|
|
41
|
+
// and Linux retries it for around 127 seconds. Every rate-limited request
|
|
42
|
+
// blocked for that long before being allowed through, so the documented
|
|
43
|
+
// "a cache outage must not become an outage" turned into one anyway.
|
|
44
|
+
socket.setTimeout(this.#timeoutMs, () => {
|
|
45
|
+
socket.destroy(new RedisError(`redis did not accept a connection within ${this.#timeoutMs}ms`));
|
|
46
|
+
});
|
|
47
|
+
socket.on('data', (chunk) => {
|
|
48
|
+
this.#buffer = Buffer.concat([this.#buffer, chunk]);
|
|
49
|
+
this.#drain();
|
|
50
|
+
});
|
|
51
|
+
const fail = (cause) => {
|
|
52
|
+
// Everything in flight is lost with the connection. Rejecting rather than
|
|
53
|
+
// hanging is what lets the callers fall back on their own terms.
|
|
54
|
+
const pending = this.#pending;
|
|
55
|
+
this.#pending = [];
|
|
56
|
+
this.#socket = undefined;
|
|
57
|
+
this.#buffer = Buffer.alloc(0);
|
|
58
|
+
for (const p of pending)
|
|
59
|
+
p.reject(new RedisError('the redis connection closed', { cause }));
|
|
60
|
+
};
|
|
61
|
+
socket.on('error', fail);
|
|
62
|
+
// Guarded on identity: without it a late `close` from a socket that has
|
|
63
|
+
// already been replaced clears `#socket` and rejects the commands in flight
|
|
64
|
+
// on the *new* connection.
|
|
65
|
+
socket.on('close', () => {
|
|
66
|
+
if (this.#socket === socket || this.#socket === undefined)
|
|
67
|
+
fail(undefined);
|
|
68
|
+
});
|
|
69
|
+
await once(socket, 'connect');
|
|
70
|
+
// The deadline was for connecting. Replies have their own, per command.
|
|
71
|
+
socket.setTimeout(0);
|
|
72
|
+
this.#socket = socket;
|
|
73
|
+
// Auth and database come from the URL, so a deployment configures them in
|
|
74
|
+
// one place rather than in three.
|
|
75
|
+
const password = decodeURIComponent(this.#url.password);
|
|
76
|
+
const username = decodeURIComponent(this.#url.username);
|
|
77
|
+
if (password !== '') {
|
|
78
|
+
await this.#send(username === '' ? ['AUTH', password] : ['AUTH', username, password]);
|
|
79
|
+
}
|
|
80
|
+
const db = this.#url.pathname.replace(/^\//, '');
|
|
81
|
+
if (db !== '' && db !== '0')
|
|
82
|
+
await this.#send(['SELECT', db]);
|
|
83
|
+
}
|
|
84
|
+
async #ensure() {
|
|
85
|
+
if (this.#socket !== undefined)
|
|
86
|
+
return;
|
|
87
|
+
// One connect attempt shared by every caller that arrives while it runs.
|
|
88
|
+
this.#connecting ??= this.#connectOnce().finally(() => {
|
|
89
|
+
this.#connecting = undefined;
|
|
90
|
+
});
|
|
91
|
+
await this.#connecting;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Parse whatever complete replies are in the buffer and hand them to the
|
|
95
|
+
* waiting callers, in order. RESP2 answers in the order it was asked, which
|
|
96
|
+
* is what makes a queue of resolvers correct without request ids.
|
|
97
|
+
*/
|
|
98
|
+
#drain() {
|
|
99
|
+
for (;;) {
|
|
100
|
+
const parsed = parse(this.#buffer, 0);
|
|
101
|
+
if (parsed === undefined)
|
|
102
|
+
return;
|
|
103
|
+
this.#buffer = this.#buffer.subarray(parsed.next);
|
|
104
|
+
const waiter = this.#pending.shift();
|
|
105
|
+
if (waiter === undefined)
|
|
106
|
+
continue;
|
|
107
|
+
if (parsed.error !== undefined)
|
|
108
|
+
waiter.reject(new RedisError(parsed.error));
|
|
109
|
+
else
|
|
110
|
+
waiter.resolve(parsed.value);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
#send(parts) {
|
|
114
|
+
const socket = this.#socket;
|
|
115
|
+
if (socket === undefined)
|
|
116
|
+
return Promise.reject(new RedisError('not connected'));
|
|
117
|
+
return new Promise((resolve, reject) => {
|
|
118
|
+
const timer = setTimeout(() => {
|
|
119
|
+
reject(new RedisError(`redis did not answer ${parts[0]} within ${this.#timeoutMs}ms`));
|
|
120
|
+
}, this.#timeoutMs);
|
|
121
|
+
this.#pending.push({
|
|
122
|
+
resolve: (v) => {
|
|
123
|
+
clearTimeout(timer);
|
|
124
|
+
resolve(v);
|
|
125
|
+
},
|
|
126
|
+
reject: (e) => {
|
|
127
|
+
clearTimeout(timer);
|
|
128
|
+
reject(e);
|
|
129
|
+
},
|
|
130
|
+
});
|
|
131
|
+
socket.write(encode(parts));
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
/** One command. Reconnects once if the connection went away between calls. */
|
|
135
|
+
async command(...parts) {
|
|
136
|
+
await this.#ensure();
|
|
137
|
+
try {
|
|
138
|
+
return await this.#send(parts);
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
if (!(error instanceof RedisError) || this.#socket !== undefined)
|
|
142
|
+
throw error;
|
|
143
|
+
await this.#ensure();
|
|
144
|
+
return this.#send(parts);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Several commands written together, answered in order.
|
|
149
|
+
*
|
|
150
|
+
* Not a transaction — no MULTI. The callers here want one round trip, not
|
|
151
|
+
* atomicity across commands, and saying so is better than implying more.
|
|
152
|
+
*/
|
|
153
|
+
async pipeline(...commands) {
|
|
154
|
+
await this.#ensure();
|
|
155
|
+
const socket = this.#socket;
|
|
156
|
+
if (socket === undefined)
|
|
157
|
+
throw new RedisError('not connected');
|
|
158
|
+
const replies = commands.map((parts) => new Promise((resolve, reject) => {
|
|
159
|
+
const timer = setTimeout(() => reject(new RedisError(`redis did not answer ${parts[0]} in time`)), this.#timeoutMs);
|
|
160
|
+
this.#pending.push({
|
|
161
|
+
resolve: (v) => {
|
|
162
|
+
clearTimeout(timer);
|
|
163
|
+
resolve(v);
|
|
164
|
+
},
|
|
165
|
+
reject: (e) => {
|
|
166
|
+
clearTimeout(timer);
|
|
167
|
+
reject(e);
|
|
168
|
+
},
|
|
169
|
+
});
|
|
170
|
+
}));
|
|
171
|
+
socket.write(commands.map(encode).join(''));
|
|
172
|
+
return Promise.all(replies);
|
|
173
|
+
}
|
|
174
|
+
async ping() {
|
|
175
|
+
try {
|
|
176
|
+
return (await this.command('PING')) === 'PONG';
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
close() {
|
|
183
|
+
this.#socket?.destroy();
|
|
184
|
+
this.#socket = undefined;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/** One RESP2 value, or undefined when the buffer does not hold a whole one. */
|
|
188
|
+
function parse(buffer, at) {
|
|
189
|
+
if (at >= buffer.length)
|
|
190
|
+
return undefined;
|
|
191
|
+
const end = buffer.indexOf('\r\n', at);
|
|
192
|
+
if (end === -1)
|
|
193
|
+
return undefined;
|
|
194
|
+
const kind = String.fromCharCode(buffer[at]);
|
|
195
|
+
const body = buffer.toString('utf8', at + 1, end);
|
|
196
|
+
const after = end + 2;
|
|
197
|
+
switch (kind) {
|
|
198
|
+
case '+':
|
|
199
|
+
return { value: body, next: after };
|
|
200
|
+
case '-':
|
|
201
|
+
return { value: null, next: after, error: body };
|
|
202
|
+
case ':':
|
|
203
|
+
return { value: Number(body), next: after };
|
|
204
|
+
case '$': {
|
|
205
|
+
const length = Number(body);
|
|
206
|
+
if (length === -1)
|
|
207
|
+
return { value: null, next: after };
|
|
208
|
+
const stop = after + length;
|
|
209
|
+
if (buffer.length < stop + 2)
|
|
210
|
+
return undefined;
|
|
211
|
+
return { value: buffer.toString('utf8', after, stop), next: stop + 2 };
|
|
212
|
+
}
|
|
213
|
+
case '*': {
|
|
214
|
+
const count = Number(body);
|
|
215
|
+
if (count === -1)
|
|
216
|
+
return { value: null, next: after };
|
|
217
|
+
const items = [];
|
|
218
|
+
let cursor = after;
|
|
219
|
+
for (let i = 0; i < count; i++) {
|
|
220
|
+
const item = parse(buffer, cursor);
|
|
221
|
+
if (item === undefined)
|
|
222
|
+
return undefined;
|
|
223
|
+
// An error inside an array belongs to that element; the callers here
|
|
224
|
+
// never send commands that can produce one, so it becomes a null.
|
|
225
|
+
items.push(item.error === undefined ? item.value : null);
|
|
226
|
+
cursor = item.next;
|
|
227
|
+
}
|
|
228
|
+
return { value: items, next: cursor };
|
|
229
|
+
}
|
|
230
|
+
default:
|
|
231
|
+
return { value: null, next: after, error: `unrecognised reply type ${kind}` };
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=redis.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.js","sourceRoot":"","sources":["../redis.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAe,MAAM,UAAU,CAAA;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AA6BlC,MAAM,IAAI,GAAG,MAAM,CAAA;AAEnB,mFAAmF;AACnF,SAAS,MAAM,CAAC,KAAwB;IACtC,IAAI,GAAG,GAAG,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,EAAE,CAAA;IACnC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,GAAG,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,EAAE,CAAA;IAC1E,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,OAAO,UAAW,SAAQ,KAAK;IACnC,YAAY,OAAe,EAAE,OAA6B;QACxD,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACvB,IAAI,CAAC,IAAI,GAAG,YAAY,CAAA;IAC1B,CAAC;CACF;AAED,MAAM,OAAO,KAAK;IACP,IAAI,CAAK;IACT,UAAU,CAAQ;IAE3B,OAAO,CAAoB;IAC3B,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACzB,uEAAuE;IACvE,QAAQ,GAAoE,EAAE,CAAA;IAC9E,WAAW,CAA2B;IAEtC,YAAY,OAAqB;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAChC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAA;IAC7C,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,MAAM,GAAG,OAAO,CAAC;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ;YACxB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;SACrC,CAAC,CAAA;QACF,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QAEvB,yEAAyE;QACzE,0EAA0E;QAC1E,EAAE;QACF,wEAAwE;QACxE,4EAA4E;QAC5E,4EAA4E;QAC5E,0EAA0E;QAC1E,wEAAwE;QACxE,qEAAqE;QACrE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,4CAA4C,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,CAAA;QACjG,CAAC,CAAC,CAAA;QAEF,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAClC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAA;YACnD,IAAI,CAAC,MAAM,EAAE,CAAA;QACf,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE;YAC9B,0EAA0E;YAC1E,iEAAiE;YACjE,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAA;YAC7B,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;YAClB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAA;YACxB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,OAAO;gBAAE,CAAC,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7F,CAAC,CAAA;QACD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACxB,wEAAwE;QACxE,4EAA4E;QAC5E,2BAA2B;QAC3B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACtB,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;gBAAE,IAAI,CAAC,SAAS,CAAC,CAAA;QAC5E,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAC7B,wEAAwE;QACxE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACpB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QAErB,0EAA0E;QAC1E,kCAAkC;QAClC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACvD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACvD,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAA;QACvF,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAChD,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,GAAG;YAAE,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,OAAM;QACtC,yEAAyE;QACzE,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACpD,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;QAC9B,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAED;;;;OAIG;IACH,MAAM;QACJ,SAAS,CAAC;YACR,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;YACrC,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAM;YAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAEjD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;YACpC,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAQ;YAClC,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;gBAAE,MAAM,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;;gBACtE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAwB;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAA;QAC3B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,eAAe,CAAC,CAAC,CAAA;QAEhF,OAAO,IAAI,OAAO,CAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5C,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,CAAC,IAAI,UAAU,CAAC,wBAAwB,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,CAAA;YACxF,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;YAEnB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjB,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;oBACb,YAAY,CAAC,KAAK,CAAC,CAAA;oBACnB,OAAO,CAAC,CAAC,CAAC,CAAA;gBACZ,CAAC;gBACD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;oBACZ,YAAY,CAAC,KAAK,CAAC,CAAA;oBACnB,MAAM,CAAC,CAAC,CAAC,CAAA;gBACX,CAAC;aACF,CAAC,CAAA;YACF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;QAC7B,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,8EAA8E;IAC9E,KAAK,CAAC,OAAO,CAAC,GAAG,KAAe;QAC9B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;gBAAE,MAAM,KAAK,CAAA;YAC7E,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;YACpB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAG,QAAwC;QACxD,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAA;QAC3B,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,IAAI,UAAU,CAAC,eAAe,CAAC,CAAA;QAE/D,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAC1B,CAAC,KAAK,EAAE,EAAE,CACR,IAAI,OAAO,CAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,wBAAwB,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,EACxE,IAAI,CAAC,UAAU,CAChB,CAAA;YACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjB,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;oBACb,YAAY,CAAC,KAAK,CAAC,CAAA;oBACnB,OAAO,CAAC,CAAC,CAAC,CAAA;gBACZ,CAAC;gBACD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;oBACZ,YAAY,CAAC,KAAK,CAAC,CAAA;oBACnB,MAAM,CAAC,CAAC,CAAC,CAAA;gBACX,CAAC;aACF,CAAC,CAAA;QACJ,CAAC,CAAC,CACL,CAAA;QAED,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QAC3C,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAA;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAA;QACvB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAA;IAC1B,CAAC;CACF;AAQD,+EAA+E;AAC/E,SAAS,KAAK,CAAC,MAAc,EAAE,EAAU;IACvC,IAAI,EAAE,IAAI,MAAM,CAAC,MAAM;QAAE,OAAO,SAAS,CAAA;IAEzC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACtC,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,SAAS,CAAA;IAEhC,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAW,CAAC,CAAA;IACtD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAA;IACjD,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;IAErB,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,GAAG;YACN,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;QACrC,KAAK,GAAG;YACN,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;QAClD,KAAK,GAAG;YACN,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;QAC7C,KAAK,GAAG,CAAC,CAAC,CAAC;YACT,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;YAC3B,IAAI,MAAM,KAAK,CAAC,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;YACtD,MAAM,IAAI,GAAG,KAAK,GAAG,MAAM,CAAA;YAC3B,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,GAAG,CAAC;gBAAE,OAAO,SAAS,CAAA;YAC9C,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,EAAE,CAAA;QACxE,CAAC;QACD,KAAK,GAAG,CAAC,CAAC,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;YAC1B,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;YACrD,MAAM,KAAK,GAAY,EAAE,CAAA;YACzB,IAAI,MAAM,GAAG,KAAK,CAAA;YAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;gBAClC,IAAI,IAAI,KAAK,SAAS;oBAAE,OAAO,SAAS,CAAA;gBACxC,qEAAqE;gBACrE,kEAAkE;gBAClE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;gBACxD,MAAM,GAAG,IAAI,CAAC,IAAI,CAAA;YACpB,CAAC;YACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;QACvC,CAAC;QACD;YACE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,2BAA2B,IAAI,EAAE,EAAE,CAAA;IACjF,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The state of a layer's move to a different embedding model.
|
|
3
|
+
*
|
|
4
|
+
* Shared between the API, which starts one and reports on it, and the worker,
|
|
5
|
+
* which does the work — so the shape is defined once rather than agreed twice.
|
|
6
|
+
*
|
|
7
|
+
* ## What is built, and what is not
|
|
8
|
+
*
|
|
9
|
+
* `docs/architecture.md` specifies six steps. Four are here: add the new named
|
|
10
|
+
* vector, reindex existing documents in bounded batches, switch `vector_name`
|
|
11
|
+
* atomically, keep search available throughout.
|
|
12
|
+
*
|
|
13
|
+
* Two are not, and are named rather than quietly skipped:
|
|
14
|
+
*
|
|
15
|
+
* - **The Recall@10 gate.** Step 4 checks the new index against "the layer's
|
|
16
|
+
* reference query set" before switching. There is no reference query set
|
|
17
|
+
* anywhere in this product — not a table, not an endpoint, not a
|
|
18
|
+
* configuration file — so building the gate would mean inventing the surface
|
|
19
|
+
* it reads from, and a gate that checks against an empty set is a gate that
|
|
20
|
+
* passes. Until it exists, a switch is a decision the operator makes by
|
|
21
|
+
* starting the reindex, and `docs/architecture.md` says so.
|
|
22
|
+
* - **Dropping the old vector after a rollback window.** Step 6. The old vector
|
|
23
|
+
* stays. It costs storage and it is what makes a rollback a one-statement
|
|
24
|
+
* change, which is worth more than the space while this path is new.
|
|
25
|
+
*/
|
|
26
|
+
/** What `layers.reindex_state` holds. The migration's check constraint enforces it. */
|
|
27
|
+
export interface ReindexState {
|
|
28
|
+
readonly status: 'running' | 'complete' | 'failed';
|
|
29
|
+
/**
|
|
30
|
+
* Which half of the migration is happening.
|
|
31
|
+
*
|
|
32
|
+
* `copying` is org-wide and cheap per point: the organization's collection is
|
|
33
|
+
* rebuilt with room for the new model and every point moves across carrying
|
|
34
|
+
* the vectors it already had. **No embeddings are computed.** It exists
|
|
35
|
+
* because a named vector cannot be added to a live Qdrant collection, so the
|
|
36
|
+
* only way to get one is to create a collection that already has it.
|
|
37
|
+
*
|
|
38
|
+
* `embedding` is per layer and is the expensive half: chunk text is embedded
|
|
39
|
+
* with the new model into the slot the copy created.
|
|
40
|
+
*
|
|
41
|
+
* The split is the whole point of the design. The cheap half happens once for
|
|
42
|
+
* the organization; the expensive half happens per layer, whenever the
|
|
43
|
+
* operator wants, with search running normally throughout.
|
|
44
|
+
*/
|
|
45
|
+
readonly phase: 'copying' | 'embedding';
|
|
46
|
+
/**
|
|
47
|
+
* The named vector being written. Also the answer to "reindexed onto what",
|
|
48
|
+
* which is why it is required in every state including `failed` — a failed
|
|
49
|
+
* reindex leaves points behind and the name is how they are found.
|
|
50
|
+
*/
|
|
51
|
+
readonly shadowVector: string;
|
|
52
|
+
/** The provider whose model produces it. */
|
|
53
|
+
readonly providerId: string;
|
|
54
|
+
readonly startedAt: string;
|
|
55
|
+
readonly finishedAt?: string;
|
|
56
|
+
/** Documents that had no shadow vector when the reindex began. */
|
|
57
|
+
readonly total: number;
|
|
58
|
+
/** Documents that have one now. Can exceed nothing; see `progress`. */
|
|
59
|
+
readonly done: number;
|
|
60
|
+
readonly failed: number;
|
|
61
|
+
readonly error?: string;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* How far along, as a ratio, for `nacre_reindex_progress_ratio`.
|
|
65
|
+
*
|
|
66
|
+
* Clamped at 1 rather than allowed past it. `total` is counted once when the
|
|
67
|
+
* reindex starts and documents ingested afterwards are picked up by the same
|
|
68
|
+
* pass, so `done` legitimately overshoots — and a progress gauge reading 1.4
|
|
69
|
+
* makes an operator think the number is broken rather than that more work
|
|
70
|
+
* arrived. A reindex that is complete reads 1 whatever the counts say.
|
|
71
|
+
*
|
|
72
|
+
* Zero documents is 1, not 0. An empty layer is finished, and a gauge that sits
|
|
73
|
+
* at zero forever for a layer with nothing in it is the stuck-alert shape.
|
|
74
|
+
*/
|
|
75
|
+
export declare function reindexProgress(state: ReindexState): number;
|
|
76
|
+
/** The JSON shape stored in the column. Snake case, like every other jsonb here. */
|
|
77
|
+
export declare function toStateJson(state: ReindexState): Record<string, unknown>;
|
|
78
|
+
/**
|
|
79
|
+
* Read it back, or `undefined` for a column that holds nothing usable.
|
|
80
|
+
*
|
|
81
|
+
* Undefined rather than a thrown error or a default: a layer with no reindex
|
|
82
|
+
* and a layer whose state cannot be parsed both mean "there is nothing to
|
|
83
|
+
* report", and the caller has no different action to take. The check constraint
|
|
84
|
+
* added in `0013` is what makes the second case not happen.
|
|
85
|
+
*/
|
|
86
|
+
export declare function fromStateJson(raw: unknown): ReindexState | undefined;
|
|
87
|
+
//# sourceMappingURL=reindex.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reindex.d.ts","sourceRoot":"","sources":["../reindex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,uFAAuF;AACvF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;IAClD;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,WAAW,CAAA;IACvC;;;;OAIG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,4CAA4C;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,kEAAkE;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,uEAAuE;IACvE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAQ3D;AAED,oFAAoF;AACpF,wBAAgB,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAaxE;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,OAAO,GAAG,YAAY,GAAG,SAAS,CAsBpE"}
|