@nacre.work/api 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/__tests__/admin-surface.test.d.ts +2 -0
- package/dist/__tests__/admin-surface.test.d.ts.map +1 -0
- package/dist/__tests__/admin-surface.test.js +431 -0
- package/dist/__tests__/admin-surface.test.js.map +1 -0
- package/dist/__tests__/init.test.d.ts +2 -0
- package/dist/__tests__/init.test.d.ts.map +1 -0
- package/dist/__tests__/init.test.js +125 -0
- package/dist/__tests__/init.test.js.map +1 -0
- package/dist/__tests__/service-keys.test.d.ts +2 -0
- package/dist/__tests__/service-keys.test.d.ts.map +1 -0
- package/dist/__tests__/service-keys.test.js +204 -0
- package/dist/__tests__/service-keys.test.js.map +1 -0
- package/dist/adapters.d.ts +197 -0
- package/dist/adapters.d.ts.map +1 -0
- package/dist/adapters.js +672 -0
- package/dist/adapters.js.map +1 -0
- package/dist/auth.d.ts +42 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +106 -0
- package/dist/auth.js.map +1 -0
- package/dist/errors.d.ts +43 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +101 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -0
- package/dist/init.d.ts +49 -0
- package/dist/init.d.ts.map +1 -0
- package/dist/init.js +201 -0
- package/dist/init.js.map +1 -0
- package/dist/main.d.ts +2 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +113 -0
- package/dist/main.js.map +1 -0
- package/dist/server.d.ts +239 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +543 -0
- package/dist/server.js.map +1 -0
- package/dist/service-keys.d.ts +82 -0
- package/dist/service-keys.d.ts.map +1 -0
- package/dist/service-keys.js +158 -0
- package/dist/service-keys.js.map +1 -0
- package/package.json +40 -0
package/dist/adapters.js
ADDED
|
@@ -0,0 +1,672 @@
|
|
|
1
|
+
import { aclTags, buildFilter, effectivePrincipals, loadGrants, loadScopeTree, PostgresGroupGraph, referenceAllows, resolve, VectorStore, vectorName, withOrg, } from '@nacre.work/core';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
export class PostgresDocuments {
|
|
4
|
+
pool;
|
|
5
|
+
role;
|
|
6
|
+
constructor(pool, role) {
|
|
7
|
+
this.pool = pool;
|
|
8
|
+
this.role = role;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Undefined for absent, for another organization's, and for one this caller
|
|
12
|
+
* has no grant reaching — one answer for all three.
|
|
13
|
+
*
|
|
14
|
+
* The query is scoped three ways on purpose: `withOrg` sets the row-level
|
|
15
|
+
* security context, `org_id` is named in the WHERE clause anyway, and the
|
|
16
|
+
* plan decides whether the layer it landed in is reachable. Any one alone
|
|
17
|
+
* would do for tenancy; only the third does anything about permissions
|
|
18
|
+
* inside a tenant, and it is the one that was missing.
|
|
19
|
+
*/
|
|
20
|
+
async read(auth, documentId) {
|
|
21
|
+
const orgId = auth.orgId;
|
|
22
|
+
// A malformed id must not reach Postgres as a cast error — an error
|
|
23
|
+
// distinguishable from "not found" is an oracle for the id format.
|
|
24
|
+
if (!/^[0-9a-f-]{36}$/i.test(documentId))
|
|
25
|
+
return undefined;
|
|
26
|
+
return withOrg(this.pool, orgId, async (client) => {
|
|
27
|
+
// The plan first. Reading the row and then checking is the same
|
|
28
|
+
// information leak with extra steps if the check is ever forgotten,
|
|
29
|
+
// and the layer a document sits in is what the grant is on.
|
|
30
|
+
const plan = resolve(await contextFor(client, auth), 'read');
|
|
31
|
+
if (plan.kind === 'none')
|
|
32
|
+
return undefined;
|
|
33
|
+
const { rows } = await client.query(`SELECT d.id, d.title, d.layer_id, l.slug AS layer, d.status, d.chunk_count, d.updated_at
|
|
34
|
+
FROM documents d
|
|
35
|
+
JOIN layers l ON l.id = d.layer_id AND l.org_id = d.org_id
|
|
36
|
+
WHERE d.org_id = $1 AND d.id = $2 AND d.deleted_at IS NULL`, [orgId, documentId]);
|
|
37
|
+
const row = rows[0];
|
|
38
|
+
if (row === undefined)
|
|
39
|
+
return undefined;
|
|
40
|
+
// `all` is org_admin, which reaches everything by rule 3. Otherwise the
|
|
41
|
+
// document has to sit in a layer the plan reached, or be named
|
|
42
|
+
// explicitly by a document-scoped grant — and not be denied by one.
|
|
43
|
+
if (plan.kind === 'scoped') {
|
|
44
|
+
if (plan.deniedDocs.includes(documentId))
|
|
45
|
+
return undefined;
|
|
46
|
+
const reachable = plan.layers.includes(row.layer_id) || plan.extraDocs.includes(documentId);
|
|
47
|
+
if (!reachable)
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
document_id: row.id,
|
|
52
|
+
layer: row.layer,
|
|
53
|
+
title: row.title,
|
|
54
|
+
status: row.status,
|
|
55
|
+
chunk_count: Number(row.chunk_count),
|
|
56
|
+
updated_at: row.updated_at.toISOString(),
|
|
57
|
+
};
|
|
58
|
+
}, this.role === undefined ? {} : { role: this.role });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
export class NacreSearchService {
|
|
62
|
+
deps;
|
|
63
|
+
constructor(deps) {
|
|
64
|
+
this.deps = deps;
|
|
65
|
+
}
|
|
66
|
+
async search(auth, query, topK) {
|
|
67
|
+
const slug = await this.deps.orgSlug(auth.orgId);
|
|
68
|
+
if (slug === undefined) {
|
|
69
|
+
// The token names an organization that is not there. Rule I3: a
|
|
70
|
+
// permission that cannot be evaluated denies.
|
|
71
|
+
return [];
|
|
72
|
+
}
|
|
73
|
+
const plan = await withOrg(this.deps.pool, auth.orgId, async (client) => {
|
|
74
|
+
const graph = await PostgresGroupGraph.load(client, auth.orgId);
|
|
75
|
+
const principals = effectivePrincipals(auth.principal, graph);
|
|
76
|
+
const grants = await loadGrants(client, auth.orgId, principals);
|
|
77
|
+
const tree = await loadScopeTree(client, auth.orgId, grants.filter((g) => g.scope.type === 'document').map((g) => g.scope.id));
|
|
78
|
+
return resolve({ orgId: auth.orgId, role: auth.role, principals, grants, tree }, 'read');
|
|
79
|
+
}, this.deps.role === undefined ? {} : { role: this.deps.role });
|
|
80
|
+
// No access at all. Returning early rather than querying is not an
|
|
81
|
+
// optimization — buildFilter refuses this plan, and it refuses it because
|
|
82
|
+
// there is no filter that means "nothing" without meaning "everything" by
|
|
83
|
+
// one mistake.
|
|
84
|
+
if (plan.kind === 'none')
|
|
85
|
+
return [];
|
|
86
|
+
const [vector] = await this.deps.embedder.embed([query]);
|
|
87
|
+
if (vector === undefined)
|
|
88
|
+
throw new Error('the embedder returned no vector for the query');
|
|
89
|
+
const hits = await this.deps.vectors.search({
|
|
90
|
+
orgId: auth.orgId,
|
|
91
|
+
orgSlug: slug,
|
|
92
|
+
plan,
|
|
93
|
+
branches: [{ kind: 'dense', using: this.deps.vectorName, vector }],
|
|
94
|
+
// Passed through. Asking for more and trimming is a post-filter that also
|
|
95
|
+
// costs more.
|
|
96
|
+
topK,
|
|
97
|
+
});
|
|
98
|
+
// Invariant I1, checked again on the way out. Raises rather than filtering:
|
|
99
|
+
// silently dropping the row would hide the bug that produced it.
|
|
100
|
+
const checked = VectorStore.assertTenant(auth.orgId, hits);
|
|
101
|
+
if (checked.length === 0)
|
|
102
|
+
return [];
|
|
103
|
+
return this.hydrate(auth.orgId, checked);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Turn hits into results a caller can read.
|
|
107
|
+
*
|
|
108
|
+
* The vector store holds no text — chunks live in Postgres, and the payload
|
|
109
|
+
* carries only what the filter needs. Returning the payload as the result,
|
|
110
|
+
* which is what happened before this existed, gave a caller identifiers and a
|
|
111
|
+
* score and nothing to show a user, and shipped `acl_tags` and `acl_version`
|
|
112
|
+
* along with it.
|
|
113
|
+
*
|
|
114
|
+
* This is a join and not a filter. The permitted set was decided inside the
|
|
115
|
+
* index traversal and is not narrowed here — with one exception, stated
|
|
116
|
+
* rather than left implicit: a chunk whose document has since been tombstoned
|
|
117
|
+
* in Postgres has no row to join to and does not appear. That is invariant I5
|
|
118
|
+
* enforced a second time on a different store, the same way row-level
|
|
119
|
+
* security backs up invariant I1, and it can only ever remove a document that
|
|
120
|
+
* is already meant to be gone.
|
|
121
|
+
*
|
|
122
|
+
* The order the index returned is kept. Postgres has no reason to preserve
|
|
123
|
+
* it, and re-sorting by anything else here would be a reranking step nobody
|
|
124
|
+
* asked for.
|
|
125
|
+
*/
|
|
126
|
+
async hydrate(orgId, hits) {
|
|
127
|
+
const chunkIds = hits.map((h) => String(h.payload.chunk_id ?? h.id));
|
|
128
|
+
const rows = await withOrg(this.deps.pool, orgId, async (client) => {
|
|
129
|
+
const { rows } = await client.query(
|
|
130
|
+
// point_id, not chunks.id. The payload's `chunk_id` is the vector
|
|
131
|
+
// store's point identifier — `chunks` keeps both, and they are
|
|
132
|
+
// different values. Joining on the wrong one matches nothing and
|
|
133
|
+
// returns an empty result set, which looks exactly like a permission
|
|
134
|
+
// working correctly.
|
|
135
|
+
`SELECT c.point_id AS chunk_id, c.document_id AS doc_id, l.slug AS layer, d.title, c.text
|
|
136
|
+
FROM chunks c
|
|
137
|
+
JOIN documents d ON d.id = c.document_id AND d.org_id = c.org_id
|
|
138
|
+
JOIN layers l ON l.id = d.layer_id AND l.org_id = d.org_id
|
|
139
|
+
WHERE c.org_id = $1
|
|
140
|
+
AND c.point_id = ANY($2::uuid[])
|
|
141
|
+
AND d.deleted_at IS NULL
|
|
142
|
+
AND l.deleted_at IS NULL`, [orgId, chunkIds]);
|
|
143
|
+
return rows;
|
|
144
|
+
}, this.deps.role === undefined ? {} : { role: this.deps.role });
|
|
145
|
+
const byId = new Map(rows.map((r) => [r.chunk_id, r]));
|
|
146
|
+
return hits
|
|
147
|
+
.map((hit) => {
|
|
148
|
+
const row = byId.get(String(hit.payload.chunk_id ?? hit.id));
|
|
149
|
+
if (row === undefined)
|
|
150
|
+
return undefined;
|
|
151
|
+
return {
|
|
152
|
+
chunk_id: row.chunk_id,
|
|
153
|
+
doc_id: row.doc_id,
|
|
154
|
+
layer: row.layer,
|
|
155
|
+
title: row.title,
|
|
156
|
+
score: hit.score,
|
|
157
|
+
text: row.text,
|
|
158
|
+
};
|
|
159
|
+
})
|
|
160
|
+
.filter((r) => r !== undefined);
|
|
161
|
+
}
|
|
162
|
+
/** Exposed so the filter can be inspected in tests without a vector store. */
|
|
163
|
+
static filterFor(orgId, plan) {
|
|
164
|
+
return buildFilter(orgId, plan);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* The audit sink.
|
|
169
|
+
*
|
|
170
|
+
* Writes with the application role, which has INSERT and SELECT on
|
|
171
|
+
* `audit_events` and neither UPDATE nor DELETE — revoked at the database level
|
|
172
|
+
* by migration 0002, so a bug here cannot rewrite history and neither can
|
|
173
|
+
* anyone holding these credentials.
|
|
174
|
+
*/
|
|
175
|
+
export class PostgresAudit {
|
|
176
|
+
pool;
|
|
177
|
+
role;
|
|
178
|
+
constructor(pool, role) {
|
|
179
|
+
this.pool = pool;
|
|
180
|
+
this.role = role;
|
|
181
|
+
}
|
|
182
|
+
async write(event) {
|
|
183
|
+
const [actorType, actorId] = event.actor.split(':');
|
|
184
|
+
await withOrg(this.pool, event.orgId, async (client) => {
|
|
185
|
+
await client.query(`INSERT INTO audit_events
|
|
186
|
+
(org_id, actor_type, actor_id, actor_label, action, surface, target, result, detail, request_id)
|
|
187
|
+
VALUES ($1,$2,$3,$4,$5,'api','{}'::jsonb,$6,$7,$8)`, [
|
|
188
|
+
event.orgId,
|
|
189
|
+
actorType ?? 'unknown',
|
|
190
|
+
/^[0-9a-f-]{36}$/i.test(actorId ?? '') ? actorId : null,
|
|
191
|
+
event.actor,
|
|
192
|
+
event.action,
|
|
193
|
+
event.result,
|
|
194
|
+
JSON.stringify(event.detail),
|
|
195
|
+
event.requestId,
|
|
196
|
+
]);
|
|
197
|
+
}, this.role === undefined ? {} : { role: this.role });
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* An OpenAI-compatible embedding client.
|
|
202
|
+
*
|
|
203
|
+
* "Bring your own model" means this has to work against anything speaking that
|
|
204
|
+
* shape, so the only thing assumed is `POST /embeddings` returning
|
|
205
|
+
* `{ data: [{ embedding: number[] }] }`.
|
|
206
|
+
*/
|
|
207
|
+
export class HttpEmbedder {
|
|
208
|
+
endpoint;
|
|
209
|
+
model;
|
|
210
|
+
dimensions;
|
|
211
|
+
constructor(endpoint, model, dimensions) {
|
|
212
|
+
this.endpoint = endpoint;
|
|
213
|
+
this.model = model;
|
|
214
|
+
this.dimensions = dimensions;
|
|
215
|
+
}
|
|
216
|
+
async embed(texts) {
|
|
217
|
+
if (texts.length === 0)
|
|
218
|
+
return [];
|
|
219
|
+
const response = await fetch(new URL('/embeddings', this.endpoint), {
|
|
220
|
+
method: 'POST',
|
|
221
|
+
headers: { 'content-type': 'application/json' },
|
|
222
|
+
body: JSON.stringify({ model: this.model, input: texts }),
|
|
223
|
+
});
|
|
224
|
+
if (!response.ok) {
|
|
225
|
+
throw new Error(`the embedding endpoint answered ${response.status}`);
|
|
226
|
+
}
|
|
227
|
+
const body = (await response.json());
|
|
228
|
+
const vectors = (body.data ?? []).map((d) => d.embedding);
|
|
229
|
+
if (vectors.length !== texts.length) {
|
|
230
|
+
// Same reason as in the ingest pipeline: a short or reordered batch
|
|
231
|
+
// attaches the wrong vector to the wrong text, and nothing downstream can
|
|
232
|
+
// tell.
|
|
233
|
+
throw new Error(`the embedding endpoint returned ${vectors.length} vectors for ${texts.length} inputs`);
|
|
234
|
+
}
|
|
235
|
+
return vectors.map((v, i) => {
|
|
236
|
+
if (!Array.isArray(v) || v.length !== this.dimensions || !v.every((n) => typeof n === 'number')) {
|
|
237
|
+
throw new Error(`vector ${i} is not ${this.dimensions} numbers; the endpoint and ` +
|
|
238
|
+
'NACRE_DEFAULT_EMBEDDING_DIM disagree, and the index would be built wrong');
|
|
239
|
+
}
|
|
240
|
+
return v;
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
export { aclTags };
|
|
245
|
+
export class NacreIngest {
|
|
246
|
+
deps;
|
|
247
|
+
constructor(deps) {
|
|
248
|
+
this.deps = deps;
|
|
249
|
+
}
|
|
250
|
+
get pool() {
|
|
251
|
+
return this.deps.pool;
|
|
252
|
+
}
|
|
253
|
+
get scope() {
|
|
254
|
+
return this.deps.role === undefined ? {} : { role: this.deps.role };
|
|
255
|
+
}
|
|
256
|
+
/** The layers this caller may write to, by slug. */
|
|
257
|
+
async writableLayer(client, auth, layerSlug) {
|
|
258
|
+
const graph = await PostgresGroupGraph.load(client, auth.orgId);
|
|
259
|
+
const principals = effectivePrincipals(auth.principal, graph);
|
|
260
|
+
const grants = await loadGrants(client, auth.orgId, principals);
|
|
261
|
+
const tree = await loadScopeTree(client, auth.orgId, grants.filter((g) => g.scope.type === 'document').map((g) => g.scope.id));
|
|
262
|
+
const plan = resolve({ orgId: auth.orgId, role: auth.role, principals, grants, tree }, 'write');
|
|
263
|
+
if (plan.kind === 'none')
|
|
264
|
+
return undefined;
|
|
265
|
+
const { rows } = await client.query(`SELECT id FROM layers WHERE org_id = $1 AND slug = $2 AND deleted_at IS NULL`, [auth.orgId, layerSlug]);
|
|
266
|
+
const id = rows[0]?.id;
|
|
267
|
+
if (id === undefined)
|
|
268
|
+
return undefined;
|
|
269
|
+
// A layer that exists but is not writable and a layer that does not exist
|
|
270
|
+
// return the same thing, so the caller cannot tell them apart.
|
|
271
|
+
if (plan.kind === 'all')
|
|
272
|
+
return id;
|
|
273
|
+
return plan.layers.includes(id) ? id : undefined;
|
|
274
|
+
}
|
|
275
|
+
async queue(auth, request) {
|
|
276
|
+
return withOrg(this.pool, auth.orgId, async (client) => {
|
|
277
|
+
const layerId = await this.writableLayer(client, auth, request.layer);
|
|
278
|
+
if (layerId === undefined)
|
|
279
|
+
return undefined;
|
|
280
|
+
const source = request.content !== undefined ? request.content : request.url;
|
|
281
|
+
const sourceType = request.content !== undefined ? 'inline' : 'url';
|
|
282
|
+
const hash = createHash('sha256').update(source, 'utf8').digest('hex');
|
|
283
|
+
const { rows } = await client.query(`INSERT INTO documents
|
|
284
|
+
(org_id, layer_id, external_id, source_type, source_ref, title, content_hash, status)
|
|
285
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,'pending')
|
|
286
|
+
ON CONFLICT (layer_id, external_id) DO UPDATE SET
|
|
287
|
+
-- Only touch the row when the content actually changed. A repeat
|
|
288
|
+
-- with the same bytes must not re-queue work: every client that
|
|
289
|
+
-- times out retries, and re-embedding is what that costs.
|
|
290
|
+
source_ref = CASE WHEN documents.content_hash IS DISTINCT FROM EXCLUDED.content_hash
|
|
291
|
+
THEN EXCLUDED.source_ref ELSE documents.source_ref END,
|
|
292
|
+
title = COALESCE(EXCLUDED.title, documents.title),
|
|
293
|
+
content_hash = EXCLUDED.content_hash,
|
|
294
|
+
status = CASE WHEN documents.content_hash IS DISTINCT FROM EXCLUDED.content_hash
|
|
295
|
+
THEN 'pending' ELSE documents.status END,
|
|
296
|
+
deleted_at = NULL,
|
|
297
|
+
updated_at = now()
|
|
298
|
+
RETURNING id, content_hash, status`, [
|
|
299
|
+
auth.orgId,
|
|
300
|
+
layerId,
|
|
301
|
+
request.externalId,
|
|
302
|
+
sourceType,
|
|
303
|
+
source,
|
|
304
|
+
request.title ?? null,
|
|
305
|
+
`sha256:${hash}`,
|
|
306
|
+
]);
|
|
307
|
+
const row = rows[0];
|
|
308
|
+
if (row === undefined)
|
|
309
|
+
throw new Error('the document upsert returned no row');
|
|
310
|
+
return {
|
|
311
|
+
documentId: row.id,
|
|
312
|
+
jobId: row.id,
|
|
313
|
+
unchanged: row.status === 'indexed',
|
|
314
|
+
};
|
|
315
|
+
}, this.scope);
|
|
316
|
+
}
|
|
317
|
+
async remove(auth, documentId) {
|
|
318
|
+
if (!/^[0-9a-f-]{36}$/i.test(documentId))
|
|
319
|
+
return false;
|
|
320
|
+
const slug = await this.deps.orgSlug(auth.orgId);
|
|
321
|
+
// The token names an organization that is not there. Refusing rather than
|
|
322
|
+
// deleting only in Postgres: a tombstone we cannot mirror into the index is
|
|
323
|
+
// a document that stays searchable while the API reports it gone.
|
|
324
|
+
if (slug === undefined)
|
|
325
|
+
return false;
|
|
326
|
+
return withOrg(this.pool, auth.orgId, async (client) => {
|
|
327
|
+
const graph = await PostgresGroupGraph.load(client, auth.orgId);
|
|
328
|
+
const principals = effectivePrincipals(auth.principal, graph);
|
|
329
|
+
const grants = await loadGrants(client, auth.orgId, principals);
|
|
330
|
+
const tree = await loadScopeTree(client, auth.orgId, [documentId]);
|
|
331
|
+
const plan = resolve({ orgId: auth.orgId, role: auth.role, principals, grants, tree }, 'write');
|
|
332
|
+
if (plan.kind === 'none')
|
|
333
|
+
return false;
|
|
334
|
+
const { rows } = await client.query(`SELECT layer_id FROM documents
|
|
335
|
+
WHERE org_id = $1 AND id = $2 AND deleted_at IS NULL`, [auth.orgId, documentId]);
|
|
336
|
+
const layerId = rows[0]?.layer_id;
|
|
337
|
+
if (layerId === undefined)
|
|
338
|
+
return false;
|
|
339
|
+
if (plan.kind === 'scoped') {
|
|
340
|
+
// deniedDocs first. `resolve` populates it precisely so a deny beats
|
|
341
|
+
// an allow at any depth (rule 5), and checking only `layers` let a
|
|
342
|
+
// document inside an allowed layer be deleted despite an explicit
|
|
343
|
+
// deny on it. The read path already honoured this — buildFilter emits
|
|
344
|
+
// must_not from the same list — so this was the write path drifting
|
|
345
|
+
// away from the read path rather than a rule nobody had implemented.
|
|
346
|
+
if (plan.deniedDocs.includes(documentId))
|
|
347
|
+
return false;
|
|
348
|
+
if (!plan.layers.includes(layerId) && !plan.extraDocs.includes(documentId))
|
|
349
|
+
return false;
|
|
350
|
+
}
|
|
351
|
+
// The index first, then the row — the reverse of ingest, and the order
|
|
352
|
+
// is the whole point.
|
|
353
|
+
//
|
|
354
|
+
// `deleted = false` is what the pre-filter tests, and that flag lives on
|
|
355
|
+
// the points. Writing only the Postgres tombstone leaves the document in
|
|
356
|
+
// every answer until the collector reaches it, which is precisely the
|
|
357
|
+
// window invariant I5 forbids; the collector runs on a schedule measured
|
|
358
|
+
// in minutes, so this is not a narrow race.
|
|
359
|
+
//
|
|
360
|
+
// Postgres-first also fails in the unrecoverable direction: the row says
|
|
361
|
+
// deleted, so nothing queues the document for retagging or re-ingest,
|
|
362
|
+
// and the only job that would ever touch its points again is the sweep —
|
|
363
|
+
// which purges rather than repairs. Index-first fails the other way: the
|
|
364
|
+
// points are marked, the row is not, the caller sees an error and
|
|
365
|
+
// retries, and in the meantime the document is already invisible.
|
|
366
|
+
await this.deps.tombstone.tombstone(slug, documentId);
|
|
367
|
+
// A tombstone, not a delete, on this side too. Physical removal of the
|
|
368
|
+
// points is the collector's job, and `deleted_at` is what puts the
|
|
369
|
+
// document in its queue.
|
|
370
|
+
await client.query(`UPDATE documents SET deleted_at = now(), updated_at = now()
|
|
371
|
+
WHERE org_id = $1 AND id = $2`, [auth.orgId, documentId]);
|
|
372
|
+
return true;
|
|
373
|
+
}, this.scope);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Ingest job status.
|
|
378
|
+
*
|
|
379
|
+
* The job id is the document id — ingest writes a row and the worker moves its
|
|
380
|
+
* status, so there is no separate job table to drift out of step with it. That
|
|
381
|
+
* also means the visibility rule is the document's: absent and another
|
|
382
|
+
* organization's answer identically.
|
|
383
|
+
*/
|
|
384
|
+
export class PostgresJobs {
|
|
385
|
+
pool;
|
|
386
|
+
role;
|
|
387
|
+
constructor(pool, role) {
|
|
388
|
+
this.pool = pool;
|
|
389
|
+
this.role = role;
|
|
390
|
+
}
|
|
391
|
+
async read(auth, jobId) {
|
|
392
|
+
const orgId = auth.orgId;
|
|
393
|
+
if (!/^[0-9a-f-]{36}$/i.test(jobId))
|
|
394
|
+
return undefined;
|
|
395
|
+
return withOrg(this.pool, orgId, async (client) => {
|
|
396
|
+
// A job id is a document id, so this is the document read with a
|
|
397
|
+
// different projection — and needs the same check. The status and the
|
|
398
|
+
// error string are both facts about a document the caller may not be
|
|
399
|
+
// allowed to know exists.
|
|
400
|
+
const plan = resolve(await contextFor(client, auth), 'read');
|
|
401
|
+
if (plan.kind === 'none')
|
|
402
|
+
return undefined;
|
|
403
|
+
const { rows } = await client.query(`SELECT id, status, error, chunk_count, layer_id FROM documents
|
|
404
|
+
WHERE org_id = $1 AND id = $2 AND deleted_at IS NULL`, [orgId, jobId]);
|
|
405
|
+
const row = rows[0];
|
|
406
|
+
if (row === undefined)
|
|
407
|
+
return undefined;
|
|
408
|
+
if (plan.kind === 'scoped') {
|
|
409
|
+
if (plan.deniedDocs.includes(jobId))
|
|
410
|
+
return undefined;
|
|
411
|
+
if (!plan.layers.includes(row.layer_id) && !plan.extraDocs.includes(jobId))
|
|
412
|
+
return undefined;
|
|
413
|
+
}
|
|
414
|
+
const status = (['queued', 'parsing', 'embedding', 'indexed', 'failed'].includes(row.status)
|
|
415
|
+
? row.status
|
|
416
|
+
: row.status === 'pending'
|
|
417
|
+
? 'queued'
|
|
418
|
+
: 'queued');
|
|
419
|
+
// Coarse on purpose. A percentage that moves smoothly would have to be
|
|
420
|
+
// written by the pipeline on every chunk, which is a write per chunk to
|
|
421
|
+
// make a progress bar prettier.
|
|
422
|
+
const progress = status === 'indexed' ? 1 : status === 'failed' ? 0 : 0.5;
|
|
423
|
+
return {
|
|
424
|
+
jobId: row.id,
|
|
425
|
+
documentId: row.id,
|
|
426
|
+
status,
|
|
427
|
+
progress,
|
|
428
|
+
...(row.error === null ? {} : { error: row.error }),
|
|
429
|
+
};
|
|
430
|
+
}, this.role === undefined ? {} : { role: this.role });
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
/** The per-request permission context, loaded once and asked several questions. */
|
|
434
|
+
async function contextFor(client, auth) {
|
|
435
|
+
const graph = await PostgresGroupGraph.load(client, auth.orgId);
|
|
436
|
+
const principals = effectivePrincipals(auth.principal, graph);
|
|
437
|
+
const grants = await loadGrants(client, auth.orgId, principals);
|
|
438
|
+
const tree = await loadScopeTree(client, auth.orgId, grants.filter((g) => g.scope.type === 'document').map((g) => g.scope.id));
|
|
439
|
+
return { orgId: auth.orgId, role: auth.role, principals, grants, tree };
|
|
440
|
+
}
|
|
441
|
+
export class PostgresLayers {
|
|
442
|
+
pool;
|
|
443
|
+
role;
|
|
444
|
+
constructor(pool, role) {
|
|
445
|
+
this.pool = pool;
|
|
446
|
+
this.role = role;
|
|
447
|
+
}
|
|
448
|
+
get scope() {
|
|
449
|
+
return this.role === undefined ? {} : { role: this.role };
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* The layers this caller may read.
|
|
453
|
+
*
|
|
454
|
+
* Narrowed by the plan in SQL rather than fetched and filtered afterwards.
|
|
455
|
+
* This is a listing rather than a search, so a post-filter here would not
|
|
456
|
+
* break invariant I2 — but writing it the other way keeps one habit for both
|
|
457
|
+
* and removes the question of which endpoints are allowed to be sloppy.
|
|
458
|
+
*/
|
|
459
|
+
async list(auth) {
|
|
460
|
+
return withOrg(this.pool, auth.orgId, async (client) => {
|
|
461
|
+
const plan = resolve(await contextFor(client, auth), 'read');
|
|
462
|
+
if (plan.kind === 'none')
|
|
463
|
+
return [];
|
|
464
|
+
// The count comes from the same statement. It is what a catalog is
|
|
465
|
+
// for — an agent choosing where to search, or a person deciding
|
|
466
|
+
// whether a layer is worth opening — and the MCP catalog has always
|
|
467
|
+
// carried it, so a REST client had to be told to use MCP for that.
|
|
468
|
+
//
|
|
469
|
+
// It counts live documents in the layer, not documents this caller may
|
|
470
|
+
// read: layer-scoped grants are all this build has, so anyone who can
|
|
471
|
+
// see the layer can read everything in it. The moment document-level
|
|
472
|
+
// grants exist — a commercial module — this becomes a number that
|
|
473
|
+
// discloses more than the caller can reach, and it has to be recomputed
|
|
474
|
+
// per caller or dropped.
|
|
475
|
+
const projection = `l.id, l.slug, l.name, l.workspace_id, l.description,
|
|
476
|
+
(SELECT count(*) FROM documents d
|
|
477
|
+
WHERE d.layer_id = l.id AND d.deleted_at IS NULL) AS document_count`;
|
|
478
|
+
const { rows } = plan.kind === 'all'
|
|
479
|
+
? await client.query(`SELECT ${projection} FROM layers l
|
|
480
|
+
WHERE l.org_id = $1 AND l.deleted_at IS NULL ORDER BY l.slug`, [auth.orgId])
|
|
481
|
+
: await client.query(`SELECT ${projection} FROM layers l
|
|
482
|
+
WHERE l.org_id = $1 AND l.deleted_at IS NULL AND l.id = ANY($2::uuid[])
|
|
483
|
+
ORDER BY l.slug`, [auth.orgId, [...plan.layers]]);
|
|
484
|
+
return rows.map((r) => ({
|
|
485
|
+
id: r.id,
|
|
486
|
+
slug: r.slug,
|
|
487
|
+
name: r.name,
|
|
488
|
+
workspaceId: r.workspace_id,
|
|
489
|
+
description: r.description ?? '',
|
|
490
|
+
documentCount: Number(r.document_count),
|
|
491
|
+
}));
|
|
492
|
+
}, this.scope);
|
|
493
|
+
}
|
|
494
|
+
async create(auth, input) {
|
|
495
|
+
if (!/^[0-9a-f-]{36}$/i.test(input.workspaceId))
|
|
496
|
+
return { kind: 'denied' };
|
|
497
|
+
return withOrg(this.pool, auth.orgId, async (client) => {
|
|
498
|
+
const context = await contextFor(client, auth);
|
|
499
|
+
// `referenceAllows` and not `resolve`: the question is "may they
|
|
500
|
+
// administer this one workspace", which is what the reference answers
|
|
501
|
+
// directly. Reading it out of a flattened plan would mean inferring a
|
|
502
|
+
// workspace permission from the layers it happens to contain — and an
|
|
503
|
+
// empty workspace contains none.
|
|
504
|
+
if (!referenceAllows(context, { type: 'workspace', id: input.workspaceId }, 'admin')) {
|
|
505
|
+
return { kind: 'denied' };
|
|
506
|
+
}
|
|
507
|
+
// Checked after the permission, so a caller who may not administer the
|
|
508
|
+
// workspace gets the same answer whether or not it exists.
|
|
509
|
+
const { rows: workspaces } = await client.query(`SELECT id FROM workspaces WHERE org_id = $1 AND id = $2 AND deleted_at IS NULL`, [auth.orgId, input.workspaceId]);
|
|
510
|
+
if (workspaces[0] === undefined)
|
|
511
|
+
return { kind: 'denied' };
|
|
512
|
+
const { rows: providers } = await client.query(`SELECT id, model, dimensions FROM embedding_providers
|
|
513
|
+
WHERE org_id = $1 OR org_id IS NULL ORDER BY org_id NULLS LAST LIMIT 1`, [auth.orgId]);
|
|
514
|
+
const provider = providers[0];
|
|
515
|
+
if (provider === undefined) {
|
|
516
|
+
throw new Error('no embedding provider is configured; a layer cannot be created without one');
|
|
517
|
+
}
|
|
518
|
+
// Derived, never a literal. The worker writes points to the vector this
|
|
519
|
+
// column names and search looks for the one the configuration derives;
|
|
520
|
+
// a layer created with a placeholder here indexes into a vector the
|
|
521
|
+
// collection does not have, which fails every upsert with `Bad
|
|
522
|
+
// Request` and would return nothing even if it did not.
|
|
523
|
+
const vector = vectorName(provider.model, provider.dimensions);
|
|
524
|
+
const { rows } = await client.query(`INSERT INTO layers (org_id, workspace_id, slug, name, provider_id, vector_name)
|
|
525
|
+
VALUES ($1,$2,$3,$4,$5,$6)
|
|
526
|
+
ON CONFLICT DO NOTHING
|
|
527
|
+
RETURNING id, slug, name, workspace_id, description`, [auth.orgId, input.workspaceId, input.slug, input.name, provider.id, vector]);
|
|
528
|
+
const row = rows[0];
|
|
529
|
+
// ON CONFLICT DO NOTHING returns nothing, and by here the caller has
|
|
530
|
+
// already proved admin on the workspace — so the only way to get here
|
|
531
|
+
// is a slug already in use.
|
|
532
|
+
if (row === undefined)
|
|
533
|
+
return { kind: 'conflict' };
|
|
534
|
+
return {
|
|
535
|
+
kind: 'created',
|
|
536
|
+
layer: {
|
|
537
|
+
id: row.id,
|
|
538
|
+
slug: row.slug,
|
|
539
|
+
name: row.name,
|
|
540
|
+
workspaceId: row.workspace_id,
|
|
541
|
+
description: row.description ?? '',
|
|
542
|
+
// Just created, so this is a fact rather than a query.
|
|
543
|
+
documentCount: 0,
|
|
544
|
+
},
|
|
545
|
+
};
|
|
546
|
+
}, this.scope);
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
export class PostgresGrants {
|
|
550
|
+
pool;
|
|
551
|
+
role;
|
|
552
|
+
constructor(pool, role) {
|
|
553
|
+
this.pool = pool;
|
|
554
|
+
this.role = role;
|
|
555
|
+
}
|
|
556
|
+
get scope() {
|
|
557
|
+
return this.role === undefined ? {} : { role: this.role };
|
|
558
|
+
}
|
|
559
|
+
/**
|
|
560
|
+
* Every grant in the organization, for a caller who may administer it.
|
|
561
|
+
*
|
|
562
|
+
* Not narrowed to the caller's own grants. Someone who can issue a grant on a
|
|
563
|
+
* scope can already see who else holds one there, and a partial list is worse
|
|
564
|
+
* than none — an administrator who cannot see an existing grant revokes the
|
|
565
|
+
* wrong thing.
|
|
566
|
+
*/
|
|
567
|
+
async list(auth) {
|
|
568
|
+
return withOrg(this.pool, auth.orgId, async (client) => {
|
|
569
|
+
const context = await contextFor(client, auth);
|
|
570
|
+
const plan = resolve(context, 'admin');
|
|
571
|
+
if (plan.kind === 'none')
|
|
572
|
+
return [];
|
|
573
|
+
const { rows } = await client.query(`SELECT id, principal_type, principal_id, scope_type, scope_id, permission, effect, source
|
|
574
|
+
FROM grants WHERE org_id = $1 ORDER BY created_at`, [auth.orgId]);
|
|
575
|
+
return rows
|
|
576
|
+
.filter((r) => referenceAllows(context, { type: r.scope_type, id: r.scope_id }, 'admin'))
|
|
577
|
+
.map((r) => ({
|
|
578
|
+
id: r.id,
|
|
579
|
+
principalType: r.principal_type,
|
|
580
|
+
principalId: r.principal_id,
|
|
581
|
+
scopeType: r.scope_type,
|
|
582
|
+
scopeId: r.scope_id,
|
|
583
|
+
permission: r.permission,
|
|
584
|
+
effect: r.effect,
|
|
585
|
+
source: r.source,
|
|
586
|
+
}));
|
|
587
|
+
}, this.scope);
|
|
588
|
+
}
|
|
589
|
+
async issue(auth, input) {
|
|
590
|
+
if (!/^[0-9a-f-]{36}$/i.test(input.scopeId) || !/^[0-9a-f-]{36}$/i.test(input.principalId)) {
|
|
591
|
+
return undefined;
|
|
592
|
+
}
|
|
593
|
+
return withOrg(this.pool, auth.orgId, async (client) => {
|
|
594
|
+
const context = await contextFor(client, auth);
|
|
595
|
+
// Admin on the scope being granted, not admin in general. Otherwise
|
|
596
|
+
// anyone holding admin on one layer could grant themselves another.
|
|
597
|
+
if (!referenceAllows(context, { type: input.scopeType, id: input.scopeId }, 'admin')) {
|
|
598
|
+
return undefined;
|
|
599
|
+
}
|
|
600
|
+
const { rows } = await client.query(`INSERT INTO grants
|
|
601
|
+
(org_id, principal_type, principal_id, scope_type, scope_id, permission, effect, source)
|
|
602
|
+
VALUES ($1,$2,$3,$4,$5,$6,'allow','api')
|
|
603
|
+
ON CONFLICT (org_id, principal_type, principal_id, scope_type, scope_id, permission)
|
|
604
|
+
DO UPDATE SET effect = 'allow'
|
|
605
|
+
RETURNING id, effect, source`, [
|
|
606
|
+
auth.orgId,
|
|
607
|
+
input.principalType,
|
|
608
|
+
input.principalId,
|
|
609
|
+
input.scopeType,
|
|
610
|
+
input.scopeId,
|
|
611
|
+
input.permission,
|
|
612
|
+
]);
|
|
613
|
+
const row = rows[0];
|
|
614
|
+
if (row === undefined)
|
|
615
|
+
return undefined;
|
|
616
|
+
return {
|
|
617
|
+
id: row.id,
|
|
618
|
+
principalType: input.principalType,
|
|
619
|
+
principalId: input.principalId,
|
|
620
|
+
scopeType: input.scopeType,
|
|
621
|
+
scopeId: input.scopeId,
|
|
622
|
+
permission: input.permission,
|
|
623
|
+
effect: row.effect,
|
|
624
|
+
source: row.source,
|
|
625
|
+
};
|
|
626
|
+
}, this.scope);
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* Withdraw a grant.
|
|
630
|
+
*
|
|
631
|
+
* A real DELETE, not an effect flip. Setting `effect = 'deny'` would look
|
|
632
|
+
* equivalent and is not: a deny is a commercial capability this build refuses
|
|
633
|
+
* to issue, and it beats an allow at any depth — so a "revocation" written
|
|
634
|
+
* that way would also suppress a grant the principal holds through a group or
|
|
635
|
+
* a parent scope. Removing the row leaves whatever else they were given.
|
|
636
|
+
*
|
|
637
|
+
* Admin on the grant's own scope, resolved from the row rather than from
|
|
638
|
+
* anything the caller sent. Reading the scope first and checking it after is
|
|
639
|
+
* the only order that works: the caller names an id, and the permission
|
|
640
|
+
* question is about what that id points at.
|
|
641
|
+
*/
|
|
642
|
+
async revoke(auth, grantId) {
|
|
643
|
+
if (!/^[0-9a-f-]{36}$/i.test(grantId))
|
|
644
|
+
return false;
|
|
645
|
+
return withOrg(this.pool, auth.orgId, async (client) => {
|
|
646
|
+
const { rows } = await client.query(`SELECT scope_type, scope_id FROM grants WHERE org_id = $1 AND id = $2`, [auth.orgId, grantId]);
|
|
647
|
+
const row = rows[0];
|
|
648
|
+
// Absent, another organization's, and one whose scope this caller
|
|
649
|
+
// cannot administer all answer the same. Rule 4 does not stop applying
|
|
650
|
+
// because the object is a grant rather than a document.
|
|
651
|
+
if (row === undefined)
|
|
652
|
+
return false;
|
|
653
|
+
const context = await contextFor(client, auth);
|
|
654
|
+
const scope = {
|
|
655
|
+
type: row.scope_type,
|
|
656
|
+
id: row.scope_id,
|
|
657
|
+
};
|
|
658
|
+
if (!referenceAllows(context, scope, 'admin'))
|
|
659
|
+
return false;
|
|
660
|
+
const result = await client.query(`DELETE FROM grants WHERE org_id = $1 AND id = $2`, [
|
|
661
|
+
auth.orgId,
|
|
662
|
+
grantId,
|
|
663
|
+
]);
|
|
664
|
+
// The trigger on `grants` bumps groups_version, which is what makes the
|
|
665
|
+
// worker recompute the payload tags and what the propagation gauge then
|
|
666
|
+
// measures. Nothing here has to do that by hand, and nothing here
|
|
667
|
+
// should: a second place that bumps it is a second place to forget.
|
|
668
|
+
return (result.rowCount ?? 0) > 0;
|
|
669
|
+
}, this.scope);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
//# sourceMappingURL=adapters.js.map
|