@anchrd/intel-api 0.12.5 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/cloudflare/cloudflare.js +1 -0
- package/dist/adapters/cloudflare-api/cloudflare-api.js +153 -44
- package/dist/adapters/db/db-indexing.js +79 -0
- package/dist/adapters/db/db.js +66 -27
- package/dist/adapters/openid/openid.js +70 -10
- package/dist/adapters/semantic-index/semantic-index.js +97 -17
- package/dist/adapters/semantic-index/semantic-index.types.d.ts +20 -1
- package/dist/bundle/bundle.js +46 -1
- package/dist/http/http.js +4 -0
- package/dist/indexing/indexing.js +177 -17
- package/dist/indexing/indexing.types.d.ts +1 -0
- package/dist/mcp/mcp.js +48 -34
- package/dist/nodes/board/board.d.ts +46 -0
- package/dist/nodes/board/board.js +475 -8
- package/dist/nodes/board/board.types.d.ts +7 -0
- package/dist/nodes/document-links/document-links.js +12 -1
- package/dist/nodes/nodes.js +152 -16
- package/dist/nodes/nodes.types.d.ts +61 -3
- package/dist/tools/tools.js +7 -1
- package/migrations/0009_no_context_policy.sql +15 -0
- package/migrations/0017_a_vector_per_card.sql +38 -0
- package/migrations/0018_no_context_policy_at_last.sql +90 -0
- package/package.json +2 -2
|
@@ -259,6 +259,7 @@ export default {
|
|
|
259
259
|
content: createContentStore(env.CONTENT),
|
|
260
260
|
semantic: env.AI && env.SEARCH ? createSemanticIndex({ ai: env.AI, index: env.SEARCH }) : undefined,
|
|
261
261
|
converter: env.AI ? createDocumentConverter(env.AI) : undefined,
|
|
262
|
+
hash: async (content) => await sha256Hex(crypto, content),
|
|
262
263
|
now: () => new Date(),
|
|
263
264
|
});
|
|
264
265
|
for (const message of batch.messages) {
|
|
@@ -19,6 +19,33 @@ const ApiOrigin = "https://api.cloudflare.com/client/v4";
|
|
|
19
19
|
*/
|
|
20
20
|
const MaxPages = 20;
|
|
21
21
|
const PerPage = 50;
|
|
22
|
+
/**
|
|
23
|
+
* ⚠️ Cloudflare's ceiling on `/ai/models/search`, and a DIFFERENT number from `PerPage` above —
|
|
24
|
+
* the limit belongs to the endpoint, not to the account. Neither may be copied onto the other's
|
|
25
|
+
* call: 50 on the model catalog halves it, 100 on the log is refused outright.
|
|
26
|
+
*
|
|
27
|
+
* And the two fail in opposite ways, of which this is the worse one. The log endpoint REFUSES with
|
|
28
|
+
* `HTTP 400 Number must be less than or equal to 50` — loud, and found in a day (#294). This one
|
|
29
|
+
* IGNORES: measured against the live account (#297), `per_page=200` and `per_page=1000` both answer
|
|
30
|
+
* `HTTP 200` with no error and `result_info.per_page: 100`.
|
|
31
|
+
*
|
|
32
|
+
* ⚠️ `result_info.total_count` cannot be used to notice a short answer either. The same account
|
|
33
|
+
* reports `total_count: 286` and returns 61 entries on page 1, with page 2 empty — a reader that
|
|
34
|
+
* paginated on that figure would loop over empty pages and call the result partial. The truthful
|
|
35
|
+
* signal is a page that came back FULL, which the log reader already uses and this one does not
|
|
36
|
+
* yet (anchrd/intel#330).
|
|
37
|
+
*/
|
|
38
|
+
const ModelPerPage = 100;
|
|
39
|
+
/**
|
|
40
|
+
* How many pages of the model catalog are read before the reader gives up.
|
|
41
|
+
*
|
|
42
|
+
* ⚠️ It is a runaway brake and not a window, which is the opposite of `MaxPages` above. The log has
|
|
43
|
+
* more calls than anybody wants to read; the catalog is finite and small — 61 entries on this
|
|
44
|
+
* account — so the loop normally ends on the first short page and this number is never reached.
|
|
45
|
+
* What it guards against is an endpoint that answers a full page forever, and a loop inside a
|
|
46
|
+
* request nobody is watching.
|
|
47
|
+
*/
|
|
48
|
+
const ModelMaxPages = 10;
|
|
22
49
|
/**
|
|
23
50
|
* The gateway's log entry, read tolerantly.
|
|
24
51
|
*
|
|
@@ -142,56 +169,138 @@ export function createCloudflareApi(deps) {
|
|
|
142
169
|
}
|
|
143
170
|
return {
|
|
144
171
|
async gatewayCalls(query) {
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
172
|
+
const logPath = `/accounts/${encodeURIComponent(deps.accountId)}/ai-gateway/gateways/${encodeURIComponent(deps.gatewayId)}/logs`;
|
|
173
|
+
/**
|
|
174
|
+
* One window of the log, optionally cut to this agent by the gateway itself.
|
|
175
|
+
*
|
|
176
|
+
* The encoding Cloudflare documents nowhere, measured against the live gateway (#274):
|
|
177
|
+
* `filters` is a URL-encoded JSON array, and `value` is an ARRAY even for a single value.
|
|
178
|
+
*
|
|
179
|
+
* filters=[{"key":"metadata.value","operator":"eq","value":["<id>"]}] → 200, filtered
|
|
180
|
+
* value as a scalar string → 400 `Expected array, received string`
|
|
181
|
+
* filters as a JSON object → 400 `Expected array, received object`
|
|
182
|
+
* filters[0][key]=… (brackets) and filters.0.key=… (dots) → 200, SILENTLY IGNORED
|
|
183
|
+
*
|
|
184
|
+
* ⚠️ The last line is why this was never guessed at: two of the five spellings answer 200
|
|
185
|
+
* with the whole unfiltered window, which reads exactly like a filter that matched
|
|
186
|
+
* everything. Two filters combine with AND (measured with `metadata.key` plus `runId`).
|
|
187
|
+
*/
|
|
188
|
+
async function read(pages, filtered) {
|
|
189
|
+
const calls = [];
|
|
190
|
+
let partial = false;
|
|
191
|
+
for (let index = 1; index <= pages; index += 1) {
|
|
192
|
+
const body = await get(logPath, {
|
|
193
|
+
page: String(index),
|
|
194
|
+
per_page: String(PerPage),
|
|
195
|
+
start_date: query.since.toISOString(),
|
|
196
|
+
end_date: query.until.toISOString(),
|
|
197
|
+
order_by: "created_at",
|
|
198
|
+
order_by_direction: "desc",
|
|
199
|
+
...(filtered
|
|
200
|
+
? {
|
|
201
|
+
filters: JSON.stringify([
|
|
202
|
+
{ key: "metadata.value", operator: "eq", value: [query.agentId] },
|
|
203
|
+
]),
|
|
204
|
+
}
|
|
205
|
+
: {}),
|
|
177
206
|
});
|
|
207
|
+
const parsed = LogResponse.safeParse(body);
|
|
208
|
+
// A shape this reader cannot make sense of is a failure, not an empty window: an empty
|
|
209
|
+
// window reads as "this agent cost nothing".
|
|
210
|
+
if (!parsed.success) {
|
|
211
|
+
throw new IntelError(502, "cloudflare_api_unreadable", "The Cloudflare API answered in a shape this version does not understand");
|
|
212
|
+
}
|
|
213
|
+
const entries = parsed.data.result ?? [];
|
|
214
|
+
// ⚠️ The local check stays even under the server-side filter, and it is no longer belt
|
|
215
|
+
// and braces: it is the only thing that could notice the filter matching the wrong rows.
|
|
216
|
+
for (const entry of entries) {
|
|
217
|
+
const metadata = readMetadata(entry.metadata);
|
|
218
|
+
if (metadata.agentId !== query.agentId)
|
|
219
|
+
continue;
|
|
220
|
+
calls.push({
|
|
221
|
+
runId: typeof metadata.runId === "string" ? metadata.runId : null,
|
|
222
|
+
model: entry.model ?? "",
|
|
223
|
+
cost: entry.cost ?? 0,
|
|
224
|
+
at: entry.created_at ?? query.until.toISOString(),
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
if (entries.length < PerPage)
|
|
228
|
+
return { calls, partial: false };
|
|
229
|
+
partial = index === pages;
|
|
178
230
|
}
|
|
179
|
-
|
|
180
|
-
return { calls, partial: false };
|
|
181
|
-
partial = page === MaxPages;
|
|
231
|
+
return { calls, partial };
|
|
182
232
|
}
|
|
183
|
-
|
|
233
|
+
const answer = await read(MaxPages, true);
|
|
234
|
+
/**
|
|
235
|
+
* ⚠️ Anything at all is taken at face value, and that is a decision with a hole in it. A
|
|
236
|
+
* filter that degraded PARTLY — matching some of this agent's rows and not others — returns
|
|
237
|
+
* here, and the under-count is then handed out as `partial: false`, which claims to be a
|
|
238
|
+
* total. The probe below cannot see that case, because the probe is only reached when the
|
|
239
|
+
* filtered read found nothing. It is the price of not reading the whole log twice on every
|
|
240
|
+
* request, and it is worth knowing before somebody reads `partial: false` as "complete".
|
|
241
|
+
*/
|
|
242
|
+
if (answer.calls.length > 0)
|
|
243
|
+
return answer;
|
|
244
|
+
/**
|
|
245
|
+
* ⚠️ Zero is the ONE answer this reader may not take at face value, and the reason is the
|
|
246
|
+
* way `filters` fails. Measured: `cached` filtered to `["false"]` answers 200 with zero rows
|
|
247
|
+
* although every row in that window carries `cached: false` — a value the index does not
|
|
248
|
+
* match empties the page instead of being refused. So the day metadata stops being indexed
|
|
249
|
+
* the way it is today, this would report "cost nothing" for an agent that spent money, under
|
|
250
|
+
* `status: "read"`. That is precisely the sentence #251 exists for.
|
|
251
|
+
*
|
|
252
|
+
* One unfiltered page is the probe: if the newest calls contain any of this agent's, the
|
|
253
|
+
* filter is lying and the honest answer is that the log could not be read. It costs one
|
|
254
|
+
* request and only in the empty case, which is a fresh agent or a quiet window — and in the
|
|
255
|
+
* case it replaces, a quiet agent on a busy gateway, it costs 2 requests where reading the
|
|
256
|
+
* whole log unfiltered used to cost 20.
|
|
257
|
+
*
|
|
258
|
+
* ⚠️ It is a probe and not a proof: an agent whose only calls are older than the newest
|
|
259
|
+
* `PerPage` of the whole gateway is invisible to it, and that case still reports zero. What
|
|
260
|
+
* it buys is that a filter which stopped working ENTIRELY cannot pass as silence.
|
|
261
|
+
*
|
|
262
|
+
* ⚠️ A probe that cannot run leaves the answer standing, deliberately. The read this is
|
|
263
|
+
* checking SUCCEEDED and said "no calls"; letting a second opinion that never arrived turn
|
|
264
|
+
* that into `unreadable` would report an outage for the most ordinary state there is — a
|
|
265
|
+
* fresh agent — because one extra request met a 429. The guard is a second opinion, not the
|
|
266
|
+
* answer.
|
|
267
|
+
*/
|
|
268
|
+
const probe = await read(1, false).catch(() => null);
|
|
269
|
+
if (probe === null || probe.calls.length === 0)
|
|
270
|
+
return answer;
|
|
271
|
+
throw new IntelError(502, "cloudflare_api_unreadable", "The Cloudflare API answered no calls for this agent while its own log holds some");
|
|
184
272
|
},
|
|
185
273
|
async workersAiModels() {
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
274
|
+
const models = [];
|
|
275
|
+
for (let index = 1; index <= ModelMaxPages; index += 1) {
|
|
276
|
+
const body = await get(`/accounts/${encodeURIComponent(deps.accountId)}/ai/models/search`, {
|
|
277
|
+
page: String(index),
|
|
278
|
+
per_page: String(ModelPerPage),
|
|
279
|
+
hide_experimental: "true",
|
|
280
|
+
});
|
|
281
|
+
const parsed = ModelResponse.safeParse(body);
|
|
282
|
+
if (!parsed.success) {
|
|
283
|
+
throw new IntelError(502, "cloudflare_api_unreadable", "The Cloudflare API answered in a shape this version does not understand");
|
|
284
|
+
}
|
|
285
|
+
const entries = parsed.data.result ?? [];
|
|
286
|
+
models.push(...entries.map(readModel));
|
|
287
|
+
// ⚠️ A SHORT page is the end, and a full one is not a reason to believe the answer was
|
|
288
|
+
// complete (#330). Before this the reader took page one and stopped: a model past position
|
|
289
|
+
// 100 was simply absent from the select, `liveStatus` still said `read`, and nothing said
|
|
290
|
+
// the list was shorter than the account. A missing entry has no `source` to mark stale
|
|
291
|
+
// with, so the per-entry labelling that covers a fallback cannot cover this at all.
|
|
292
|
+
//
|
|
293
|
+
// ⚠️ `result_info.total_count` is NOT the test, which is why it is not parsed. Measured on
|
|
294
|
+
// the live account (#297): it reports 286 while page one returns 61 and page two is empty.
|
|
295
|
+
// A reader that paginated on that figure would loop over empty pages and call the result
|
|
296
|
+
// partial. The full page is the only honest signal.
|
|
297
|
+
if (entries.length < ModelPerPage)
|
|
298
|
+
return models;
|
|
193
299
|
}
|
|
194
|
-
|
|
300
|
+
// Ten full pages is not a big account, it is an endpoint that stopped ending. Refusing is the
|
|
301
|
+
// honest answer: the catalog service falls back to its built-in table and the screen says the
|
|
302
|
+
// figures may be old — better than a list that is silently missing whatever came after.
|
|
303
|
+
throw new IntelError(502, "cloudflare_api_unreadable", "The Cloudflare API kept answering full pages of models past the read limit");
|
|
195
304
|
},
|
|
196
305
|
};
|
|
197
306
|
}
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
// How much of a chunk is kept as the passage a searcher is shown. The same 480 characters
|
|
2
|
+
// `hydrateVisibleCitations` cuts out of the full-text row, so a citation reads the same length
|
|
3
|
+
// whether it came from a card's own vector or from the fallback beside it.
|
|
4
|
+
const maxPassageCharacters = 480;
|
|
1
5
|
function mapTarget(row, contentKeys) {
|
|
2
6
|
return {
|
|
3
7
|
nodeId: row.node_id,
|
|
@@ -68,6 +72,81 @@ export function createNodeIndexRepository(db) {
|
|
|
68
72
|
.bind(target.versionId, target.nodeId, chunk.title, chunk.text, target.nodeId, target.versionId)),
|
|
69
73
|
]);
|
|
70
74
|
},
|
|
75
|
+
/**
|
|
76
|
+
* ⚠️ `archived_at IS NOT NULL` carries this on its own, and it is the whole guard
|
|
77
|
+
* (anchrd/intel#348). `getTarget` above answers null for two reasons and only one of them means
|
|
78
|
+
* "purge": the other is a queue message for a version a later save superseded, and Cloudflare
|
|
79
|
+
* Queues deliver at least once, so those arrive as a matter of course. Acting on one would empty
|
|
80
|
+
* the vector index of a live board behind the back of the save that had just filled it — and
|
|
81
|
+
* nothing would report it, because a search that finds less is not an error.
|
|
82
|
+
*
|
|
83
|
+
* ⚠️ Deliberately NOT `current_version_id = v.id` as well, although that is what would make this
|
|
84
|
+
* the mirror image of `getTarget`. Symmetry is not a reason to write a condition: once the node
|
|
85
|
+
* IS archived, every version of it says the same thing about that node's vectors, so the extra
|
|
86
|
+
* clause could only ever refuse a purge that was right. A condition no test can turn red is a
|
|
87
|
+
* condition the next reader has to guess the purpose of.
|
|
88
|
+
*/
|
|
89
|
+
async archivedNodeId(versionId) {
|
|
90
|
+
const row = await db
|
|
91
|
+
.prepare(`SELECT n.id AS node_id
|
|
92
|
+
FROM node_versions v
|
|
93
|
+
JOIN nodes n ON n.id = v.node_id
|
|
94
|
+
WHERE v.id = ? AND n.archived_at IS NOT NULL`)
|
|
95
|
+
.bind(versionId)
|
|
96
|
+
.first();
|
|
97
|
+
return row?.node_id ?? null;
|
|
98
|
+
},
|
|
99
|
+
async listVectors(nodeId) {
|
|
100
|
+
const result = await db
|
|
101
|
+
.prepare(`SELECT chunk_key, fingerprint, passage FROM node_vectors WHERE node_id = ?`)
|
|
102
|
+
.bind(nodeId)
|
|
103
|
+
.all();
|
|
104
|
+
return (result.results ?? []).map((row) => ({
|
|
105
|
+
chunkKey: row.chunk_key,
|
|
106
|
+
fingerprint: row.fingerprint,
|
|
107
|
+
passage: row.passage,
|
|
108
|
+
}));
|
|
109
|
+
},
|
|
110
|
+
/**
|
|
111
|
+
* The record of every vector this node has, replacing the record it had.
|
|
112
|
+
*
|
|
113
|
+
* ⚠️ The same delete-then-insert in ONE batch as `replace` above, and the same guard in front of
|
|
114
|
+
* every statement: a pass for a version that is no longer current, or for a node that has been
|
|
115
|
+
* archived meanwhile, writes nothing at all rather than half a record. What must not happen is
|
|
116
|
+
* the delete landing without the inserts — the next pass would then re-embed the whole board,
|
|
117
|
+
* which is exactly the cost anchrd/intel#301 exists to avoid.
|
|
118
|
+
*/
|
|
119
|
+
async replaceVectors(target, records) {
|
|
120
|
+
await db.batch([
|
|
121
|
+
db
|
|
122
|
+
.prepare(`DELETE FROM node_vectors WHERE node_id = ? AND EXISTS (
|
|
123
|
+
SELECT 1 FROM nodes
|
|
124
|
+
WHERE id = ? AND current_version_id = ? AND archived_at IS NULL
|
|
125
|
+
)`)
|
|
126
|
+
.bind(target.nodeId, target.nodeId, target.versionId),
|
|
127
|
+
...records.map((record) => db
|
|
128
|
+
.prepare(`INSERT INTO node_vectors (node_id, chunk_key, version_id, fingerprint, passage)
|
|
129
|
+
SELECT ?, ?, ?, ?, ? WHERE EXISTS (
|
|
130
|
+
SELECT 1 FROM nodes
|
|
131
|
+
WHERE id = ? AND current_version_id = ? AND archived_at IS NULL
|
|
132
|
+
)`)
|
|
133
|
+
.bind(target.nodeId, record.chunkKey, target.versionId, record.fingerprint, record.passage.slice(0, maxPassageCharacters), target.nodeId, target.versionId)),
|
|
134
|
+
]);
|
|
135
|
+
},
|
|
136
|
+
/**
|
|
137
|
+
* The record of one node, gone — because its vectors are (anchrd/intel#348).
|
|
138
|
+
*
|
|
139
|
+
* ⚠️ No guard at all, and the absence is deliberate in both directions. There is no
|
|
140
|
+
* `archived_at IS NOT NULL`: the state this write exists to prevent is a row that outlives the
|
|
141
|
+
* vector it names, and a node restored between the deletion and this statement would keep
|
|
142
|
+
* exactly that — a record claiming vectors that are gone, which the next pass reads as "already
|
|
143
|
+
* embedded" and skips, leaving those cards silently unfindable. And there is no version
|
|
144
|
+
* condition: `archivedNodeId` has already established which node this is, from that same
|
|
145
|
+
* version, and a second reading of a moving row is a second answer rather than a safer one.
|
|
146
|
+
*/
|
|
147
|
+
async deleteVectors(nodeId) {
|
|
148
|
+
await db.prepare("DELETE FROM node_vectors WHERE node_id = ?").bind(nodeId).run();
|
|
149
|
+
},
|
|
71
150
|
async markIndexed(versionId, occurredAt) {
|
|
72
151
|
await db
|
|
73
152
|
.prepare(`UPDATE node_index_state
|
package/dist/adapters/db/db.js
CHANGED
|
@@ -289,20 +289,17 @@ export function createNodeRepository(deps) {
|
|
|
289
289
|
* ⚠️ `nodes` arrive parents before children: `parent_id REFERENCES nodes(id)` is checked as
|
|
290
290
|
* each row lands, and a child before its parent would abort the batch that is otherwise valid.
|
|
291
291
|
* The service owns that order; this only writes what it is handed.
|
|
292
|
-
*
|
|
293
|
-
* `context_policy` is the dead column migration 0009 explains; 'relevant' is the fixed value
|
|
294
|
-
* `insertNode` writes for the same reason.
|
|
295
292
|
*/
|
|
296
293
|
async importTree(input) {
|
|
297
294
|
const statements = [
|
|
298
295
|
deps.db
|
|
299
296
|
.prepare(`INSERT INTO nodes (
|
|
300
|
-
id, parent_id, kind, title, description,
|
|
297
|
+
id, parent_id, kind, title, description, owner_id,
|
|
301
298
|
current_version_id, created_at, updated_at, archived_at
|
|
302
299
|
) SELECT
|
|
303
300
|
json_extract(value, '$.id'), json_extract(value, '$.parentId'),
|
|
304
301
|
json_extract(value, '$.kind'), json_extract(value, '$.title'),
|
|
305
|
-
json_extract(value, '$.description'),
|
|
302
|
+
json_extract(value, '$.description'),
|
|
306
303
|
json_extract(value, '$.ownerId'), json_extract(value, '$.currentVersionId'),
|
|
307
304
|
json_extract(value, '$.createdAt'), json_extract(value, '$.updatedAt'), NULL
|
|
308
305
|
FROM json_each(?)`)
|
|
@@ -458,14 +455,10 @@ export function createNodeRepository(deps) {
|
|
|
458
455
|
try {
|
|
459
456
|
await deps.db.batch([
|
|
460
457
|
deps.db
|
|
461
|
-
// ⚠️ `context_policy` is dead and is written anyway (#76). The column is NOT NULL
|
|
462
|
-
// without a DEFAULT and D1 will not let it be dropped — migration 0009 carries the
|
|
463
|
-
// reason. The fixed value is the price; nothing reads it, and the contract no longer
|
|
464
|
-
// knows the field. When the column goes (anchrd/intel#86), this line goes with it.
|
|
465
458
|
.prepare(`INSERT INTO nodes (
|
|
466
|
-
id, parent_id, kind, title, description,
|
|
459
|
+
id, parent_id, kind, title, description, owner_id,
|
|
467
460
|
current_version_id, created_at, updated_at, archived_at
|
|
468
|
-
) VALUES (?, ?, ?, ?, ?,
|
|
461
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
469
462
|
.bind(node.id, node.parentId, node.kind, node.title, node.description, node.ownerId, node.currentVersionId, node.createdAt, node.updatedAt, node.archivedAt),
|
|
470
463
|
deps.db
|
|
471
464
|
.prepare(`INSERT INTO idempotency_keys (
|
|
@@ -1144,32 +1137,46 @@ export function createNodeRepository(deps) {
|
|
|
1144
1137
|
match: "lexical",
|
|
1145
1138
|
}));
|
|
1146
1139
|
},
|
|
1147
|
-
async hydrateVisibleCitations(actor,
|
|
1148
|
-
|
|
1149
|
-
|
|
1140
|
+
async hydrateVisibleCitations(actor, hits, scopeId) {
|
|
1141
|
+
// One entry per node: the caller has already picked the best-scoring chunk of each, so a
|
|
1142
|
+
// board that answered with four cards arrives as the one card that answered best.
|
|
1143
|
+
const unique = [...new Map(hits.map((hit) => [hit.nodeId, hit])).values()].slice(0, 100);
|
|
1144
|
+
if (unique.length === 0)
|
|
1150
1145
|
return [];
|
|
1151
1146
|
const rows = [];
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1147
|
+
// Two bindings per hit now instead of one, so the batch is halved and then some: the actor's
|
|
1148
|
+
// predicate and the optional scope are bound in front of them, and D1 counts every one of
|
|
1149
|
+
// them against the same per-statement ceiling.
|
|
1150
|
+
for (let offset = 0; offset < unique.length; offset += 40) {
|
|
1151
|
+
const batch = unique.slice(offset, offset + 40);
|
|
1152
|
+
const pairs = batch.map(() => "(?, ?)").join(", ");
|
|
1155
1153
|
const result = await deps.db
|
|
1156
|
-
.prepare(`${visibleCte}${scopeCte(scopeId)}
|
|
1154
|
+
.prepare(`${visibleCte}${scopeCte(scopeId)},
|
|
1155
|
+
hits(node_id, chunk_key) AS (VALUES ${pairs})
|
|
1157
1156
|
SELECT n.id AS node_id, v.id AS version_id, n.title,
|
|
1158
|
-
|
|
1157
|
+
-- ⚠️ The card that answered, and only otherwise the first passage of the node
|
|
1158
|
+
-- (anchrd/intel#301). Every vector written before #301 is named by the bare node id
|
|
1159
|
+
-- and has no row here, so the fallback is not defensive coding but the exact
|
|
1160
|
+
-- behaviour those vectors had — a board found through one of them still answers,
|
|
1161
|
+
-- with the passage it always answered with, until the next indexing pass.
|
|
1162
|
+
COALESCE(vector.passage, substr(node_fts.content, 1, 480)) AS passage,
|
|
1159
1163
|
n.updated_at AS freshness
|
|
1160
|
-
FROM
|
|
1164
|
+
FROM hits
|
|
1165
|
+
JOIN nodes n ON n.id = hits.node_id
|
|
1161
1166
|
JOIN node_versions v ON v.id = n.current_version_id
|
|
1162
1167
|
JOIN node_fts ON node_fts.version_id = v.id
|
|
1168
|
+
LEFT JOIN node_vectors vector
|
|
1169
|
+
ON vector.node_id = hits.node_id AND vector.chunk_key = hits.chunk_key
|
|
1163
1170
|
JOIN allowed ON allowed.id = n.id${scopeJoin(scopeId)}
|
|
1164
|
-
WHERE n.
|
|
1165
|
-
.bind(...readBindings(actor), ...scopeBindings(scopeId), ...
|
|
1171
|
+
WHERE n.archived_at IS NULL`)
|
|
1172
|
+
.bind(...readBindings(actor), ...scopeBindings(scopeId), ...batch.flatMap((hit) => [hit.nodeId, hit.chunkKey]))
|
|
1166
1173
|
.all();
|
|
1167
1174
|
rows.push(...(result.results ?? []));
|
|
1168
1175
|
}
|
|
1169
|
-
const order = new Map(
|
|
1170
|
-
// The same one-per-node rule as the lexical half above, for the same reason: a
|
|
1171
|
-
//
|
|
1172
|
-
//
|
|
1176
|
+
const order = new Map(unique.map((hit, index) => [hit.nodeId, index]));
|
|
1177
|
+
// The same one-per-node rule as the lexical half above, for the same reason: a citation names
|
|
1178
|
+
// a NODE, and a board holds one full-text row per task (#285), so the join above still
|
|
1179
|
+
// produces one row per card even though only one of them was asked about.
|
|
1173
1180
|
const deduped = dedupedByNode(rows);
|
|
1174
1181
|
deduped.sort((left, right) => (order.get(left.node_id) ?? 0) - (order.get(right.node_id) ?? 0));
|
|
1175
1182
|
return deduped.map((row) => ({
|
|
@@ -1183,10 +1190,42 @@ export function createNodeRepository(deps) {
|
|
|
1183
1190
|
match: "semantic",
|
|
1184
1191
|
}));
|
|
1185
1192
|
},
|
|
1193
|
+
async invalidateVectors() {
|
|
1194
|
+
/**
|
|
1195
|
+
* ⚠️ The fingerprints are cleared and the ROWS are kept, which is not a detail. A record row
|
|
1196
|
+
* says two things at once: "this chunk was embedded from this text" and "this vector exists".
|
|
1197
|
+
* Only the first is stale after a rebuild is asked for; deleting the row would throw the
|
|
1198
|
+
* second away, and then a card removed between this call and that board's pass would leave a
|
|
1199
|
+
* vector nothing could ever name again — matching questions and answering with a card that is
|
|
1200
|
+
* not on the board.
|
|
1201
|
+
*
|
|
1202
|
+
* The empty string is a value no digest produces, so every chunk compares as changed and the
|
|
1203
|
+
* next pass embeds all of them. `node_fts` is deliberately not touched: its rows are
|
|
1204
|
+
* overwritten by the pass that rewrites them, and emptying it would take the lexical half of
|
|
1205
|
+
* the installation offline for as long as the queue needs to work through the tree.
|
|
1206
|
+
*/
|
|
1207
|
+
await deps.db.prepare("UPDATE node_vectors SET fingerprint = ''").run();
|
|
1208
|
+
},
|
|
1209
|
+
/**
|
|
1210
|
+
* ⚠️ Archived nodes are IN this walk since anchrd/intel#348, and the omission of
|
|
1211
|
+
* `archived_at IS NULL` is the point. The pass an archived node gets is a purge, not an index —
|
|
1212
|
+
* `getTarget` refuses it and `archivedNodeId` says why — so including them is what makes
|
|
1213
|
+
* `reindex` the way back for a vector that is in the index with nothing left to name it.
|
|
1214
|
+
*
|
|
1215
|
+
* Three of those exist and none of them has another repair: everything archived before #348,
|
|
1216
|
+
* which nothing ever swept; a node whose `archive` wrote its row and then failed to reach the
|
|
1217
|
+
* queue; and a pass that upserted vectors in the moment somebody archived the node underneath
|
|
1218
|
+
* it. Without this line an administrator's rebuild walked straight past all three, and the only
|
|
1219
|
+
* remaining answer would have been to delete the whole Vectorize index by hand.
|
|
1220
|
+
*
|
|
1221
|
+
* It does mean `queued` counts them. That is still what the number says — versions handed to the
|
|
1222
|
+
* queue — and a rebuild that silently skipped part of the tree is the failure `reindex` exists
|
|
1223
|
+
* to prevent.
|
|
1224
|
+
*/
|
|
1186
1225
|
async listCurrentVersionIds(input) {
|
|
1187
1226
|
const result = await deps.db
|
|
1188
1227
|
.prepare(`SELECT current_version_id AS id FROM nodes
|
|
1189
|
-
WHERE current_version_id IS NOT NULL
|
|
1228
|
+
WHERE current_version_id IS NOT NULL
|
|
1190
1229
|
AND (? IS NULL OR current_version_id > ?)
|
|
1191
1230
|
ORDER BY id LIMIT ?`)
|
|
1192
1231
|
.bind(input.after, input.after, input.limit)
|
|
@@ -24,9 +24,56 @@ function metadataCandidates(resourceUrl, challenge) {
|
|
|
24
24
|
function resourceMetadataChallenge(header) {
|
|
25
25
|
return header?.match(/resource_metadata="([^"]+)"/i)?.[1] ?? null;
|
|
26
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* ⚠️ `issuer` is REQUIRED by RFC 8414 §2 and by OpenID Connect Discovery, and it is read here for one
|
|
29
|
+
* reason: it is the only thing that says WHO a metadata document describes. Two of the URLs below are
|
|
30
|
+
* bare-root guesses, and a bare root under a multi-tenant authorization server answers for a
|
|
31
|
+
* different tenant — a perfectly valid document about somebody else.
|
|
32
|
+
*/
|
|
27
33
|
const AuthorizationServerMetadata = z.looseObject({
|
|
34
|
+
issuer: z.string().nullish(),
|
|
28
35
|
scopes_supported: z.array(z.string()).nullish(),
|
|
29
36
|
});
|
|
37
|
+
/**
|
|
38
|
+
* Where an authorization server may publish its metadata, in the order worth asking.
|
|
39
|
+
*
|
|
40
|
+
* ⚠️ One candidate was never a decision, it was an omission — `metadataCandidates` above tries three
|
|
41
|
+
* and `withAlgorithmFallback` tries two. A server that publishes only OIDC discovery read as "names
|
|
42
|
+
* no scopes", which is #97 at a different server class and with the same silent symptom (#303).
|
|
43
|
+
*
|
|
44
|
+
* ⚠️ The three spellings are not interchangeable, and the third is the one that is easy to miss.
|
|
45
|
+
* RFC 8414 §3.1 INSERTS the issuer's path between the well-known prefix and nothing else; OpenID
|
|
46
|
+
* Connect Discovery 1.0 APPENDS its suffix to the issuer instead. A realm-style issuer
|
|
47
|
+
* (`https://as.example/realms/x`) publishes at `…/realms/x/.well-known/openid-configuration` and at
|
|
48
|
+
* neither of the other two — which is exactly the form `client.discovery` asks for further down this
|
|
49
|
+
* file, so the same server was reachable for registration and invisible here.
|
|
50
|
+
*/
|
|
51
|
+
function authorizationServerCandidates(issuer) {
|
|
52
|
+
const url = new URL(issuer);
|
|
53
|
+
const path = url.pathname === "/" ? "" : url.pathname.replace(/\/+$/, "");
|
|
54
|
+
return [
|
|
55
|
+
// RFC 8414 with the path inserted. First, because it is the shape the MCP portals publish.
|
|
56
|
+
`${url.origin}/.well-known/oauth-authorization-server${path}`,
|
|
57
|
+
// The same document at the bare root, for an issuer WITH a path that publishes it there anyway.
|
|
58
|
+
`${url.origin}/.well-known/oauth-authorization-server`,
|
|
59
|
+
// OpenID Connect Discovery 1.0, appended to the issuer — the realm-style form.
|
|
60
|
+
`${url.origin}${path}/.well-known/openid-configuration`,
|
|
61
|
+
// The same document read the RFC 8414 way, and at the bare root.
|
|
62
|
+
`${url.origin}/.well-known/openid-configuration${path}`,
|
|
63
|
+
`${url.origin}/.well-known/openid-configuration`,
|
|
64
|
+
].filter((value, index, values) => values.indexOf(value) === index);
|
|
65
|
+
}
|
|
66
|
+
/** Two issuer identifiers are the same server, compared the way a URL says so and not as strings. */
|
|
67
|
+
function sameIssuer(published, wanted) {
|
|
68
|
+
if (!published)
|
|
69
|
+
return false;
|
|
70
|
+
try {
|
|
71
|
+
return new URL(published).href.replace(/\/$/, "") === new URL(wanted).href.replace(/\/$/, "");
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
30
77
|
/**
|
|
31
78
|
* The scopes this authorization server says it understands, or `null` when it names none.
|
|
32
79
|
*
|
|
@@ -41,16 +88,29 @@ const AuthorizationServerMetadata = z.looseObject({
|
|
|
41
88
|
* Intel then asked it for `offline_access`, a scope it never published (#97).
|
|
42
89
|
*/
|
|
43
90
|
async function publishedScopes(fetcher, issuer) {
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
91
|
+
for (const candidate of authorizationServerCandidates(issuer)) {
|
|
92
|
+
const response = await fetcher(candidate, {
|
|
93
|
+
signal: AbortSignal.timeout(RequestTimeoutMs),
|
|
94
|
+
headers: { accept: "application/json" },
|
|
95
|
+
}).catch(() => null);
|
|
96
|
+
if (!response?.ok)
|
|
97
|
+
continue;
|
|
98
|
+
const metadata = AuthorizationServerMetadata.safeParse(await response.json().catch(() => null));
|
|
99
|
+
if (!metadata.success)
|
|
100
|
+
continue;
|
|
101
|
+
// ⚠️ The document has to be ABOUT this issuer, and that check is what makes the bare-root
|
|
102
|
+
// candidates safe to ask at all. Under a multi-tenant server the root answers for a different
|
|
103
|
+
// tenant, and taking its `scopes_supported` would ask THIS server for a scope it never published
|
|
104
|
+
// — the very rule this function exists to keep, broken by the fallback added to protect it.
|
|
105
|
+
// The resource loop further down does the same thing for the same reason.
|
|
106
|
+
if (!sameIssuer(metadata.data.issuer, issuer))
|
|
107
|
+
continue;
|
|
108
|
+
// ⚠️ A document that answered ABOUT THIS SERVER and named no scopes ends the search. Reading on
|
|
109
|
+
// would let the next candidate answer a question this server has already answered — "I publish
|
|
110
|
+
// none" — and the documents of one issuer may disagree about a field RFC 8414 makes optional.
|
|
111
|
+
return metadata.data.scopes_supported ?? null;
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
54
114
|
}
|
|
55
115
|
/**
|
|
56
116
|
* Of the scopes Intel would like, the ones this server published — joined, or absent.
|