@anchrd/intel-api 0.13.0 → 0.15.0

package/dist/indexing/indexing.js

@@ -1,7 +1,13 @@
-import { BlockNoteDocument, BlockNoteMediaType, BoardDocument } from "@anchrd/intel-contract";
-import { upgradeStoredBoard } from "../nodes/board/board.js";
+import { BlockNoteDocument, BlockNoteMediaType } from "@anchrd/intel-contract";
 export class PermanentIndexingError extends Error {
 }
+// The key under which a node with exactly one vector is filed — since #390 that is every kind. It
+// is the bare node id in the index itself, which is why nothing written before anchrd/intel#301 has
+// to be renamed or embedded again.
+const wholeNodeChunkKey = "";
+// What the pass hands to `purgeVectors` at its end: the names it has just written, in case the node
+// was archived while it worked and the record of them was refused (anchrd/intel#348).
+const writtenKeys = (chunks) => chunks.map((chunk) => chunk.key);
 function indexText(mediaType, content) {
     if (mediaType !== BlockNoteMediaType)
         return content;
@@ -15,43 +21,114 @@ function indexText(mediaType, content) {
     }
 }
 /**
- * One passage per task, never one per board (#285).
+ * What one chunk is embedded as (anchrd/intel#301).
  *
- * ⚠️ This is the whole reason a board is chunked at all. "Where do I stand with X" has to land on a
- * CARD: indexed as one blob, a board of three hundred tasks matches on any of them and answers with
- * whichever thousand characters the snippet function happened to cut — text from a task that may
- * have nothing to do with the question. Each chunk carries the task's own title, which is the
- * column the FTS table weights highest.
+ * The node's title in front of a whole-node chunk, which is the exact string the semantic adapter
+ * used to compose on its own side. It moved here because the fingerprint has to cover what is
+ * embedded and nothing else: a prefix added downstream would sit outside the comparison and change
+ * nothing when it changed. The vector it produces therefore keeps the same name AND the same
+ * content it had before this — an upgrading installation loses no answer, though its first pass
+ * over any node does embed once more, because `node_vectors` starts empty and has no fingerprint to
+ * compare against. That cost is one embedding per node, paid when that node is next saved or when
+ * an administrator runs `reindex`, and never per card.
  *
- * ⚠️ The status list is not indexed. "Backlog" and "Done" appear on every board in the
- * installation, so they are the words most likely to match and the least likely to mean anything.
+ * ⚠️ Only the WHOLE-node chunk is prefixed, and the distinction is what kept a rename cheap while
+ * a node could hold many chunks: a per-chunk prefix would have put the node's title into every
+ * fingerprint, so renaming it once cost one embedding per chunk.
  */
-function boardChunks(content) {
-    let parsed;
-    try {
-        parsed = JSON.parse(content);
-    }
-    catch (error) {
-        throw new PermanentIndexingError(`Node version content is not valid JSON: ${error instanceof Error ? error.message : "unknown parse failure"}`);
-    }
-    /**
-     * ⚠️ Upgraded first, exactly like `parseStoredBoard` — this is the SECOND place a stored board
-     * body is parsed, and it reads bodies nobody has rewritten since (anchrd/intel#318).
-     *
-     * `reindex` walks every current version in the installation, so an old board arrives here as it
-     * was written: without `terminal` (#311) or with a repeated `dependsOn`. Without the upgrade the
-     * refusal below is PERMANENT — the version is marked failed and never retried — so that board
-     * would simply stop being findable, quietly, while its screen still worked. A rule added at one
-     * parse site and not the other is how a fix looks complete and is not.
-     */
-    const board = BoardDocument.safeParse(upgradeStoredBoard(parsed));
-    if (!board.success) {
-        throw new PermanentIndexingError("Node version content is not a valid board document");
+function embeddedText(target, chunk) {
+    return chunk.key === wholeNodeChunkKey ? `${target.title}\n\n${chunk.text}` : chunk.text;
+}
+/**
+ * The vectors this node needs, minus the ones it already has (anchrd/intel#301).
+ *
+ * ⚠️ This is the answer to "an unchanged chunk must not be embedded again". A save writes the
+ * whole document, so without this every save would re-embed everything it holds. The fingerprint of
+ * the exact text that was embedded is what tells an untouched chunk from a changed one.
+ *
+ * ⚠️ The comparison is against D1 and deliberately not against Vectorize. Vectorize writes are
+ * asynchronous — a vector upserted a moment ago is not readable yet — so a check made there would
+ * answer about the save before last and re-embed a node that had just been saved twice.
+ *
+ * ⚠️ The record is written only after the upsert returned, in the caller. A pass that dies in
+ * between leaves the record short, so the next one embeds again; the opposite order would leave a
+ * record claiming a vector nobody ever wrote, and nothing would ever notice.
+ */
+async function replaceVectors(deps, target, chunks) {
+    const semantic = deps.semantic;
+    if (!semantic)
+        return;
+    const stored = await deps.repository.listVectors(target.nodeId);
+    const known = new Map(stored.map((record) => [record.chunkKey, record.fingerprint]));
+    const records = [];
+    const changed = [];
+    for (const chunk of chunks) {
+        const text = embeddedText(target, chunk);
+        const fingerprint = await deps.hash(text);
+        if (known.get(chunk.key) !== fingerprint)
+            changed.push({ key: chunk.key, text });
+        records.push({ chunkKey: chunk.key, fingerprint, passage: chunk.text });
     }
-    return board.data.tasks.map((task) => ({
-        title: task.title,
-        text: [task.title, ...task.labels, task.description].filter(Boolean).join("\n\n"),
-    }));
+    const wanted = new Set(chunks.map((chunk) => chunk.key));
+    const stale = stored.map((record) => record.chunkKey).filter((key) => !wanted.has(key));
+    // ⚠️ The one-time sweep of the vector this node had BEFORE it was chunked. An empty record with a
+    // chunked node means this is the first pass since anchrd/intel#301 (or since `reindex` emptied
+    // the record), and the whole-node vector written under the bare node id is still sitting there —
+    // matching questions and answering with the wrong card's passage. Once a record exists this
+    // branch is never taken again, so it costs one call per node rather than one per save.
+    if (stored.length === 0 && !wanted.has(wholeNodeChunkKey))
+        stale.push(wholeNodeChunkKey);
+    await semantic.upsert(target, changed);
+    if (stale.length > 0)
+        await semantic.remove(target.nodeId, stale);
+    await deps.repository.replaceVectors(target, records);
+}
+/**
+ * Everything an archived node had in the vector index, taken out of it (anchrd/intel#348).
+ *
+ * ⚠️ The ORDER is the whole safety argument, and it is the exact mirror of the one above. There the
+ * record is written last, because a row claiming a vector nobody wrote would make the next pass skip
+ * that card forever. Here the record is DELETED last, because the row is the only thing that knows
+ * what a vector is called: drop it first and the vectors stay in the index under names nothing left
+ * anywhere can produce. A pass that dies in between leaves vectors already gone and a record that
+ * still names them — which costs the next pass one embedding per card and nothing else, because the
+ * queue redelivers this message and the second run asks Vectorize to forget names it has already
+ * forgotten. Both failures are repaired by repeating; neither ends in a wrong answer.
+ *
+ * ⚠️ The bare node id goes with them whether or not it is in the record. That name is the ONE vector
+ * a node can hold without any record of it — everything written before anchrd/intel#301 is filed
+ * under it — and an archived node has no later pass in which the one-time sweep in `replaceVectors`
+ * could find it. The standing cost of that is one `deleteByIds` of a single name on a redelivered
+ * message, which Vectorize answers without complaint for an id it no longer holds.
+ *
+ * ⚠️ `alsoNamed` is what makes this run after a NORMAL pass too, and it closes the one hole a purge
+ * driven from `archive` alone cannot. A pass that has already upserted its vectors when somebody
+ * archives the node writes no record at all — `replaceVectors` refuses on `archived_at IS NULL` —
+ * so those vectors would sit in the index with nothing anywhere able to name them, and no message
+ * left to repair it. The pass knows the names it just wrote, so it hands them over.
+ *
+ * ⚠️ Without a semantic index configured nothing happens at all, `node_vectors` included. The rows
+ * can only have been written while one WAS configured, so a deployment that has temporarily lost the
+ * binding must not take the record away — that would leave the vectors behind it unnameable, which
+ * is the failure this whole function exists to avoid.
+ *
+ * ⚠️ One race is left and it is the harmless direction: a restore that lands between the read below
+ * and the deletion loses the vectors that restore's own pass had just written. The node keeps
+ * answering out of the lexical half — `hydrateVisibleCitations` falls back on the full-text passage
+ * when there is no `node_vectors` row — and the next save or `reindex` puts the vectors back. The
+ * record is gone with them, so nothing claims otherwise in the meantime.
+ */
+async function purgeVectors(deps, versionId, alsoNamed = []) {
+    const semantic = deps.semantic;
+    if (!semantic)
+        return;
+    const nodeId = await deps.repository.archivedNodeId(versionId);
+    if (nodeId === null)
+        return;
+    const recorded = (await deps.repository.listVectors(nodeId)).map((record) => record.chunkKey);
+    const keys = [...new Set([wholeNodeChunkKey, ...recorded, ...alsoNamed])];
+    await semantic.remove(nodeId, keys);
+    await deps.repository.deleteVectors(nodeId);
 }
 async function readCanonical(deps, target) {
     if (target.kind === "attachment") {
@@ -67,20 +144,13 @@ export function createIndexing(deps) {
     return {
         async index(versionId) {
             const target = await deps.repository.getTarget(versionId);
-            if (!target)
-                return;
-            // ⚠️ An agent's body is a definition, not something to read: node IDs, a cron line and a model
-            // name (ADR-0005 §4). Indexing it verbatim would fill the index with identifiers and, worse,
-            // put them into the passage a searcher is shown — the definition names nodes the searcher may
-            // have no access to, and a snippet is the one place their IDs would become visible without
-            // the tree being asked. What makes an agent findable is what a person wrote about it, so the
-            // text is its description, and the FTS table indexes the title on its own (#139).
-            if (target.kind === "agent") {
-                await deps.repository.replace(target, [
-                    { title: target.title, text: target.description ?? "" },
-                ]);
-                await deps.semantic?.replace(target, target.description ?? "");
-                await deps.repository.markIndexed(versionId, deps.now().toISOString());
+            // ⚠️ No target has meant "nothing to do" since the pass existed, and for a superseded version
+            // it still does. For an ARCHIVED node it never did: its vectors stayed in the index, invisible
+            // because every citation is hydrated through a join on `nodes` — and costing places in a
+            // candidate list that Vectorize caps at 100 for everybody (anchrd/intel#348). `purgeVectors`
+            // is the one that tells the two apart; this call cannot.
+            if (!target) {
+                await purgeVectors(deps, versionId);
                 return;
             }
             // An attachment is one object of bytes; text kinds may be several, because a table's content
@@ -100,12 +170,20 @@ export function createIndexing(deps) {
                 if (text === undefined) {
                     throw new PermanentIndexingError(`No document converter is configured for ${target.mediaType}`);
                 }
-                // ⚠️ The semantic index stays per NODE while the lexical one gains a row per task. Vectorize
-                // is keyed by node id (`semantic-index.ts`), so a board would need its own key space there
-                // before it could carry one vector per card — a bigger change than #285, and one nothing
-                // asks for yet: the lexical half is what "find the card" needs today.
-                await deps.repository.replace(target, target.kind === "board" ? boardChunks(text) : [{ title: target.title, text }]);
-                await deps.semantic?.replace(target, text);
+                // ⚠️ One chunk per node since #390, and the LIST is what stays. Both indexes read it, so
+                // they cannot come to disagree about what a node is made of — which is the property that
+                // let a board be one passage and one vector per card (#285, anchrd/intel#301) without
+                // either half being told about it separately.
+                const chunks = [{ key: wholeNodeChunkKey, title: target.title, text }];
+                await deps.repository.replace(target, chunks);
+                await replaceVectors(deps, target, chunks);
+                // ⚠️ Asked again, at the END of a pass that started on a live node (anchrd/intel#348). The
+                // node may have been archived while this pass was embedding, and then `replaceVectors` above
+                // wrote no record at all — its statements refuse on `archived_at IS NULL`. The vectors are in
+                // the index either way, so without this they would stay there with nothing anywhere able to
+                // name them and no message left to try again: the archive's own pass has an empty record to
+                // read from. One extra `SELECT` per pass buys that, and on a live node it answers null.
+                await purgeVectors(deps, versionId, writtenKeys(chunks));
                 await deps.repository.markIndexed(versionId, deps.now().toISOString());
             }
             catch (error) {

package/dist/indexing/indexing.types.d.ts

@@ -6,5 +6,6 @@ export interface IndexingDeps {
     content: ContentStore;
     semantic?: SemanticIndex | undefined;
     converter?: DocumentConverter | undefined;
+    hash(content: string): Promise<string>;
     now(): Date;
 }

package/dist/intel/intel.js

@@ -72,9 +72,6 @@ export function createIntel(deps) {
         flows: deps.flows,
         tools: deps.tools,
         bundle: deps.bundle,
-        agents: deps.agents,
-        agentCosts: deps.agentCosts,
-        models: deps.models,
         resource,
         resourceMetadataUrl,
         ...(deps.auth ? { auth: deps.auth } : {}),
@@ -87,13 +84,11 @@ export function createIntel(deps) {
         }
         return await handleMcp(context.req.raw, {
             authorization,
-            // ⚠️ The very token this call was authorized with, so the agent tools reach the runtime as the
-            // caller and not as Intel. There is no cookie path on `/mcp`, so `bearer` is the same value
-            // `authorize` just used — read again rather than passed out of it, because a function that
-            // returned a credential beside its answer would be one somebody logs.
+            // ⚠️ The very token this call was authorized with, so a tool that has to speak to Gate does
+            // it as the caller and not as Intel. There is no cookie path on `/mcp`, so `bearer` is the
+            // same value `authorize` just used — read again rather than passed out of it, because a
+            // function that returned a credential beside its answer would be one somebody logs.
             bearer: bearer(context.req.raw.headers) ?? "",
-            agents: deps.agents,
-            agentCosts: deps.agentCosts,
             flows: deps.flows,
             nodes: deps.nodes,
             tools: deps.tools,

package/dist/intel/intel.types.d.ts

@@ -1,10 +1,7 @@
 import type { GateClient } from "@anchrd/gate-sdk";
-import type { AgentCostsService } from "../agent-costs/agent-costs.types.js";
-import type { AgentRuntimeService } from "../agent-runtime/agent-runtime.types.js";
 import type { BrowserAuth } from "../auth/auth.types.js";
 import type { BundleService } from "../bundle/bundle.types.js";
 import type { FlowService } from "../flows/flows.types.js";
-import type { ModelCatalogService } from "../model-catalog/model-catalog.types.js";
 import type { NodeService } from "../nodes/nodes.types.js";
 import type { ToolService } from "../tools/tools.types.js";
 export interface IntelDeps {
@@ -15,8 +12,5 @@ export interface IntelDeps {
     flows: FlowService;
     tools: ToolService;
     bundle: BundleService;
-    agents: AgentRuntimeService;
-    agentCosts: AgentCostsService;
-    models: ModelCatalogService;
     auth?: BrowserAuth;
 }