@anchrd/intel-api 0.13.0 → 0.15.0

package/dist/adapters/db/db.js

@@ -5,7 +5,7 @@ const grantColumns = `id, node_id, principal_type, principal_id, verb, expires_a
   created_by, created_at`;
 const linkColumns = `link.id, link.source_node_id, link.target_node_id, link.relation,
   link.origin, link.label, link.created_by, link.created_at`;
-// The first row of each node, in the order they arrive. A chunked kind — a board, whose index holds
+// The first row of each node, in the order they arrive. A chunked kind — one whose index holds
 // one row per task (#285) — answers a query once per matching card; a citation names a node, so the
 // best of them is the one that gets to speak for it.
 function dedupedByNode(rows) {
@@ -146,66 +146,7 @@ export function createNodeRepository(deps) {
     JOIN allowed ON allowed.id = n.id
     WHERE n.archived_at IS NOT NULL
     ORDER BY n.archived_at DESC, lower(n.title), n.id`;
-    /**
-     * The agents this actor may see, over the whole tree or under one folder (#139).
-     *
-     * ⚠️ The same `allowed` set every other read joins against, not a second predicate written for
-     * agents. An agent is a node, and the moment its list answers from its own rule it can answer
-     * differently from `list` about the very same row.
-     *
-     * Three scopes, matching the flow list: no clause at all for the whole tree, `IS NULL` for the
-     * root level, and one folder by ID.
-     */
-    const visibleAgents = (clause, includeArchived) => `${visibleCte}
-    SELECT ${nodeColumns}
-    FROM nodes n
-    JOIN allowed ON allowed.id = n.id
-    WHERE n.kind = 'agent' ${includeArchived ? "" : "AND n.archived_at IS NULL"} ${clause}
-    ORDER BY lower(n.title), n.id`;
-    const agentScopeOf = (parentId) => parentId === undefined
-        ? { clause: "", bindings: [] }
-        : parentId === null
-            ? { clause: "AND n.parent_id IS NULL", bindings: [] }
-            : { clause: "AND n.parent_id = ?", bindings: [parentId] };
     return {
-        async listVisibleAgents(actor, input) {
-            const scope = agentScopeOf(input.parentId);
-            const result = await deps.db
-                .prepare(visibleAgents(scope.clause, input.includeArchived === true))
-                .bind(...readBindings(actor), ...scope.bindings)
-                .all();
-            return (result.results ?? []).map(mapNode);
-        },
-        /**
-         * The R2 keys of the current definitions of every agent in this folder's subtree that this
-         * actor may see — the input to the share warning (#139, ADR-0005 §2).
-         *
-         * ⚠️ Keys, never bodies and never titles. Whether a referenced node may be NAMED to whoever is
-         * sharing is the tree's question and is asked afterwards through `getVisible`, exactly as the
-         * flow side does it. Answering it here would be the second rule that #17 and #19 got wrong.
-         *
-         * ⚠️ Agents this actor may not see contribute nothing, not even a count. Their references would
-         * otherwise be attributed to a folder this actor administers, which discloses that the agent is
-         * there at all.
-         */
-        async listVisibleAgentDefinitionKeys(actor, folderId) {
-            const result = await deps.db
-                .prepare(`${visibleCte},
-          scope(id) AS (
-            SELECT id FROM nodes WHERE id = ?
-            UNION
-            SELECT child.id FROM nodes child JOIN scope ON child.parent_id = scope.id
-          )
-          SELECT version.content_key AS content_key
-          FROM nodes n
-          JOIN allowed ON allowed.id = n.id
-          JOIN node_versions version ON version.id = n.current_version_id
-          WHERE n.kind = 'agent' AND n.archived_at IS NULL AND n.id IN (SELECT id FROM scope)
-          ORDER BY n.id`)
-                .bind(...readBindings(actor), folderId)
-                .all();
-            return (result.results ?? []).map((row) => row.content_key);
-        },
         async listVisible(actor, input) {
             const result = await deps.db
                 .prepare(input.archivedOnly ? archivedEverywhere : visibleChildren(false))
@@ -289,20 +230,17 @@ export function createNodeRepository(deps) {
          * ⚠️ `nodes` arrive parents before children: `parent_id REFERENCES nodes(id)` is checked as
          * each row lands, and a child before its parent would abort the batch that is otherwise valid.
          * The service owns that order; this only writes what it is handed.
-         *
-         * `context_policy` is the dead column migration 0009 explains; 'relevant' is the fixed value
-         * `insertNode` writes for the same reason.
          */
         async importTree(input) {
             const statements = [
                 deps.db
                     .prepare(`INSERT INTO nodes (
-            id, parent_id, kind, title, description, context_policy, owner_id,
+            id, parent_id, kind, title, description, owner_id,
             current_version_id, created_at, updated_at, archived_at
           ) SELECT
             json_extract(value, '$.id'), json_extract(value, '$.parentId'),
             json_extract(value, '$.kind'), json_extract(value, '$.title'),
-            json_extract(value, '$.description'), 'relevant',
+            json_extract(value, '$.description'),
             json_extract(value, '$.ownerId'), json_extract(value, '$.currentVersionId'),
             json_extract(value, '$.createdAt'), json_extract(value, '$.updatedAt'), NULL
           FROM json_each(?)`)
@@ -380,7 +318,7 @@ export function createNodeRepository(deps) {
          *
          * ⚠️ The idempotency row points at the VERSION, and the audit event points at the node, so the
          * two are joined through the version id the snapshot writer puts into the metadata. That
-         * metadata is where a board operation says which task it touched and how many it removed —
+         * metadata is where a snapshot says which positions it touched and how many it removed —
          * facts a replay cannot recompute from the stored document, because the document is the state
          * AFTER the write. Without this, repeating a delete would answer with a count nobody measured.
          */
@@ -442,30 +380,15 @@ export function createNodeRepository(deps) {
                 return null;
             return row.resource_id.startsWith("1:");
         },
-        // ⚠️ Its own statement rather than a column on the node read: four of the five kinds can never
-        // have an Application, and the two joins every node read already carries are not going to grow
-        // a third for a question only agents ask (#182).
-        async agentApplicationId(nodeId) {
-            const row = await deps.db
-                .prepare("SELECT application_id FROM agent_applications WHERE node_id = ?")
-                .bind(nodeId)
-                .first();
-            return row?.application_id ?? null;
-        },
         async insertNode(input) {
             const node = input.node;
-            const application = input.application;
             try {
                 await deps.db.batch([
                     deps.db
-                        // ⚠️ `context_policy` is dead and is written anyway (#76). The column is NOT NULL
-                        // without a DEFAULT and D1 will not let it be dropped — migration 0009 carries the
-                        // reason. The fixed value is the price; nothing reads it, and the contract no longer
-                        // knows the field. When the column goes (anchrd/intel#86), this line goes with it.
                         .prepare(`INSERT INTO nodes (
-            id, parent_id, kind, title, description, context_policy, owner_id,
+            id, parent_id, kind, title, description, owner_id,
             current_version_id, created_at, updated_at, archived_at
-          ) VALUES (?, ?, ?, ?, ?, 'relevant', ?, ?, ?, ?, ?)`)
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
                         .bind(node.id, node.parentId, node.kind, node.title, node.description, node.ownerId, node.currentVersionId, node.createdAt, node.updatedAt, node.archivedAt),
                     deps.db
                         .prepare(`INSERT INTO idempotency_keys (
@@ -476,23 +399,7 @@ export function createNodeRepository(deps) {
                         .prepare(`INSERT INTO audit_events (
             id, actor_id, action, resource_type, resource_id, metadata_json, occurred_at
           ) VALUES (?, ?, 'node.create', 'node', ?, ?, ?)`)
-                        .bind(input.auditId, input.actorId, node.id, 
-                    // ⚠️ The Application ID is deliberately NOT in this metadata, and the key is not
-                    // within reach of it at all. An audit event is read back by people and by the
-                    // reindexer; what belongs in it is that an agent was created, not the name of the
-                    // principal behind it — the mapping row below is where that is kept, once (D27).
-                    JSON.stringify({ kind: node.kind, parentId: node.parentId }), node.createdAt),
-                    // Last in the batch, and in the SAME batch: D1 runs a batch as one transaction, so the
-                    // mapping cannot end up without its node or the node without its mapping. An agent whose
-                    // principal is recorded nowhere is one that archiving would quietly fail to switch off.
-                    ...(application
-                        ? [
-                            deps.db
-                                .prepare(`INSERT INTO agent_applications (node_id, application_id, created_at)
-                     VALUES (?, ?, ?)`)
-                                .bind(node.id, application.id, node.createdAt),
-                        ]
-                        : []),
+                        .bind(input.auditId, input.actorId, node.id, JSON.stringify({ kind: node.kind, parentId: node.parentId }), node.createdAt),
                 ]);
             }
             catch (error) {
@@ -839,7 +746,7 @@ export function createNodeRepository(deps) {
                     .bind(version.id, version.nodeId, version.sequence, version.contentKey, version.mediaType, version.contentHash, version.size, 
                 // ⚠️ Taken from the version rather than written in as `'snapshot'`. A table's mutation
                 // marks a snapshot, because reading a table starts at the newest one and everything
-                // before it is history (#135). A board's version carries the whole board by
+                // before it is history (#135). A snapshot version carries the whole document by
                 // construction, so it marks nothing — the same `null` a document's version carries, and
                 // for the same reason (#285).
                 version.segment, version.createdBy, version.createdAt, version.nodeId, input.baseVersionId),
@@ -875,7 +782,7 @@ export function createNodeRepository(deps) {
         },
         async listGrants(resourceId) {
             const result = await deps.db
-                .prepare(`SELECT ${grantColumns} FROM tree_grants
+                .prepare(`SELECT ${grantColumns} FROM node_grants
           WHERE node_id = ?
           ORDER BY principal_type, principal_id, verb`)
                 .bind(resourceId)
@@ -894,7 +801,7 @@ export function createNodeRepository(deps) {
             JOIN ancestors child ON child.parent_id = parent.id
           )
           SELECT 1 AS reaches
-          FROM tree_grants grant_row
+          FROM node_grants grant_row
           JOIN ancestors ON ancestors.id = grant_row.node_id
           WHERE grant_row.id <> ?
             AND grant_row.principal_type = 'organization'
@@ -973,17 +880,6 @@ export function createNodeRepository(deps) {
                     .bind(input.auditId, input.actorId, input.sourceNodeId, JSON.stringify({ targetNodeIds: input.links.map((link) => link.targetNodeId) }), input.occurredAt),
             ]);
         },
-        async recordAgentKeyRotation(input) {
-            // ⚠️ No node row moves, so there is nothing to batch this with — the event IS the write. The
-            // key is not among the columns and must not be added to them: audit is metadata, and a row
-            // that carried the credential would outlive every other copy of it.
-            await deps.db
-                .prepare(`INSERT INTO audit_events (
-          id, actor_id, action, resource_type, resource_id, metadata_json, occurred_at
-        ) VALUES (?, ?, 'agent.rotate_key', 'node', ?, ?, ?)`)
-                .bind(input.auditId, input.actorId, input.nodeId, JSON.stringify({ applicationId: input.applicationId }), input.occurredAt)
-                .run();
-        },
         async graphVisible(actor, input) {
             const nodeResult = await deps.db
                 .prepare(`${visibleCte}
@@ -1028,7 +924,7 @@ export function createNodeRepository(deps) {
                     deps.db
                         // The verb is part of the key, so re-granting the same verb only refreshes its expiry
                         // and never turns one verb into another.
-                        .prepare(`INSERT INTO tree_grants (
+                        .prepare(`INSERT INTO node_grants (
               id, node_id, principal_type, principal_id, verb,
               expires_at, created_by, created_at
             ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
@@ -1038,7 +934,7 @@ export function createNodeRepository(deps) {
                     deps.db
                         .prepare(`INSERT INTO idempotency_keys (
               actor_id, operation, idempotency_key, resource_id, created_at
-            ) SELECT ?, 'node.share', ?, id, ? FROM tree_grants
+            ) SELECT ?, 'node.share', ?, id, ? FROM node_grants
               WHERE node_id = ? AND principal_type = ? AND principal_id = ? AND verb = ?`)
                         .bind(input.actorId, input.idempotencyKey, grant.createdAt, grant.resourceId, principalType, principalId, grant.verb),
                     deps.db
@@ -1048,7 +944,7 @@ export function createNodeRepository(deps) {
                         .bind(input.auditId, input.actorId, grant.resourceId, JSON.stringify({ principalType, verb: grant.verb }), grant.createdAt),
                 ]);
                 const stored = await deps.db
-                    .prepare(`SELECT ${grantColumns} FROM tree_grants
+                    .prepare(`SELECT ${grantColumns} FROM node_grants
             WHERE node_id = ? AND principal_type = ? AND principal_id = ? AND verb = ?`)
                     .bind(grant.resourceId, principalType, principalId, grant.verb)
                     .first();
@@ -1064,7 +960,7 @@ export function createNodeRepository(deps) {
                     .first();
                 if (replayed) {
                     const row = await deps.db
-                        .prepare(`SELECT ${grantColumns} FROM tree_grants WHERE id = ?`)
+                        .prepare(`SELECT ${grantColumns} FROM node_grants WHERE id = ?`)
                         .bind(replayed.resource_id)
                         .first();
                     if (row)
@@ -1081,11 +977,11 @@ export function createNodeRepository(deps) {
               actor_id, operation, idempotency_key, resource_id, created_at
             ) SELECT ?, 'node.revoke', ?,
               (CASE WHEN EXISTS (
-                SELECT 1 FROM tree_grants WHERE id = ? AND node_id = ?
+                SELECT 1 FROM node_grants WHERE id = ? AND node_id = ?
               ) THEN '1:' ELSE '0:' END) || ?, ?`)
                         .bind(input.actorId, input.idempotencyKey, input.grantId, input.resourceId, input.grantId, input.occurredAt),
                     deps.db
-                        .prepare("DELETE FROM tree_grants WHERE id = ? AND node_id = ?")
+                        .prepare("DELETE FROM node_grants WHERE id = ? AND node_id = ?")
                         .bind(input.grantId, input.resourceId),
                     deps.db
                         .prepare(`INSERT INTO audit_events (
@@ -1128,10 +1024,10 @@ export function createNodeRepository(deps) {
           LIMIT ?`)
                 .bind(...readBindings(actor), ...scopeBindings(input.scopeId), query, input.limit)
                 .all();
-            // ⚠️ One citation per node, and the FIRST is the one kept. A board holds one index row per
-            // task (#285), so a query matching three cards of the same board arrives here three times;
+            // ⚠️ One citation per node, and the FIRST is the one kept. The index is CHUNKED — a node
+            // may hold several rows (#285) — so a query matching three of them arrives here three times;
             // the rows come back in rank order, so the survivor is the best-matching card and its own
-            // passage is what the reader is shown. Without this, one busy board could fill a whole page
+            // passage is what the reader is shown. Without this, one busy node could fill a whole page
             // of results and push every other node off it.
             return dedupedByNode(result.results ?? []).map((row) => ({
                 nodeId: row.node_id,
@@ -1144,32 +1040,46 @@ export function createNodeRepository(deps) {
                 match: "lexical",
             }));
         },
-        async hydrateVisibleCitations(actor, nodeIds, scopeId) {
-            const uniqueIds = [...new Set(nodeIds)].slice(0, 100);
-            if (uniqueIds.length === 0)
+        async hydrateVisibleCitations(actor, hits, scopeId) {
+            // One entry per node: the caller has already picked the best-scoring chunk of each, so a
+            // node that answered with four passages arrives as the one passage that answered best.
+            const unique = [...new Map(hits.map((hit) => [hit.nodeId, hit])).values()].slice(0, 100);
+            if (unique.length === 0)
                 return [];
             const rows = [];
-            for (let offset = 0; offset < uniqueIds.length; offset += 96) {
-                const ids = uniqueIds.slice(offset, offset + 96);
-                const placeholders = ids.map(() => "?").join(", ");
+            // Two bindings per hit now instead of one, so the batch is halved and then some: the actor's
+            // predicate and the optional scope are bound in front of them, and D1 counts every one of
+            // them against the same per-statement ceiling.
+            for (let offset = 0; offset < unique.length; offset += 40) {
+                const batch = unique.slice(offset, offset + 40);
+                const pairs = batch.map(() => "(?, ?)").join(", ");
                 const result = await deps.db
-                    .prepare(`${visibleCte}${scopeCte(scopeId)}
+                    .prepare(`${visibleCte}${scopeCte(scopeId)},
+            hits(node_id, chunk_key) AS (VALUES ${pairs})
             SELECT n.id AS node_id, v.id AS version_id, n.title,
-              substr(node_fts.content, 1, 480) AS passage,
+              -- ⚠️ The card that answered, and only otherwise the first passage of the node
+              -- (anchrd/intel#301). Every vector written before #301 is named by the bare node id
+              -- and has no row here, so the fallback is not defensive coding but the exact
+              -- behaviour those vectors had — a board found through one of them still answers,
+              -- with the passage it always answered with, until the next indexing pass.
+              COALESCE(vector.passage, substr(node_fts.content, 1, 480)) AS passage,
               n.updated_at AS freshness
-            FROM nodes n
+            FROM hits
+            JOIN nodes n ON n.id = hits.node_id
             JOIN node_versions v ON v.id = n.current_version_id
             JOIN node_fts ON node_fts.version_id = v.id
+            LEFT JOIN node_vectors vector
+              ON vector.node_id = hits.node_id AND vector.chunk_key = hits.chunk_key
             JOIN allowed ON allowed.id = n.id${scopeJoin(scopeId)}
-            WHERE n.id IN (${placeholders}) AND n.archived_at IS NULL`)
-                    .bind(...readBindings(actor), ...scopeBindings(scopeId), ...ids)
+            WHERE n.archived_at IS NULL`)
+                    .bind(...readBindings(actor), ...scopeBindings(scopeId), ...batch.flatMap((hit) => [hit.nodeId, hit.chunkKey]))
                     .all();
                 rows.push(...(result.results ?? []));
             }
-            const order = new Map(uniqueIds.map((id, index) => [id, index]));
-            // The same one-per-node rule as the lexical half above, for the same reason: a semantic hit
-            // is a hit on a NODE (the vector index is keyed by node id), so hydrating it must not turn one
-            // hit into one row per task.
+            const order = new Map(unique.map((hit, index) => [hit.nodeId, index]));
+            // The same one-per-node rule as the lexical half above, for the same reason: a citation names
+            // a NODE, and a board holds one full-text row per task (#285), so the join above still
+            // produces one row per card even though only one of them was asked about.
             const deduped = dedupedByNode(rows);
             deduped.sort((left, right) => (order.get(left.node_id) ?? 0) - (order.get(right.node_id) ?? 0));
             return deduped.map((row) => ({
@@ -1183,10 +1093,42 @@ export function createNodeRepository(deps) {
                 match: "semantic",
             }));
         },
+        async invalidateVectors() {
+            /**
+             * ⚠️ The fingerprints are cleared and the ROWS are kept, which is not a detail. A record row
+             * says two things at once: "this chunk was embedded from this text" and "this vector exists".
+             * Only the first is stale after a rebuild is asked for; deleting the row would throw the
+             * second away, and then a card removed between this call and that board's pass would leave a
+             * vector nothing could ever name again — matching questions and answering with a card that is
+             * not on the board.
+             *
+             * The empty string is a value no digest produces, so every chunk compares as changed and the
+             * next pass embeds all of them. `node_fts` is deliberately not touched: its rows are
+             * overwritten by the pass that rewrites them, and emptying it would take the lexical half of
+             * the installation offline for as long as the queue needs to work through the tree.
+             */
+            await deps.db.prepare("UPDATE node_vectors SET fingerprint = ''").run();
+        },
+        /**
+         * ⚠️ Archived nodes are IN this walk since anchrd/intel#348, and the omission of
+         * `archived_at IS NULL` is the point. The pass an archived node gets is a purge, not an index —
+         * `getTarget` refuses it and `archivedNodeId` says why — so including them is what makes
+         * `reindex` the way back for a vector that is in the index with nothing left to name it.
+         *
+         * Three of those exist and none of them has another repair: everything archived before #348,
+         * which nothing ever swept; a node whose `archive` wrote its row and then failed to reach the
+         * queue; and a pass that upserted vectors in the moment somebody archived the node underneath
+         * it. Without this line an administrator's rebuild walked straight past all three, and the only
+         * remaining answer would have been to delete the whole Vectorize index by hand.
+         *
+         * It does mean `queued` counts them. That is still what the number says — versions handed to the
+         * queue — and a rebuild that silently skipped part of the tree is the failure `reindex` exists
+         * to prevent.
+         */
         async listCurrentVersionIds(input) {
             const result = await deps.db
                 .prepare(`SELECT current_version_id AS id FROM nodes
-          WHERE current_version_id IS NOT NULL AND archived_at IS NULL
+          WHERE current_version_id IS NOT NULL
             AND (? IS NULL OR current_version_id > ?)
           ORDER BY id LIMIT ?`)
                 .bind(input.after, input.after, input.limit)

package/dist/adapters/semantic-index/semantic-index.js

@@ -1,15 +1,48 @@
 const defaultModel = "@cf/baai/bge-m3";
 const maxEmbeddingCharacters = 24_000;
-function embedding(result) {
+// How much of one embedding request this adapter is willing to be: at most this many texts, and at
+// most this many characters across them. Both bounds are ours rather than a number Cloudflare
+// publishes — what the model documents is the shape of `text`, not a batch size — so they are set
+// well inside anything plausible. The point is that a board with three hundred cards costs a
+// handful of requests on its first pass instead of three hundred round trips.
+const maxBatchTexts = 25;
+const maxBatchCharacters = 96_000;
+// Vectorize takes at most 1000 vectors in one call from a Worker (platform limits). A board may
+// hold 5000 tasks, so the write is cut rather than sent whole and refused.
+const maxVectorsPerCall = 1000;
+/**
+ * How a chunk is named in the vector index (anchrd/intel#301).
+ *
+ * ⚠️ `""` keeps the bare node id, which is what every vector written before this was called. That
+ * is the whole reason the separator sits on the SUFFIX side: nothing an installation already holds
+ * changes its name, so this ships without re-embedding a tree.
+ *
+ * ⚠️ Reading it back splits at the FIRST separator, and that direction matters. A node id is minted
+ * here (`ulid`) and cannot contain a `#`; a task id can, because a bundle import carries the task
+ * ids written in the file. Splitting at the first `#` therefore always recovers the node id exactly
+ * and leaves the rest to the key — and even a key that never matched a card costs a hit that
+ * hydrates to nothing, never a citation for a node the searcher did not match.
+ */
+const chunkSeparator = "#";
+function vectorId(nodeId, key) {
+    return key === "" ? nodeId : `${nodeId}${chunkSeparator}${key}`;
+}
+function embeddings(result, expected) {
     if (typeof result !== "object" ||
         result === null ||
         !("data" in result) ||
         !Array.isArray(result.data) ||
-        !Array.isArray(result.data[0]) ||
-        !result.data[0].every((value) => typeof value === "number")) {
+        // ⚠️ The count is part of the check. The rows come back in the order the texts went out and
+        // nothing else identifies them, so a short answer would silently pair every embedding after the
+        // gap with the wrong card — a search that answers confidently with the neighbouring task.
+        result.data.length !== expected ||
+        // ⚠️ And every row has to hold something. An empty one passes "is an array of numbers" and would
+        // be upserted under a real card's name as a vector that matches nothing — indexed, recorded as
+        // fingerprinted, and never looked at again until somebody edits that card.
+        !result.data.every((row) => Array.isArray(row) && row.length > 0 && row.every((value) => typeof value === "number"))) {
         throw new Error("The embedding provider returned an invalid response");
     }
-    return result.data[0];
+    return result.data;
 }
 function matches(result) {
     if (typeof result !== "object" ||
@@ -27,27 +60,74 @@ function matches(result) {
             typeof match.score !== "number") {
             return [];
         }
-        return [{ nodeId: match.id, score: Math.max(0, Math.min(1, match.score)) }];
+        const separator = match.id.indexOf(chunkSeparator);
+        return [
+            {
+                nodeId: separator < 0 ? match.id : match.id.slice(0, separator),
+                chunkKey: separator < 0 ? "" : match.id.slice(separator + 1),
+                score: Math.max(0, Math.min(1, match.score)),
+            },
+        ];
     });
 }
+function batched(chunks) {
+    const batches = [];
+    let current = [];
+    let characters = 0;
+    for (const chunk of chunks) {
+        if (current.length > 0 &&
+            (current.length >= maxBatchTexts || characters >= maxBatchCharacters)) {
+            batches.push(current);
+            current = [];
+            characters = 0;
+        }
+        current.push(chunk);
+        characters += chunk.text.length;
+    }
+    if (current.length > 0)
+        batches.push(current);
+    return batches;
+}
 export function createSemanticIndex(deps) {
     const model = deps.model ?? defaultModel;
-    async function embed(text) {
-        return embedding(await deps.ai.run(model, { text: [text] }));
+    async function embed(texts) {
+        return embeddings(await deps.ai.run(model, { text: texts }), texts.length);
     }
     return {
-        async replace(target, content) {
-            const text = `${target.title}\n\n${content}`.slice(0, maxEmbeddingCharacters);
-            await deps.index.upsert([
-                {
-                    id: target.nodeId,
-                    values: await embed(text),
-                    metadata: { versionId: target.versionId },
-                },
-            ]);
+        async upsert(target, chunks) {
+            const vectors = [];
+            for (const batch of batched(chunks)) {
+                const values = await embed(batch.map((chunk) => chunk.text.slice(0, maxEmbeddingCharacters)));
+                batch.forEach((chunk, index) => {
+                    const embedded = values[index];
+                    // The index is what makes this reachable at all — `embeddings` has already refused a short
+                    // answer and an empty row, so this is the type system's question rather than the
+                    // provider's. It throws anyway rather than skipping: the alternative to a missing card is
+                    // never a shorter list, it is a card recorded as fingerprinted with no vector behind it.
+                    if (!embedded)
+                        throw new Error("The embedding provider returned an invalid response");
+                    vectors.push({
+                        id: vectorId(target.nodeId, chunk.key),
+                        values: embedded,
+                        metadata: { versionId: target.versionId },
+                    });
+                });
+            }
+            for (let offset = 0; offset < vectors.length; offset += maxVectorsPerCall) {
+                await deps.index.upsert(vectors.slice(offset, offset + maxVectorsPerCall));
+            }
+        },
+        async remove(nodeId, keys) {
+            const ids = keys.map((key) => vectorId(nodeId, key));
+            for (let offset = 0; offset < ids.length; offset += maxVectorsPerCall) {
+                await deps.index.deleteByIds(ids.slice(offset, offset + maxVectorsPerCall));
+            }
         },
         async search(query, limit) {
-            const result = await deps.index.query(await embed(query), {
+            const [vector] = await embed([query]);
+            if (!vector)
+                throw new Error("The embedding provider returned an invalid response");
+            const result = await deps.index.query(vector, {
                 topK: Math.max(1, Math.min(100, limit)),
                 returnMetadata: "none",
             });

package/dist/adapters/semantic-index/semantic-index.types.d.ts

@@ -12,17 +12,36 @@ export interface VectorizeBinding {
             versionId: string;
         };
     }>): Promise<unknown>;
+    deleteByIds(ids: string[]): Promise<unknown>;
     query(vector: number[], options: {
         topK: number;
         returnMetadata: "none";
     }): Promise<unknown>;
 }
+/**
+ * One vector's worth of a node (anchrd/intel#301).
+ *
+ * ⚠️ `key` is `""` for a node that carries exactly one vector — every kind but `board` — and the
+ * vector is then named by the bare node id it has always been named by. A board card carries its
+ * task id, and its vector is `<node id>#<task id>`.
+ *
+ * `text` is embedded verbatim. Composing it — the node's title in front of a document's body, the
+ * card's own title in front of a card — belongs to the indexing pass rather than here, because the
+ * same string has to be fingerprinted there to decide whether this vector needs making at all. A
+ * title prefixed on this side would sit outside that fingerprint and change nothing when it changed.
+ */
+export interface SemanticChunk {
+    key: string;
+    text: string;
+}
 export interface SemanticHit {
     nodeId: string;
+    chunkKey: string;
     score: number;
 }
 export interface SemanticIndex {
-    replace(target: NodeIndexTarget, content: string): Promise<void>;
+    upsert(target: NodeIndexTarget, chunks: SemanticChunk[]): Promise<void>;
+    remove(nodeId: string, keys: string[]): Promise<void>;
     search(query: string, limit: number): Promise<SemanticHit[]>;
 }
 export interface SemanticIndexDeps {