@assuredloop/cli 0.2.0 → 0.2.1

package/README.md

@@ -28,13 +28,13 @@ It needs Node 24 or newer and git 2.31 or newer. Install the npm package
 
     npm install --global @assuredloop/cli
 
-The recommended install adds the optional package `@assuredloop/search`
-beside it, for search at level 2 (see "Search"):
+The recommended install adds the embedding model's library beside it, for
+search at level 2 (see "Search"). `al` names the library as an optional
+dependency, so npm installs it only when you ask:
 
-    npm install --global @assuredloop/cli @assuredloop/search
+    npm install --global @assuredloop/cli @huggingface/transformers@4.3.1
 
 To pin a version, name it: `npm install --global @assuredloop/cli@<version>`.
-Give both packages the same version.
 
 The package carries the skill, at
 `$(npm root -g)/@assuredloop/cli/skills/assuredloop/SKILL.md`. Add one line to
@@ -49,7 +49,7 @@ the skill at the pinned version into the project, for example as
 `.claude/skills/assuredloop/SKILL.md`. Name that path in the AGENTS.md line,
 and say in the commit which version it came from.
 
-Search at level 2 needs the optional package `@assuredloop/search` (see
+Search at level 2 needs the library `@huggingface/transformers` 4.3.1 (see
 "Search"). Install it in the project, or beside `al`.
 
 ## The model in one page
@@ -176,7 +176,7 @@ itself (2, then 1, then 0), and says which level answered and why.
 |---|---|---|
 | 0 | a scan of the export | nothing |
 | 1 | a local full-text index (SQLite FTS5, BM25), with the ID in its own exact column | Node 24 or newer (`node:sqlite`) |
-| 2 | level 1, plus a small local embedding model; word and vector results are fused | the package `@assuredloop/search` |
+| 2 | level 1, plus a small local embedding model; word and vector results are fused | the library `@huggingface/transformers` 4.3.1, in the project or beside `al` |
 
 - The default query is the current system: the spec text at the selected
   commit. `--change <name>` adds that request's open change spec, its
@@ -199,7 +199,9 @@ itself (2, then 1, then 0), and says which level answered and why.
   brings it up to date before it answers; `--rebuild` builds it again.
 - Level 2 runs a small local model (bge-small-en-v1.5, pinned by its
   revision). The model download is the only data that leaves the machine.
-  `al` itself makes no network call.
+  `al` itself makes no network call. Without the library, `al search`
+  answers at level 1, and a `Not known` line says how to add level 2. Only
+  `al search` loads the library.
 
 ## One central repo, and output repos
 
@@ -299,15 +301,13 @@ in this repository.
 
 ## Releasing
 
-1. Bump `version` in `package.json` and in `packages/search/package.json` to
-   the same version in a PR, and merge it.
+1. Bump `version` in `package.json` in a PR, and merge it.
 2. Publish a GitHub Release tagged `v<version>` on that commit. Mark it a
    pre-release to publish under npm's `next` tag.
 
 `.github/workflows/publish.yml` does the rest: it runs the tests and
-`al check --strict`, checks that the tag matches both `package.json` files,
-and publishes @assuredloop/cli, then @assuredloop/search, to npm. It skips a
-package whose version is already on npm, so a failed run can run again. It
-stores no npm token: npm trusts the workflow by name, and adds a provenance
-statement saying which commit built the package. npm must trust the workflow
-for each package before its first publish.
+`al check --strict`, checks that the tag matches `package.json`, and
+publishes @assuredloop/cli to npm. It skips a version that is already on npm,
+so a failed run can run again. It stores no npm token: npm trusts the
+workflow by name, and adds a provenance statement saying which commit built
+the package.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@assuredloop/cli",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "AssuredLoop keeps an AI focused on one change: the al command gathers and checks a change's requirement, spec, decisions, code and tests.",
   "license": "MIT",
   "repository": {
@@ -33,5 +33,13 @@
   },
   "dependencies": {
     "yaml": "2.9.1"
+  },
+  "peerDependencies": {
+    "@huggingface/transformers": "4.3.1"
+  },
+  "peerDependenciesMeta": {
+    "@huggingface/transformers": {
+      "optional": true
+    }
   }
 }

package/src/v4/level2.js

@@ -0,0 +1,154 @@
+// Level 2 of `al search` (design.md 11): the embedding model and the vectors.
+// `al search` gives it the rows to embed and a query, and it answers with the
+// row keys ranked by similarity; search.js fuses that list with its full-text
+// list.
+//
+// The vectors live beside al's index, in <dir>/vectors.sqlite. Only new or
+// changed rows are embedded; a change of model embeds everything again.
+// The model's library, @huggingface/transformers, is an optional peer
+// dependency: it is found by normal Node resolution (the project first, then
+// beside al) and imported only at the first embed, so no other command and no
+// search without words loads it.
+import { mkdirSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+const BATCH = 32;
+
+// node:sqlite is imported here, not at load, so a Node without it still
+// loads this file and falls back by search.js's own check.
+async function open(dir) {
+  const { DatabaseSync } = await import('node:sqlite');
+  mkdirSync(dir, { recursive: true });
+  const db = new DatabaseSync(join(dir, 'vectors.sqlite'));
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT);
+    CREATE TABLE IF NOT EXISTS vectors (key TEXT PRIMARY KEY, sha256 TEXT NOT NULL, vec BLOB NOT NULL);
+  `);
+  return db;
+}
+
+// A vector as unit length, so a dot product is the cosine.
+function unit(v) {
+  const a = Float32Array.from(v);
+  let n = 0;
+  for (const x of a) n += x * x;
+  n = Math.sqrt(n) || 1;
+  for (let i = 0; i < a.length; i++) a[i] /= n;
+  return a;
+}
+const blob = (a) => Buffer.from(a.buffer, a.byteOffset, a.byteLength);
+const vector = (b) => new Float32Array(b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength));
+
+async function embedAll(embed, texts) {
+  const out = [];
+  for (let i = 0; i < texts.length; i += BATCH) {
+    const part = texts.slice(i, i + BATCH);
+    const vs = await embed(part);
+    if (!Array.isArray(vs) || vs.length !== part.length) throw new Error(`the model gave ${vs?.length ?? 'no'} vectors for ${part.length} texts`);
+    out.push(...vs.map(unit));
+  }
+  if (out.some((v) => v.length !== out[0].length)) throw new Error('the model gave vectors of different lengths');
+  return out;
+}
+
+// A level-2 object: `model` names it in al's output; `embed(texts)` gives one
+// vector per text. `queryPrefix` is put before a query only.
+function createLevel2({ model, embed, queryPrefix = '' }) {
+  if (typeof model !== 'string' || !model) throw new Error('createLevel2 needs a model name');
+  if (typeof embed !== 'function') throw new Error('createLevel2 needs an embed function');
+  return {
+    model,
+
+    // Makes the stored vectors match `rows` ([{ key, sha256, text }]):
+    // embeds the new and changed rows and removes the others.
+    async refresh({ dir, rows }) {
+      const db = await open(dir);
+      try {
+        if (db.prepare("SELECT v FROM meta WHERE k = 'model'").get()?.v !== model) {
+          db.exec('DELETE FROM vectors');
+          db.prepare("INSERT OR REPLACE INTO meta (k, v) VALUES ('model', ?)").run(model);
+        }
+        const have = new Map(db.prepare('SELECT key, sha256 FROM vectors').all().map((x) => [x.key, x.sha256]));
+        const want = new Map(rows.map((r) => [r.key, r]));
+        const todo = rows.filter((r) => have.get(r.key) !== r.sha256);
+        const vs = await embedAll(embed, todo.map((r) => r.text));
+        db.exec('BEGIN');
+        const del = db.prepare('DELETE FROM vectors WHERE key = ?');
+        let removed = 0;
+        for (const k of have.keys()) if (!want.has(k)) { del.run(k); removed += 1; }
+        const put = db.prepare('INSERT OR REPLACE INTO vectors (key, sha256, vec) VALUES (?, ?, ?)');
+        todo.forEach((r, i) => put.run(r.key, r.sha256, blob(vs[i])));
+        db.exec('COMMIT');
+        return { embedded: todo.length, removed };
+      } finally {
+        db.close();
+      }
+    },
+
+    // The `keys` ranked by similarity to `query`, best first, at most `k`.
+    // A plain scan of the stored vectors: no vector database is needed at
+    // this size (design.md 11).
+    async rank({ dir, query, keys, k }) {
+      const [q] = await embedAll(embed, [`${queryPrefix}${query}`]);
+      const db = await open(dir);
+      try {
+        const wanted = new Set(keys);
+        const scored = [];
+        for (const { key, vec } of db.prepare('SELECT key, vec FROM vectors').all()) {
+          if (!wanted.has(key)) continue;
+          const v = vector(vec);
+          if (v.length !== q.length) throw new Error('the stored vectors and the query differ in length');
+          let s = 0;
+          for (let i = 0; i < v.length; i++) s += v[i] * q[i];
+          scored.push([key, s]);
+        }
+        scored.sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+        return scored.slice(0, k).map(([key]) => key);
+      } finally {
+        db.close();
+      }
+    },
+  };
+}
+
+export const LIBRARY = '@huggingface/transformers';
+const VERSION = '4.3.1';
+export const INSTALL = `npm install --global @assuredloop/cli ${LIBRARY}@${VERSION}`;
+
+// The library's path by normal Node resolution: from the project, then
+// beside al. Null when neither has it.
+export function findLibrary(top) {
+  for (const from of [join(top, 'package.json'), import.meta.url]) {
+    try { return createRequire(from).resolve(LIBRARY); } catch { /* not here */ }
+  }
+  return null;
+}
+
+// The real model: bge-small-en-v1.5 (MIT), the ONNX port, pinned by revision.
+const MODEL = 'Xenova/bge-small-en-v1.5';
+const REVISION = 'ea104dacec62c0de699686887e3f920caeb4f3e3';
+
+// Level 2 with the library at `path`, which is imported at the first embed.
+export function level2(path) {
+  let extractor = null;
+  async function embed(texts) {
+    if (!extractor) {
+      // The path is the library's require entry; its exports may come as a default.
+      const mod = await import(pathToFileURL(path).href);
+      const pipeline = mod.pipeline ?? mod.default?.pipeline;
+      if (typeof pipeline !== 'function') throw new Error(`${path} has no pipeline function`);
+      extractor = await pipeline('feature-extraction', MODEL, { revision: REVISION, dtype: 'q8' });
+    }
+    // bge takes the first token's vector (CLS pooling).
+    const out = await extractor(texts, { pooling: 'cls', normalize: true });
+    return out.tolist();
+  }
+  return createLevel2({
+    model: `${MODEL}@${REVISION.slice(0, 12)}`,
+    embed,
+    // bge's instruction for a short query that looks for passages.
+    queryPrefix: 'Represent this sentence for searching relevant passages: ',
+  });
+}

package/src/v4/search.js

@@ -1,13 +1,10 @@
 // al search (design.md 11): the current system, a change's context and the
 // history, at the strongest level installed. Level 0 scans the export; level
 // 1 is a SQLite FTS5 index of it (BM25) with the ID in its own exact column;
-// level 2 fuses level 1 with the ranking of the optional package
-// @assuredloop/search (reciprocal rank fusion). `al` holds no model and no
-// vectors: the package does, and returns a ranked list.
+// level 2 fuses level 1 with the ranking of level2.js (reciprocal rank
+// fusion), which needs the optional peer dependency @huggingface/transformers.
 import { mkdirSync, rmSync } from 'node:fs';
-import { createRequire } from 'node:module';
 import { isAbsolute, join } from 'node:path';
-import { pathToFileURL } from 'node:url';
 import { parse } from 'yaml';
 import { Fail, isName } from './base.js';
 import { git } from './git.js';
@@ -39,22 +36,13 @@ async function sqlite() {
   }
 }
 
-// The optional package by normal Node resolution: from the project, then
-// beside al. Null with the reason when neither has it.
+// Level 2 when its library is installed, or null with the reason. level2.js
+// is loaded only here, and the library only at its first embed.
 async function level2(top) {
-  for (const from of [join(top, 'package.json'), import.meta.url]) {
-    let path;
-    try { path = createRequire(from).resolve('@assuredloop/search'); } catch { continue; }
-    try {
-      const mod = await import(pathToFileURL(path).href);
-      const l2 = mod.default;
-      if (!l2 || typeof l2.refresh !== 'function' || typeof l2.rank !== 'function') return { reason: `${path} is not a level 2 package (no refresh and rank)` };
-      return { l2 };
-    } catch (e) {
-      return { reason: `@assuredloop/search did not load: ${e.message.split('\n')[0]}` };
-    }
-  }
-  return { reason: 'level 2 is not installed: no @assuredloop/search in the project or beside al' };
+  const { findLibrary, level2: create, LIBRARY, INSTALL } = await import('./level2.js');
+  const path = findLibrary(top);
+  if (!path) return { reason: `level 2 is not installed: no ${LIBRARY} in the project or beside al`, install: `level 2: ${LIBRARY} is not installed; to add level 2, run ${INSTALL}` };
+  return { l2: create(path) };
 }
 
 // The folder of the index: <git dir>/assuredloop, never in the working tree.
@@ -352,7 +340,7 @@ export async function search({ top, args, opts }) {
   let lite = null;
   if (level === 2) {
     found2 = await level2(top);
-    if (!found2.l2) { falls.push(found2.reason); level = 1; }
+    if (!found2.l2) { falls.push(found2.reason); notKnown.push(found2.install); level = 1; }
   }
   if (level >= 1) {
     lite = await sqlite();