@aigne/afs-index 1.12.0-beta.5 → 1.12.0-beta.7

package/dist/embed.cjs

@@ -2,6 +2,16 @@ let ufo = require("ufo");
 
 //#region src/embed.ts
 const DEFAULT_EMBED_TIMEOUT_MS = 2e3;
+/**
+* Minimum meaningful character count for a PASSAGE (index-time) embedding.
+* Empty/whitespace text is always skipped; below this the text carries no
+* semantic signal and would only produce a near-mean "hub" vector that sits at
+* moderate cosine (~0.5) to EVERY query — an indexed empty/near-empty summary
+* then surfaces for every semantic search (issue #1396). Kept low (2) so real
+* short titles like "AI" / "Go" still embed; only 0–1 char content-free records
+* are dropped.
+*/
+const MIN_PASSAGE_EMBED_CHARS = 2;
 function embeddingTimeoutMs() {
 	const raw = (globalThis.process?.env)?.AFS_INDEX_EMBED_TIMEOUT_MS;
 	const parsed = raw ? Number(raw) : DEFAULT_EMBED_TIMEOUT_MS;
@@ -25,6 +35,9 @@ function createEmbedder(opts) {
 	return async (text, type = "passage") => {
 		const { embeddingEnabled, llmProvider, embedRoot } = opts;
 		if (!text || !embeddingEnabled || !llmProvider || !embedRoot?.exec) return null;
+		const trimmed = text.trim();
+		if (!trimmed) return null;
+		if (type === "passage" && trimmed.length < MIN_PASSAGE_EMBED_CHARS) return null;
 		const safeText = text.length > 6e3 ? text.slice(0, 6e3) : text;
 		try {
 			const result = await withTimeout(embedRoot.exec((0, ufo.joinURL)(llmProvider, ".actions", "embed"), {

package/dist/embed.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"embed.d.cts","names":[],"sources":["../src/embed.ts"],"mappings":";;;UAYiB,eAAA;EAMI;EAJnB,gBAAA;EAOU;EALV,WAAA;;EAEA,SAAA,GAAY,OAAA;AAAA;AAAA,KAGF,QAAA,IAAY,IAAA,WAAe,IAAA,2BAA+B,OAAA;;;;AA8BtE;iBAAgB,cAAA,CAAe,IAAA,EAAM,eAAA,GAAkB,QAAA"}
+{"version":3,"file":"embed.d.cts","names":[],"sources":["../src/embed.ts"],"mappings":";;;UAYiB,eAAA;EAMI;EAJnB,gBAAA;EAOU;EALV,WAAA;;EAEA,SAAA,GAAY,OAAA;AAAA;AAAA,KAGF,QAAA,IAAY,IAAA,WAAe,IAAA,2BAA+B,OAAA;;;;AAyCtE;iBAAgB,cAAA,CAAe,IAAA,EAAM,eAAA,GAAkB,QAAA"}

package/dist/embed.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"embed.d.mts","names":[],"sources":["../src/embed.ts"],"mappings":";;;UAYiB,eAAA;EAMI;EAJnB,gBAAA;EAOU;EALV,WAAA;;EAEA,SAAA,GAAY,OAAA;AAAA;AAAA,KAGF,QAAA,IAAY,IAAA,WAAe,IAAA,2BAA+B,OAAA;;;;AA8BtE;iBAAgB,cAAA,CAAe,IAAA,EAAM,eAAA,GAAkB,QAAA"}
+{"version":3,"file":"embed.d.mts","names":[],"sources":["../src/embed.ts"],"mappings":";;;UAYiB,eAAA;EAMI;EAJnB,gBAAA;EAOU;EALV,WAAA;;EAEA,SAAA,GAAY,OAAA;AAAA;AAAA,KAGF,QAAA,IAAY,IAAA,WAAe,IAAA,2BAA+B,OAAA;;;;AAyCtE;iBAAgB,cAAA,CAAe,IAAA,EAAM,eAAA,GAAkB,QAAA"}

package/dist/embed.mjs

@@ -2,6 +2,16 @@ import { joinURL } from "ufo";
 
 //#region src/embed.ts
 const DEFAULT_EMBED_TIMEOUT_MS = 2e3;
+/**
+* Minimum meaningful character count for a PASSAGE (index-time) embedding.
+* Empty/whitespace text is always skipped; below this the text carries no
+* semantic signal and would only produce a near-mean "hub" vector that sits at
+* moderate cosine (~0.5) to EVERY query — an indexed empty/near-empty summary
+* then surfaces for every semantic search (issue #1396). Kept low (2) so real
+* short titles like "AI" / "Go" still embed; only 0–1 char content-free records
+* are dropped.
+*/
+const MIN_PASSAGE_EMBED_CHARS = 2;
 function embeddingTimeoutMs() {
 	const raw = (globalThis.process?.env)?.AFS_INDEX_EMBED_TIMEOUT_MS;
 	const parsed = raw ? Number(raw) : DEFAULT_EMBED_TIMEOUT_MS;
@@ -25,6 +35,9 @@ function createEmbedder(opts) {
 	return async (text, type = "passage") => {
 		const { embeddingEnabled, llmProvider, embedRoot } = opts;
 		if (!text || !embeddingEnabled || !llmProvider || !embedRoot?.exec) return null;
+		const trimmed = text.trim();
+		if (!trimmed) return null;
+		if (type === "passage" && trimmed.length < MIN_PASSAGE_EMBED_CHARS) return null;
 		const safeText = text.length > 6e3 ? text.slice(0, 6e3) : text;
 		try {
 			const result = await withTimeout(embedRoot.exec(joinURL(llmProvider, ".actions", "embed"), {

package/dist/embed.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"embed.mjs","names":[],"sources":["../src/embed.ts"],"sourcesContent":["/**\n * Shared passage/query embedder for the semantic index (Epic #932).\n *\n * Both the query path (`AFSIndex.autoEmbed`) and the two index paths — the\n * `index` action AND the changelog follower — need to turn text into a vector\n * via the SAME `<llmProvider>/.actions/embed` call. Extracted here so the follower\n * (and the CF cron's `rebuildFollowerFromCursor`, which has no `AFSIndex` handle)\n * can embed identically without duplicating the request/response shape handling.\n */\nimport type { AFSRoot } from \"@aigne/afs\";\nimport { joinURL } from \"ufo\";\n\nexport interface EmbedderOptions {\n  /** Master switch — when false the embedder is a no-op (returns null). */\n  embeddingEnabled?: boolean;\n  /** AFS path of the embed model node, e.g. `/dev/ai/hubs/workers-ai/models/bge-m3`. */\n  llmProvider?: string;\n  /** AFS root whose `exec` reaches `<llmProvider>/.actions/embed` (a `/dev/ai` mount). */\n  embedRoot?: AFSRoot;\n}\n\nexport type Embedder = (text?: string, type?: \"query\" | \"passage\") => Promise<number[] | null>;\n\nconst DEFAULT_EMBED_TIMEOUT_MS = 2_000;\n\nfunction embeddingTimeoutMs(): number {\n  const env = (globalThis as { process?: { env?: Record<string, string | undefined> } }).process\n    ?.env;\n  const raw = env?.AFS_INDEX_EMBED_TIMEOUT_MS;\n  const parsed = raw ? Number(raw) : DEFAULT_EMBED_TIMEOUT_MS;\n  return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_EMBED_TIMEOUT_MS;\n}\n\nasync function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  try {\n    return await Promise.race([\n      promise,\n      new Promise<null>((resolve) => {\n        timer = setTimeout(() => resolve(null), timeoutMs);\n      }),\n    ]);\n  } finally {\n    if (timer) clearTimeout(timer);\n  }\n}\n\n/**\n * Build an embedder. Returns null (never throws) when embedding is disabled,\n * unconfigured, or the inference call fails — callers degrade to FTS+anchor.\n */\nexport function createEmbedder(opts: EmbedderOptions): Embedder {\n  return async (text, type = \"passage\") => {\n    const { embeddingEnabled, llmProvider, embedRoot } = opts;\n    if (!text || !embeddingEnabled || !llmProvider || !embedRoot?.exec) {\n      return null;\n    }\n    // text-embedding-3-* / bge-m3 have ~8K-token contexts; ~6 chars/token, so\n    // 6000 chars stays well inside the cap so a verbose body doesn't 400.\n    const safeText = text.length > 6000 ? text.slice(0, 6000) : text;\n    try {\n      // Superset args: legacy/E5 `{ text, type }` → `{ embedding }`, and\n      // AigneHub/OpenAI `{ input }` → `{ embeddings: [[...]] }`. Accept either.\n      const result = await withTimeout(\n        embedRoot.exec(\n          joinURL(llmProvider, \".actions\", \"embed\"),\n          {\n            text: safeText,\n            input: safeText,\n            type,\n          },\n          {},\n        ),\n        embeddingTimeoutMs(),\n      );\n      if (!result || !result.success || !result.data) return null;\n      const data = result.data as { embedding?: number[]; embeddings?: number[][] };\n      return data.embedding ?? data.embeddings?.[0] ?? null;\n    } catch {\n      return null;\n    }\n  };\n}\n"],"mappings":";;;AAuBA,MAAM,2BAA2B;AAEjC,SAAS,qBAA6B;CAGpC,MAAM,OAFO,WAA0E,SACnF,MACa;CACjB,MAAM,SAAS,MAAM,OAAO,IAAI,GAAG;AACnC,QAAO,OAAO,SAAS,OAAO,IAAI,SAAS,IAAI,SAAS;;AAG1D,eAAe,YAAe,SAAqB,WAAsC;CACvF,IAAI;AACJ,KAAI;AACF,SAAO,MAAM,QAAQ,KAAK,CACxB,SACA,IAAI,SAAe,YAAY;AAC7B,WAAQ,iBAAiB,QAAQ,KAAK,EAAE,UAAU;IAClD,CACH,CAAC;WACM;AACR,MAAI,MAAO,cAAa,MAAM;;;;;;;AAQlC,SAAgB,eAAe,MAAiC;AAC9D,QAAO,OAAO,MAAM,OAAO,cAAc;EACvC,MAAM,EAAE,kBAAkB,aAAa,cAAc;AACrD,MAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,eAAe,CAAC,WAAW,KAC5D,QAAO;EAIT,MAAM,WAAW,KAAK,SAAS,MAAO,KAAK,MAAM,GAAG,IAAK,GAAG;AAC5D,MAAI;GAGF,MAAM,SAAS,MAAM,YACnB,UAAU,KACR,QAAQ,aAAa,YAAY,QAAQ,EACzC;IACE,MAAM;IACN,OAAO;IACP;IACD,EACD,EAAE,CACH,EACD,oBAAoB,CACrB;AACD,OAAI,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,OAAO,KAAM,QAAO;GACvD,MAAM,OAAO,OAAO;AACpB,UAAO,KAAK,aAAa,KAAK,aAAa,MAAM;UAC3C;AACN,UAAO"}
+{"version":3,"file":"embed.mjs","names":[],"sources":["../src/embed.ts"],"sourcesContent":["/**\n * Shared passage/query embedder for the semantic index (Epic #932).\n *\n * Both the query path (`AFSIndex.autoEmbed`) and the two index paths — the\n * `index` action AND the changelog follower — need to turn text into a vector\n * via the SAME `<llmProvider>/.actions/embed` call. Extracted here so the follower\n * (and the CF cron's `rebuildFollowerFromCursor`, which has no `AFSIndex` handle)\n * can embed identically without duplicating the request/response shape handling.\n */\nimport type { AFSRoot } from \"@aigne/afs\";\nimport { joinURL } from \"ufo\";\n\nexport interface EmbedderOptions {\n  /** Master switch — when false the embedder is a no-op (returns null). */\n  embeddingEnabled?: boolean;\n  /** AFS path of the embed model node, e.g. `/dev/ai/hubs/workers-ai/models/bge-m3`. */\n  llmProvider?: string;\n  /** AFS root whose `exec` reaches `<llmProvider>/.actions/embed` (a `/dev/ai` mount). */\n  embedRoot?: AFSRoot;\n}\n\nexport type Embedder = (text?: string, type?: \"query\" | \"passage\") => Promise<number[] | null>;\n\nconst DEFAULT_EMBED_TIMEOUT_MS = 2_000;\n\n/**\n * Minimum meaningful character count for a PASSAGE (index-time) embedding.\n * Empty/whitespace text is always skipped; below this the text carries no\n * semantic signal and would only produce a near-mean \"hub\" vector that sits at\n * moderate cosine (~0.5) to EVERY query — an indexed empty/near-empty summary\n * then surfaces for every semantic search (issue #1396). Kept low (2) so real\n * short titles like \"AI\" / \"Go\" still embed; only 0–1 char content-free records\n * are dropped.\n */\nconst MIN_PASSAGE_EMBED_CHARS = 2;\n\nfunction embeddingTimeoutMs(): number {\n  const env = (globalThis as { process?: { env?: Record<string, string | undefined> } }).process\n    ?.env;\n  const raw = env?.AFS_INDEX_EMBED_TIMEOUT_MS;\n  const parsed = raw ? Number(raw) : DEFAULT_EMBED_TIMEOUT_MS;\n  return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_EMBED_TIMEOUT_MS;\n}\n\nasync function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  try {\n    return await Promise.race([\n      promise,\n      new Promise<null>((resolve) => {\n        timer = setTimeout(() => resolve(null), timeoutMs);\n      }),\n    ]);\n  } finally {\n    if (timer) clearTimeout(timer);\n  }\n}\n\n/**\n * Build an embedder. Returns null (never throws) when embedding is disabled,\n * unconfigured, or the inference call fails — callers degrade to FTS+anchor.\n */\nexport function createEmbedder(opts: EmbedderOptions): Embedder {\n  return async (text, type = \"passage\") => {\n    const { embeddingEnabled, llmProvider, embedRoot } = opts;\n    if (!text || !embeddingEnabled || !llmProvider || !embedRoot?.exec) {\n      return null;\n    }\n    // Skip content-free text (issue #1396): bge-m3 / e5 map empty/whitespace(-only)\n    // input to a near-mean \"hub\" vector that sits at moderate cosine (~0.5) to\n    // EVERY query, so an indexed empty summary surfaces for every semantic search\n    // (the observed \"完全没相关性\" junk). Whitespace-only is dropped on BOTH paths;\n    // the PASSAGE (index-time) path additionally enforces a minimum meaningful\n    // length so a title-less record never writes a stored hub vector. The QUERY\n    // path keeps short-but-real queries (e.g. \"AI\") — a thin query only yields a\n    // weak query vector (gated later by `embeddingFloor`) and never pollutes the\n    // index.\n    const trimmed = text.trim();\n    if (!trimmed) return null;\n    if (type === \"passage\" && trimmed.length < MIN_PASSAGE_EMBED_CHARS) return null;\n    // text-embedding-3-* / bge-m3 have ~8K-token contexts; ~6 chars/token, so\n    // 6000 chars stays well inside the cap so a verbose body doesn't 400.\n    const safeText = text.length > 6000 ? text.slice(0, 6000) : text;\n    try {\n      // Superset args: legacy/E5 `{ text, type }` → `{ embedding }`, and\n      // AigneHub/OpenAI `{ input }` → `{ embeddings: [[...]] }`. Accept either.\n      const result = await withTimeout(\n        embedRoot.exec(\n          joinURL(llmProvider, \".actions\", \"embed\"),\n          {\n            text: safeText,\n            input: safeText,\n            type,\n          },\n          {},\n        ),\n        embeddingTimeoutMs(),\n      );\n      if (!result || !result.success || !result.data) return null;\n      const data = result.data as { embedding?: number[]; embeddings?: number[][] };\n      return data.embedding ?? data.embeddings?.[0] ?? null;\n    } catch {\n      return null;\n    }\n  };\n}\n"],"mappings":";;;AAuBA,MAAM,2BAA2B;;;;;;;;;;AAWjC,MAAM,0BAA0B;AAEhC,SAAS,qBAA6B;CAGpC,MAAM,OAFO,WAA0E,SACnF,MACa;CACjB,MAAM,SAAS,MAAM,OAAO,IAAI,GAAG;AACnC,QAAO,OAAO,SAAS,OAAO,IAAI,SAAS,IAAI,SAAS;;AAG1D,eAAe,YAAe,SAAqB,WAAsC;CACvF,IAAI;AACJ,KAAI;AACF,SAAO,MAAM,QAAQ,KAAK,CACxB,SACA,IAAI,SAAe,YAAY;AAC7B,WAAQ,iBAAiB,QAAQ,KAAK,EAAE,UAAU;IAClD,CACH,CAAC;WACM;AACR,MAAI,MAAO,cAAa,MAAM;;;;;;;AAQlC,SAAgB,eAAe,MAAiC;AAC9D,QAAO,OAAO,MAAM,OAAO,cAAc;EACvC,MAAM,EAAE,kBAAkB,aAAa,cAAc;AACrD,MAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,eAAe,CAAC,WAAW,KAC5D,QAAO;EAWT,MAAM,UAAU,KAAK,MAAM;AAC3B,MAAI,CAAC,QAAS,QAAO;AACrB,MAAI,SAAS,aAAa,QAAQ,SAAS,wBAAyB,QAAO;EAG3E,MAAM,WAAW,KAAK,SAAS,MAAO,KAAK,MAAM,GAAG,IAAK,GAAG;AAC5D,MAAI;GAGF,MAAM,SAAS,MAAM,YACnB,UAAU,KACR,QAAQ,aAAa,YAAY,QAAQ,EACzC;IACE,MAAM;IACN,OAAO;IACP;IACD,EACD,EAAE,CACH,EACD,oBAAoB,CACrB;AACD,OAAI,CAAC,UAAU,CAAC,OAAO,WAAW,CAAC,OAAO,KAAM,QAAO;GACvD,MAAM,OAAO,OAAO;AACpB,UAAO,KAAK,aAAa,KAAK,aAAa,MAAM;UAC3C;AACN,UAAO"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/afs-index",
-  "version": "1.12.0-beta.5",
+  "version": "1.12.0-beta.7",
   "description": "AIGNE AFS Index Provider — unified structured indexing with anchor, FTS, and embedding search",
   "license": "UNLICENSED",
   "publishConfig": {
@@ -36,9 +36,9 @@
     "drizzle-orm": "^0.45.1",
     "ufo": "^1.6.1",
     "zod": "^4.0.0",
-    "@aigne/afs": "^1.12.0-beta.5",
-    "@aigne/afs-logger": "^1.12.0-beta.5",
-    "@aigne/afs-sqlite": "^1.12.0-beta.5"
+    "@aigne/afs": "^1.12.0-beta.7",
+    "@aigne/afs-logger": "^1.12.0-beta.7",
+    "@aigne/afs-sqlite": "^1.12.0-beta.7"
   },
   "devDependencies": {
     "@types/bun": "^1.3.6",
@@ -46,7 +46,7 @@
     "rimraf": "^6.1.2",
     "tsdown": "0.20.0-beta.3",
     "typescript": "5.9.2",
-    "@aigne/afs-testing": "1.12.0-beta.5",
+    "@aigne/afs-testing": "1.12.0-beta.7",
     "@aigne/typescript-config": "0.0.0"
   },
   "scripts": {