@zosmaai/pi-llm-wiki 0.12.1 → 0.12.2

package/extensions/llm-wiki/lib/qmd-store.ts

@@ -0,0 +1,222 @@
+import { join } from "node:path";
+import { createStore, type HybridQueryResult, type QMDStore, type SearchResult } from "@tobilu/qmd";
+
+/**
+ * Package-private normalized adapter over the pinned @tobilu/qmd SDK.
+ *
+ * This is the ONLY production module allowed to import @tobilu/qmd. It hides
+ * SDK-specific types, collection config, model identity, and close behavior so
+ * the rest of the extension never touches QMD internals or tables directly.
+ */
+
+export const QMD_PACKAGE_VERSION = "2.5.3";
+export const QMD_DEFAULT_MODELS = {
+  embed: "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf",
+  generate: "hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf",
+  rerank: "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf",
+} as const;
+
+export interface QmdResolvedModels {
+  embed: string;
+  generate: string;
+  rerank: string;
+}
+
+export interface QmdStoreUpdateResult {
+  collections: number;
+  indexed: number;
+  updated: number;
+  unchanged: number;
+  removed: number;
+  needsEmbedding: number;
+}
+
+export interface QmdStoreEmbedResult {
+  docsProcessed: number;
+  chunksEmbedded: number;
+  errors: number;
+  durationMs: number;
+}
+
+export interface QmdStoreStatus {
+  totalDocuments: number;
+  needsEmbedding: number;
+  hasVectorIndex: boolean;
+  canonicalDocuments: number;
+  evidenceDocuments: number;
+}
+
+export interface QmdIndexStore {
+  update(
+    onProgress?: (progress: {
+      collection: string;
+      file: string;
+      current: number;
+      total: number;
+    }) => void,
+  ): Promise<QmdStoreUpdateResult>;
+  embed(options: {
+    force: boolean;
+    onProgress?: (progress: {
+      chunksEmbedded: number;
+      totalChunks: number;
+      errors: number;
+    }) => void;
+  }): Promise<QmdStoreEmbedResult>;
+  status(): Promise<QmdStoreStatus>;
+  /** lexical: store.searchLex(query, { limit }) — BM25 only, no model load. */
+  searchLex(query: string, limit?: number): Promise<QmdSearchHit[]>;
+  /** hybrid + adaptive-initial: typed lex/vec queries, NO LLM expansion, NO rerank. */
+  searchTyped(query: string, limit?: number): Promise<QmdSearchHit[]>;
+  /** adaptive-uncertain + quality: plain query, LLM expansion + rerank, with intent. */
+  searchExpanded(
+    query: string,
+    intent: string | undefined,
+    limit?: number,
+  ): Promise<QmdSearchHit[]>;
+  close(): Promise<void>;
+}
+
+/**
+ * Normalized, SDK-free retrieval hit. `score` is 0..1 relative to this result
+ * list's max (within-store confidence only — never comparable across stores).
+ */
+export interface QmdSearchHit {
+  /** Mirror collection the hit came from ("canonical" | "evidence"). */
+  collection: "canonical" | "evidence";
+  /** Mirror-relative file, e.g. "qmd://canonical/concepts/rag.md". */
+  file: string;
+  title: string;
+  /** 0..1, normalized to this result list's max. Within-store confidence only. */
+  score: number;
+  source: "fts" | "vec";
+  /** Best chunk body when the SDK returned one (hybrid results). */
+  body?: string;
+}
+
+/**
+ * Mirror collection a hit came from: the SDK reports virtual paths
+ * "qmd://<collection>/<path>.md", and the mirror layout is
+ * ".../documents/<role>/<pageId>.md". Either form resolves the role.
+ */
+function roleFromFile(file: string): QmdSearchHit["collection"] {
+  return /(?:^|\/)documents\/canonical\/|^qmd:\/\/canonical\//.test(file)
+    ? "canonical"
+    : "evidence";
+}
+
+type RawHit = Omit<QmdSearchHit, "score"> & { raw: number };
+
+/** Normalize raw within-store scores to 0..1 against this list's max. */
+function withNormalizedScores(hits: RawHit[]): QmdSearchHit[] {
+  const max = Math.max(...hits.map((h) => h.raw), 1e-9);
+  return hits.map(({ raw, ...rest }) => ({ ...rest, score: raw / max }));
+}
+
+const mapLexHit = (r: SearchResult): RawHit => ({
+  collection: roleFromFile(r.filepath),
+  file: r.filepath,
+  title: r.title,
+  source: r.source,
+  raw: r.score,
+});
+
+const mapHybridHit = (r: HybridQueryResult): RawHit => ({
+  collection: roleFromFile(r.file),
+  file: r.file,
+  title: r.title,
+  source: "fts" as const,
+  body: r.bestChunk,
+  raw: r.score,
+});
+
+export type QmdStoreFactory = (input: {
+  dbPath: string;
+  documentsPath: string;
+}) => Promise<QmdIndexStore>;
+
+/**
+ * Open a QMD index store over the mirror documents directory, with two
+ * non-overlapping collections (canonical and evidence).
+ */
+export async function openQmdIndexStore(input: {
+  dbPath: string;
+  documentsPath: string;
+}): Promise<QmdIndexStore> {
+  const store: QMDStore = await createStore({
+    dbPath: input.dbPath,
+    config: {
+      global_context: "Validated LLM Wiki knowledge",
+      collections: {
+        canonical: {
+          path: join(input.documentsPath, "canonical"),
+          pattern: "**/*.md",
+          context: { "/": "Reusable conclusions, entities, requirements, and procedures" },
+        },
+        evidence: {
+          path: join(input.documentsPath, "evidence"),
+          pattern: "**/*.md",
+          context: { "/": "Source evidence, observations, trajectories, and unpromoted notes" },
+        },
+      },
+    },
+  });
+
+  return {
+    update: (onProgress) => store.update({ onProgress }),
+    embed: ({ force, onProgress }) => store.embed({ force, chunkStrategy: "regex", onProgress }),
+    status: async () => {
+      const status = await store.getStatus();
+      const counts = Object.fromEntries(
+        status.collections.map((collection) => [collection.name, collection.documents]),
+      );
+      return {
+        totalDocuments: status.totalDocuments,
+        needsEmbedding: status.needsEmbedding,
+        hasVectorIndex: status.hasVectorIndex,
+        canonicalDocuments: counts.canonical ?? 0,
+        evidenceDocuments: counts.evidence ?? 0,
+      };
+    },
+    close: () => store.close(),
+    searchLex: async (query, limit = 40) =>
+      withNormalizedScores((await store.searchLex(query, { limit })).map(mapLexHit)),
+    searchTyped: async (query, limit = 10) =>
+      withNormalizedScores(
+        (
+          await store.search({
+            queries: [
+              { type: "lex", query },
+              { type: "vec", query },
+            ],
+            rerank: false,
+            candidateLimit: 40,
+            limit,
+            explain: true,
+          })
+        ).map(mapHybridHit),
+      ),
+    searchExpanded: async (query, intent, limit = 10) =>
+      withNormalizedScores(
+        (
+          await store.search({
+            query,
+            intent,
+            rerank: true,
+            candidateLimit: 40,
+            limit,
+            explain: true,
+          })
+        ).map(mapHybridHit),
+      ),
+  };
+}
+
+/** Resolve model identities from env, defaulting to pinned models. No downloads. */
+export function resolveQmdModels(env: NodeJS.ProcessEnv = process.env): QmdResolvedModels {
+  return {
+    embed: env.QMD_EMBED_MODEL?.trim() || QMD_DEFAULT_MODELS.embed,
+    generate: env.QMD_GENERATE_MODEL?.trim() || QMD_DEFAULT_MODELS.generate,
+    rerank: env.QMD_RERANK_MODEL?.trim() || QMD_DEFAULT_MODELS.rerank,
+  };
+}

package/extensions/llm-wiki/lib/recall.ts

@@ -148,11 +148,13 @@ function queryTerms(query: string): string[] {
   if (compact && compact !== normalized) terms.push(compact);
 
   for (const part of normalized.split(/\s+/)) {
-    if (part.length >= 2) terms.push(part);
+    if (part.length >= 2 && !STOPWORDS.has(part)) terms.push(part);
   }
 
   const latinRuns = normalized.match(/[a-z0-9]{2,}/g) ?? [];
-  terms.push(...latinRuns);
+  for (const run of latinRuns) {
+    if (!STOPWORDS.has(run)) terms.push(run);
+  }
 
   const cjkRuns =
     normalized.match(/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]+/gu) ?? [];
@@ -168,9 +170,36 @@ function queryTerms(query: string): string[] {
   return unique(terms).slice(0, 30);
 }
 
+/** Matches any CJK (Han / Hiragana / Katakana) character. */
+const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;
+
+/**
+ * Whether `haystack` contains `term`.
+ *
+ * Root-cause fix for issue #223. Previously this was a raw substring
+ * containment, so a 2-char query token like "to" or "so" matched thousands of
+ * unrelated words ("to" ⊂ "history"/"story"; "so" ⊂ "person"/"wilson"), and
+ * weighted field stacking pushed that junk past the auto-injection gate.
+ *
+ * Now:
+ * - CJK terms (Han/Kana) have no whitespace word boundaries, so they still
+ *   match by compact substring — a CJK bigram matches inside a longer glued
+ *   CJK run, preserving the multilingual recall the vaults depend on.
+ * - Latin/ASCII terms require a WHOLE-WORD match: a spaceless haystack is a
+ *   single word (term must equal it), and a spaced haystack must contain the
+ *   term as a whole whitespace-delimited token. "to" still matches the word
+ *   "to" and "Go" still matches "Go", but no longer "history"/"story"/"person".
+ */
 function includesTerm(haystack: string, term: string): boolean {
   if (!haystack || !term) return false;
-  return haystack.includes(term) || compactText(haystack).includes(compactText(term));
+  const termCompact = compactText(term);
+  if (CJK_RE.test(termCompact)) {
+    return compactText(haystack).includes(termCompact);
+  }
+  if (!/\s/.test(haystack)) {
+    return compactText(haystack) === termCompact;
+  }
+  return haystack.split(/\s+/).some((t) => t === term || t === termCompact);
 }
 
 function scoreField(value: unknown, terms: string[], weight: number): number {
@@ -238,6 +267,51 @@ const STOPWORDS = new Set([
   "type",
   "used",
   "using",
+  // Common English function words (issue #223). These must not become scoring
+  // terms, or a natural-language prompt full of "to"/"so"/"the"/"did" pushes
+  // unrelated pages past the auto-injection gate. 2-letter acronyms that are
+  // NOT stopwords ("go", "pi", "sso") still match, as intended.
+  "to",
+  "so",
+  "did",
+  "do",
+  "does",
+  "and",
+  "or",
+  "but",
+  "a",
+  "an",
+  "i",
+  "it",
+  "its",
+  "is",
+  "are",
+  "was",
+  "be",
+  "am",
+  "for",
+  "on",
+  "at",
+  "by",
+  "as",
+  "if",
+  "in",
+  "of",
+  "up",
+  "out",
+  "not",
+  "no",
+  "can",
+  "has",
+  "had",
+  "me",
+  "my",
+  "you",
+  "your",
+  "we",
+  "our",
+  "us",
+  "too",
 ]);
 
 // ─── Chunk-Level Indexing ────────────────────────────

package/extensions/llm-wiki/lib/runtime.ts

@@ -25,7 +25,20 @@ import { loadTaskConfig, noticesEnabled, TASK_DEFAULTS, type TaskConfig } from "
  */
 
 export type ResolveResult =
-  | { ok: true; model: unknown; apiKey: string; headers?: Record<string, string> }
+  | {
+      ok: true;
+      model: unknown;
+      apiKey: string;
+      headers?: Record<string, string | null>;
+      /**
+       * Stream function for the model's API (issue #222): the provider's own
+       * `streamSimple` when the model belongs to an extension-registered
+       * provider whose api pi-ai's default stream path cannot resolve.
+       */
+      streamFn?: unknown;
+      /** Provider-scoped env from auth resolution (pi >= 0.85). */
+      env?: Record<string, string>;
+    }
   | { ok: false; reason: string };
 
 type NotifyLevel = "info" | "warning" | "error";
@@ -36,9 +49,23 @@ export interface ResolveCtx {
   model: unknown;
   modelRegistry: {
     find(provider: string, id: string): unknown;
-    getApiKeyAndHeaders(
-      model: unknown,
-    ): Promise<{ ok: boolean; apiKey?: string; headers?: Record<string, string> }>;
+    getApiKeyAndHeaders(model: unknown): Promise<{
+      ok: boolean;
+      apiKey?: string;
+      headers?: Record<string, string | null>;
+      /** Auth-provided endpoint redirect (pi >= 0.85); beats the catalogue baseUrl. */
+      baseUrl?: string;
+      /** Provider-scoped env values (pi >= 0.85); must reach the stream options. */
+      env?: Record<string, string>;
+    }>;
+    /**
+     * Registered extension provider config (issue #222). Optional: pi < 0.85
+     * has no such method (it registers provider streamSimples directly into
+     * pi-ai's registry instead), in which case the default stream path is used.
+     */
+    getRegisteredProviderConfig?(
+      provider: string,
+    ): { api?: string; streamSimple?: unknown } | undefined;
   };
   hasUI: boolean;
   ui?: { notify: Notify };
@@ -137,7 +164,32 @@ export class Runtime {
       const provider = (model as { provider?: string }).provider ?? "unknown";
       return { ok: false, reason: `no API key for provider "${provider}"` };
     }
-    return { ok: true, model, apiKey: auth.apiKey ?? "", headers: auth.headers };
+    // Extension-registered providers (issue #222): pi-ai's default stream path
+    // may not be able to resolve their api (e.g. claude-bridge on pi 0.85+), so
+    // surface the provider's own streamSimple for the sub-agent stream. On pi
+    // < 0.85 the registry method is absent (?.) and the default path — which
+    // already knows extension streamSimples — is used instead.
+    const modelProvider = (model as { provider?: string }).provider;
+    const registered = modelProvider
+      ? ctx.modelRegistry.getRegisteredProviderConfig?.(modelProvider)
+      : undefined;
+    const streamFn =
+      registered?.streamSimple && registered.api === (model as { api?: string }).api
+        ? registered.streamSimple
+        : undefined;
+    // Auth can redirect the endpoint (e.g. GitHub Copilot business vs
+    // individual accounts) and/or carry provider-scoped env values (pi >=
+    // 0.85). Streaming against the catalogue values yields 421 Misdirected
+    // Request and the synthesis silently produces nothing (issue #222).
+    const authedModel = auth.baseUrl ? { ...(model as object), baseUrl: auth.baseUrl } : model;
+    return {
+      ok: true,
+      model: authedModel,
+      apiKey: auth.apiKey ?? "",
+      headers: auth.headers,
+      env: auth.env,
+      streamFn,
+    };
   }
 
   /**

package/extensions/llm-wiki/lib/subagent.ts

@@ -2,19 +2,46 @@ import {
   type AgentContext,
   type AgentLoopConfig,
   type AgentTool,
-  agentLoop,
+  runAgentLoop,
+  type StreamFn,
 } from "@earendil-works/pi-agent-core";
 import type { Api, Message, Model } from "@earendil-works/pi-ai";
 
+let cachedDefaultStreamFn: StreamFn | undefined;
+
+/**
+ * The default pi-ai stream function (dispatches to registered API providers).
+ * It moved from the package root to the `./compat` subpath in pi-ai 0.85, so
+ * it is resolved lazily — a static import of either path breaks the other pi
+ * version at load time. Cached after the first resolution.
+ */
+async function resolveDefaultStreamFn(): Promise<StreamFn> {
+  if (cachedDefaultStreamFn) return cachedDefaultStreamFn;
+  const root = await import("@earendil-works/pi-ai");
+  const rootFn = (root as { streamSimple?: StreamFn }).streamSimple;
+  if (rootFn) {
+    cachedDefaultStreamFn = rootFn;
+    return rootFn;
+  }
+  // pi-ai >= 0.85 exposes streamSimple via the ./compat subpath. The
+  // specifier is a variable so static tooling (vite in vitest, jiti in pi)
+  // cannot resolve a subpath that does not exist in pi < 0.85; at runtime
+  // this branch is only reached when the root import lacks streamSimple.
+  const compatSpecifier = "@earendil-works/pi-ai/compat";
+  const compat = await import(compatSpecifier);
+  cachedDefaultStreamFn = (compat as { streamSimple: StreamFn }).streamSimple;
+  return cachedDefaultStreamFn;
+}
+
 /**
  * Thin sub-agent runner for the LLM Wiki background lane (issue #64, part of #63).
  *
- * Wraps `agentLoop` so background tasks (ingest synthesis, topic inference,
+ * Wraps the agent loop so background tasks (ingest synthesis, topic inference,
  * etc.) can run a focused, single-purpose agent on a resolved model with its
  * own system prompt and tools — mirroring pi-observational-memory's
  * `runObserver`. The caller drives behavior entirely through `tools`
  * (tool-side effects accumulate results); this wrapper just drives the loop to
- * completion and drains its event stream.
+ * completion.
  *
  * This is infrastructure: it makes no wiki-specific decisions. Concrete
  * background workers (issues #65, #66) supply the prompts and tools.
@@ -22,7 +49,8 @@ import type { Api, Message, Model } from "@earendil-works/pi-ai";
 export interface RunSubAgentArgs<TApi extends Api = Api> {
   model: Model<TApi>;
   apiKey: string;
-  headers?: Record<string, string>;
+  /** Auth-provided request headers; pi >= 0.85 may carry null (unset) values. */
+  headers?: Record<string, string | null>;
   /** System prompt that defines the sub-agent's role. */
   systemPrompt: string;
   /** The user-turn instruction/payload to process. */
@@ -32,6 +60,15 @@ export interface RunSubAgentArgs<TApi extends Api = Api> {
   /** Max output tokens per model call. Default 4096. */
   maxTokens?: number;
   signal?: AbortSignal;
+  /**
+   * Stream function for the model's API (issue #222). Providers registered by
+   * extensions through `pi.registerProvider()` may not be resolvable by pi-ai's
+   * default stream path; the caller (Runtime.resolveModel) supplies the
+   * provider's own `streamSimple` when the model belongs to such a provider.
+   */
+  streamFn?: StreamFn;
+  /** Provider-scoped env from auth resolution (issue #222; pi >= 0.85). */
+  env?: Record<string, string>;
 }
 
 /**
@@ -40,11 +77,26 @@ export interface RunSubAgentArgs<TApi extends Api = Api> {
  * Returns nothing useful directly — by design, results are collected by the
  * `tools` the caller passes (their `execute` accumulates into caller-owned
  * state). This keeps the runner generic across every background task type.
+ *
+ * Rejections from the loop (provider errors, auth failures, a streamFn that
+ * throws) reject this promise, so `BackgroundRuntime.launchTask`'s try/catch
+ * degrades them to a warning toast.
  */
 export async function runSubAgent<TApi extends Api = Api>(
   args: RunSubAgentArgs<TApi>,
 ): Promise<void> {
-  const { model, apiKey, headers, systemPrompt, userPrompt, tools, maxTokens, signal } = args;
+  const {
+    model,
+    apiKey,
+    headers,
+    systemPrompt,
+    userPrompt,
+    tools,
+    maxTokens,
+    signal,
+    streamFn,
+    env,
+  } = args;
 
   const text = userPrompt.trim();
   if (!text) return;
@@ -72,11 +124,22 @@ export async function runSubAgent<TApi extends Api = Api>(
     convertToLlm: (msgs) => msgs as Message[],
     toolExecution: "sequential",
     ...(reasoning ? { reasoning: "high" as const } : {}),
+    // Provider-scoped env from auth resolution (issue #222). pi-agent-core
+    // spreads the config into the stream options, where pi-ai >= 0.85 honors
+    // it; the conditional spread keeps this compiling on pi < 0.85.
+    ...(env ? { env } : {}),
   };
 
-  const stream = agentLoop(prompts, context, config, signal);
-  for await (const _event of stream) {
-    // Drain events; tool `execute` callbacks collect results caller-side.
-  }
-  await stream.result();
+  // Drive the loop directly instead of agentLoop(): agentLoop() wraps the
+  // loop in a detached promise (`void runAgentLoop(...).then(...)` with no
+  // .catch), so a rejection from the stream path — e.g. "No API provider
+  // registered for api: X" for a model from an extension-registered provider
+  // — escaped as an uncaughtException and killed the whole pi process while
+  // this function's stream drain hung forever (issue #222). runAgentLoop is
+  // the same loop with the rejection propagating to THIS promise.
+  // pi >= 0.85 requires streamFn explicitly (its internal fallback throws
+  // unless the host configured a default), so we always pass one: the
+  // provider-specific function when available, else pi-ai's default.
+  const activeStreamFn = streamFn ?? (await resolveDefaultStreamFn());
+  await runAgentLoop(prompts, context, config, async () => {}, signal, activeStreamFn);
 }