chainlesschain 0.166.34 → 0.166.37

package/src/lib/evolution/skill-candidate-registry.js

@@ -4,6 +4,7 @@ import path from "node:path";
 import { types as utilTypes } from "node:util";
 import { getHomeDir } from "../paths.js";
 import { ensurePrivateDirectory, ensurePrivateFile } from "../secure-fs.js";
+import { withEvolutionFileIdentity } from "./evolution-file-identity.js";
 import {
   verifySkillDependencyLock,
   verifySkillRuntimeManifest,
@@ -1793,6 +1794,15 @@ export class SkillCandidateRegistry {
   }
 
   _initializeTenantMarker() {
+    return withEvolutionFileIdentity(
+      this._fs,
+      this._markerPath,
+      (_full, sameStable) =>
+        this.#initializeTenantMarkerWithIdentity(sameStable),
+    );
+  }
+
+  #initializeTenantMarkerWithIdentity(sameStable) {
     const existing = lstatOrNull(this._fs, this._markerPath);
     if (existing) {
       const verified = this._readAndVerifyTenantMarker();
@@ -1832,7 +1842,6 @@ export class SkillCandidateRegistry {
           "candidate tenant marker temporary file is unsafe",
         );
       }
-      const writtenIdentity = entryIdentity(written);
       this._fs.closeSync(descriptor);
       descriptor = null;
       if (this._secure) {
@@ -1847,7 +1856,7 @@ export class SkillCandidateRegistry {
         staged.isSymbolicLink() ||
         Number(staged.nlink) !== 1 ||
         staged.size !== bytes.length ||
-        entryIdentity(staged) !== writtenIdentity ||
+        !sameStable(staged, written) ||
         !samePath(realpath(this._fs, temporaryPath), temporaryPath)
       ) {
         throw registryError(
@@ -1870,7 +1879,7 @@ export class SkillCandidateRegistry {
         !linked.isFile() ||
         linked.isSymbolicLink() ||
         Number(linked.nlink) !== 2 ||
-        entryIdentity(linked) !== writtenIdentity
+        entryIdentity(linked) !== entryIdentity(staged)
       ) {
         throw registryError(
           "SKILL_CANDIDATE_STORE_UNSAFE",
@@ -1901,6 +1910,24 @@ export class SkillCandidateRegistry {
   }
 
   _readBoundedRegularFile(filePath, maximum, code, label) {
+    return withEvolutionFileIdentity(this._fs, filePath, (samePathHandle) =>
+      this.#readBoundedRegularFileWithIdentity(
+        filePath,
+        maximum,
+        code,
+        label,
+        samePathHandle,
+      ),
+    );
+  }
+
+  #readBoundedRegularFileWithIdentity(
+    filePath,
+    maximum,
+    code,
+    label,
+    samePathHandle,
+  ) {
     let descriptor = null;
     try {
       const before = this._fs.lstatSync(filePath);
@@ -1925,7 +1952,7 @@ export class SkillCandidateRegistry {
       if (
         !opened.isFile() ||
         Number(opened.nlink) !== 1 ||
-        entryIdentity(opened) !== entryIdentity(before) ||
+        !samePathHandle(before, opened) ||
         opened.size !== before.size
       ) {
         throw registryError(code, `${label} changed while it was opened`);
@@ -1941,12 +1968,12 @@ export class SkillCandidateRegistry {
         !afterPath.isFile() ||
         afterPath.isSymbolicLink() ||
         Number(afterPath.nlink) !== 1 ||
-        entryIdentity(afterPath) !== entryIdentity(opened) ||
+        !samePathHandle(afterPath, after) ||
         !samePath(realpath(this._fs, filePath), filePath)
       ) {
         throw registryError(code, `${label} changed while it was read`);
       }
-      return { bytes, identity: entryIdentity(opened) };
+      return { bytes, identity: entryIdentity(before) };
     } finally {
       if (descriptor !== null) this._fs.closeSync(descriptor);
     }
@@ -2637,6 +2664,14 @@ export class SkillCandidateRegistry {
         "candidate create accepts only input; admission context is registry-owned",
       );
     }
+    return withEvolutionFileIdentity(
+      this._fs,
+      this._markerPath,
+      (_full, sameStable) => this.#createWithIdentity(input, sameStable),
+    );
+  }
+
+  #createWithIdentity(input, sameStable) {
     this._assertNoMixedTenantArtifacts();
     const verificationContext = this._resolveAdmissionContext(input);
     const candidate = buildSkillCandidateDraft(input, verificationContext);
@@ -2684,7 +2719,6 @@ export class SkillCandidateRegistry {
           { candidateId: candidate.candidateId, commitState: "not-committed" },
         );
       }
-      const writtenIdentity = entryIdentity(written);
       this._fs.closeSync(descriptor);
       descriptor = null;
       if (this._secure) {
@@ -2699,7 +2733,7 @@ export class SkillCandidateRegistry {
         staged.isSymbolicLink() ||
         Number(staged.nlink) !== 1 ||
         staged.size !== bytes.length ||
-        entryIdentity(staged) !== writtenIdentity ||
+        !sameStable(staged, written) ||
         !samePath(realpath(this._fs, temporaryPath), temporaryPath)
       ) {
         throw registryError(
@@ -2742,7 +2776,7 @@ export class SkillCandidateRegistry {
         linked.isSymbolicLink() ||
         Number(linked.nlink) !== 2 ||
         linked.size !== bytes.length ||
-        entryIdentity(linked) !== writtenIdentity ||
+        entryIdentity(linked) !== entryIdentity(staged) ||
         !samePath(realpath(this._fs, filePath), filePath)
       ) {
         throw registryError(
@@ -2759,7 +2793,7 @@ export class SkillCandidateRegistry {
         finalized.isSymbolicLink() ||
         Number(finalized.nlink) !== 1 ||
         finalized.size !== bytes.length ||
-        entryIdentity(finalized) !== writtenIdentity
+        entryIdentity(finalized) !== entryIdentity(staged)
       ) {
         throw registryError(
           "SKILL_CANDIDATE_COMMIT_UNKNOWN",
@@ -2794,7 +2828,7 @@ export class SkillCandidateRegistry {
         );
       }
       if (
-        stored.identity !== writtenIdentity ||
+        stored.identity !== entryIdentity(staged) ||
         verified.tenantId !== this.tenantId ||
         verified.candidateId !== candidate.candidateId ||
         !stored.bytes.equals(bytes) ||

package/src/lib/evolution/skill-release-registry.js

@@ -5,6 +5,7 @@ import path from "node:path";
 import { types as utilTypes } from "node:util";
 import { getHomeDir } from "../paths.js";
 import { ensurePrivateDirectory, ensurePrivateFile } from "../secure-fs.js";
+import { withEvolutionFileIdentity } from "./evolution-file-identity.js";
 import {
   SKILL_CANDIDATE_MIGRATION_RECEIPT_SCHEMA,
   SKILL_CANDIDATE_MIGRATION_RECORD_SCHEMA,
@@ -2543,6 +2544,26 @@ function lstatOrNull(fsImpl, target) {
 }
 
 function readBoundedSingleLinkFile(fsImpl, filePath, maximum, code, label) {
+  return withEvolutionFileIdentity(fsImpl, filePath, (samePathHandle) =>
+    readBoundedSingleLinkFileWithIdentity(
+      fsImpl,
+      filePath,
+      maximum,
+      code,
+      label,
+      samePathHandle,
+    ),
+  );
+}
+
+function readBoundedSingleLinkFileWithIdentity(
+  fsImpl,
+  filePath,
+  maximum,
+  code,
+  label,
+  samePathHandle,
+) {
   let descriptor = null;
   try {
     const before = fsImpl.lstatSync(filePath);
@@ -2564,7 +2585,7 @@ function readBoundedSingleLinkFile(fsImpl, filePath, maximum, code, label) {
     if (
       !opened.isFile() ||
       Number(opened.nlink) !== 1 ||
-      identity(opened) !== identity(before) ||
+      !samePathHandle(before, opened) ||
       opened.size !== before.size
     ) {
       throw failure(code, `${label} changed while opening`);
@@ -2580,12 +2601,12 @@ function readBoundedSingleLinkFile(fsImpl, filePath, maximum, code, label) {
       !afterPath.isFile() ||
       afterPath.isSymbolicLink() ||
       Number(afterPath.nlink) !== 1 ||
-      identity(afterPath) !== identity(opened) ||
+      !samePathHandle(afterPath, after) ||
       !samePath(realpath(fsImpl, filePath), filePath)
     ) {
       throw failure(code, `${label} changed while reading`);
     }
-    return { bytes, identity: identity(opened) };
+    return { bytes, identity: identity(before) };
   } finally {
     if (descriptor !== null) fsImpl.closeSync(descriptor);
   }
@@ -2786,8 +2807,7 @@ export class SkillReleaseRegistry {
         readActive: SkillReleaseRegistry.prototype.readActive.bind(this),
         readState: SkillReleaseRegistry.prototype.readState.bind(this),
         readRelease: SkillReleaseRegistry.prototype.readRelease.bind(this),
-        readInventory:
-          SkillReleaseRegistry.prototype.readInventory.bind(this),
+        readInventory: SkillReleaseRegistry.prototype.readInventory.bind(this),
         matchesTransactionLedger: (value) => value === transactionLedger,
       }),
     );
@@ -2870,6 +2890,15 @@ export class SkillReleaseRegistry {
   }
 
   #initializeTenantMarker() {
+    return withEvolutionFileIdentity(
+      this.#fs,
+      this.#markerPath,
+      (_full, sameStable) =>
+        this.#initializeTenantMarkerWithIdentity(sameStable),
+    );
+  }
+
+  #initializeTenantMarkerWithIdentity(sameStable) {
     const existing = lstatOrNull(this.#fs, this.#markerPath);
     if (existing) {
       const verified = this.#readAndVerifyTenantMarker();
@@ -2905,7 +2934,6 @@ export class SkillReleaseRegistry {
           "release tenant marker temporary file is unsafe",
         );
       }
-      const writtenIdentity = identity(written);
       this.#fs.closeSync(descriptor);
       descriptor = null;
       if (this.#secure) {
@@ -2920,7 +2948,7 @@ export class SkillReleaseRegistry {
         staged.isSymbolicLink() ||
         Number(staged.nlink) !== 1 ||
         staged.size !== bytes.length ||
-        identity(staged) !== writtenIdentity ||
+        !sameStable(staged, written) ||
         !samePath(realpath(this.#fs, temporaryPath), temporaryPath)
       ) {
         throw failure(
@@ -2943,7 +2971,7 @@ export class SkillReleaseRegistry {
         !linked.isFile() ||
         linked.isSymbolicLink() ||
         Number(linked.nlink) !== 2 ||
-        identity(linked) !== writtenIdentity
+        identity(linked) !== identity(staged)
       ) {
         throw failure(
           "SKILL_RELEASE_STORE_UNSAFE",
@@ -3229,6 +3257,12 @@ export class SkillReleaseRegistry {
       throw failure("SKILL_RELEASE_STORE_UNSAFE", "temporary name is invalid");
     }
     const filePath = this.#path("staging", fileName);
+    return withEvolutionFileIdentity(this.#fs, filePath, (_full, sameStable) =>
+      this.#writeTemporaryWithIdentity(filePath, value, sameStable),
+    );
+  }
+
+  #writeTemporaryWithIdentity(filePath, value, sameStable) {
     const bytes = serialize(value);
     let descriptor = null;
     let writtenIdentity = null;
@@ -3248,7 +3282,7 @@ export class SkillReleaseRegistry {
           "temporary write was incomplete",
         );
       }
-      writtenIdentity = identity(written);
+      writtenIdentity = written;
     } finally {
       if (descriptor !== null) this.#fs.closeSync(descriptor);
     }
@@ -3263,7 +3297,7 @@ export class SkillReleaseRegistry {
       !staged.isFile() ||
       staged.isSymbolicLink() ||
       Number(staged.nlink) !== 1 ||
-      identity(staged) !== writtenIdentity ||
+      !sameStable(staged, writtenIdentity) ||
       staged.size !== bytes.length ||
       !samePath(realpath(this.#fs, filePath), filePath)
     ) {
@@ -4006,8 +4040,8 @@ export class SkillReleaseRegistry {
     }
     const afterReleases = this.#readInventoryNames("artifacts", releasePattern);
     const afterStates = this.#readInventoryNames("active", statePattern);
-    const afterStateDigests = afterStates.map((name) =>
-      this.readState(name.slice(0, -".json".length)).stateDigest,
+    const afterStateDigests = afterStates.map(
+      (name) => this.readState(name.slice(0, -".json".length)).stateDigest,
     );
     if (
       canonicalJson(beforeReleases) !== canonicalJson(afterReleases) ||

package/src/lib/llm-connection-probe.js

@@ -1,5 +1,10 @@
 import { formatProviderResponseError } from "./provider-http-error.js";
 
+// Reasoning models spend the same output allowance on thinking before the
+// final answer. A 16-token probe can therefore receive HTTP 200 with only
+// reasoning_content and finish_reason=length from an otherwise usable model.
+const PROBE_OUTPUT_TOKENS = 1024;
+
 export async function probeLlmConnection(
   target,
   { fetchImpl = fetch, timeoutMs = 20_000 } = {},
@@ -13,6 +18,7 @@ export async function probeLlmConnection(
       model: target.model,
       prompt: "Say hi in one word.",
       stream: false,
+      options: { num_predict: PROBE_OUTPUT_TOKENS },
     };
   } else {
     if (!target.apiKey) throw new Error("API key required");
@@ -22,7 +28,7 @@ export async function probeLlmConnection(
       headers["anthropic-version"] = "2023-06-01";
       body = {
         model: target.model,
-        max_tokens: 16,
+        max_tokens: PROBE_OUTPUT_TOKENS,
         messages: [{ role: "user", content: "Say hi in one word." }],
       };
     } else if (target.provider === "gemini") {
@@ -30,7 +36,7 @@ export async function probeLlmConnection(
       headers["x-goog-api-key"] = target.apiKey;
       body = {
         contents: [{ parts: [{ text: "Say hi in one word." }] }],
-        generationConfig: { maxOutputTokens: 16 },
+        generationConfig: { maxOutputTokens: PROBE_OUTPUT_TOKENS },
       };
     } else {
       url = `${base}/chat/completions`;
@@ -38,7 +44,7 @@ export async function probeLlmConnection(
       body = {
         model: target.model,
         messages: [{ role: "user", content: "Say hi in one word." }],
-        max_tokens: 16,
+        max_tokens: PROBE_OUTPUT_TOKENS,
       };
     }
   }
@@ -67,13 +73,24 @@ export async function probeLlmConnection(
               .join("")
           : target.provider === "gemini"
             ? data.candidates?.[0]?.content?.parts
+                ?.filter((p) => p.thought !== true)
                 ?.map((p) => p.text || "")
                 .join("")
             : data.choices?.[0]?.message?.content;
-    if (typeof text !== "string" || !text.trim())
+    if (typeof text !== "string" || !text.trim()) {
+      const reachedLimit =
+        data.choices?.[0]?.finish_reason === "length" ||
+        data.candidates?.[0]?.finishReason === "MAX_TOKENS" ||
+        data.stop_reason === "max_tokens" ||
+        data.done_reason === "length";
+      if (reachedLimit)
+        throw new Error(
+          "The model responded but exhausted the test output limit before a final answer; this does not mean the connection settings are invalid",
+        );
       throw new Error(
         "The endpoint returned no model text; check the protocol and model",
       );
+    }
     return text.trim().slice(0, 100);
   } catch (error) {
     if (controller.signal.aborted)

package/src/lib/personal-data-hub-wiring.js

@@ -467,6 +467,7 @@ async function initHub() {
 
   // Phase 8 — EntityResolver pipeline
   const entityResolver = new EntityResolver({ vault });
+  let resolverEmbeddingStage = null;
   // Plan A v0.1 — in-APK Android cc has no Ollama on localhost:11434.
   // Every embedding call would TCP-timeout (measured: ~60s extra per sync
   // on Xiaomi 24115RA8EC 2026-05-21). Detect Termux $PREFIX for our APK
@@ -504,6 +505,7 @@ async function initHub() {
         vault,
       });
       entityResolver._embeddingStage = embeddingStage.asStageFn();
+      resolverEmbeddingStage = embeddingStage;
     }
   } catch (_err) {
     // Fall back to rule-only — registry still works
@@ -1071,6 +1073,23 @@ async function initHub() {
     alipayAccountsPath,
     entityResolver,
     aichatAccountsStore,
+    async drainResolver(options = {}, factory = null) {
+      if (factory === null) return entityResolver.drain(options);
+      const { createGovernedHubResolver } =
+        await import("./evolution/governed-hub-resolver.js");
+      const scoped = createGovernedHubResolver(
+        {
+          resolver: entityResolver,
+          llm,
+          embeddingStage: resolverEmbeddingStage,
+          EntityResolver,
+          EmbeddingStage: EntityResolverEmbeddingStage,
+          LLMStage: EntityResolverLLMStage,
+        },
+        factory,
+      );
+      return scoped.drain(options);
+    },
     aiChatAdapter,
     aichatWizard,
     aichatHealthChecker,
@@ -2091,6 +2110,36 @@ export async function getHub() {
   return _initPromise;
 }
 
+export async function getGovernedAnalysisHub(evolutionCompositionFactory) {
+  const hub = await getHub();
+  const { createGovernedHubLlm } =
+    await import("./evolution/governed-hub-llm.js");
+  const llm = createGovernedHubLlm(hub.engine.llm, evolutionCompositionFactory);
+  const engine = new AnalysisEngine({
+    vault: hub.engine.vault,
+    llm,
+    ragRetriever: hub.engine.ragRetriever,
+    maxFacts: hub.engine.maxFacts,
+    maxQueryLimit: hub.engine.maxQueryLimit,
+    systemPrompt: hub.engine.systemPrompt,
+  });
+  const { runGovernedHubSkill } =
+    await import("./evolution/governed-hub-skill.js");
+  return Object.freeze({
+    ...hub,
+    llm,
+    engine,
+    runSkill: (name, options = {}) =>
+      runGovernedHubSkill(
+        hub,
+        evolutionCompositionFactory,
+        runAnalysisSkill,
+        name,
+        options,
+      ),
+  });
+}
+
 // ─── Minimal hub bootstrap for read-only / LLM-free commands ────────────
 //
 // 2026-05-27 — `cc hub retrieve-context` cold-start was 90s+ on Android

package/src/lib/provider-stream.js

@@ -6,37 +6,77 @@
 
 import { BUILT_IN_PROVIDERS } from "./llm-providers.js";
 
-export async function* ollamaTokenStream({ baseUrl, model, prompt, signal }) {
-  const res = await fetch(`${baseUrl}/api/generate`, {
+async function* responseLines(body) {
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  let ended = false;
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) {
+        ended = true;
+        buffer += decoder.decode();
+        if (buffer) yield buffer;
+        return;
+      }
+      buffer += decoder.decode(value, { stream: true });
+      let newline;
+      while ((newline = buffer.indexOf("\n")) >= 0) {
+        const line = buffer.slice(0, newline);
+        buffer = buffer.slice(newline + 1);
+        yield line;
+      }
+    }
+  } finally {
+    try {
+      if (!ended) await reader.cancel();
+    } catch {
+      // Preserve the original parser/transport failure during cleanup.
+    } finally {
+      reader.releaseLock();
+    }
+  }
+}
+
+export async function* ollamaTokenStream({
+  baseUrl,
+  model,
+  prompt,
+  messages,
+  signal,
+  requireCompletion = false,
+}) {
+  const res = await fetch(`${baseUrl}/api/${messages ? "chat" : "generate"}`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ model, prompt, stream: true }),
+    body: JSON.stringify(
+      messages
+        ? { model, messages, stream: true }
+        : { model, prompt, stream: true },
+    ),
     signal,
   });
   if (!res.ok || !res.body) {
     throw new Error(`Ollama ${res.status} ${res.statusText}`);
   }
-  const reader = res.body.getReader();
-  const decoder = new TextDecoder();
-  let buf = "";
-  while (true) {
-    const { value, done } = await reader.read();
-    if (done) break;
-    buf += decoder.decode(value, { stream: true });
-    let nl;
-    while ((nl = buf.indexOf("\n")) >= 0) {
-      const line = buf.slice(0, nl).trim();
-      buf = buf.slice(nl + 1);
-      if (!line) continue;
-      try {
-        const obj = JSON.parse(line);
-        if (obj.response) yield obj.response;
-        if (obj.done) return;
-      } catch {
-        /* skip malformed */
-      }
+  for await (const raw of responseLines(res.body)) {
+    const line = raw.trim();
+    if (!line) continue;
+    try {
+      const obj = JSON.parse(line);
+      if (requireCompletion && obj.error)
+        throw new Error("Ollama stream returned an error");
+      const delta = messages ? obj.message?.content : obj.response;
+      if (delta) yield delta;
+      if (obj.done) return;
+    } catch (error) {
+      if (requireCompletion) throw error;
+      /* skip malformed */
     }
   }
+  if (requireCompletion)
+    throw new Error("Ollama stream ended before its completion marker");
 }
 
 export async function* openAIStream({
@@ -44,7 +84,9 @@ export async function* openAIStream({
   apiKey,
   model,
   prompt,
+  messages,
   signal,
+  requireCompletion = false,
 }) {
   const res = await fetch(`${baseUrl}/chat/completions`, {
     method: "POST",
@@ -54,7 +96,7 @@ export async function* openAIStream({
     },
     body: JSON.stringify({
       model,
-      messages: [{ role: "user", content: prompt }],
+      messages: messages ?? [{ role: "user", content: prompt }],
       stream: true,
     }),
     signal,
@@ -62,29 +104,24 @@ export async function* openAIStream({
   if (!res.ok || !res.body) {
     throw new Error(`${res.status} ${res.statusText}`);
   }
-  const reader = res.body.getReader();
-  const decoder = new TextDecoder();
-  let buf = "";
-  while (true) {
-    const { value, done } = await reader.read();
-    if (done) break;
-    buf += decoder.decode(value, { stream: true });
-    const lines = buf.split("\n");
-    buf = lines.pop() || "";
-    for (const raw of lines) {
-      const line = raw.trim();
-      if (!line || !line.startsWith("data:")) continue;
-      const payload = line.slice(5).trim();
-      if (payload === "[DONE]") return;
-      try {
-        const obj = JSON.parse(payload);
-        const delta = obj?.choices?.[0]?.delta?.content;
-        if (delta) yield delta;
-      } catch {
-        /* skip */
-      }
+  for await (const raw of responseLines(res.body)) {
+    const line = raw.trim();
+    if (!line || !line.startsWith("data:")) continue;
+    const payload = line.slice(5).trim();
+    if (payload === "[DONE]") return;
+    try {
+      const obj = JSON.parse(payload);
+      if (requireCompletion && obj.error)
+        throw new Error("OpenAI stream returned an error");
+      const delta = obj?.choices?.[0]?.delta?.content;
+      if (delta) yield delta;
+    } catch (error) {
+      if (requireCompletion) throw error;
+      /* skip */
     }
   }
+  if (requireCompletion)
+    throw new Error("OpenAI stream ended before its completion marker");
 }
 
 /**
@@ -92,13 +129,23 @@ export async function* openAIStream({
  * Throws on unsupported provider / missing API key.
  */
 export function buildProviderSource(provider, opts = {}) {
-  const { model, baseUrl, apiKey, prompt, signal } = opts;
+  const {
+    model,
+    baseUrl,
+    apiKey,
+    prompt,
+    messages,
+    signal,
+    requireCompletion,
+  } = opts;
   if (provider === "ollama") {
     return ollamaTokenStream({
       baseUrl: baseUrl || "http://localhost:11434",
       model: model || "qwen2:7b",
       prompt,
+      messages,
       signal,
+      requireCompletion,
     });
   }
   const def = BUILT_IN_PROVIDERS[provider];
@@ -115,7 +162,9 @@ export function buildProviderSource(provider, opts = {}) {
     apiKey: finalKey,
     model: model || def.models[0],
     prompt,
+    messages,
     signal,
+    requireCompletion,
   });
 }