chainlesschain 0.166.34 → 0.166.37

package/src/lib/evolution/evolution-witness-segments.js

@@ -0,0 +1,330 @@
+import path from "node:path";
+import { ensurePrivateDirectory } from "../secure-fs.js";
+
+export const EVOLUTION_SEGMENTED_WITNESS_SCHEMA =
+  "chainlesschain.evolution-file-witness-store/v2";
+const SEGMENT_SCHEMA = "chainlesschain.evolution-file-witness-segment/v1";
+export const EVOLUTION_WITNESS_SEGMENT_RECORDS = 256;
+const MAX_RECORDS = 262_144;
+const MAX_SEGMENTS = MAX_RECORDS / EVOLUTION_WITNESS_SEGMENT_RECORDS;
+const MAX_CACHE_BYTES = 16 * 1024 * 1024;
+const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+const HEAD_KEYS = new Set([
+  "schema",
+  "current",
+  "segments",
+  "history",
+  "discardedAnchors",
+]);
+const SEGMENT_KEYS = new Set(["schema", "history", "discardedAnchors"]);
+
+function exact(value, keys, label) {
+  if (
+    !value ||
+    typeof value !== "object" ||
+    Array.isArray(value) ||
+    Object.keys(value).length !== keys.size ||
+    Object.keys(value).some((key) => !keys.has(key))
+  ) {
+    throw new Error(`${label} fields are invalid`);
+  }
+}
+
+function matchesDiscard(entry, snapshot) {
+  return (
+    snapshot &&
+    (entry.anchorDigest === snapshot.anchorDigest ||
+      entry.headDigest === snapshot.headDigest ||
+      entry.segmentDigest === snapshot.segmentDigest)
+  );
+}
+
+/** Internal storage adapter. Record signatures and trust policy stay with the witness. */
+export function createEvolutionWitnessSegments({
+  filePath,
+  fsImpl,
+  readBytes,
+  writeBytes,
+  confirmDurable,
+  validateStore,
+  legacySchema,
+  canonical,
+  hash,
+  encodeRecord,
+  maximumHistoryBytes,
+}) {
+  const segmentDirectory = `${filePath}.segments-v2`;
+  const secureOptions = { deps: { fs: fsImpl }, failIfUnavailable: true };
+  const metadata = new WeakMap();
+  const cache = new Map();
+  let cacheBytes = 0;
+
+  const segmentPath = (digest) => {
+    if (!DIGEST.test(digest || ""))
+      throw new Error("witness segment digest is invalid");
+    return path.join(segmentDirectory, `${digest.slice(7)}.json`);
+  };
+
+  const assertDirectory = () => {
+    ensurePrivateDirectory(segmentDirectory, secureOptions);
+    const observed = fsImpl.lstatSync(segmentDirectory);
+    if (!observed.isDirectory() || observed.isSymbolicLink()) {
+      throw new Error("witness segment directory is invalid");
+    }
+  };
+
+  const validateSegment = (segment, previous, trustEpoch) => {
+    exact(segment, SEGMENT_KEYS, "witness segment");
+    if (
+      segment.schema !== SEGMENT_SCHEMA ||
+      !Array.isArray(segment.history) ||
+      segment.history.length !== EVOLUTION_WITNESS_SEGMENT_RECORDS
+    ) {
+      throw new Error("witness segment length or schema is invalid");
+    }
+    validateStore(
+      {
+        schema: legacySchema,
+        current: segment.history.at(-1),
+        history: segment.history,
+        discardedAnchors: segment.discardedAnchors,
+      },
+      trustEpoch,
+      previous,
+    );
+    return segment;
+  };
+
+  const observe = (records, discardedAnchors, observations) => {
+    for (const query of observations.discards) {
+      if (
+        discardedAnchors.some((entry) => matchesDiscard(entry, query.snapshot))
+      ) {
+        query.matched = true;
+      }
+    }
+    for (const query of observations.records) {
+      const firstGeneration = records[0].generation;
+      const index = query.checkpoint.generation - firstGeneration;
+      if (Number.isSafeInteger(index) && index >= 0 && index < records.length) {
+        const record = records[index];
+        if (record.witnessDigest === query.checkpoint.witnessDigest)
+          query.record = record;
+      }
+    }
+  };
+
+  const remember = (digest, segment, trustEpoch) => {
+    if (trustEpoch === null) return;
+    // A cached decision belongs only to bytes rehashed on this read and a
+    // current verifier epoch. Copy through a small encoding so no parser
+    // substring can retain a whole source segment.
+    const encoded = canonical({
+      first: segment.history[0],
+      last: segment.history.at(-1),
+      discardedAnchors: segment.discardedAnchors,
+    });
+    const bytes = 6 * encoded.length;
+    const existing = cache.get(digest);
+    if (existing) cacheBytes -= existing.bytes;
+    cache.delete(digest);
+    if (cache.size >= MAX_SEGMENTS || cacheBytes + bytes > MAX_CACHE_BYTES)
+      return;
+    cache.set(digest, { ...JSON.parse(encoded), trustEpoch, bytes });
+    cacheBytes += bytes;
+  };
+
+  const load = (head, trustEpoch, options = {}) => {
+    exact(head, HEAD_KEYS, "segmented witness store");
+    if (
+      head.schema !== EVOLUTION_SEGMENTED_WITNESS_SCHEMA ||
+      !Array.isArray(head.segments) ||
+      head.segments.length < 1 ||
+      head.segments.length >= MAX_SEGMENTS ||
+      !Array.isArray(head.history) ||
+      head.history.length < 1 ||
+      head.history.length > EVOLUTION_WITNESS_SEGMENT_RECORDS ||
+      head.segments.length * EVOLUTION_WITNESS_SEGMENT_RECORDS +
+        head.history.length >
+        MAX_RECORDS
+    ) {
+      throw new Error("segmented witness store capacity or schema is invalid");
+    }
+    assertDirectory();
+    const seen = new Set();
+    const observations = {
+      discards: (options.discards || [])
+        .filter(Boolean)
+        .map((snapshot) => ({ snapshot, matched: false })),
+      records: (options.records || []).map((checkpoint) => ({
+        checkpoint,
+        record: null,
+      })),
+    };
+    let previous = null;
+    let storedBytes = options.headBytes;
+    if (
+      !Number.isSafeInteger(storedBytes) ||
+      storedBytes < 1 ||
+      storedBytes > maximumHistoryBytes
+    ) {
+      throw new Error("witness history exceeds its configured maximum size");
+    }
+    for (const digest of head.segments) {
+      if (seen.has(digest)) throw new Error("witness segment is repeated");
+      seen.add(digest);
+      const bytes = readBytes(
+        segmentPath(digest),
+        maximumHistoryBytes - storedBytes,
+      );
+      storedBytes += bytes.length;
+      if (hash(bytes) !== digest)
+        throw new Error("witness segment bytes changed");
+      const cached = cache.get(digest);
+      const canReuse =
+        cached && trustEpoch !== null && cached.trustEpoch === trustEpoch;
+      const needsRecord = observations.records.some(
+        ({ checkpoint }) =>
+          canReuse &&
+          checkpoint.generation >= cached.first.generation &&
+          checkpoint.generation <= cached.last.generation,
+      );
+      if (canReuse) {
+        if (
+          (!previous && cached.first.generation !== 0) ||
+          (previous &&
+            (cached.first.generation !== previous.generation + 1 ||
+              cached.first.previousWitnessDigest !== previous.witnessDigest))
+        ) {
+          throw new Error("witness segment history is not contiguous");
+        }
+        for (const query of observations.discards) {
+          if (
+            cached.discardedAnchors.some((entry) =>
+              matchesDiscard(entry, query.snapshot),
+            )
+          ) {
+            query.matched = true;
+          }
+        }
+        if (needsRecord) {
+          const parsed = JSON.parse(
+            new TextDecoder("utf-8", { fatal: true }).decode(bytes),
+          );
+          observe(parsed.history, parsed.discardedAnchors, observations);
+        }
+        previous = cached.last;
+        continue;
+      }
+      const parsed = JSON.parse(
+        new TextDecoder("utf-8", { fatal: true }).decode(bytes),
+      );
+      const segment = validateSegment(parsed, previous, trustEpoch);
+      observe(segment.history, segment.discardedAnchors, observations);
+      remember(digest, segment, trustEpoch);
+      previous = segment.history.at(-1);
+    }
+    const store = validateStore(
+      {
+        schema: legacySchema,
+        current: head.current,
+        history: head.history,
+        discardedAnchors: head.discardedAnchors,
+      },
+      trustEpoch,
+      previous,
+    );
+    observe(store.history, store.discardedAnchors, observations);
+    metadata.set(store, {
+      segments: [...head.segments],
+      observations,
+      tailPreviousDiscardDigest: previous.discardAccumulatorDigest,
+      segmentBytes: storedBytes - options.headBytes,
+    });
+    return store;
+  };
+
+  const publish = (store) => {
+    const existing = metadata.get(store);
+    const segments = [...(existing?.segments || [])];
+    if (
+      segments.length * EVOLUTION_WITNESS_SEGMENT_RECORDS +
+        store.history.length >
+      MAX_RECORDS
+    ) {
+      throw new Error("witness history exceeds its configured record capacity");
+    }
+    assertDirectory();
+    let history = store.history;
+    let discardedAnchors = store.discardedAnchors;
+    let segmentBytes = existing?.segmentBytes ?? 0;
+    let accumulator =
+      existing?.tailPreviousDiscardDigest ??
+      history[0].discardAccumulatorDigest;
+    while (history.length > EVOLUTION_WITNESS_SEGMENT_RECORDS) {
+      const records = history.slice(0, EVOLUTION_WITNESS_SEGMENT_RECORDS);
+      let discardCount = 0;
+      // The first record's predecessor may belong to the preceding segment.
+      // The caller carries the exact predecessor accumulator when loading.
+      for (const record of records) {
+        if (record.discardAccumulatorDigest !== accumulator) discardCount++;
+        accumulator = record.discardAccumulatorDigest;
+      }
+      const part = {
+        schema: SEGMENT_SCHEMA,
+        history: records,
+        discardedAnchors: discardedAnchors.slice(0, discardCount),
+      };
+      const bytes = Buffer.from(`${canonical(part)}\n`, "utf8");
+      segmentBytes += bytes.length;
+      if (segmentBytes > maximumHistoryBytes) {
+        throw new Error("witness history exceeds its configured maximum size");
+      }
+      const digest = hash(bytes);
+      const destination = segmentPath(digest);
+      if (fsImpl.existsSync(destination)) {
+        if (!readBytes(destination).equals(bytes)) {
+          throw new Error(
+            "existing witness segment differs from its content address",
+          );
+        }
+        // A previous process may have stopped after rename and before fsync of
+        // the directory. Reuse requires that durability boundary again.
+        confirmDurable(destination, bytes);
+      } else {
+        // The witness CAS lock serializes publication. Only complete, fsynced
+        // bytes are renamed into this immutable content address before HEAD.
+        writeBytes(destination, bytes);
+      }
+      segments.push(digest);
+      history = history.slice(EVOLUTION_WITNESS_SEGMENT_RECORDS);
+      discardedAnchors = discardedAnchors.slice(discardCount);
+    }
+    const encoded = `{"current":${encodeRecord(store.current)},"discardedAnchors":${canonical(discardedAnchors)},"history":[${history.map(encodeRecord).join(",")}],"schema":${JSON.stringify(EVOLUTION_SEGMENTED_WITNESS_SCHEMA)},"segments":${canonical(segments)}}`;
+    const headBytes = Buffer.from(`${encoded}\n`, "utf8");
+    if (segmentBytes + headBytes.length > maximumHistoryBytes) {
+      throw new Error("witness history exceeds its configured maximum size");
+    }
+    writeBytes(filePath, headBytes);
+    return store.current;
+  };
+
+  return {
+    load,
+    publish,
+    handles: (store) =>
+      metadata.has(store) ||
+      store.history.length > EVOLUTION_WITNESS_SEGMENT_RECORDS,
+    isSegmented: (store) => metadata.has(store),
+    isDiscarded: (store, snapshot) =>
+      metadata
+        .get(store)
+        ?.observations.discards.find((query) => query.snapshot === snapshot)
+        ?.matched === true,
+    lookup: (store, checkpoint) =>
+      metadata
+        .get(store)
+        ?.observations.records.find((query) => query.checkpoint === checkpoint)
+        ?.record ?? null,
+  };
+}

package/src/lib/evolution/governed-hub-llm.js

@@ -0,0 +1,56 @@
+import { prepareGovernedModelTurn } from "./governed-model-turn.js";
+
+// A scoped wrapper preserves the Hub's independent local/cloud consent check.
+// Never replace llm on the process-wide cached Hub or its analysis engine.
+export function createGovernedHubLlm(llm, factory) {
+  if (
+    !llm ||
+    typeof llm.chat !== "function" ||
+    typeof llm.isLocal !== "boolean"
+  ) {
+    throw new TypeError(
+      "A Hub model client with declared locality is required",
+    );
+  }
+  if (typeof factory !== "function")
+    throw new TypeError("A Hub evolution factory is required");
+  const identity = { name: llm.name, isLocal: llm.isLocal, chat: llm.chat };
+  const assertIdentity = () => {
+    if (
+      llm.name !== identity.name ||
+      llm.isLocal !== identity.isLocal ||
+      llm.chat !== identity.chat
+    ) {
+      throw new Error(
+        "Hub model identity changed; reopen the analysis invocation",
+      );
+    }
+  };
+  const chat = identity.chat.bind(llm);
+  return Object.freeze({
+    get name() {
+      assertIdentity();
+      return identity.name;
+    },
+    get isLocal() {
+      assertIdentity();
+      return identity.isLocal;
+    },
+    async chat(messages, options = {}) {
+      assertIdentity();
+      const turn = await prepareGovernedModelTurn(factory, {
+        mode: "hub-analysis",
+        messages,
+        signal: options.signal,
+      });
+      assertIdentity();
+      const response = await chat(turn.messages, options);
+      assertIdentity();
+      if (!response || typeof response.text !== "string") {
+        throw new Error("Hub model response must contain text");
+      }
+      await turn.complete(response.text);
+      return response;
+    },
+  });
+}

package/src/lib/evolution/governed-hub-resolver.js

@@ -0,0 +1,65 @@
+import { prepareGovernedModelTurn } from "./governed-model-turn.js";
+import { createGovernedHubLlm } from "./governed-hub-llm.js";
+
+export function createGovernedHubResolver(ports, factory) {
+  if (typeof factory !== "function")
+    throw new TypeError("A resolver evolution factory is required");
+  const {
+    resolver,
+    llm,
+    embeddingStage,
+    EntityResolver,
+    EmbeddingStage,
+    LLMStage,
+  } = ports;
+  // Rebuild stages for each drain. Neither the shared resolver nor its embedding
+  // cache may acquire the authority of an individual WebSocket request.
+  const embedding =
+    embeddingStage &&
+    new EmbeddingStage({
+      vault: resolver.vault,
+      embedFn: async (profile) => {
+        const turn = await prepareGovernedModelTurn(factory, {
+          mode: "hub-embedding",
+          messages: [{ role: "user", content: profile }],
+        });
+        // Chat projection prepends host provenance. Embed only the projected
+        // profile: per-Run evidence IDs must not change semantic similarity.
+        const profileMessage = turn.messages[1];
+        if (
+          turn.messages.length !== 2 ||
+          turn.messages[0].role !== "system" ||
+          profileMessage?.role !== "user" ||
+          typeof profileMessage.content !== "string"
+        )
+          throw new Error(
+            "Projected embedding input must contain one text message",
+          );
+        const vector = await embeddingStage._embed(profileMessage.content);
+        if (
+          (!Array.isArray(vector) && !(vector instanceof Float32Array)) ||
+          vector.length === 0 ||
+          !Array.from(vector).every(Number.isFinite)
+        )
+          throw new Error(
+            "Embedding response must contain a nonempty finite vector",
+          );
+        await turn.complete(JSON.stringify(Array.from(vector)));
+        return vector;
+      },
+    });
+  const model =
+    resolver._llmStage &&
+    new LLMStage({
+      llm: createGovernedHubLlm(llm, factory),
+      acceptNonLocal: false,
+    });
+  return new EntityResolver({
+    vault: resolver.vault,
+    candidateLimit: resolver._candidateLimit,
+    embeddingHighThreshold: resolver._embeddingHighThreshold,
+    embeddingLowThreshold: resolver._embeddingLowThreshold,
+    embeddingStage: embedding ? embedding.asStageFn() : null,
+    llmStage: model ? model.asStageFn() : null,
+  });
+}

package/src/lib/evolution/governed-hub-skill.js

@@ -0,0 +1,36 @@
+import { createGovernedHubLlm } from "./governed-hub-llm.js";
+
+// Analysis skills treat commentary failures as optional. A governed invocation
+// must still surface a failed model turn after the SDK returns its report.
+export async function runGovernedHubSkill(
+  hub,
+  factory,
+  runSkill,
+  name,
+  options,
+) {
+  const model = createGovernedHubLlm(hub.llm, factory);
+  let failure;
+  let failed = false;
+  const llm = Object.freeze({
+    get name() {
+      return model.name;
+    },
+    get isLocal() {
+      return model.isLocal;
+    },
+    async chat(messages, chatOptions) {
+      if (failed) throw failure;
+      try {
+        return await model.chat(messages, chatOptions);
+      } catch (error) {
+        failure = error;
+        failed = true;
+        throw error;
+      }
+    },
+  });
+  const result = await runSkill({ vault: hub.vault, llm }, name, options);
+  if (failed) throw failure;
+  return result;
+}

package/src/lib/evolution/governed-model-turn.js

@@ -0,0 +1,88 @@
+import { randomUUID } from "node:crypto";
+import { types } from "node:util";
+import { captureAgentEvolutionRuntimeComposition } from "./agent-evolution-runtime-composition-brand.js";
+import { captureAgentEvolutionIngress } from "./agent-evolution-ingress.js";
+
+export function readEvolutionCompositionFactory(dependencies = {}) {
+  if (
+    !dependencies ||
+    typeof dependencies !== "object" ||
+    Array.isArray(dependencies) ||
+    types.isProxy(dependencies)
+  ) {
+    throw new TypeError("Model turn dependencies must be a plain object");
+  }
+  const property = Object.getOwnPropertyDescriptor(
+    dependencies,
+    "evolutionCompositionFactory",
+  );
+  if (
+    property &&
+    (!Object.hasOwn(property, "value") || typeof property.value !== "function")
+  ) {
+    throw new TypeError(
+      "evolutionCompositionFactory must be a function data property",
+    );
+  }
+  return property?.value ?? null;
+}
+
+// Only a host-owned factory can supply authority. Client messages supply input,
+// never a run identifier or composition to borrow from another request.
+export async function prepareGovernedModelTurn(
+  factory,
+  { mode, messages, signal },
+) {
+  signal?.throwIfAborted();
+  if (factory === null)
+    return {
+      governed: false,
+      messages,
+      complete: async () => {
+        signal?.throwIfAborted();
+      },
+    };
+  if (typeof factory !== "function")
+    throw new TypeError("Invalid evolution composition factory");
+  const runId = `${mode}-${randomUUID()}`;
+  const composition = captureAgentEvolutionRuntimeComposition(
+    await factory(
+      Object.freeze({ mode, runId, taskId: runId, cwd: process.cwd() }),
+    ),
+  );
+  const ingress = captureAgentEvolutionIngress(composition.evolutionIngress);
+  if (
+    composition.runId !== runId ||
+    ingress.runId !== runId ||
+    composition.tenantId !== ingress.tenantId
+  ) {
+    throw new Error(
+      "Model evolution composition is not bound to the requested Run",
+    );
+  }
+  signal?.throwIfAborted();
+  await ingress.start();
+  await ingress.ingestUserPrompt({ messages, source: mode });
+  const prepared = await ingress.prepareModelRequest({ messages, tools: [] });
+  signal?.throwIfAborted();
+  return Object.freeze({
+    governed: true,
+    messages: prepared.messages,
+    async complete(content) {
+      signal?.throwIfAborted();
+      await ingress.ingestAgentEvent({ type: "response-complete", content });
+      signal?.throwIfAborted();
+      await ingress.complete();
+    },
+  });
+}
+
+export async function* governModelTokenSource(source, turn, signal) {
+  let content = "";
+  for await (const delta of source) {
+    signal?.throwIfAborted();
+    content += delta;
+    yield delta;
+  }
+  await turn.complete(content);
+}

package/src/lib/evolution/progressive-canary-external-watchdog.js

@@ -218,6 +218,7 @@ export function createProgressiveCanaryHeartbeatAuthority({
   const authority = Object.freeze({
     planDigest: plan.planDigest,
     async issue({ sequence, stage, activeStateDigest } = {}) {
+      const issuedAt = timestamp(Number(now()), "heartbeat clock");
       const payload = {
         schema: PROGRESSIVE_CANARY_HOST_HEARTBEAT_SCHEMA,
         planDigest: plan.planDigest,
@@ -228,15 +229,16 @@ export function createProgressiveCanaryHeartbeatAuthority({
         sequence: timestamp(sequence, "heartbeat sequence"),
         stage: id(stage, "heartbeat stage"),
         activeStateDigest: digest(activeStateDigest, "activeStateDigest"),
-        issuedAt: timestamp(Number(now()), "heartbeat clock"),
-        expiresAt: Number(now()) + plan.leaseDurationMs,
+        issuedAt,
+        expiresAt: timestamp(
+          issuedAt + plan.leaseDurationMs,
+          "heartbeat expiry",
+        ),
         authorityId: plan.heartbeatAuthority.id,
         authorityRevision: plan.heartbeatAuthority.revision,
         handlerDigest: plan.heartbeatAuthority.handlerDigest,
         publicKeySpkiDigest: plan.heartbeatAuthority.publicKeySpkiDigest,
       };
-      if (payload.expiresAt !== payload.issuedAt + plan.leaseDurationMs)
-        throw new Error("heartbeat clock changed during issuance");
       const signature = await attestor(Object.freeze(structuredClone(payload)));
       const core = { ...payload, signature };
       return verify(