@vellumai/assistant 0.8.6 → 0.8.7

package/src/memory/v3/edge-learning-job.ts

@@ -1,160 +0,0 @@
-/**
- * Memory v3 — `memory_v3_edge_learning` job (fast lane, no LLM).
- *
- * Reconciles the raw co-activation log (`memory_v3_coactivation`, migration
- * 262) into the weighted learned-edge graph (`memory_v3_auto_edges`, migration
- * 263). One pass does three things:
- *
- *   1. **Decay** — multiplicatively age all existing auto-edge weights toward
- *      zero on a half-life schedule (the rich-get-richer counterweight: an edge
- *      that stops being reinforced fades, so a once-hot pair can't dominate the
- *      adjacency forever).
- *   2. **Reinforce** — for each recent co-activation whose `used` flag is set,
- *      bump the `source → target` weight. *Used-only*: we learn associations
- *      that proved load-bearing for a turn, not pairs that merely surfaced
- *      together. The watermark checkpoint advances so each co-activation is
- *      counted once.
- *   3. **Propose** — surface the top-weight auto-edges as advisory promotion
- *      *candidates* for the assistant to ratify into curated `edges:` during
- *      consolidation. This job PROPOSES; it never auto-writes page frontmatter.
- *      Diversity counterweight: candidates are capped and a single source's
- *      out-edges are bounded so one hub can't monopolize the slate.
- *
- * Decay runs before reinforce so a fresh reinforcement isn't immediately aged
- * by the same pass. The job is idempotent in effect: re-running with no new
- * co-activations only decays (which is itself elapsed-time-bounded).
- */
-
-import { getLogger } from "../../util/logger.js";
-import { getMemoryCheckpoint, setMemoryCheckpoint } from "../checkpoints.js";
-import type { DrizzleDb } from "../db-connection.js";
-import { getDb } from "../db-connection.js";
-import type { MemoryJob } from "../jobs-store.js";
-import {
-  type AutoEdgeRow,
-  decay,
-  reinforce,
-  topByWeight,
-} from "./auto-edges.js";
-import { readCoactivations } from "./coactivation-store.js";
-
-const log = getLogger("memory-v3-edge-learning");
-
-/**
- * Half-life of auto-edge weight decay. Matches the v2 injection-score cadence
- * (3 days) — a pair reinforced 3 days ago and never since contributes half its
- * weight, 6 days ago a quarter.
- */
-export const EDGE_DECAY_HALF_LIFE_MS = 3 * 24 * 60 * 60 * 1000;
-
-/** Max promotion candidates surfaced per run (the diversity cap). */
-export const MAX_PROMOTION_CANDIDATES = 20;
-
-/** Max candidates contributed by any single source slug (anti-hub diversity). */
-export const MAX_CANDIDATES_PER_SOURCE = 3;
-
-/**
- * Minimum weight for an auto-edge to be eligible for promotion. A pair must
- * accrue more than a single reinforcement (which decays away) before it's worth
- * proposing as a curated edge.
- */
-export const PROMOTION_WEIGHT_FLOOR = 1.5;
-
-/** Checkpoint key for the high-water mark of reconciled co-activations. */
-const WATERMARK_KEY = "memory_v3_edge_learning:coactivation_watermark";
-
-/** Summary of one edge-learning pass, returned for the dispatcher log + tests. */
-export interface EdgeLearningResult {
-  /** Used co-activations reinforced this pass. */
-  reinforced: number;
-  /** Co-activations skipped because `used` was falsy. */
-  skippedUnused: number;
-  /** Auto-edges pruned by decay (fell below the floor). */
-  pruned: number;
-  /** Advisory promotion candidates, heaviest first, after the diversity cap. */
-  candidates: AutoEdgeRow[];
-}
-
-/**
- * Run one edge-learning pass against `database`. Pure of LLM and workspace I/O —
- * the whole pass is bounded DB work, hence the fast lane.
- */
-export function runEdgeLearning(
-  database: DrizzleDb,
-  now = Date.now(),
-): EdgeLearningResult {
-  // 1. Decay first so this pass's reinforcements aren't immediately aged.
-  const pruned = decay(database, now, EDGE_DECAY_HALF_LIFE_MS);
-
-  // 2. Reinforce from co-activations newer than the watermark. The watermark is
-  //    a created_at boundary; `since` is inclusive so we nudge it forward by 1ms
-  //    to avoid re-counting the boundary row.
-  const watermark = parseInt(getMemoryCheckpoint(WATERMARK_KEY) ?? "0", 10);
-  const since = watermark > 0 ? watermark + 1 : undefined;
-  const coactivations = readCoactivations(database, since);
-
-  let reinforced = 0;
-  let skippedUnused = 0;
-  let maxCreatedAt = watermark;
-  for (const row of coactivations) {
-    if (row.createdAt > maxCreatedAt) maxCreatedAt = row.createdAt;
-    // Reinforce usefulness, not mere retrieval: skip co-activations the loop
-    // (or a later usefulness reconciliation) did not mark as used.
-    if (!row.used) {
-      skippedUnused += 1;
-      continue;
-    }
-    reinforce(database, row.sourceSlug, row.targetSlug, now);
-    reinforced += 1;
-  }
-  if (maxCreatedAt > watermark) {
-    setMemoryCheckpoint(WATERMARK_KEY, String(maxCreatedAt));
-  }
-
-  // 3. Propose promotion candidates: heaviest auto-edges above the floor, capped
-  //    overall and per-source so a single hub can't monopolize the slate.
-  const candidates = selectPromotionCandidates(
-    topByWeight(database, MAX_PROMOTION_CANDIDATES * MAX_CANDIDATES_PER_SOURCE),
-  );
-
-  log.info(
-    {
-      reinforced,
-      skippedUnused,
-      pruned,
-      candidateCount: candidates.length,
-    },
-    "v3 edge learning complete",
-  );
-
-  return { reinforced, skippedUnused, pruned, candidates };
-}
-
-/**
- * Apply the weight floor and the overall / per-source diversity caps to a
- * weight-sorted list of auto-edges. Input must already be sorted heaviest-first
- * (as {@link topByWeight} returns).
- */
-function selectPromotionCandidates(sorted: AutoEdgeRow[]): AutoEdgeRow[] {
-  const out: AutoEdgeRow[] = [];
-  const perSource = new Map<string, number>();
-  for (const edge of sorted) {
-    if (out.length >= MAX_PROMOTION_CANDIDATES) break;
-    if (edge.weight < PROMOTION_WEIGHT_FLOOR) continue;
-    const count = perSource.get(edge.sourceSlug) ?? 0;
-    if (count >= MAX_CANDIDATES_PER_SOURCE) continue;
-    perSource.set(edge.sourceSlug, count + 1);
-    out.push(edge);
-  }
-  return out;
-}
-
-/**
- * Job handler for `memory_v3_edge_learning`. Thin wrapper over
- * {@link runEdgeLearning} so the heavy lifting (and its tests) live in one
- * place. The job carries no payload — it always reconciles the whole recent
- * co-activation log.
- */
-export function memoryV3EdgeLearningJob(_job: MemoryJob): EdgeLearningResult {
-  return runEdgeLearning(getDb());
-}

package/src/memory/v3/edges.ts

@@ -1,286 +0,0 @@
-/**
- * Memory v3 — Curated edge-expansion lane.
- *
- * Given a set of confident seed slugs, pull their 1–2 hop *outgoing*
- * neighborhood from the curated `edges:` graph (each concept page's
- * frontmatter `edges:` list, surfaced by v2's `getEdgeIndex`). This is a
- * provider-free, read-only structural expansion — no LLM, no scoring. It
- * answers "given that we're confident about A, what does A's curated graph
- * say we should also pull in?".
- *
- * The optional `extraAdjacency` parameter is the seam a later PR uses to inject
- * above-threshold *weighted auto-edges* (edges the system learned, not ones a
- * human curated) WITHOUT modifying this module. When supplied, it is treated as
- * additional out-edges merged with the curated graph during traversal: the
- * effective out-neighborhood of a node is `curated[node] ∪ extraAdjacency[node]`.
- *
- * The result is the union of every seed's reachable neighborhood (`pulled`,
- * with the full seed set excluded — a seed reachable from another seed is still
- * a seed, not a neighbor) plus a per-seed `EdgeExpansion[]` trace so a harness
- * can attribute each pulled slug to the seed it came from.
- */
-
-import { getEdgeIndex, getReachable } from "../v2/edge-index.js";
-import type { EdgeExpansion } from "../v2/harness/trace.js";
-
-/** Default hop budget. The design calls for a 1–2 hop walk; 2 is the ceiling. */
-const DEFAULT_HOPS = 2;
-
-// ---------------------------------------------------------------------------
-// Expansion bounds
-// ---------------------------------------------------------------------------
-//
-// The seed set handed to this lane is the *union* of every upstream lane's
-// candidates (scouts + dense filter + tree-walk descent), so on a mature corpus
-// it can run to one-to-two-thousand slugs. Expanding all of them — each to its
-// full 1–2 hop neighborhood — pulls in tens of thousands of slugs, which inflates
-// the downstream selection gate's candidate pool toward the entire corpus and
-// drowns the high-confidence signal. The expansion is also purely structural
-// (no scoring), so a larger pool buys no precision; it only adds gate cost.
-//
-// These bounds keep the lane's output proportional to a curated neighborhood
-// rather than the corpus. They are module-level constants (mirroring
-// `LANE_QUERY_LIMIT` in `scouts.ts`) rather than config fields so the lane stays
-// self-contained — the caps protect the gate regardless of how callers compose
-// the seed union. Truncation is deterministic (sorted slugs) so a given seed
-// graph always yields the same bounded result.
-
-/**
- * Cap on the number of distinct seeds expanded. Seeds are processed in
- * first-seen order; once this many have been expanded the rest are dropped. The
- * upstream lanes already rank their hits, so the earliest seeds are the most
- * confident — truncating the long tail costs little recall.
- */
-const MAX_SEEDS_EXPANDED = 150;
-
-/**
- * Cap on how many slugs a single seed may contribute. A hub node in a dense
- * curated graph can reach a large 2-hop neighborhood on its own; without this a
- * single highly-connected seed could dominate the union. The per-seed
- * neighborhood is truncated to its lexicographically-first slugs.
- */
-const MAX_PULLS_PER_SEED = 32;
-
-/**
- * Default ceiling on the size of the unioned `pulled` set. This is the
- * load-bearing bound: it caps the lane's contribution to the gate's pool no
- * matter how many seeds or how dense the graph. Once reached, no further seeds
- * are expanded. Overridable per call via {@link ExpandEdgesArgs.maxTotalPulls}
- * (the loop wires it to `memory.v3.edges.maxPulls`).
- */
-const MAX_TOTAL_PULLS = 400;
-
-// ---------------------------------------------------------------------------
-// Seed ranking
-// ---------------------------------------------------------------------------
-
-/**
- * Lane-trust order used to rank seeds *before* the {@link MAX_SEEDS_EXPANDED}
- * cap (lower = expanded first). The seed union arrives in lane order (hot
- * first), so without reordering the seed budget is spent on the hot recency
- * lane and the LLM-vetted tree/dense seeds — the most query-relevant — are the
- * ones truncated. Ranking tree > dense > sparse > hot spends the budget on
- * relevance first. A seed whose lane is unknown/absent (or edge-pulled) ranks
- * last. Requires `laneBySlug`; without it the incoming order is kept.
- */
-const SEED_LANE_RANK: Readonly<Record<string, number>> = {
-  tree: 0,
-  dense: 1,
-  sparse: 2,
-  hot: 3,
-};
-const SEED_LANE_RANK_LAST = 4;
-
-function seedRank(
-  slug: string,
-  laneBySlug: ReadonlyMap<string, string>,
-): number {
-  const lane = laneBySlug.get(slug);
-  return lane !== undefined
-    ? (SEED_LANE_RANK[lane] ?? SEED_LANE_RANK_LAST)
-    : SEED_LANE_RANK_LAST;
-}
-
-export interface ExpandEdgesArgs {
-  workspaceDir: string;
-  /** Confident seed slugs to expand from. */
-  seeds: Iterable<string>;
-  /**
-   * Per-seed lane provenance (slug → first lane that surfaced it). When present,
-   * seeds are ranked by lane trust ({@link SEED_LANE_RANK}) before the seed cap,
-   * so the budget is spent on query-relevant seeds rather than recency. Typed as
-   * a plain string map to avoid importing the loop's `LaneSource` (circular).
-   */
-  laneBySlug?: ReadonlyMap<string, string>;
-  /** Hop budget for the outgoing walk. Defaults to {@link DEFAULT_HOPS}. */
-  hops?: number;
-  /**
-   * Extra *outgoing* adjacency (`from → Set<to>`) merged with the curated graph
-   * during traversal. The injection seam for learned weighted auto-edges; this
-   * module never reads or thresholds weights itself — the caller pre-filters to
-   * above-threshold edges before passing them in.
-   */
-  extraAdjacency?: ReadonlyMap<string, ReadonlySet<string>>;
-  /**
-   * Hard ceiling on the unioned `pulled` set — the lane's contribution to the
-   * gate pool. Defaults to {@link MAX_TOTAL_PULLS} when omitted or not a finite
-   * number ≥ 0.
-   */
-  maxTotalPulls?: number;
-}
-
-export interface ExpandEdgesResult {
-  /** Union of every seed's reachable neighborhood, seeds excluded. */
-  pulled: Set<string>;
-  /** Per-seed attribution: which slugs each seed pulled in. */
-  expansions: EdgeExpansion[];
-}
-
-/**
- * BFS the outgoing neighborhood of `seed` within `hops`, walking the union of
- * the curated `outgoing` adjacency and any `extraAdjacency`. Mirrors v2's
- * `getReachable` semantics — start excluded, bounded by `hops` and a visited
- * set so cycles can't loop — but over a merged adjacency view.
- */
-function reachableMerged(
-  curated: ReadonlyMap<string, ReadonlySet<string>>,
-  extra: ReadonlyMap<string, ReadonlySet<string>>,
-  seed: string,
-  hops: number,
-): Set<string> {
-  const result = new Set<string>();
-  if (hops <= 0) return result;
-
-  const visited = new Set<string>([seed]);
-  let frontier: string[] = [seed];
-
-  for (let depth = 0; depth < hops && frontier.length > 0; depth++) {
-    const next: string[] = [];
-    for (const node of frontier) {
-      const curatedNeighbors = curated.get(node);
-      const extraNeighbors = extra.get(node);
-      for (const neighbors of [curatedNeighbors, extraNeighbors]) {
-        if (!neighbors) continue;
-        for (const neighbor of neighbors) {
-          if (visited.has(neighbor)) continue;
-          visited.add(neighbor);
-          result.add(neighbor);
-          next.push(neighbor);
-        }
-      }
-    }
-    frontier = next;
-  }
-
-  return result;
-}
-
-/**
- * Expand a set of confident seed slugs to their 1–2 hop curated neighborhood.
- *
- * Each expanded seed produces one `EdgeExpansion { from, pulled }` entry (sorted
- * slugs for deterministic output); no seed ever appears in `pulled` — not its
- * own entry and not another seed's, even when one seed is a neighbor of another
- * (the seeds-excluded contract). The top-level `pulled` set is the union across
- * all expanded seeds — a slug pulled by more than one seed appears once there
- * but in each contributing seed's expansion.
- *
- * Bounded by {@link MAX_SEEDS_EXPANDED}, {@link MAX_PULLS_PER_SEED}, and
- * {@link MAX_TOTAL_PULLS} so a large seed union or a dense graph can't balloon
- * the downstream gate's pool — see the bounds block above. Seeds past the count
- * cap, or any seed reached after the union ceiling is full, are skipped entirely
- * and get no expansion entry; every entry that is emitted lists exactly the
- * slugs that seed contributed to the bounded union, so the trace stays faithful.
- *
- * Provider-free and read-only: the only I/O is `getEdgeIndex`, which reads
- * concept-page frontmatter from disk (and caches module-locally in v2).
- */
-export async function expandEdges(
-  args: ExpandEdgesArgs,
-): Promise<ExpandEdgesResult> {
-  const {
-    workspaceDir,
-    seeds,
-    hops = DEFAULT_HOPS,
-    extraAdjacency,
-    laneBySlug,
-  } = args;
-
-  // Per-call override of the union ceiling, falling back to the default constant
-  // for an omitted or invalid (negative/NaN) value.
-  const maxTotal =
-    typeof args.maxTotalPulls === "number" &&
-    Number.isFinite(args.maxTotalPulls) &&
-    args.maxTotalPulls >= 0
-      ? args.maxTotalPulls
-      : MAX_TOTAL_PULLS;
-
-  const index = await getEdgeIndex(workspaceDir);
-  const pulled = new Set<string>();
-  const expansions: EdgeExpansion[] = [];
-
-  // Rank seeds by lane trust before the seed cap so the budget goes to the most
-  // query-relevant seeds (tree/dense/sparse) rather than recency (hot), which
-  // leads the incoming candidate order. `Array.prototype.sort` is stable, so
-  // seeds within a tier keep their candidate order. Without `laneBySlug` the
-  // incoming order is preserved.
-  const orderedSeeds = [...seeds];
-  if (laneBySlug) {
-    orderedSeeds.sort(
-      (a, b) => seedRank(a, laneBySlug) - seedRank(b, laneBySlug),
-    );
-  }
-
-  // De-dupe seeds while preserving (ranked) order for a stable trace. The full
-  // de-duped seed set is also the exclusion set: a seed reachable from another
-  // seed is itself a confident hit, not a neighbor, so it must never land in
-  // `pulled` (the seeds-excluded contract) — `getReachable` only excludes the
-  // walk's own start node, so a B reachable from A would otherwise leak in.
-  const seedSet = new Set<string>();
-  for (const seed of orderedSeeds) seedSet.add(seed);
-
-  const seenSeeds = new Set<string>();
-
-  for (const seed of orderedSeeds) {
-    if (seenSeeds.has(seed)) continue;
-    seenSeeds.add(seed);
-
-    // Bound the number of seeds expanded, and stop once the union is full —
-    // the remaining seeds would only inflate the gate's pool. Checked before
-    // doing any per-seed work so an oversized seed set is cheap to truncate.
-    if (seenSeeds.size > MAX_SEEDS_EXPANDED || pulled.size >= maxTotal) {
-      break;
-    }
-
-    const reachable = extraAdjacency
-      ? reachableMerged(index.outgoing, extraAdjacency, seed, hops)
-      : getReachable(index, seed, hops, "out");
-
-    // Drop any other seed from this seed's neighborhood (seeds-excluded
-    // contract) and split the rest into slugs already in the union vs. fresh
-    // ones. The per-seed fan-out cap and the union's remaining headroom apply
-    // only to fresh slugs, so a slot is never spent on a duplicate an earlier
-    // seed already pulled — at the cap that would silently drop a unique
-    // neighbor. Sorting first keeps truncation deterministic. The trace lists
-    // every reached slug that made it into the bounded union (fresh admissions
-    // plus the duplicates this seed also reaches), so it stays a faithful
-    // attribution while the budget is reserved for new recall.
-    const sorted = [...reachable].sort();
-    const alreadyPulled = sorted.filter(
-      (slug) => !seedSet.has(slug) && pulled.has(slug),
-    );
-    const fresh = sorted.filter(
-      (slug) => !seedSet.has(slug) && !pulled.has(slug),
-    );
-
-    const remaining = maxTotal - pulled.size;
-    const perSeedCap = Math.min(MAX_PULLS_PER_SEED, remaining);
-    const admitted = fresh.slice(0, perSeedCap);
-    for (const slug of admitted) pulled.add(slug);
-
-    const seedPulled = [...alreadyPulled, ...admitted].sort();
-    expansions.push({ from: seed, pulled: seedPulled });
-  }
-
-  return { pulled, expansions };
-}