@vellumai/assistant 0.5.5 → 0.5.7

package/src/memory/archive-store.ts

@@ -1,400 +0,0 @@
-import { createHash } from "node:crypto";
-
-import { eq } from "drizzle-orm";
-import { v4 as uuid } from "uuid";
-
-import { estimateTextTokens } from "../context/token-estimator.js";
-import { getLogger } from "../util/logger.js";
-import { getDb, rawChanges } from "./db.js";
-import { enqueueMemoryJob, type MemoryJobType } from "./jobs-store.js";
-import {
-  memoryChunks,
-  memoryEpisodes,
-  memoryObservations,
-} from "./schema.js";
-
-const log = getLogger("memory-archive-store");
-
-// ── Content hashing ─────────────────────────────────────────────────
-
-/**
- * Compute a SHA-256 content hash for a chunk's content, scoped by scopeId.
- * Used for idempotent upserts — if the hash already exists within the same
- * scope, the chunk is skipped.
- */
-export function computeChunkContentHash(
-  scopeId: string,
-  content: string,
-): string {
-  return createHash("sha256").update(`${scopeId}|${content}`).digest("hex");
-}
-
-/**
- * Compute a SHA-256 hash of the observation content, scoped by scopeId.
- * Used for idempotent chunk deduplication.
- */
-export function computeObservationContentHash(
-  scopeId: string,
-  content: string,
-): string {
-  return createHash("sha256").update(`${scopeId}|${content}`).digest("hex");
-}
-
-// ── Token estimation ────────────────────────────────────────────────
-
-/**
- * Rough token count estimate based on character length.
- * Uses the common ~4 chars/token heuristic for English text.
- */
-export function estimateTokens(text: string): number {
-  return Math.max(1, Math.ceil(text.length / 4));
-}
-
-// ── Chunk upsert ────────────────────────────────────────────────────
-
-export interface UpsertChunkInput {
-  /** Scope for memory isolation. Defaults to "default". */
-  scopeId?: string;
-  /** FK to the parent observation. */
-  observationId: string;
-  /** The chunk text to embed and recall. */
-  content: string;
-  /** Optional pre-computed token estimate. If omitted, estimated from content length. */
-  tokenEstimate?: number;
-}
-
-export interface UpsertChunkResult {
-  chunkId: string;
-  /** True if a new row was inserted; false if an existing row matched the content hash. */
-  inserted: boolean;
-}
-
-/**
- * Idempotently upsert a chunk into the archive. If a chunk with the same
- * (scopeId, contentHash) already exists, the insert is skipped and the
- * existing row's id is returned. Otherwise a new row is inserted and an
- * `embed_chunk` job is enqueued.
- */
-export function upsertChunk(input: UpsertChunkInput): UpsertChunkResult {
-  const scopeId = input.scopeId ?? "default";
-  const contentHash = computeChunkContentHash(scopeId, input.content);
-  const tokenEstimate = input.tokenEstimate ?? estimateTokens(input.content);
-  const db = getDb();
-
-  // Check for an existing chunk with the same content hash in this scope
-  const existing = db
-    .select({ id: memoryChunks.id })
-    .from(memoryChunks)
-    .where(eq(memoryChunks.contentHash, contentHash))
-    .get();
-
-  if (existing) {
-    log.debug(
-      { scopeId, contentHash, existingId: existing.id },
-      "Chunk already exists, skipping insert",
-    );
-    return { chunkId: existing.id, inserted: false };
-  }
-
-  const chunkId = uuid();
-  const now = Date.now();
-
-  db.insert(memoryChunks)
-    .values({
-      id: chunkId,
-      scopeId,
-      observationId: input.observationId,
-      content: input.content,
-      tokenEstimate,
-      contentHash,
-      createdAt: now,
-    })
-    .run();
-
-  // Enqueue an embedding job for the new chunk
-  enqueueMemoryJob("embed_chunk", {
-    chunkId,
-    scopeId,
-  });
-
-  log.debug(
-    { chunkId, scopeId, contentHash },
-    "Inserted new chunk and enqueued embed_chunk job",
-  );
-
-  return { chunkId, inserted: true };
-}
-
-/**
- * Upsert multiple chunks for a single observation. Returns results for
- * each input in the same order.
- */
-export function upsertChunks(inputs: UpsertChunkInput[]): UpsertChunkResult[] {
-  return inputs.map((input) => upsertChunk(input));
-}
-
-// ── Chunk queries ───────────────────────────────────────────────────
-
-/**
- * Get a chunk by its ID.
- */
-export function getChunkById(
-  chunkId: string,
-): typeof memoryChunks.$inferSelect | undefined {
-  const db = getDb();
-  return db
-    .select()
-    .from(memoryChunks)
-    .where(eq(memoryChunks.id, chunkId))
-    .get();
-}
-
-/**
- * Get all chunks for a given observation.
- */
-export function getChunksByObservationId(
-  observationId: string,
-): Array<typeof memoryChunks.$inferSelect> {
-  const db = getDb();
-  return db
-    .select()
-    .from(memoryChunks)
-    .where(eq(memoryChunks.observationId, observationId))
-    .all();
-}
-
-// ── Episode insertion helpers ───────────────────────────────────────
-
-export interface InsertEpisodeParams {
-  scopeId?: string;
-  conversationId: string;
-  title: string;
-  summary: string;
-  tokenEstimate: number;
-  source?: string;
-  startAt: number;
-  endAt: number;
-}
-
-/**
- * Insert an episode row produced by conversation compaction.
- * Compaction episodes summarize a contiguous block of turns that was
- * compressed to free context-window space.
- *
- * An `embed_episode` job is enqueued automatically so the episode
- * becomes searchable via vector recall.
- */
-export function insertCompactionEpisode(params: InsertEpisodeParams): {
-  episodeId: string;
-  jobId: string;
-} {
-  return insertEpisodeAndEnqueue(params);
-}
-
-/**
- * Insert an episode row produced by resolution (end-of-conversation)
- * summarization. Resolution episodes capture the full narrative arc
- * of a completed conversation.
- *
- * An `embed_episode` job is enqueued automatically so the episode
- * becomes searchable via vector recall.
- */
-export function insertResolutionEpisode(params: InsertEpisodeParams): {
-  episodeId: string;
-  jobId: string;
-} {
-  return insertEpisodeAndEnqueue(params);
-}
-
-// ── Internal (episode) ──────────────────────────────────────────────
-
-function insertEpisodeAndEnqueue(params: InsertEpisodeParams): {
-  episodeId: string;
-  jobId: string;
-} {
-  const db = getDb();
-  const episodeId = uuid();
-  const now = Date.now();
-
-  db.insert(memoryEpisodes)
-    .values({
-      id: episodeId,
-      scopeId: params.scopeId ?? "default",
-      conversationId: params.conversationId,
-      title: params.title,
-      summary: params.summary,
-      tokenEstimate: params.tokenEstimate,
-      source: params.source ?? null,
-      startAt: params.startAt,
-      endAt: params.endAt,
-      createdAt: now,
-      updatedAt: now,
-    })
-    .run();
-
-  const jobId = enqueueMemoryJob("embed_episode" satisfies MemoryJobType, {
-    episodeId,
-  });
-
-  log.debug(
-    { episodeId, jobId, conversationId: params.conversationId },
-    "Inserted episode and enqueued embed job",
-  );
-
-  return { episodeId, jobId };
-}
-
-// ── Observation types ───────────────────────────────────────────────
-
-export interface InsertObservationParams {
-  conversationId: string;
-  messageId?: string | null;
-  role: string;
-  content: string;
-  scopeId?: string;
-  modality?: string;
-  source?: string | null;
-}
-
-export interface InsertedObservation {
-  observationId: string;
-  chunkId: string | null;
-  contentHash: string;
-  embeddingJobId: string | null;
-}
-
-// ── Observation insert helpers ──────────────────────────────────────
-
-/**
- * Insert a raw observation row and its associated chunk. If a chunk with the
- * same content hash already exists in the scope, the chunk insert is skipped
- * (idempotent dual-write safety). An `embed_observation` job is enqueued when
- * a new chunk is created.
- *
- * Returns the observation ID, chunk ID (null if deduplicated), content hash,
- * and embedding job ID (null if no new chunk was created).
- */
-export function insertObservation(
-  params: InsertObservationParams,
-): InsertedObservation {
-  const db = getDb();
-  const now = Date.now();
-  const scopeId = params.scopeId ?? "default";
-  const modality = params.modality ?? "text";
-
-  const observationId = uuid();
-  const contentHash = computeObservationContentHash(scopeId, params.content);
-
-  // Insert the observation row
-  db.insert(memoryObservations)
-    .values({
-      id: observationId,
-      scopeId,
-      conversationId: params.conversationId,
-      messageId: params.messageId ?? null,
-      role: params.role,
-      content: params.content,
-      modality,
-      source: params.source ?? null,
-      createdAt: now,
-    })
-    .run();
-
-  // Attempt to insert the chunk — the unique index on (scope_id, content_hash)
-  // will cause a conflict if this content was already chunked. We use
-  // onConflictDoNothing to silently skip the duplicate.
-  const chunkId = uuid();
-  const tokenEstimate = estimateTextTokens(params.content);
-
-  db.insert(memoryChunks)
-    .values({
-      id: chunkId,
-      scopeId,
-      observationId,
-      content: params.content,
-      tokenEstimate,
-      contentHash,
-      createdAt: now,
-    })
-    .onConflictDoNothing({
-      target: [memoryChunks.scopeId, memoryChunks.contentHash],
-    })
-    .run();
-
-  // Drizzle bun:sqlite .run() returns void; use rawChanges() to detect no-ops
-  const chunkInserted = rawChanges() > 0;
-
-  let embeddingJobId: string | null = null;
-  if (chunkInserted) {
-    embeddingJobId = enqueueMemoryJob("embed_observation", {
-      observationId,
-      chunkId,
-    });
-    log.debug(
-      { observationId, chunkId, contentHash, embeddingJobId },
-      "Inserted observation with new chunk, enqueued embed job",
-    );
-  } else {
-    log.debug(
-      { observationId, contentHash },
-      "Inserted observation, chunk deduplicated by content hash",
-    );
-  }
-
-  return {
-    observationId,
-    chunkId: chunkInserted ? chunkId : null,
-    contentHash,
-    embeddingJobId,
-  };
-}
-
-/**
- * Insert multiple observations in a single transaction.
- * Returns the results for each insertion.
- */
-export function insertObservations(
-  observations: InsertObservationParams[],
-): InsertedObservation[] {
-  const db = getDb();
-  const results: InsertedObservation[] = [];
-  db.transaction((tx) => {
-    // We don't use `tx` directly for individual inserts because
-    // insertObservation uses getDb() internally. Instead, the transaction
-    // wrapper ensures atomicity at the SQLite level.
-    void tx;
-    for (const obs of observations) {
-      results.push(insertObservation(obs));
-    }
-  });
-  return results;
-}
-
-/**
- * Look up an observation by ID.
- */
-export function getObservation(
-  observationId: string,
-): typeof memoryObservations.$inferSelect | undefined {
-  const db = getDb();
-  return db
-    .select()
-    .from(memoryObservations)
-    .where(eq(memoryObservations.id, observationId))
-    .get();
-}
-
-/**
- * Look up a chunk by observation ID. Returns the first chunk linked to the
- * given observation, or undefined if none exists.
- */
-export function getChunkByObservationId(
-  observationId: string,
-): typeof memoryChunks.$inferSelect | undefined {
-  const db = getDb();
-  return db
-    .select()
-    .from(memoryChunks)
-    .where(eq(memoryChunks.observationId, observationId))
-    .get();
-}

package/src/memory/brief-formatting.ts

@@ -1,33 +0,0 @@
-/**
- * Shared formatting helpers for rendering capped markdown sections
- * inside the `<memory_brief>` wrapper.
- */
-
-export interface BriefEntry {
-  /** One-line markdown bullet text (no leading `- `). */
-  text: string;
-}
-
-/**
- * Render a titled markdown section with a capped number of bullet entries.
- *
- * Returns `null` when `entries` is empty so callers can easily omit absent
- * sections.  The output is a markdown string like:
- *
- * ```
- * ### Time-Relevant Context
- * - Meeting with Alice in 2 hours
- * - Quarterly review deadline tomorrow
- * ```
- */
-export function renderBriefSection(
-  title: string,
-  entries: BriefEntry[],
-  maxEntries: number,
-): string | null {
-  if (entries.length === 0) return null;
-
-  const capped = entries.slice(0, maxEntries);
-  const bullets = capped.map((e) => `- ${e.text}`).join("\n");
-  return `### ${title}\n${bullets}`;
-}

package/src/memory/brief-open-loops.ts

@@ -1,266 +0,0 @@
-/**
- * Open-loop brief compiler.
- *
- * Merges reducer-created open_loops rows with live task queue (work items)
- * and follow-up state into a ranked, deduplicated list of bullets for the
- * memory brief.
- *
- * Ranking tiers (highest first):
- *   1. Overdue     — dueAt in the past
- *   2. Due ≤ 24 h  — dueAt within the next 24 hours
- *   3. Due ≤ 7 d   — dueAt within the next 7 days
- *   4. High-priority / blocked — work items at priority tier 0 or
- *      follow-ups with status "nudged"
- *   5. Recently touched — updatedAt within the last 48 hours
- *
- * After ranked items, at most ONE low-salience loop is resurfaced via
- * deterministic pseudo-random sampling seeded by `scopeId + userMessageId`.
- * The resurfaced loop's `surfacedAt` is updated so it won't repeat
- * immediately.
- */
-
-import { and, eq } from "drizzle-orm";
-
-import {
-  type BriefFollowUp,
-  getPendingAndOverdueFollowUps,
-} from "../followups/followup-store.js";
-import {
-  type ActionableWorkItem,
-  getActionableWorkItems,
-} from "../tasks/task-store.js";
-import { getDb } from "./db.js";
-import { updateLastSurfacedAt } from "./reducer-store.js";
-import { openLoops } from "./schema.js";
-
-// ── Constants ─────────────────────────────────────────────────────────
-
-const MS_24H = 24 * 60 * 60 * 1000;
-const MS_7D = 7 * 24 * 60 * 60 * 1000;
-const MS_48H = 48 * 60 * 60 * 1000;
-
-// ── Types ─────────────────────────────────────────────────────────────
-
-export interface OpenLoopBullet {
-  /** Dedupe key — one of `loop:<id>`, `work:<id>`, or `followup:<id>`. */
-  key: string;
-  summary: string;
-  tier: number; // 1–5, lower = higher priority
-  source: "loop" | "work_item" | "followup";
-  sourceId: string;
-}
-
-export interface OpenLoopBriefResult {
-  /** Bullets ordered by tier then recency. */
-  bullets: OpenLoopBullet[];
-  /** ID of the resurfaced low-salience loop, if any. */
-  resurfacedLoopId: string | null;
-}
-
-interface OpenLoopRow {
-  id: string;
-  scopeId: string;
-  summary: string;
-  status: string;
-  source: string;
-  dueAt: number | null;
-  surfacedAt: number | null;
-  createdAt: number;
-  updatedAt: number;
-}
-
-// ── Deterministic hash ────────────────────────────────────────────────
-
-/**
- * Simple 32-bit FNV-1a hash for deterministic pseudo-random selection.
- * Not cryptographic — only needs to be well-distributed for sampling.
- */
-function fnv1a(input: string): number {
-  let h = 0x811c9dc5;
-  for (let i = 0; i < input.length; i++) {
-    h ^= input.charCodeAt(i);
-    h = Math.imul(h, 0x01000193);
-  }
-  return h >>> 0; // unsigned 32-bit
-}
-
-// ── Tier assignment ───────────────────────────────────────────────────
-
-function assignLoopTier(loop: OpenLoopRow, now: number): number {
-  if (loop.dueAt != null) {
-    if (loop.dueAt <= now) return 1; // overdue
-    if (loop.dueAt <= now + MS_24H) return 2; // due within 24h
-    if (loop.dueAt <= now + MS_7D) return 3; // due within 7d
-  }
-  if (now - loop.updatedAt <= MS_48H) return 5; // recently touched
-  return 6; // low salience — candidate for resurfacing
-}
-
-function assignWorkItemTier(item: ActionableWorkItem, now: number): number {
-  if (item.priorityTier === 0) return 4; // high priority
-  if (item.status === "awaiting_review") return 4;
-  if (now - item.updatedAt <= MS_48H) return 5;
-  return 6;
-}
-
-function assignFollowUpTier(fu: BriefFollowUp, now: number): number {
-  if (fu.expectedResponseBy != null && fu.expectedResponseBy <= now) return 1;
-  if (fu.expectedResponseBy != null && fu.expectedResponseBy <= now + MS_24H)
-    return 2;
-  if (fu.expectedResponseBy != null && fu.expectedResponseBy <= now + MS_7D)
-    return 3;
-  if (fu.status === "nudged") return 4;
-  if (now - fu.updatedAt <= MS_48H) return 5;
-  return 6;
-}
-
-// ── Compiler ──────────────────────────────────────────────────────────
-
-/**
- * Compile the open-loop section of the memory brief.
- *
- * @param scopeId   Memory scope (e.g. assistant instance ID)
- * @param userMessageId  Current user message ID — used as part of the
- *                       resurfacing seed so the selection is deterministic
- *                       per turn but varies across turns.
- * @param now       Current epoch-ms timestamp (injectable for testing).
- */
-export function compileOpenLoopBrief(
-  scopeId: string,
-  userMessageId: string,
-  now: number = Date.now(),
-): OpenLoopBriefResult {
-  // 1. Gather data from all three sources
-  const loops = getOpenLoopsForScope(scopeId);
-  const workItems = getActionableWorkItems();
-  const followUps = getPendingAndOverdueFollowUps();
-
-  // 2. Convert to bullets with tier assignment
-  const bullets: OpenLoopBullet[] = [];
-  const seenKeys = new Set<string>();
-
-  // Loops first (they are the authoritative open-loop source)
-  for (const loop of loops) {
-    const key = `loop:${loop.id}`;
-    if (seenKeys.has(key)) continue;
-    seenKeys.add(key);
-    bullets.push({
-      key,
-      summary: loop.summary,
-      tier: assignLoopTier(loop, now),
-      source: "loop",
-      sourceId: loop.id,
-    });
-  }
-
-  // Work items — skip if already represented by a loop with matching summary
-  const loopSummaries = new Set(loops.map((l) => l.summary.toLowerCase()));
-  for (const item of workItems) {
-    const key = `work:${item.id}`;
-    if (seenKeys.has(key)) continue;
-    // Deduplicate against loop summaries
-    if (loopSummaries.has(item.title.toLowerCase())) continue;
-    seenKeys.add(key);
-    bullets.push({
-      key,
-      summary: item.title,
-      tier: assignWorkItemTier(item, now),
-      source: "work_item",
-      sourceId: item.id,
-    });
-  }
-
-  // Follow-ups — skip if already represented by a loop
-  for (const fu of followUps) {
-    const key = `followup:${fu.id}`;
-    if (seenKeys.has(key)) continue;
-    const fuSummary =
-      `Awaiting reply on ${fu.channel} (${fu.conversationId})`.toLowerCase();
-    if (loopSummaries.has(fuSummary)) continue;
-    seenKeys.add(key);
-    bullets.push({
-      key,
-      summary: `Awaiting reply on ${fu.channel} (${fu.conversationId})`,
-      tier: assignFollowUpTier(fu, now),
-      source: "followup",
-      sourceId: fu.id,
-    });
-  }
-
-  // 3. Split into ranked (tiers 1–5) and low-salience (tier 6)
-  const ranked: OpenLoopBullet[] = [];
-  const lowSalience: OpenLoopBullet[] = [];
-
-  for (const b of bullets) {
-    if (b.tier <= 5) {
-      ranked.push(b);
-    } else {
-      lowSalience.push(b);
-    }
-  }
-
-  // Sort ranked: by tier ascending, then by source priority (loop > work > followup)
-  ranked.sort((a, b) => {
-    if (a.tier !== b.tier) return a.tier - b.tier;
-    return sourcePriority(a.source) - sourcePriority(b.source);
-  });
-
-  // 4. Deterministic resurfacing of ONE low-salience loop
-  let resurfacedLoopId: string | null = null;
-
-  if (lowSalience.length > 0) {
-    // Only consider loops from the open_loops table for resurfacing
-    // (work items and follow-ups have their own lifecycle)
-    const resurfaceCandidates = lowSalience.filter((b) => b.source === "loop");
-
-    if (resurfaceCandidates.length > 0) {
-      // Sort candidates deterministically by key for stable ordering
-      resurfaceCandidates.sort((a, b) => a.key.localeCompare(b.key));
-
-      const seed = `${scopeId}:${userMessageId}`;
-      const hash = fnv1a(seed);
-      const idx = hash % resurfaceCandidates.length;
-      const picked = resurfaceCandidates[idx];
-
-      // Promote picked to tier 5 and add to ranked output
-      ranked.push({ ...picked, tier: 5 });
-      resurfacedLoopId = picked.sourceId;
-
-      // Update surfacedAt so it is deprioritised on subsequent turns
-      updateLastSurfacedAt(picked.sourceId, now);
-    }
-  }
-
-  // Re-sort after potential resurfaced bullet insertion
-  ranked.sort((a, b) => {
-    if (a.tier !== b.tier) return a.tier - b.tier;
-    return sourcePriority(a.source) - sourcePriority(b.source);
-  });
-
-  return {
-    bullets: ranked,
-    resurfacedLoopId,
-  };
-}
-
-// ── Internals ─────────────────────────────────────────────────────────
-
-function sourcePriority(source: "loop" | "work_item" | "followup"): number {
-  switch (source) {
-    case "loop":
-      return 0;
-    case "work_item":
-      return 1;
-    case "followup":
-      return 2;
-  }
-}
-
-function getOpenLoopsForScope(scopeId: string): OpenLoopRow[] {
-  const db = getDb();
-  return db
-    .select()
-    .from(openLoops)
-    .where(and(eq(openLoops.scopeId, scopeId), eq(openLoops.status, "open")))
-    .all() as OpenLoopRow[];
-}