@vellumai/assistant 0.3.0

package/src/memory/search/semantic.ts

@@ -0,0 +1,121 @@
+import { eq } from 'drizzle-orm';
+import { getDb } from '../db.js';
+import { getQdrantClient } from '../qdrant-client.js';
+import {
+  memoryItems,
+  memoryItemSources,
+  memorySegments,
+  memorySummaries,
+} from '../schema.js';
+import type { Candidate } from './types.js';
+import { computeRecencyScore } from './ranking.js';
+
+export async function semanticSearch(
+  queryVector: number[],
+  _provider: string,
+  _model: string,
+  limit: number,
+  excludedMessageIds: string[] = [],
+  scopeIds?: string[],
+): Promise<Candidate[]> {
+  if (limit <= 0) return [];
+
+  const qdrant = getQdrantClient();
+
+  // Overfetch to account for items filtered out post-query (invalidated, excluded, etc.)
+  const fetchLimit = limit * 2;
+  const results = await qdrant.searchWithFilter(
+    queryVector,
+    fetchLimit,
+    ['item', 'summary', 'segment'],
+    excludedMessageIds,
+  );
+
+  const db = getDb();
+  const candidates: Candidate[] = [];
+  for (const result of results) {
+    const { payload, score } = result;
+    const semantic = mapCosineToUnit(score);
+    const createdAt = payload.created_at ?? Date.now();
+
+    if (payload.target_type === 'item') {
+      // Validate the backing memory item is still active and has non-excluded evidence
+      const item = db.select().from(memoryItems).where(eq(memoryItems.id, payload.target_id)).get();
+      if (!item || item.status !== 'active' || item.invalidAt !== null) continue;
+      if (scopeIds && !scopeIds.includes(item.scopeId)) continue;
+      const sources = db.select().from(memoryItemSources)
+        .where(eq(memoryItemSources.memoryItemId, payload.target_id)).all();
+      if (sources.length === 0) continue;
+      if (excludedMessageIds.length > 0) {
+        const nonExcluded = sources.filter((s) => !excludedMessageIds.includes(s.messageId));
+        if (nonExcluded.length === 0) continue;
+      }
+      candidates.push({
+        key: `item:${payload.target_id}`,
+        type: 'item',
+        id: payload.target_id,
+        source: 'semantic',
+        text: `${item.subject}: ${item.statement}`,
+        kind: item.kind,
+        confidence: item.confidence,
+        importance: item.importance ?? 0.5,
+        createdAt: item.lastSeenAt,
+        lexical: 0,
+        semantic,
+        recency: computeRecencyScore(item.lastSeenAt),
+        finalScore: 0,
+      });
+    } else if (payload.target_type === 'summary') {
+      if (scopeIds) {
+        const summary = db.select().from(memorySummaries).where(eq(memorySummaries.id, payload.target_id)).get();
+        if (!summary || !scopeIds.includes(summary.scopeId)) continue;
+      }
+      candidates.push({
+        key: `summary:${payload.target_id}`,
+        type: 'summary',
+        id: payload.target_id,
+        source: 'semantic',
+        text: payload.text.replace(/^\[[^\]]+\]\s*/, ''),
+        kind: payload.kind === 'global' ? 'global_summary' : 'conversation_summary',
+        confidence: 0.6,
+        importance: 0.6,
+        createdAt: payload.last_seen_at ?? createdAt,
+        lexical: 0,
+        semantic,
+        recency: computeRecencyScore(payload.last_seen_at ?? createdAt),
+        finalScore: 0,
+      });
+    } else {
+      if (scopeIds) {
+        const segment = db.select().from(memorySegments).where(eq(memorySegments.id, payload.target_id)).get();
+        if (!segment || !scopeIds.includes(segment.scopeId)) continue;
+      }
+      candidates.push({
+        key: `segment:${payload.target_id}`,
+        type: 'segment',
+        id: payload.target_id,
+        source: 'semantic',
+        text: payload.text,
+        kind: 'segment',
+        confidence: 0.55,
+        importance: 0.5,
+        createdAt,
+        lexical: 0,
+        semantic,
+        recency: computeRecencyScore(createdAt),
+        finalScore: 0,
+      });
+    }
+    if (candidates.length >= limit) break;
+  }
+  return candidates;
+}
+
+export function mapCosineToUnit(value: number): number {
+  return Math.max(0, Math.min(1, (value + 1) / 2));
+}
+
+export function isQdrantConnectionError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  return /ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENETUNREACH|fetch failed/i.test(err.message);
+}

package/src/memory/search/types.ts

@@ -0,0 +1,137 @@
+export type CandidateType = 'segment' | 'item' | 'summary';
+export type CandidateSource =
+  | 'lexical'
+  | 'semantic'
+  | 'recency'
+  | 'entity_direct'
+  | 'entity_relation'
+  | 'item_direct';
+
+export interface Candidate {
+  key: string;
+  type: CandidateType;
+  id: string;
+  source: CandidateSource;
+  text: string;
+  kind: string;
+  confidence: number;
+  importance: number;
+  createdAt: number;
+  lexical: number;
+  semantic: number;
+  recency: number;
+  finalScore: number;
+}
+
+export interface MemoryRecallCandiateDebug {
+  key: string;
+  type: CandidateType;
+  kind: string;
+  finalScore: number;
+  lexical: number;
+  semantic: number;
+  recency: number;
+}
+
+export interface MemoryRecallResult {
+  enabled: boolean;
+  degraded: boolean;
+  reason?: string;
+  provider?: string;
+  model?: string;
+  lexicalHits: number;
+  semanticHits: number;
+  recencyHits: number;
+  entityHits: number;
+  relationSeedEntityCount: number;
+  relationTraversedEdgeCount: number;
+  relationNeighborEntityCount: number;
+  relationExpandedItemCount: number;
+  earlyTerminated: boolean;
+  mergedCount: number;
+  selectedCount: number;
+  rerankApplied: boolean;
+  injectedTokens: number;
+  injectedText: string;
+  latencyMs: number;
+  topCandidates: MemoryRecallCandiateDebug[];
+}
+
+/**
+ * Override the global scope policy for a single retrieval call.
+ * Private threads use this to guarantee they always read from their own
+ * scope AND fall back to 'default', regardless of what the global config says.
+ */
+export interface ScopePolicyOverride {
+  /** The primary scope to query (e.g. a private thread's scope ID). */
+  scopeId: string;
+  /** When true, results from the 'default' scope are included alongside
+   *  the primary scope. Equivalent to 'allow_global_fallback' behavior
+   *  but controlled per-call instead of globally. */
+  fallbackToDefault: boolean;
+}
+
+export interface MemoryRecallOptions {
+  excludeMessageIds?: string[];
+  signal?: AbortSignal;
+  scopeId?: string;
+  /**
+   * When set, overrides both `scopeId` and the global `scopePolicy` config
+   * for this retrieval call. Designed for private threads that need to
+   * guarantee private+default fallback independent of global settings.
+   */
+  scopePolicyOverride?: ScopePolicyOverride;
+  maxInjectTokensOverride?: number;
+}
+
+export interface CollectedCandidates {
+  lexical: Candidate[];
+  recency: Candidate[];
+  semantic: Candidate[];
+  entity: Candidate[];
+  relationSeedEntityCount: number;
+  relationTraversedEdgeCount: number;
+  relationNeighborEntityCount: number;
+  relationExpandedItemCount: number;
+  earlyTerminated: boolean;
+  merged: Candidate[];
+}
+
+export interface EntitySearchResult {
+  candidates: Candidate[];
+  relationSeedEntityCount: number;
+  relationTraversedEdgeCount: number;
+  relationNeighborEntityCount: number;
+  relationExpandedItemCount: number;
+}
+
+export interface MatchedEntityRow {
+  id: string;
+  name: string;
+  type: string;
+  aliases: string | null;
+  mention_count: number;
+}
+
+export interface ItemMetadata {
+  accessCount: number;
+  lastUsedAt: number | null;
+  verificationState: string;
+}
+
+export interface MemorySearchResult {
+  id: string;
+  type: CandidateType;
+  kind: string;
+  text: string;
+  confidence: number;
+  importance: number;
+  createdAt: number;
+  finalScore: number;
+  /** Per-source scores for provenance/debugging */
+  scores: {
+    lexical: number;
+    semantic: number;
+    recency: number;
+  };
+}

package/src/memory/segmenter.ts

@@ -0,0 +1,68 @@
+import { estimateTextTokens } from '../context/token-estimator.js';
+
+const CHARS_PER_TOKEN_APPROX = 4;
+const MIN_SEGMENT_CHARS = 256;
+
+export interface SegmentedText {
+  segmentIndex: number;
+  text: string;
+  tokenEstimate: number;
+}
+
+export function segmentText(
+  text: string,
+  targetTokens: number,
+  overlapTokens: number,
+): SegmentedText[] {
+  const normalized = normalizeInput(text);
+  if (normalized.length === 0) return [];
+
+  const maxChars = Math.max(MIN_SEGMENT_CHARS, targetTokens * CHARS_PER_TOKEN_APPROX);
+  const overlapChars = Math.min(Math.max(0, overlapTokens * CHARS_PER_TOKEN_APPROX), Math.floor(maxChars / 2));
+
+  if (normalized.length <= maxChars) {
+    return [{
+      segmentIndex: 0,
+      text: normalized,
+      tokenEstimate: estimateTextTokens(normalized),
+    }];
+  }
+
+  const segments: SegmentedText[] = [];
+  let start = 0;
+  let index = 0;
+  while (start < normalized.length) {
+    let end = Math.min(normalized.length, start + maxChars);
+
+    if (end < normalized.length) {
+      // Prefer cutting on a whitespace boundary to avoid splitting words.
+      const boundary = normalized.lastIndexOf(' ', end);
+      if (boundary > start + Math.floor(maxChars * 0.6)) {
+        end = boundary;
+      }
+    }
+
+    const chunk = normalized.slice(start, end).trim();
+    if (chunk.length > 0) {
+      segments.push({
+        segmentIndex: index,
+        text: chunk,
+        tokenEstimate: estimateTextTokens(chunk),
+      });
+      index += 1;
+    }
+
+    if (end >= normalized.length) break;
+    start = Math.max(start + 1, end - overlapChars);
+  }
+
+  return segments;
+}
+
+function normalizeInput(text: string): string {
+  return text
+    .replace(/\r\n/g, '\n')
+    .replace(/\t/g, '  ')
+    .replace(/[ ]{2,}/g, ' ')
+    .trim();
+}

package/src/memory/shared-app-links-store.ts

@@ -0,0 +1,138 @@
+/**
+ * Store for cloud-shared app link records.
+ *
+ * Each record holds a .vellumapp zip bundle keyed by a short, shareable token.
+ */
+
+import { eq, lte, or, and, isNull } from 'drizzle-orm';
+import { randomUUID, randomBytes } from 'node:crypto';
+import { getDb } from './db.js';
+import { sharedAppLinks } from './schema.js';
+import type { AppManifest } from '../bundler/manifest.js';
+
+export interface SharedAppLinkRecord {
+  id: string;
+  shareToken: string;
+  bundleData: Buffer;
+  bundleSizeBytes: number;
+  manifestJson: string;
+  downloadCount: number;
+  createdAt: number;
+  expiresAt: number | null;
+}
+
+const SHARE_LINK_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
+
+function generateShareToken(): string {
+  // 16 bytes = 128 bits of entropy (matches UUID standard), base64url-encoded
+  // to a 22-character URL-safe string.
+  return randomBytes(16).toString('base64url');
+}
+
+/** Delete all rows whose effective expiry has passed (including legacy NULL rows). */
+function sweepExpiredLinks(): void {
+  const db = getDb();
+  const now = Date.now();
+  db.delete(sharedAppLinks)
+    .where(or(
+      lte(sharedAppLinks.expiresAt, now),
+      // Legacy rows created before TTL was added have expiresAt = NULL;
+      // treat them as expired once createdAt + TTL has passed.
+      and(isNull(sharedAppLinks.expiresAt), lte(sharedAppLinks.createdAt, now - SHARE_LINK_TTL_MS)),
+    ))
+    .run();
+}
+
+export function createSharedAppLink(
+  bundleData: Buffer,
+  manifest: AppManifest,
+): { id: string; shareToken: string } {
+  sweepExpiredLinks();
+
+  const db = getDb();
+  const id = randomUUID();
+  const shareToken = generateShareToken();
+  const now = Date.now();
+
+  db.insert(sharedAppLinks)
+    .values({
+      id,
+      shareToken,
+      bundleData,
+      bundleSizeBytes: bundleData.length,
+      manifestJson: JSON.stringify(manifest),
+      downloadCount: 0,
+      createdAt: now,
+      expiresAt: now + SHARE_LINK_TTL_MS,
+    })
+    .run();
+
+  return { id, shareToken };
+}
+
+export function getSharedAppLink(shareToken: string): SharedAppLinkRecord | null {
+  const db = getDb();
+  const row = db
+    .select()
+    .from(sharedAppLinks)
+    .where(eq(sharedAppLinks.shareToken, shareToken))
+    .get();
+
+  if (!row) return null;
+
+  const effectiveExpiry = row.expiresAt ?? row.createdAt + SHARE_LINK_TTL_MS;
+  if (effectiveExpiry < Date.now()) return null;
+
+  return {
+    id: row.id,
+    shareToken: row.shareToken,
+    bundleData: row.bundleData as Buffer,
+    bundleSizeBytes: row.bundleSizeBytes,
+    manifestJson: row.manifestJson,
+    downloadCount: row.downloadCount,
+    createdAt: row.createdAt,
+    expiresAt: row.expiresAt,
+  };
+}
+
+export function deleteSharedAppLink(id: string): boolean {
+  const db = getDb();
+  const existing = db
+    .select({ id: sharedAppLinks.id })
+    .from(sharedAppLinks)
+    .where(eq(sharedAppLinks.id, id))
+    .get();
+
+  if (!existing) return false;
+
+  db.delete(sharedAppLinks)
+    .where(eq(sharedAppLinks.id, id))
+    .run();
+
+  return true;
+}
+
+export function deleteSharedAppLinkByToken(shareToken: string): boolean {
+  const db = getDb();
+  const existing = db
+    .select({ id: sharedAppLinks.id })
+    .from(sharedAppLinks)
+    .where(eq(sharedAppLinks.shareToken, shareToken))
+    .get();
+
+  if (!existing) return false;
+
+  db.delete(sharedAppLinks)
+    .where(eq(sharedAppLinks.shareToken, shareToken))
+    .run();
+
+  return true;
+}
+
+export function incrementDownloadCount(shareToken: string): void {
+  const db = getDb();
+  const raw = (db as unknown as { $client: import('bun:sqlite').Database }).$client;
+  raw.prepare(
+    `UPDATE shared_app_links SET download_count = download_count + 1 WHERE share_token = ?`,
+  ).run(shareToken);
+}

package/src/memory/tool-usage-store.ts

@@ -0,0 +1,62 @@
+import { v4 as uuid } from 'uuid';
+import { desc, lt, count } from 'drizzle-orm';
+import { getDb } from './db.js';
+import { toolInvocations } from './schema.js';
+import { getLogger } from '../util/logger.js';
+
+export interface ToolInvocationRecord {
+  conversationId: string;
+  toolName: string;
+  input: string;
+  result: string;
+  decision: string;
+  riskLevel: string;
+  durationMs: number;
+}
+
+export function recordToolInvocation(record: ToolInvocationRecord): void {
+  const db = getDb();
+  db.insert(toolInvocations).values({
+    id: uuid(),
+    conversationId: record.conversationId,
+    toolName: record.toolName,
+    input: record.input,
+    result: record.result,
+    decision: record.decision,
+    riskLevel: record.riskLevel,
+    durationMs: record.durationMs,
+    createdAt: Date.now(),
+  }).run();
+}
+
+export function getRecentInvocations(limit: number) {
+  const db = getDb();
+  return db
+    .select()
+    .from(toolInvocations)
+    .orderBy(desc(toolInvocations.createdAt))
+    .limit(limit)
+    .all();
+}
+
+const log = getLogger('audit-log');
+
+/**
+ * Delete tool invocation records older than the specified number of days.
+ * Returns the number of deleted records. Does nothing if retentionDays is 0.
+ */
+export function rotateToolInvocations(retentionDays: number): number {
+  if (retentionDays <= 0) return 0;
+
+  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+  const db = getDb();
+
+  // Count before delete (Drizzle's .run() returns void on bun-sqlite)
+  const [countRow] = db.select({ value: count() }).from(toolInvocations).where(lt(toolInvocations.createdAt, cutoff)).all();
+  const toDelete = countRow?.value ?? 0;
+  if (toDelete === 0) return 0;
+
+  db.delete(toolInvocations).where(lt(toolInvocations.createdAt, cutoff)).run();
+  log.info(`Rotated ${toDelete} audit log entries older than ${retentionDays} day(s)`);
+  return toDelete;
+}

package/src/messaging/activity-analyzer.ts

@@ -0,0 +1,76 @@
+/**
+ * Activity classifier for messaging conversations.
+ *
+ * Takes Conversation[] from any provider and groups them by activity level.
+ * Works for Slack channels, Gmail senders, Discord servers — anything
+ * that maps to the Conversation type.
+ */
+
+import type { Conversation } from './provider-types.js';
+
+export type ActivityLevel = 'dead' | 'low' | 'medium' | 'high';
+
+export interface ActivityGroup {
+  level: ActivityLevel;
+  conversations: Conversation[];
+}
+
+export interface ActivitySummary {
+  platform: string;
+  totalConversations: number;
+  groups: ActivityGroup[];
+}
+
+const ONE_DAY_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Classify conversations by activity level based on last activity timestamp
+ * and unread count. Returns sorted groups from high to dead.
+ */
+export function classifyActivity(
+  conversations: Conversation[],
+  platform: string,
+  now = Date.now(),
+): ActivitySummary {
+  const groups: Record<ActivityLevel, Conversation[]> = {
+    high: [],
+    medium: [],
+    low: [],
+    dead: [],
+  };
+
+  for (const conv of conversations) {
+    const ageDays = (now - conv.lastActivityAt) / ONE_DAY_MS;
+
+    let level: ActivityLevel;
+    if (conv.unreadCount > 0 && ageDays < 1) {
+      level = 'high';
+    } else if (ageDays < 7) {
+      level = 'medium';
+    } else if (ageDays < 30) {
+      level = 'low';
+    } else {
+      level = 'dead';
+    }
+
+    groups[level].push(conv);
+  }
+
+  // Sort each group by lastActivityAt descending
+  for (const level of Object.keys(groups) as ActivityLevel[]) {
+    groups[level].sort((a, b) => b.lastActivityAt - a.lastActivityAt);
+  }
+
+  const result: ActivityGroup[] = [];
+  for (const level of ['high', 'medium', 'low', 'dead'] as ActivityLevel[]) {
+    if (groups[level].length > 0) {
+      result.push({ level, conversations: groups[level] });
+    }
+  }
+
+  return {
+    platform,
+    totalConversations: conversations.length,
+    groups: result,
+  };
+}

package/src/messaging/draft-store.ts

@@ -0,0 +1,88 @@
+/**
+ * Local draft storage for messaging.
+ *
+ * Stores drafts at ~/.vellum/workspace/data/drafts/<platform>/<id>.json
+ * Works across all platforms — Slack, Gmail, Discord, etc.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { getRootDir } from '../util/platform.js';
+
+export interface Draft {
+  id: string;
+  platform: string;
+  conversationId: string;
+  text: string;
+  threadId?: string;
+  subject?: string;
+  createdAt: number;
+  updatedAt: number;
+  metadata?: Record<string, unknown>;
+}
+
+function getDraftsDir(platform: string): string {
+  const dir = join(getRootDir(), 'workspace', 'data', 'drafts', platform);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  return dir;
+}
+
+function getDraftPath(platform: string, id: string): string {
+  return join(getDraftsDir(platform), `${id}.json`);
+}
+
+export function createDraft(opts: {
+  platform: string;
+  conversationId: string;
+  text: string;
+  threadId?: string;
+  subject?: string;
+  metadata?: Record<string, unknown>;
+}): Draft {
+  const now = Date.now();
+  const draft: Draft = {
+    id: randomUUID(),
+    platform: opts.platform,
+    conversationId: opts.conversationId,
+    text: opts.text,
+    threadId: opts.threadId,
+    subject: opts.subject,
+    createdAt: now,
+    updatedAt: now,
+    metadata: opts.metadata,
+  };
+
+  writeFileSync(getDraftPath(draft.platform, draft.id), JSON.stringify(draft, null, 2));
+  return draft;
+}
+
+export function getDraft(platform: string, id: string): Draft | null {
+  const path = getDraftPath(platform, id);
+  if (!existsSync(path)) return null;
+  return JSON.parse(readFileSync(path, 'utf-8')) as Draft;
+}
+
+export function listDrafts(platform: string): Draft[] {
+  const dir = getDraftsDir(platform);
+  const files = readdirSync(dir).filter((f) => f.endsWith('.json'));
+  const drafts: Draft[] = [];
+  for (const file of files) {
+    try {
+      const content = readFileSync(join(dir, file), 'utf-8');
+      drafts.push(JSON.parse(content) as Draft);
+    } catch {
+      // Skip malformed files
+    }
+  }
+  return drafts.sort((a, b) => b.updatedAt - a.updatedAt);
+}
+
+export function deleteDraft(platform: string, id: string): boolean {
+  const path = getDraftPath(platform, id);
+  if (!existsSync(path)) return false;
+  unlinkSync(path);
+  return true;
+}

package/src/messaging/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export { summarizeThread } from './thread-summarizer.js';
+export { triageMessage } from './triage-engine.js';