@vellumai/assistant 0.6.4 → 0.6.6

package/src/memory/qdrant-client.test.ts

@@ -0,0 +1,60 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { applyNestedDefaults } from "../config/loader.js";
+import type { AssistantConfig } from "../config/types.js";
+import { resolveQdrantUrl } from "./qdrant-client.js";
+
+const DEFAULT_CONFIG: AssistantConfig = applyNestedDefaults({});
+
+describe("resolveQdrantUrl", () => {
+  const savedPort = process.env.QDRANT_HTTP_PORT;
+  const savedUrl = process.env.QDRANT_URL;
+
+  beforeEach(() => {
+    delete process.env.QDRANT_HTTP_PORT;
+    delete process.env.QDRANT_URL;
+  });
+
+  afterEach(() => {
+    if (savedPort === undefined) delete process.env.QDRANT_HTTP_PORT;
+    else process.env.QDRANT_HTTP_PORT = savedPort;
+    if (savedUrl === undefined) delete process.env.QDRANT_URL;
+    else process.env.QDRANT_URL = savedUrl;
+  });
+
+  test("falls back to config when no env vars are set", () => {
+    expect(resolveQdrantUrl(DEFAULT_CONFIG)).toBe("http://127.0.0.1:6333");
+  });
+
+  test("honours QDRANT_URL when set", () => {
+    process.env.QDRANT_URL = "http://qdrant.example.com:6333";
+    expect(resolveQdrantUrl(DEFAULT_CONFIG)).toBe(
+      "http://qdrant.example.com:6333",
+    );
+  });
+
+  test("QDRANT_HTTP_PORT wins over QDRANT_URL", () => {
+    process.env.QDRANT_URL = "http://qdrant.example.com:6333";
+    process.env.QDRANT_HTTP_PORT = "20200";
+    expect(resolveQdrantUrl(DEFAULT_CONFIG)).toBe("http://127.0.0.1:20200");
+  });
+
+  test("QDRANT_HTTP_PORT wins over config default", () => {
+    process.env.QDRANT_HTTP_PORT = "20200";
+    expect(resolveQdrantUrl(DEFAULT_CONFIG)).toBe("http://127.0.0.1:20200");
+  });
+
+  test("respects a non-default config URL when no env is set", () => {
+    const config: AssistantConfig = {
+      ...DEFAULT_CONFIG,
+      memory: {
+        ...DEFAULT_CONFIG.memory,
+        qdrant: {
+          ...DEFAULT_CONFIG.memory.qdrant,
+          url: "http://custom-host:9999",
+        },
+      },
+    };
+    expect(resolveQdrantUrl(config)).toBe("http://custom-host:9999");
+  });
+});

package/src/memory/qdrant-client.ts

@@ -1,10 +1,35 @@
 import { QdrantClient as QdrantRestClient } from "@qdrant/js-client-rest";
 import { v4 as uuid } from "uuid";
 
+import { getQdrantHttpPortEnv, getQdrantUrlEnv } from "../config/env.js";
+import type { AssistantConfig } from "../config/types.js";
 import { getLogger } from "../util/logger.js";
 
 const log = getLogger("qdrant-client");
 
+/**
+ * Resolve the Qdrant base URL for this process.
+ *
+ * Precedence (highest first):
+ *   1. `QDRANT_HTTP_PORT` — a locally-spawned Qdrant sidecar on 127.0.0.1:<port>.
+ *      Set by the CLI when spawning the daemon with a non-default port
+ *      (multi-local instances).
+ *   2. `QDRANT_URL` — an external Qdrant instance (K8s sidecar, remote URL).
+ *   3. `config.memory.qdrant.url` — static config (defaults to
+ *      `http://127.0.0.1:6333`).
+ *
+ * Every caller that constructs a Qdrant URL should route through this helper
+ * so the precedence stays consistent across the daemon, CLI, and any
+ * background jobs that re-read the config.
+ */
+export function resolveQdrantUrl(config: AssistantConfig): string {
+  const port = getQdrantHttpPortEnv();
+  if (port) return `http://127.0.0.1:${port}`;
+  const url = getQdrantUrlEnv();
+  if (url) return url;
+  return config.memory.qdrant.url;
+}
+
 export interface QdrantSparseVector {
   indices: number[];
   values: number[];
@@ -20,7 +45,13 @@ export interface QdrantClientConfig {
 }
 
 export interface QdrantPointPayload {
-  target_type: "segment" | "item" | "summary" | "media" | "graph_node";
+  target_type:
+    | "segment"
+    | "item"
+    | "summary"
+    | "media"
+    | "graph_node"
+    | "pkb_file";
   target_id: string;
   text: string;
   kind?: string;
@@ -496,6 +527,44 @@ export class VellumQdrantClient {
     }
   }
 
+  /**
+   * Delete all vectors matching target_type, payload path, and memory scope.
+   * Used to remove every chunk belonging to a single PKB file within a scope.
+   * The memory_scope_id filter is required — omitting it would wipe that
+   * path's chunks across every scope that happens to index the same relpath.
+   */
+  async deleteByTargetTypeAndPath(
+    targetType: string,
+    path: string,
+    memoryScopeId: string,
+  ): Promise<void> {
+    await this.ensureCollection();
+
+    const doDelete = () =>
+      this.client.delete(this.collection, {
+        wait: true,
+        filter: {
+          must: [
+            { key: "target_type", match: { value: targetType } },
+            { key: "path", match: { value: path } },
+            { key: "memory_scope_id", match: { value: memoryScopeId } },
+          ],
+        },
+      });
+
+    try {
+      await doDelete();
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        await this.ensureCollection();
+        await doDelete();
+      } else {
+        throw err;
+      }
+    }
+  }
+
   async count(): Promise<number> {
     await this.ensureCollection();
 
@@ -605,6 +674,10 @@ export class VellumQdrantClient {
         field_name: "memory_scope_id",
         field_schema: "keyword",
       }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "path",
+        field_schema: "keyword",
+      }),
     ]);
   }
 
@@ -640,6 +713,79 @@ export class VellumQdrantClient {
     });
   }
 
+  /**
+   * Scroll over every point with a given target_type and return the requested
+   * payload fields. Handles Qdrant's cursor-based pagination internally.
+   *
+   * Used by startup reconciliation (e.g. PKB filesystem diffing) to enumerate
+   * all indexed points for a target type without loading vectors.
+   */
+  async scrollByTargetType(
+    targetType: string,
+    options?: {
+      memoryScopeId?: string;
+      path?: string;
+      batchSize?: number;
+    },
+  ): Promise<Array<{ id: string; payload: Record<string, unknown> }>> {
+    await this.ensureCollection();
+
+    const batchSize = options?.batchSize ?? 256;
+    const must: Array<Record<string, unknown>> = [
+      { key: "target_type", match: { value: targetType } },
+    ];
+    if (options?.memoryScopeId) {
+      must.push({
+        key: "memory_scope_id",
+        match: { value: options.memoryScopeId },
+      });
+    }
+    if (options?.path) {
+      must.push({ key: "path", match: { value: options.path } });
+    }
+    const filter = {
+      must,
+      must_not: [{ key: "_meta", match: { value: true } }],
+    };
+
+    // Guard against a pathological pagination loop.
+    const maxIterations = 10_000;
+    const doScroll = async () => {
+      const out: Array<{ id: string; payload: Record<string, unknown> }> = [];
+      let offset: string | number | undefined = undefined;
+      for (let i = 0; i < maxIterations; i++) {
+        const result = await this.client.scroll(this.collection, {
+          filter,
+          limit: batchSize,
+          with_payload: true,
+          with_vector: false,
+          ...(offset !== undefined ? { offset } : {}),
+        });
+        for (const point of result.points) {
+          const id =
+            typeof point.id === "string" ? point.id : String(point.id);
+          const payload = (point.payload ?? {}) as Record<string, unknown>;
+          out.push({ id, payload });
+        }
+        const next = result.next_page_offset;
+        if (next == null) break;
+        offset = typeof next === "string" ? next : (next as number);
+      }
+      return out;
+    };
+
+    try {
+      return await doScroll();
+    } catch (err) {
+      if (this.isCollectionMissing(err)) {
+        this.collectionReady = false;
+        await this.ensureCollection();
+        return await doScroll();
+      }
+      throw err;
+    }
+  }
+
   private async findByTarget(
     targetType: string,
     targetId: string

package/src/memory/schema/infrastructure.ts

@@ -28,6 +28,7 @@ export const cronJobs = sqliteTable("cron_jobs", {
   reuseConversation: integer("reuse_conversation", { mode: "boolean" })
     .notNull()
     .default(false), // reuse the same conversation across runs
+  script: text("script"), // shell command for script mode (nullable, only used when mode = 'script')
   createdAt: integer("created_at").notNull(),
   updatedAt: integer("updated_at").notNull(),
 });

package/src/memory/schema/oauth.ts

@@ -20,7 +20,7 @@ export const oauthProviders = sqliteTable("oauth_providers", {
   userinfoUrl: text("userinfo_url"),
   baseUrl: text("base_url"),
   defaultScopes: text("default_scopes").notNull().default("[]"),
-  scopePolicy: text("scope_policy").notNull().default("{}"),
+  availableScopes: text("available_scopes"),
   scopeSeparator: text("scope_separator").notNull().default(" "),
   authorizeParams: text("extra_params"),
   pingUrl: text("ping_url"),
@@ -30,6 +30,9 @@ export const oauthProviders = sqliteTable("oauth_providers", {
   revokeUrl: text("revoke_url"),
   revokeBodyTemplate: text("revoke_body_template"),
   managedServiceConfigKey: text("managed_service_config_key"),
+  managedServiceIsPaid: integer("managed_service_is_paid", { mode: "boolean" })
+    .notNull()
+    .default(false),
   displayLabel: text("display_name"),
   description: text("description"),
   dashboardUrl: text("dashboard_url"),

package/src/memory/slack-thread-store.ts

@@ -62,6 +62,43 @@ export function setThreadTs(
   log.debug({ conversationId, channelId, threadTs }, "Thread mapping created");
 }
 
+/**
+ * Read-side accessor for the in-memory thread mapping. Returns the
+ * `threadTs` previously associated with this conversation via
+ * {@link setThreadTs}, or `null` if no mapping exists. Does not bump
+ * `lastUsedAt` or otherwise mutate the entry — pure lookup.
+ */
+export function getThreadTs(conversationId: string): string | null {
+  const mapping = threadMappings.get(conversationId);
+  return mapping ? mapping.threadTs : null;
+}
+
+/**
+ * Read both `threadTs` and `channelId` without mutating the entry. Used by
+ * dispatch to snapshot pre-update state so a turn that ends up rejected
+ * as already-processing can restore the in-flight turn's mapping.
+ */
+export function peekThreadMapping(
+  conversationId: string,
+): { threadTs: string; channelId: string } | null {
+  const mapping = threadMappings.get(conversationId);
+  return mapping
+    ? { threadTs: mapping.threadTs, channelId: mapping.channelId }
+    : null;
+}
+
+/**
+ * Drop any thread mapping associated with this conversation. Called on
+ * inbound Slack turns that arrive at the channel root (no `threadTs` on
+ * the callback URL) so that a stale mapping from a prior in-thread turn
+ * cannot be copied onto the outbound reply's `slackMeta`. Without this,
+ * a channel-root reply following an earlier thread turn would be
+ * persisted as if it belonged to the old thread.
+ */
+export function clearThreadTs(conversationId: string): void {
+  threadMappings.delete(conversationId);
+}
+
 /**
  * Extract the threadTs from a Slack reply callback URL, if present.
  * The gateway encodes threadTs as a query parameter on the callback URL.

package/src/messaging/providers/gmail/adapter.ts

@@ -65,6 +65,7 @@ function extractPlainTextBody(msg: GmailMessage): string {
 function mapGmailMessage(msg: GmailMessage): Message {
   const from = extractHeader(msg, "From");
   const subject = extractHeader(msg, "Subject");
+  const rfc822MessageId = extractHeader(msg, "Message-ID");
 
   // Parse sender name/email from "Name <email>" format
   const emailMatch = from.match(/<([^>]+)>/);
@@ -90,6 +91,7 @@ function mapGmailMessage(msg: GmailMessage): Message {
       subject,
       labelIds: msg.labelIds,
       snippet: msg.snippet,
+      ...(rfc822MessageId ? { rfc822MessageId } : {}),
     },
   };
 }
@@ -251,23 +253,11 @@ export const gmailMessagingProvider: MessagingProvider = {
     options?: HistoryOptions,
   ): Promise<Message[]> {
     const conn = requireConnection(connection);
-    // Get all messages in a Gmail thread
+    const thread = await gmail.getThread(conn, threadId, "full");
+    const messages = thread.messages ?? [];
+    if (!messages.length) return [];
     const limit = options?.limit ?? 50;
-    const listResult = await gmail.listMessages(
-      conn,
-      `thread:${threadId}`,
-      limit,
-    );
-
-    if (!listResult.messages?.length) return [];
-
-    const messages = await gmail.batchGetMessages(
-      conn,
-      listResult.messages.map((m) => m.id),
-      "full",
-    );
-
-    return messages.map(mapGmailMessage);
+    return messages.slice(0, limit).map(mapGmailMessage);
   },
 
   async markRead(

package/src/messaging/providers/gmail/client.ts

@@ -16,6 +16,7 @@ import type {
   GmailMessageListResponse,
   GmailModifyRequest,
   GmailProfile,
+  GmailThread,
   GmailVacationSettings,
 } from "./types.js";
 
@@ -217,6 +218,7 @@ export async function listMessages(
   maxResults = 20,
   pageToken?: string,
   labelIds?: string[],
+  signal?: AbortSignal,
 ): Promise<GmailMessageListResponse> {
   const params = new URLSearchParams();
   if (query) params.set("q", query);
@@ -230,6 +232,7 @@ export async function listMessages(
     "/messages",
     undefined,
     paramsToQuery(params),
+    signal,
   );
 }
 
@@ -256,6 +259,25 @@ export async function getMessage(
   );
 }
 
+/** Get a thread and all its messages by thread ID. */
+export async function getThread(
+  connection: OAuthConnection,
+  threadId: string,
+  format: GmailMessageFormat = "full",
+  metadataHeaders?: string[],
+): Promise<GmailThread> {
+  const params = new URLSearchParams({ format });
+  if (format === "metadata" && metadataHeaders) {
+    for (const h of metadataHeaders) params.append("metadataHeaders", h);
+  }
+  return request<GmailThread>(
+    connection,
+    `/threads/${threadId}`,
+    undefined,
+    paramsToQuery(params),
+  );
+}
+
 /**
  * Parse a single part from a multipart batch response into its HTTP status and JSON body.
  * Each part contains MIME headers, then an embedded HTTP response (status line, headers, body).

package/src/messaging/providers/gmail/types.ts

@@ -44,6 +44,13 @@ export interface GmailMessage {
   raw?: string;
 }
 
+/** Gmail thread response */
+export interface GmailThread {
+  id: string;
+  historyId?: string;
+  messages?: GmailMessage[];
+}
+
 /** Gmail label */
 export interface GmailLabel {
   id: string;

package/src/messaging/providers/slack/adapter.ts

@@ -108,7 +108,8 @@ async function runReadWithFallback<T>(
   if (connection) return call(connection);
   const auth = getReadAuth(undefined);
   const usingUserToken =
-    _cachedSlackWriteAuth !== null && _cachedSlackReadAuth !== _cachedSlackWriteAuth;
+    _cachedSlackWriteAuth !== null &&
+    _cachedSlackReadAuth !== _cachedSlackWriteAuth;
   try {
     return await call(auth);
   } catch (err) {
@@ -203,6 +204,11 @@ function mapMessage(
   channelId: string,
   senderName: string,
 ): Message {
+  // Bot-authored when Slack sets `subtype: "bot_message"` or attributes the
+  // row to a `bot_id` with no user. Backfill callers rely on this flag to
+  // avoid rehydrating assistant/bot replies as user turns.
+  const isBot =
+    msg.subtype === "bot_message" || (msg.bot_id != null && !msg.user);
   return {
     id: msg.ts,
     conversationId: channelId,
@@ -214,6 +220,7 @@ function mapMessage(
     platform: "slack",
     reactions: msg.reactions?.map((r) => ({ name: r.name, count: r.count })),
     hasAttachments: (msg.files?.length ?? 0) > 0,
+    ...(isBot ? { metadata: { isBot: true } } : {}),
   };
 }
 
@@ -234,7 +241,12 @@ export const slackProvider: MessagingProvider = {
   id: "slack",
   displayName: "Slack",
   credentialService: "slack",
-  capabilities: new Set(["reactions", "threads", "join_channel", "leave_channel"]),
+  capabilities: new Set([
+    "reactions",
+    "threads",
+    "join_channel",
+    "leave_channel",
+  ]),
 
   async isConnected(): Promise<boolean> {
     // Socket Mode: check for bot token directly in credential store.