@functional-systems/lambdadb 0.4.0 → 0.4.2

package/src/compat/qdrant/client.ts

@@ -2,6 +2,7 @@ import { LambdaDBClient, type LambdaDBClientOptions } from "../../client.js";
 import { ResourceNotFoundError } from "../../models/errors/index.js";
 import type { CreateCollectionInput } from "../../types/public.js";
 import {
+  ORIGINAL_ID_FIELD,
   mergeIndexConfigs,
   payloadSchemaToIndexConfigs,
   plainIndexConfigs,
@@ -10,6 +11,7 @@ import {
   queryVectorAndField,
   resultToScoredPoint,
   docToRecord,
+  vectorFieldName,
   vectorConfigToIndexConfigs,
   type IndexConfigs,
 } from "./conversions.js";
@@ -82,10 +84,10 @@ type QueryPointsOptions = {
   queryFilter?: models.Filter | JsonObject | null;
   query_filter?: models.Filter | JsonObject | null;
   limit?: number | undefined;
-  withPayload?: boolean | undefined;
-  with_payload?: boolean | undefined;
-  withVectors?: boolean | undefined;
-  with_vectors?: boolean | undefined;
+  withPayload?: PayloadSelector | undefined;
+  with_payload?: PayloadSelector | undefined;
+  withVectors?: VectorSelector | undefined;
+  with_vectors?: VectorSelector | undefined;
   using?: string | undefined;
   searchParams?: models.SearchParams | JsonObject | null;
   search_params?: models.SearchParams | JsonObject | null;
@@ -98,6 +100,9 @@ type QueryPointsOptions = {
   [key: string]: unknown;
 };
 
+type PayloadSelector = boolean | string[];
+type VectorSelector = boolean | string[];
+
 export class QdrantCompatClient {
   private readonly client: LambdaDBLike;
 
@@ -399,10 +404,10 @@ export class QdrantCompatClient {
     collectionName: string,
     options: {
       ids: models.PointId[];
-      withPayload?: boolean | undefined;
-      with_payload?: boolean | undefined;
-      withVectors?: boolean | undefined;
-      with_vectors?: boolean | undefined;
+      withPayload?: PayloadSelector | undefined;
+      with_payload?: PayloadSelector | undefined;
+      withVectors?: VectorSelector | undefined;
+      with_vectors?: VectorSelector | undefined;
       [key: string]: unknown;
     },
   ): Promise<models.Record[]> {
@@ -415,15 +420,19 @@ export class QdrantCompatClient {
       ...rest
     } = options;
     warnIgnored(rest);
-    const includeVectors = withVectors ?? withVectorsSnake ?? false;
-    const response = await this.client.collection(collectionName).docs.fetch({
+    const payloadSelector = normalizePayloadSelector(withPayload ?? withPayloadSnake ?? true);
+    const vectorSelector = normalizeVectorSelector(withVectors ?? withVectorsSnake ?? false);
+    const fetchBody: JsonObject = {
       ids: ids.map(String),
       consistentRead: true,
-      includeVectors,
-    });
+      includeVectors: vectorSelector !== false,
+    };
+    const fields = fieldsForSelectors(payloadSelector, vectorSelector);
+    if (fields !== undefined) fetchBody["fields"] = fields;
+    const response = await this.client.collection(collectionName).docs.fetch(fetchBody);
     return response.docs.map((doc) => docToRecord(doc, {
-      withPayload: withPayload ?? withPayloadSnake ?? true,
-      withVectors: includeVectors,
+      withPayload: payloadSelector,
+      withVectors: vectorSelector,
     }));
   }
 
@@ -512,7 +521,8 @@ export class QdrantCompatClient {
       warn("Qdrant searchParams are ignored by the LambdaDB compatibility client");
     }
 
-    const includeVectors = withVectors ?? withVectorsSnake ?? false;
+    const payloadSelector = normalizePayloadSelector(withPayload ?? withPayloadSnake ?? true);
+    const vectorSelector = normalizeVectorSelector(withVectors ?? withVectorsSnake ?? false);
     const { vector, field } = queryVectorAndField(query, using);
     const knn: JsonObject = {
       field,
@@ -522,16 +532,19 @@ export class QdrantCompatClient {
     const convertedFilter = filterToLambdaDB(queryFilter ?? queryFilterSnake);
     if (Object.keys(convertedFilter).length > 0) knn["filter"] = convertedFilter;
 
-    const response = await this.client.collection(collectionName).query({
+    const queryBody: JsonObject = {
       query: { knn },
       size: limit,
       consistentRead: true,
-      includeVectors,
-    });
+      includeVectors: vectorSelector !== false,
+    };
+    const fields = fieldsForSelectors(payloadSelector, vectorSelector);
+    if (fields !== undefined) queryBody["fields"] = fields;
+    const response = await this.client.collection(collectionName).query(queryBody);
     return new models.QueryResponse({
       points: response.docs.map((result) => resultToScoredPoint(result, {
-        withPayload: withPayload ?? withPayloadSnake ?? true,
-        withVectors: includeVectors,
+        withPayload: payloadSelector,
+        withVectors: vectorSelector,
       })),
     });
   }
@@ -549,10 +562,10 @@ export class QdrantCompatClient {
       query: unknown;
       limit?: number | undefined;
       filter?: models.Filter | JsonObject | null;
-      with_payload?: boolean | unknown[] | undefined;
-      withPayload?: boolean | unknown[] | undefined;
-      with_vector?: boolean | undefined;
-      withVectors?: boolean | undefined;
+      with_payload?: PayloadSelector | undefined;
+      withPayload?: PayloadSelector | undefined;
+      with_vector?: VectorSelector | undefined;
+      withVectors?: VectorSelector | undefined;
       params?: unknown;
       [key: string]: unknown;
     },
@@ -574,9 +587,9 @@ export class QdrantCompatClient {
     };
     if (filter !== undefined) queryOptions.queryFilter = filter;
     if (limit !== undefined) queryOptions.limit = limit;
-    const payloadSelector = payloadSelectorToBoolean(withPayload ?? withPayloadSnake);
+    const payloadSelector = normalizeOptionalPayloadSelector(withPayload ?? withPayloadSnake);
     if (payloadSelector !== undefined) queryOptions.withPayload = payloadSelector;
-    const vectorSelector = withVectors ?? withVectorSnake;
+    const vectorSelector = normalizeOptionalVectorSelector(withVectors ?? withVectorSnake);
     if (vectorSelector !== undefined) queryOptions.withVectors = vectorSelector;
     if (isObject(params)) queryOptions.searchParams = params;
     return this.queryPoints(collectionName, queryOptions);
@@ -603,10 +616,10 @@ export class QdrantCompatClient {
       scrollFilter?: models.Filter | JsonObject | null;
       scroll_filter?: models.Filter | JsonObject | null;
       limit?: number | undefined;
-      withPayload?: boolean | undefined;
-      with_payload?: boolean | undefined;
-      withVectors?: boolean | undefined;
-      with_vectors?: boolean | undefined;
+      withPayload?: PayloadSelector | undefined;
+      with_payload?: PayloadSelector | undefined;
+      withVectors?: VectorSelector | undefined;
+      with_vectors?: VectorSelector | undefined;
       [key: string]: unknown;
     } = {},
   ): Promise<[models.Record[], string | undefined]> {
@@ -624,14 +637,13 @@ export class QdrantCompatClient {
     if (scrollFilter !== undefined || scrollFilterSnake !== undefined) {
       throw new UnsupportedQdrantFeatureError("Filtered scroll is not supported in v1");
     }
-    if (withVectors === true || withVectorsSnake === true) {
-      throw new UnsupportedQdrantFeatureError("Scroll with vectors is not supported in v1");
-    }
+    const payloadSelector = normalizePayloadSelector(withPayload ?? withPayloadSnake ?? true);
+    const vectorSelector = normalizeVectorSelector(withVectors ?? withVectorsSnake ?? false);
     const response = await this.client.collection(collectionName).docs.list({ size: limit });
     return [
       response.docs.map((doc) => docToRecord(doc, {
-        withPayload: withPayload ?? withPayloadSnake ?? true,
-        withVectors: false,
+        withPayload: payloadSelector,
+        withVectors: vectorSelector,
       })),
       response.nextPageToken,
     ];
@@ -851,9 +863,48 @@ function hasSameIndexConfigs(existing: IndexConfigs, expected: IndexConfigs): bo
   );
 }
 
-function payloadSelectorToBoolean(selector: boolean | unknown[] | undefined): boolean | undefined {
-  if (Array.isArray(selector)) return true;
-  return selector;
+function normalizeOptionalPayloadSelector(selector: unknown): PayloadSelector | undefined {
+  if (selector === undefined) return undefined;
+  return normalizePayloadSelector(selector);
+}
+
+function normalizePayloadSelector(selector: unknown): PayloadSelector {
+  if (selector === true || selector === false) return selector;
+  if (Array.isArray(selector) && selector.every((item) => typeof item === "string")) {
+    return selector;
+  }
+  throw new QdrantCompatValidationError(
+    "withPayload/with_payload must be a boolean or a string field list",
+  );
+}
+
+function normalizeOptionalVectorSelector(selector: unknown): VectorSelector | undefined {
+  if (selector === undefined) return undefined;
+  return normalizeVectorSelector(selector);
+}
+
+function normalizeVectorSelector(selector: unknown): VectorSelector {
+  if (selector === true || selector === false) return selector;
+  if (Array.isArray(selector) && selector.every((item) => typeof item === "string")) {
+    return selector;
+  }
+  throw new QdrantCompatValidationError(
+    "withVectors/with_vector must be a boolean or a string vector-name list",
+  );
+}
+
+function fieldsForSelectors(
+  payloadSelector: PayloadSelector,
+  vectorSelector: VectorSelector,
+): { include: string[] } | undefined {
+  if (!Array.isArray(payloadSelector)) return undefined;
+  if (vectorSelector === true) return undefined;
+
+  const include = new Set<string>([ORIGINAL_ID_FIELD, ...payloadSelector]);
+  if (Array.isArray(vectorSelector)) {
+    for (const name of vectorSelector) include.add(vectorFieldName(name));
+  }
+  return { include: Array.from(include) };
 }
 
 function isLambdaDBLike(value: unknown): value is LambdaDBLike {

package/src/compat/qdrant/conversions.ts

@@ -96,19 +96,19 @@ export function pointToDoc(point: unknown): JsonObject {
 
 export function docToRecord(
   input: unknown,
-  options: { withPayload?: boolean; withVectors?: boolean } = {},
+  options: { withPayload?: boolean | string[]; withVectors?: boolean | string[] } = {},
 ): models.Record {
   const doc = unwrapDoc(input);
   return new models.Record({
     id: docId(doc),
-    payload: options.withPayload === false ? undefined : payloadFromDoc(doc),
-    vector: vectorFromDoc(doc, options.withVectors === true),
+    payload: payloadFromDoc(doc, options.withPayload ?? true),
+    vector: vectorFromDoc(doc, options.withVectors ?? false),
   });
 }
 
 export function resultToScoredPoint(
   result: unknown,
-  options: { withPayload?: boolean; withVectors?: boolean } = {},
+  options: { withPayload?: boolean | string[]; withVectors?: boolean | string[] } = {},
 ): models.ScoredPoint {
   const doc = unwrapDoc(result);
   const score = isObject(result) && typeof result["score"] === "number"
@@ -117,8 +117,8 @@ export function resultToScoredPoint(
   return new models.ScoredPoint({
     id: docId(doc),
     score,
-    payload: options.withPayload === false ? undefined : payloadFromDoc(doc),
-    vector: vectorFromDoc(doc, options.withVectors === true),
+    payload: payloadFromDoc(doc, options.withPayload ?? true),
+    vector: vectorFromDoc(doc, options.withVectors ?? false),
   });
 }
 
@@ -356,26 +356,34 @@ function validatePayload(payload: JsonObject): void {
   }
 }
 
-function payloadFromDoc(doc: JsonObject): JsonObject {
+function payloadFromDoc(doc: JsonObject, selector: boolean | string[]): JsonObject | undefined {
+  if (selector === false) return undefined;
+  const selectedFields = Array.isArray(selector) ? new Set(selector) : undefined;
   const payload: JsonObject = {};
   for (const [key, value] of Object.entries(doc)) {
-    if (key !== ID_FIELD && !key.startsWith(RESERVED_PREFIX)) {
+    if (
+      key !== ID_FIELD &&
+      !key.startsWith(RESERVED_PREFIX) &&
+      (selectedFields === undefined || selectedFields.has(key))
+    ) {
       payload[key] = value;
     }
   }
   return payload;
 }
 
-function vectorFromDoc(doc: JsonObject, withVectors: boolean): unknown {
-  if (!withVectors) return undefined;
+function vectorFromDoc(doc: JsonObject, selector: boolean | string[]): unknown {
+  if (selector === false) return undefined;
+  const selectedNames = Array.isArray(selector) ? new Set(selector) : undefined;
 
   const named: JsonObject = {};
   let defaultVector: unknown;
   for (const [key, value] of Object.entries(doc)) {
-    if (key === DEFAULT_VECTOR_NAME) {
+    if (key === DEFAULT_VECTOR_NAME && shouldIncludeVectorName("", key, selectedNames)) {
       defaultVector = value;
     } else if (key.startsWith(`${DEFAULT_VECTOR_NAME}_`)) {
-      named[key.slice(DEFAULT_VECTOR_NAME.length + 1)] = value;
+      const name = key.slice(DEFAULT_VECTOR_NAME.length + 1);
+      if (shouldIncludeVectorName(name, key, selectedNames)) named[name] = value;
     }
   }
 
@@ -386,6 +394,16 @@ function vectorFromDoc(doc: JsonObject, withVectors: boolean): unknown {
   return defaultVector;
 }
 
+function shouldIncludeVectorName(
+  qdrantName: string,
+  internalName: string,
+  selectedNames: Set<string> | undefined,
+): boolean {
+  return selectedNames === undefined ||
+    selectedNames.has(qdrantName) ||
+    selectedNames.has(internalName);
+}
+
 function docId(doc: JsonObject): PointId {
   const originalId = doc[ORIGINAL_ID_FIELD];
   if (typeof originalId === "string" || typeof originalId === "number") {

package/test/integration/qdrant-compat-external.test.mjs

@@ -38,7 +38,7 @@ class InMemoryDocs {
       docs: body.ids
         .map((id) => this.collection.docs.get(String(id)))
         .filter(Boolean)
-        .map((doc) => ({ collection: this.collection.name, doc })),
+        .map((doc) => ({ collection: this.collection.name, doc: projectDoc(doc, body.fields) })),
       total: body.ids.length,
       took: 0,
       isDocsInline: true,
@@ -46,8 +46,15 @@ class InMemoryDocs {
   }
 
   async delete(body) {
-    for (const id of body.ids) {
-      this.collection.docs.delete(String(id));
+    if (body.ids) {
+      for (const id of body.ids) {
+        this.collection.docs.delete(String(id));
+      }
+    }
+    if (body.filter) {
+      for (const [id, doc] of this.collection.docs.entries()) {
+        if (matchesLambdaDBFilter(doc, body.filter)) this.collection.docs.delete(id);
+      }
     }
     return { message: "ok" };
   }
@@ -56,7 +63,7 @@ class InMemoryDocs {
     return {
       docs: Array.from(this.collection.docs.values())
         .slice(0, body.size ?? 10)
-        .map((doc) => ({ collection: this.collection.name, doc })),
+        .map((doc) => ({ collection: this.collection.name, doc: projectDoc(doc, body.fields) })),
       isDocsInline: true,
       total: this.collection.docs.size,
     };
@@ -95,9 +102,10 @@ class InMemoryCollection {
         const knn = body.query.knn;
         const queryVector = knn.queryVector;
         const docs = Array.from(this.docs.values())
+          .filter((doc) => matchesLambdaDBFilter(doc, knn.filter))
           .map((doc) => ({
             collection: this.name,
-            doc,
+            doc: projectDoc(doc, body.fields),
             score: dot(queryVector, doc[knn.field]),
           }))
           .sort((a, b) => b.score - a.score)
@@ -179,6 +187,48 @@ test("langchain qdrant vector store works with QdrantCompatClient", {
   assert.equal(results[0][0].metadata.tenant, "acme");
 });
 
+test("langchain qdrant vector store supports filter search and delete", {
+  skip: shouldRun ? false : "Set LAMBDADB_RUN_EXTERNAL_INTEGRATION_TESTS=1",
+}, async () => {
+  const client = new QdrantCompatClient(new InMemoryLambdaDB());
+  const embeddings = new StaticEmbeddings();
+
+  const store = await LangChainQdrantVectorStore.fromDocuments(
+    [
+      new LangChainDocument({
+        pageContent: "alpha",
+        metadata: { tenant: "acme", group: "keep" },
+      }),
+      new LangChainDocument({
+        pageContent: "beta",
+        metadata: { tenant: "other", group: "remove" },
+      }),
+    ],
+    embeddings,
+    {
+      client,
+      collectionName: "langchain-filter-docs",
+    },
+  );
+
+  const filteredResults = await store.similaritySearchVectorWithScore([1, 0, 0], 2, {
+    must: [{ key: "content", match: { value: "alpha" } }],
+  });
+
+  assert.equal(filteredResults.length, 1);
+  assert.equal(filteredResults[0][0].pageContent, "alpha");
+
+  await store.delete({
+    filter: {
+      must: [{ key: "content", match: { value: "beta" } }],
+    },
+  });
+
+  const remainingResults = await store.similaritySearchVectorWithScore([0, 1, 0], 2);
+  assert.equal(remainingResults.length, 1);
+  assert.equal(remainingResults[0][0].pageContent, "alpha");
+});
+
 test("llamaindex qdrant vector store works with QdrantCompatClient", {
   skip: shouldRun ? false : "Set LAMBDADB_RUN_EXTERNAL_INTEGRATION_TESTS=1",
 }, async () => {
@@ -214,6 +264,42 @@ test("llamaindex qdrant vector store works with QdrantCompatClient", {
   assert.equal(results.nodes[0].getContent(), "alpha");
 });
 
+test("llamaindex qdrant vector store supports delete by filter", {
+  skip: shouldRun ? false : "Set LAMBDADB_RUN_EXTERNAL_INTEGRATION_TESTS=1",
+}, async () => {
+  const client = new QdrantCompatClient(new InMemoryLambdaDB());
+  const store = new LlamaIndexQdrantVectorStore({
+    client,
+    collectionName: "llamaindex-delete-docs",
+  });
+
+  const alpha = new TextNode({
+    text: "alpha",
+    metadata: { tenant: "acme" },
+    relationships: { SOURCE: { nodeId: "alpha-ref" } },
+  });
+  alpha.id_ = "alpha";
+  alpha.embedding = [1, 0, 0];
+
+  const beta = new TextNode({
+    text: "beta",
+    metadata: { tenant: "acme" },
+    relationships: { SOURCE: { nodeId: "beta-ref" } },
+  });
+  beta.id_ = "beta";
+  beta.embedding = [0, 1, 0];
+
+  await store.add([alpha, beta]);
+  await store.delete("alpha-ref");
+
+  const results = await store.query({
+    queryEmbedding: [1, 0, 0],
+    similarityTopK: 2,
+  });
+
+  assert.deepEqual(results.ids, ["beta"]);
+});
+
 function vectorForText(text) {
   if (text.includes("alpha")) return [1, 0, 0];
   if (text.includes("beta")) return [0, 1, 0];
@@ -224,3 +310,42 @@ function dot(left, right) {
   if (!Array.isArray(left) || !Array.isArray(right)) return 0;
   return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
 }
+
+function projectDoc(doc, fields) {
+  const include = fields?.include;
+  if (!Array.isArray(include)) return doc;
+  const projected = { id: doc.id };
+  for (const field of include) {
+    if (field in doc) projected[field] = doc[field];
+  }
+  return projected;
+}
+
+function matchesLambdaDBFilter(doc, filter) {
+  if (!filter || Object.keys(filter).length === 0) return true;
+  if (filter.queryString) return matchesQueryString(doc, filter.queryString.query);
+  if (Array.isArray(filter.bool)) return matchesBoolClauses(doc, filter.bool);
+  if (filter.bool?.clauses) return matchesBoolClauses(doc, filter.bool.clauses);
+  return true;
+}
+
+function matchesBoolClauses(doc, clauses) {
+  const shouldClauses = [];
+  for (const clause of clauses) {
+    const matched = matchesLambdaDBFilter(doc, clause);
+    if (clause.occur === "must_not") {
+      if (matched) return false;
+    } else if (clause.occur === "should") {
+      shouldClauses.push(matched);
+    } else if (!matched) {
+      return false;
+    }
+  }
+  return shouldClauses.length === 0 || shouldClauses.some(Boolean);
+}
+
+function matchesQueryString(doc, query) {
+  const [field, ...rest] = String(query).split(":");
+  const expected = rest.join(":");
+  return String(doc[field]) === expected;
+}

package/test/qdrant-compat.test.mjs

@@ -50,7 +50,13 @@ class FakeDocs {
       docs: [
         {
           collection: "docs",
-          doc: { id: "1", _qdrant_id: 1, tenant: "acme" },
+          doc: {
+            id: "1",
+            _qdrant_id: 1,
+            _qdrant_vector: [0.1, 0.2],
+            _qdrant_vector_title: [0.3, 0.4],
+            tenant: "acme",
+          },
         },
       ],
       nextPageToken: undefined,
@@ -328,6 +334,59 @@ test("queryPoints maps dense vectors, filters, and responses", async () => {
   );
 });
 
+test("payload and vector selectors map to LambdaDB fields and qdrant responses", async () => {
+  const fake = new FakeLambdaDB();
+  const collection = fake.collection("docs");
+  collection.query = async function query(body) {
+    this.queries.push(body);
+    return {
+      docs: [
+        {
+          collection: this.name,
+          score: 0.9,
+          doc: {
+            id: "1",
+            _qdrant_id: 1,
+            _qdrant_vector: [0.1, 0.2],
+            _qdrant_vector_title: [0.3, 0.4],
+            tenant: "acme",
+            hidden: "value",
+          },
+        },
+      ],
+    };
+  };
+  const client = new QdrantCompatClient(fake);
+
+  const response = await client.queryPoints("docs", {
+    query: [0.1, 0.2],
+    limit: 1,
+    withPayload: ["tenant"],
+    withVectors: ["title"],
+  });
+
+  assert.deepEqual(response.points[0].payload, { tenant: "acme" });
+  assert.deepEqual(response.points[0].vector, { title: [0.3, 0.4] });
+  assert.deepEqual(collection.queries[0].fields, {
+    include: ["_qdrant_id", "tenant", "_qdrant_vector_title"],
+  });
+  assert.equal(collection.queries[0].includeVectors, true);
+
+  const records = await client.retrieve("docs", {
+    ids: [1],
+    withPayload: ["tenant"],
+    withVectors: false,
+  });
+  assert.deepEqual(records[0].payload, { tenant: "acme" });
+  assert.equal(records[0].vector, undefined);
+  assert.deepEqual(collection.docs.fetches.at(-1), {
+    ids: ["1"],
+    consistentRead: true,
+    includeVectors: false,
+    fields: { include: ["_qdrant_id", "tenant"] },
+  });
+});
+
 test("retrieve, delete, scroll, count, and getCollection return qdrant-style objects", async () => {
   const fake = new FakeLambdaDB();
   fake.indexConfigs = {
@@ -392,8 +451,14 @@ test("retrieve, delete, scroll, count, and getCollection return qdrant-style obj
     },
   ]);
 
-  const [scrollRecords, nextOffset] = await client.scroll("docs", { limit: 3 });
+  const [scrollRecords, nextOffset] = await client.scroll("docs", {
+    limit: 3,
+    withPayload: ["tenant"],
+    withVectors: ["title"],
+  });
   assert.equal(scrollRecords[0].id, 1);
+  assert.deepEqual(scrollRecords[0].payload, { tenant: "acme" });
+  assert.deepEqual(scrollRecords[0].vector, { title: [0.3, 0.4] });
   assert.equal(nextOffset, undefined);
   assert.deepEqual(fake.collection("docs").docs.lists, [{ size: 3 }]);