@functional-systems/lambdadb 0.3.4 → 0.4.0

package/src/compat/qdrant/models.ts

@@ -0,0 +1,317 @@
+type JsonObject = { [key: string]: unknown };
+
+export const Distance = {
+  COSINE: "Cosine",
+  EUCLID: "Euclid",
+  DOT: "Dot",
+  MANHATTAN: "Manhattan",
+  Cosine: "Cosine",
+  Euclid: "Euclid",
+  Dot: "Dot",
+  Manhattan: "Manhattan",
+} as const;
+export type Distance = (typeof Distance)[keyof typeof Distance];
+
+export const PayloadSchemaType = {
+  KEYWORD: "keyword",
+  INTEGER: "integer",
+  FLOAT: "float",
+  BOOL: "bool",
+  DATETIME: "datetime",
+  TEXT: "text",
+  UUID: "uuid",
+  GEO: "geo",
+  Keyword: "keyword",
+  Integer: "integer",
+  Float: "float",
+  Bool: "bool",
+  Datetime: "datetime",
+  Text: "text",
+  Uuid: "uuid",
+  Geo: "geo",
+} as const;
+export type PayloadSchemaType =
+  (typeof PayloadSchemaType)[keyof typeof PayloadSchemaType];
+
+export const UpdateStatus = {
+  ACKNOWLEDGED: "acknowledged",
+  COMPLETED: "completed",
+} as const;
+export type UpdateStatus = (typeof UpdateStatus)[keyof typeof UpdateStatus];
+
+export type PointId = string | number;
+export type DenseVector = number[];
+export type NamedVector = {
+  [name: string]: DenseVector | SparseVector | JsonObject;
+};
+export type PointVector = DenseVector | NamedVector;
+
+function assignExtras<T extends object>(target: T, init: JsonObject): T {
+  for (const [key, value] of Object.entries(init)) {
+    if (!(key in target)) {
+      Object.assign(target, { [key]: value });
+    }
+  }
+  return target;
+}
+
+export class VectorParams {
+  size: number;
+  distance: Distance;
+
+  constructor(init: { size: number; distance?: Distance | string | undefined }) {
+    this.size = init.size;
+    this.distance = normalizeDistance(init.distance);
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class SparseVector {
+  indices: number[];
+  values: number[];
+
+  constructor(init: { indices: number[]; values: number[] }) {
+    this.indices = init.indices;
+    this.values = init.values;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class PointStruct {
+  id: PointId;
+  vector: PointVector;
+  payload: JsonObject | undefined;
+
+  constructor(init: {
+    id: PointId;
+    vector: PointVector;
+    payload?: JsonObject | null | undefined;
+  }) {
+    this.id = init.id;
+    this.vector = init.vector;
+    this.payload = init.payload ?? undefined;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class MatchValue {
+  value: unknown;
+
+  constructor(init: { value: unknown }) {
+    this.value = init.value;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class MatchAny {
+  any: unknown[];
+
+  constructor(init: { any: unknown[] }) {
+    this.any = init.any;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class MatchExcept {
+  except: unknown[];
+  except_: unknown[];
+
+  constructor(init: { except?: unknown[] | undefined; except_?: unknown[] | undefined }) {
+    this.except = init.except ?? init.except_ ?? [];
+    this.except_ = this.except;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class MatchText {
+  text: string;
+
+  constructor(init: { text: string }) {
+    this.text = init.text;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class Range {
+  gt: unknown;
+  gte: unknown;
+  lt: unknown;
+  lte: unknown;
+
+  constructor(init: {
+    gt?: unknown;
+    gte?: unknown;
+    lt?: unknown;
+    lte?: unknown;
+  }) {
+    this.gt = init.gt;
+    this.gte = init.gte;
+    this.lt = init.lt;
+    this.lte = init.lte;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export type MatchCondition =
+  | MatchValue
+  | MatchAny
+  | MatchExcept
+  | MatchText
+  | JsonObject;
+
+export class FieldCondition {
+  key: string;
+  match: MatchCondition | undefined;
+  range: Range | JsonObject | undefined;
+
+  constructor(init: {
+    key: string;
+    match?: MatchCondition | null | undefined;
+    range?: Range | JsonObject | null | undefined;
+  }) {
+    this.key = init.key;
+    this.match = init.match ?? undefined;
+    this.range = init.range ?? undefined;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class HasIdCondition {
+  hasId: PointId[];
+  has_id: PointId[];
+
+  constructor(init: { hasId?: PointId[] | undefined; has_id?: PointId[] | undefined }) {
+    this.hasId = init.hasId ?? init.has_id ?? [];
+    this.has_id = this.hasId;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export type Condition = FieldCondition | HasIdCondition | JsonObject;
+
+export class Filter {
+  must: Condition[] | undefined;
+  should: Condition[] | undefined;
+  mustNot: Condition[] | undefined;
+  must_not: Condition[] | undefined;
+
+  constructor(init: {
+    must?: Condition[] | null | undefined;
+    should?: Condition[] | null | undefined;
+    mustNot?: Condition[] | null | undefined;
+    must_not?: Condition[] | null | undefined;
+  } = {}) {
+    this.must = init.must ?? undefined;
+    this.should = init.should ?? undefined;
+    this.mustNot = init.mustNot ?? init.must_not ?? undefined;
+    this.must_not = this.mustNot;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class SearchParams {
+  hnswEf: number | undefined;
+  hnsw_ef: number | undefined;
+  exact: boolean | undefined;
+
+  constructor(init: {
+    hnswEf?: number | undefined;
+    hnsw_ef?: number | undefined;
+    exact?: boolean | undefined;
+  } = {}) {
+    this.hnswEf = init.hnswEf ?? init.hnsw_ef;
+    this.hnsw_ef = this.hnswEf;
+    this.exact = init.exact;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class ScoredPoint {
+  id: PointId;
+  version: number | undefined;
+  score: number | undefined;
+  payload: JsonObject | undefined;
+  vector: unknown;
+
+  constructor(init: {
+    id: PointId;
+    version?: number | undefined;
+    score?: number | undefined;
+    payload?: JsonObject | null | undefined;
+    vector?: unknown;
+  }) {
+    this.id = init.id;
+    this.version = init.version;
+    this.score = init.score;
+    this.payload = init.payload ?? undefined;
+    this.vector = init.vector;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class Record {
+  id: PointId;
+  payload: JsonObject | undefined;
+  vector: unknown;
+
+  constructor(init: {
+    id: PointId;
+    payload?: JsonObject | null | undefined;
+    vector?: unknown;
+  }) {
+    this.id = init.id;
+    this.payload = init.payload ?? undefined;
+    this.vector = init.vector;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class QueryResponse {
+  points: ScoredPoint[];
+
+  constructor(init: { points: ScoredPoint[] }) {
+    this.points = init.points;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class UpdateResult {
+  operationId: PointId | undefined;
+  operation_id: PointId | undefined;
+  status: UpdateStatus;
+
+  constructor(init: {
+    operationId?: PointId | undefined;
+    operation_id?: PointId | undefined;
+    status?: UpdateStatus | string | undefined;
+  } = {}) {
+    this.operationId = init.operationId ?? init.operation_id;
+    this.operation_id = this.operationId;
+    this.status = normalizeUpdateStatus(init.status);
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+export class CountResult {
+  count: number;
+
+  constructor(init: { count: number }) {
+    this.count = init.count;
+    assignExtras(this, init as JsonObject);
+  }
+}
+
+function normalizeDistance(distance: Distance | string | undefined): Distance {
+  const value = distance ?? Distance.COSINE;
+  if (value === "Cosine" || value === "cosine") return Distance.COSINE;
+  if (value === "Euclid" || value === "euclidean") return Distance.EUCLID;
+  if (value === "Dot" || value === "dot_product") return Distance.DOT;
+  if (value === "Manhattan" || value === "manhattan") return Distance.MANHATTAN;
+  return value as Distance;
+}
+
+function normalizeUpdateStatus(status: UpdateStatus | string | undefined): UpdateStatus {
+  const value = status ?? UpdateStatus.COMPLETED;
+  if (value === UpdateStatus.ACKNOWLEDGED) return UpdateStatus.ACKNOWLEDGED;
+  return UpdateStatus.COMPLETED;
+}

package/src/compat/qdrant.ts

@@ -0,0 +1,3 @@
+export { QdrantCompatClient, QdrantClient } from "./qdrant/client.js";
+export * as models from "./qdrant/models.js";
+export * from "./qdrant/errors.js";

package/test/integration/qdrant-compat-external.test.mjs

@@ -0,0 +1,226 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { Document as LangChainDocument } from "@langchain/core/documents";
+import { QdrantVectorStore as LangChainQdrantVectorStore } from "@langchain/qdrant";
+import { Settings } from "@llamaindex/core/global";
+import { TextNode } from "@llamaindex/core/schema";
+import { QdrantVectorStore as LlamaIndexQdrantVectorStore } from "@llamaindex/qdrant";
+
+import { QdrantCompatClient } from "../../dist/esm/compat/qdrant.js";
+
+const shouldRun = process.env.LAMBDADB_RUN_EXTERNAL_INTEGRATION_TESTS === "1";
+
+class StaticEmbeddings {
+  async embedDocuments(texts) {
+    return texts.map(vectorForText);
+  }
+
+  async embedQuery(text) {
+    return vectorForText(text);
+  }
+}
+
+class InMemoryDocs {
+  constructor(collection) {
+    this.collection = collection;
+  }
+
+  async upsert(body) {
+    for (const doc of body.docs) {
+      this.collection.docs.set(String(doc.id), doc);
+    }
+    return { message: "ok" };
+  }
+
+  async fetch(body) {
+    return {
+      docs: body.ids
+        .map((id) => this.collection.docs.get(String(id)))
+        .filter(Boolean)
+        .map((doc) => ({ collection: this.collection.name, doc })),
+      total: body.ids.length,
+      took: 0,
+      isDocsInline: true,
+    };
+  }
+
+  async delete(body) {
+    for (const id of body.ids) {
+      this.collection.docs.delete(String(id));
+    }
+    return { message: "ok" };
+  }
+
+  async list(body = {}) {
+    return {
+      docs: Array.from(this.collection.docs.values())
+        .slice(0, body.size ?? 10)
+        .map((doc) => ({ collection: this.collection.name, doc })),
+      isDocsInline: true,
+      total: this.collection.docs.size,
+    };
+  }
+}
+
+class InMemoryCollection {
+  constructor(owner, name) {
+    this.owner = owner;
+    this.name = name;
+    this.docs = new Map();
+    this.docApi = new InMemoryDocs(this);
+    this.indexConfigs = {};
+  }
+
+  handle() {
+    return {
+      docs: this.docApi,
+      get: async () => ({
+        collection: {
+          collectionName: this.name,
+          indexConfigs: this.indexConfigs,
+          numDocs: this.docs.size,
+          collectionStatus: "ACTIVE",
+        },
+      }),
+      update: async (body) => {
+        this.indexConfigs = body.indexConfigs;
+        return { collection: { collectionName: this.name, indexConfigs: this.indexConfigs } };
+      },
+      delete: async () => {
+        this.owner.collections.delete(this.name);
+        return { message: "ok" };
+      },
+      query: async (body) => {
+        const knn = body.query.knn;
+        const queryVector = knn.queryVector;
+        const docs = Array.from(this.docs.values())
+          .map((doc) => ({
+            collection: this.name,
+            doc,
+            score: dot(queryVector, doc[knn.field]),
+          }))
+          .sort((a, b) => b.score - a.score)
+          .slice(0, body.size ?? knn.k ?? 10);
+        return {
+          docs,
+          total: docs.length,
+          took: 0,
+          isDocsInline: true,
+        };
+      },
+    };
+  }
+}
+
+class InMemoryLambdaDB {
+  constructor() {
+    this.collections = new Map();
+  }
+
+  async listCollections() {
+    return {
+      collections: Array.from(this.collections.keys()).map((collectionName) => ({
+        collectionName,
+      })),
+    };
+  }
+
+  async createCollection(body) {
+    const collection = this.ensureCollection(body.collectionName);
+    collection.indexConfigs = body.indexConfigs ?? {};
+    return { collection: { collectionName: body.collectionName } };
+  }
+
+  collection(name) {
+    return this.ensureCollection(name).handle();
+  }
+
+  ensureCollection(name) {
+    if (!this.collections.has(name)) {
+      this.collections.set(name, new InMemoryCollection(this, name));
+    }
+    return this.collections.get(name);
+  }
+}
+
+test("langchain qdrant vector store works with QdrantCompatClient", {
+  skip: shouldRun ? false : "Set LAMBDADB_RUN_EXTERNAL_INTEGRATION_TESTS=1",
+}, async () => {
+  Settings.embedModel = {
+    getTextEmbedding: async (text) => vectorForText(text),
+    getQueryEmbedding: async (text) => vectorForText(text),
+  };
+  const client = new QdrantCompatClient(new InMemoryLambdaDB());
+  const embeddings = new StaticEmbeddings();
+
+  const store = await LangChainQdrantVectorStore.fromDocuments(
+    [
+      new LangChainDocument({
+        pageContent: "alpha",
+        metadata: { tenant: "acme" },
+      }),
+      new LangChainDocument({
+        pageContent: "beta",
+        metadata: { tenant: "acme" },
+      }),
+    ],
+    embeddings,
+    {
+      client,
+      collectionName: "langchain-docs",
+    },
+  );
+
+  const results = await store.similaritySearchVectorWithScore([1, 0, 0], 1);
+
+  assert.equal(results.length, 1);
+  assert.equal(results[0][0].pageContent, "alpha");
+  assert.equal(results[0][0].metadata.tenant, "acme");
+});
+
+test("llamaindex qdrant vector store works with QdrantCompatClient", {
+  skip: shouldRun ? false : "Set LAMBDADB_RUN_EXTERNAL_INTEGRATION_TESTS=1",
+}, async () => {
+  const client = new QdrantCompatClient(new InMemoryLambdaDB());
+  const store = new LlamaIndexQdrantVectorStore({
+    client,
+    collectionName: "llamaindex-docs",
+  });
+
+  const alpha = new TextNode({
+    text: "alpha",
+    metadata: { tenant: "acme" },
+  });
+  alpha.id_ = "alpha";
+  alpha.embedding = [1, 0, 0];
+
+  const beta = new TextNode({
+    text: "beta",
+    metadata: { tenant: "acme" },
+  });
+  beta.id_ = "beta";
+  beta.embedding = [0, 1, 0];
+
+  const ids = await store.add([alpha, beta]);
+  assert.deepEqual(ids, ["alpha", "beta"]);
+
+  const results = await store.query({
+    queryEmbedding: [1, 0, 0],
+    similarityTopK: 1,
+  });
+
+  assert.equal(results.ids[0], "alpha");
+  assert.equal(results.nodes[0].getContent(), "alpha");
+});
+
+function vectorForText(text) {
+  if (text.includes("alpha")) return [1, 0, 0];
+  if (text.includes("beta")) return [0, 1, 0];
+  return [0, 0, 1];
+}
+
+function dot(left, right) {
+  if (!Array.isArray(left) || !Array.isArray(right)) return 0;
+  return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
+}

package/test/integration/qdrant-compat-live.test.mjs

@@ -0,0 +1,122 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  QdrantCompatClient,
+  UnsupportedQdrantFeatureError,
+  models,
+} from "../../dist/esm/compat/qdrant.js";
+
+const shouldRun = process.env.LAMBDADB_RUN_LIVE_TESTS === "1";
+const projectApiKey = process.env.LAMBDADB_PROJECT_API_KEY;
+const projectName = process.env.LAMBDADB_PROJECT_NAME || "playground";
+const baseUrl = process.env.LAMBDADB_BASE_URL || "https://api.lambdadb.ai";
+
+test("qdrant compatibility live smoke", {
+  skip: shouldRun && projectApiKey
+    ? false
+    : "Set LAMBDADB_RUN_LIVE_TESTS=1 and LAMBDADB_PROJECT_API_KEY to run live tests",
+}, async () => {
+  const client = new QdrantCompatClient({
+    projectApiKey,
+    projectName,
+    baseUrl,
+  });
+  const collectionName = `qdrant-live-${Date.now()}`;
+
+  try {
+    await client.createCollection(collectionName, {
+      vectorsConfig: new models.VectorParams({
+        size: 3,
+        distance: models.Distance.COSINE,
+      }),
+      payloadSchema: {
+        tenant: models.PayloadSchemaType.KEYWORD,
+      },
+    });
+
+    assert.equal(await client.collectionExists(collectionName), true);
+
+    const upsertResult = await client.upsert(collectionName, {
+      points: [
+        new models.PointStruct({
+          id: 1,
+          vector: [1.0, 0.0, 0.0],
+          payload: { tenant: "acme", title: "alpha" },
+        }),
+        new models.PointStruct({
+          id: 2,
+          vector: [0.0, 1.0, 0.0],
+          payload: { tenant: "acme", title: "beta" },
+        }),
+        new models.PointStruct({
+          id: 3,
+          vector: [0.0, 0.0, 1.0],
+          payload: { tenant: "other", title: "gamma" },
+        }),
+      ],
+    });
+    assert.equal(upsertResult.status, models.UpdateStatus.COMPLETED);
+
+    const queryResult = await client.queryPoints(collectionName, {
+      query: [1.0, 0.0, 0.0],
+      queryFilter: new models.Filter({
+        must: [
+          new models.FieldCondition({
+            key: "tenant",
+            match: new models.MatchValue({ value: "acme" }),
+          }),
+        ],
+      }),
+      limit: 2,
+    });
+    assert.ok(queryResult.points.length > 0);
+    assert.equal(queryResult.points[0].payload?.tenant, "acme");
+
+    const records = await client.retrieve(collectionName, { ids: [1] });
+    assert.equal(records[0]?.id, 1);
+    assert.equal(records[0]?.payload?.tenant, "acme");
+
+    const deleteResult = await client.delete(collectionName, {
+      pointsSelector: [1],
+    });
+    assert.equal(deleteResult.status, models.UpdateStatus.COMPLETED);
+
+    const deleteByFilterResult = await client.delete(collectionName, {
+      pointsSelector: {
+        filter: new models.Filter({
+          must: [
+            new models.FieldCondition({
+              key: "tenant",
+              match: new models.MatchValue({ value: "other" }),
+            }),
+          ],
+        }),
+      },
+    });
+    assert.equal(deleteByFilterResult.status, models.UpdateStatus.COMPLETED);
+
+    const deletedByFilterRecords = await client.retrieve(collectionName, { ids: [3] });
+    assert.equal(deletedByFilterRecords.length, 0);
+
+    await assert.rejects(
+      client.scroll(collectionName, {
+        scrollFilter: new models.Filter({
+          must: [
+            new models.FieldCondition({
+              key: "tenant",
+              match: new models.MatchValue({ value: "acme" }),
+            }),
+          ],
+        }),
+      }),
+      UnsupportedQdrantFeatureError,
+    );
+  } finally {
+    try {
+      await client.deleteCollection(collectionName);
+    } catch (error) {
+      if (error?.name !== "ResourceNotFoundError") throw error;
+    }
+  }
+});