@kernl-sdk/turbopuffer 0.1.0

package/src/convert/document.ts

@@ -0,0 +1,95 @@
+/**
+ * Document conversion codecs.
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type {
+  FieldValue,
+  DenseVector,
+  UnknownDocument,
+} from "@kernl-sdk/retrieval";
+import type { Row } from "@turbopuffer/turbopuffer/resources/namespaces";
+
+/**
+ * Codec for converting documents to Turbopuffer Row.
+ *
+ * Documents must have an `id` field. All other fields become attributes.
+ * Undefined values are omitted.
+ */
+export const DOCUMENT: Codec<UnknownDocument, Row> = {
+  encode: (doc) => {
+    const { id, ...fields } = doc;
+
+    if (typeof id !== "string") {
+      throw new Error('Document must have a string "id" field');
+    }
+
+    const row: Row = { id };
+
+    for (const [key, val] of Object.entries(fields)) {
+      if (val !== undefined) {
+        row[key] = encodeFieldValue(val);
+      }
+    }
+
+    return row;
+  },
+
+  decode: (_row) => {
+    throw new Error("DOCUMENT.decode: not implemented");
+  },
+};
+
+/**
+ * Codec for converting document patches to Turbopuffer Row.
+ *
+ * Similar to DOCUMENT but passes through null values to unset fields.
+ */
+export const PATCH: Codec<UnknownDocument, Row> = {
+  encode: (patch) => {
+    const { id, ...fields } = patch;
+
+    if (typeof id !== "string") {
+      throw new Error('Patch must have a string "id" field');
+    }
+
+    const row: Row = { id };
+
+    for (const [key, val] of Object.entries(fields)) {
+      if (val === null) {
+        row[key] = null; // null unsets the field in Turbopuffer
+      } else if (val !== undefined) {
+        row[key] = encodeFieldValue(val);
+      }
+    }
+
+    return row;
+  },
+
+  decode: (_row) => {
+    throw new Error("PATCH.decode: not implemented");
+  },
+};
+
+/**
+ * Check if a value is a DenseVector.
+ */
+function isDenseVector(val: FieldValue): val is DenseVector {
+  return (
+    typeof val === "object" &&
+    val !== null &&
+    "kind" in val &&
+    val.kind === "vector"
+  );
+}
+
+/**
+ * Convert a FieldValue to Turbopuffer attribute value.
+ * Extracts vector values from DenseVector wrapper.
+ */
+function encodeFieldValue(val: FieldValue): unknown {
+  if (isDenseVector(val)) {
+    return val.values;
+  }
+  return val;
+}

package/src/convert/filter.ts

@@ -0,0 +1,123 @@
+/**
+ * Filter conversion codecs.
+ *
+ * Converts MongoDB-style filters to Turbopuffer filter format.
+ */
+
+import type { Filter, FieldOps, ScalarValue } from "@kernl-sdk/retrieval";
+import type { Filter as TpufFilter } from "@turbopuffer/turbopuffer/resources/custom";
+
+/**
+ * Codec for converting Filter to Turbopuffer Filter.
+ */
+export const FILTER = {
+  encode: (filter: Filter): TpufFilter => {
+    const conditions: TpufFilter[] = [];
+
+    for (const [key, value] of Object.entries(filter)) {
+      if (value === undefined) continue;
+
+      // Logical operators
+      if (key === "$and" && Array.isArray(value)) {
+        const sub = (value as Filter[]).map(FILTER.encode);
+        if (sub.length === 1) {
+          conditions.push(sub[0]);
+        } else {
+          conditions.push(["And", sub]);
+        }
+        continue;
+      }
+
+      if (key === "$or" && Array.isArray(value)) {
+        const sub = (value as Filter[]).map(FILTER.encode);
+        conditions.push(["Or", sub]);
+        continue;
+      }
+
+      if (key === "$not") {
+        conditions.push(["Not", FILTER.encode(value as Filter)]);
+        continue;
+      }
+
+      // Field-level filter
+      if (isFieldOps(value)) {
+        conditions.push(...encodeFieldOps(key, value));
+      } else {
+        // Simple equality: { field: value }
+        conditions.push([key, "Eq", value as ScalarValue]);
+      }
+    }
+
+    if (conditions.length === 0) {
+      throw new Error("Empty filter");
+    }
+
+    if (conditions.length === 1) {
+      return conditions[0];
+    }
+
+    return ["And", conditions];
+  },
+
+  decode: (_filter: TpufFilter): Filter => {
+    throw new Error("FILTER.decode: not implemented");
+  },
+};
+
+/**
+ * Check if a value is a FieldOps object (has operator keys like $eq, $gt, etc.)
+ */
+function isFieldOps(value: unknown): value is FieldOps {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return false;
+  }
+  const keys = Object.keys(value);
+  return keys.some((k) => k.startsWith("$"));
+}
+
+/**
+ * Encode field-level operators to Turbopuffer filters.
+ */
+function encodeFieldOps(field: string, ops: FieldOps): TpufFilter[] {
+  const conditions: TpufFilter[] = [];
+
+  if (ops.$eq !== undefined) {
+    conditions.push([field, "Eq", ops.$eq]);
+  }
+  if (ops.$neq !== undefined) {
+    conditions.push([field, "NotEq", ops.$neq]);
+  }
+  if (ops.$gt !== undefined) {
+    conditions.push([field, "Gt", ops.$gt]);
+  }
+  if (ops.$gte !== undefined) {
+    conditions.push([field, "Gte", ops.$gte]);
+  }
+  if (ops.$lt !== undefined) {
+    conditions.push([field, "Lt", ops.$lt]);
+  }
+  if (ops.$lte !== undefined) {
+    conditions.push([field, "Lte", ops.$lte]);
+  }
+  if (ops.$in !== undefined) {
+    conditions.push([field, "In", ops.$in]);
+  }
+  if (ops.$nin !== undefined) {
+    conditions.push([field, "NotIn", ops.$nin]);
+  }
+  if (ops.$contains !== undefined) {
+    conditions.push([field, "Contains", ops.$contains]);
+  }
+  if (ops.$startsWith !== undefined) {
+    conditions.push([field, "Glob", `${ops.$startsWith}*`]);
+  }
+  if (ops.$endsWith !== undefined) {
+    conditions.push([field, "Glob", `*${ops.$endsWith}`]);
+  }
+  if (ops.$exists !== undefined) {
+    // exists: true → NotEq null, exists: false → Eq null
+    conditions.push([field, ops.$exists ? "NotEq" : "Eq", null]);
+  }
+
+  return conditions;
+}

package/src/convert/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Type conversion codecs for Turbopuffer.
+ */
+
+export { SCALAR_TYPE, SIMILARITY, FIELD_SCHEMA, INDEX_SCHEMA } from "./schema";
+export { DOCUMENT, PATCH } from "./document";
+export { FILTER } from "./filter";
+export { QUERY, SEARCH_HIT } from "./query";

package/src/convert/query.ts

@@ -0,0 +1,151 @@
+/**
+ * Query conversion codecs.
+ *
+ * Converts the new RankingSignal-based query format to Turbopuffer params.
+ */
+
+import type {
+  SearchQuery,
+  SearchHit,
+  RankingSignal,
+  UnknownDocument,
+} from "@kernl-sdk/retrieval";
+import type {
+  Row,
+  NamespaceQueryParams,
+} from "@turbopuffer/turbopuffer/resources/namespaces";
+import type { RankBy } from "@turbopuffer/turbopuffer/resources/custom";
+
+import { FILTER } from "./filter";
+
+/**
+ * Codec for converting SearchQuery to Turbopuffer NamespaceQueryParams.
+ */
+export const QUERY = {
+  encode: (query: SearchQuery): NamespaceQueryParams => {
+    const params: NamespaceQueryParams = {};
+
+    // Build rank_by from query signals
+    const signals = query.query ?? query.max;
+    if (signals && signals.length > 0) {
+      params.rank_by = buildRankBy(signals, query.max !== undefined);
+    }
+
+    // top K
+    if (query.topK !== undefined) {
+      params.top_k = query.topK;
+    }
+
+    // filters
+    if (query.filter) {
+      params.filters = FILTER.encode(query.filter);
+    }
+
+    // include attributes
+    if (query.include !== undefined) {
+      if (typeof query.include === "boolean") {
+        params.include_attributes = query.include;
+      } else {
+        params.include_attributes = [...query.include];
+      }
+    }
+
+    return params;
+  },
+
+  decode: (_params: NamespaceQueryParams): SearchQuery => {
+    throw new Error("QUERY.decode: not implemented");
+  },
+};
+
+/**
+ * Codec for converting Turbopuffer Row to SearchHit.
+ */
+export const SEARCH_HIT = {
+  encode: <TDocument = UnknownDocument>(_hit: SearchHit<TDocument>): Row => {
+    throw new Error("SEARCH_HIT.encode: not implemented");
+  },
+
+  decode: <TDocument = UnknownDocument>(
+    row: Row,
+    index: string,
+  ): SearchHit<TDocument> => {
+    const { id, $dist, ...rest } = row;
+
+    const dist = typeof $dist === "number" ? $dist : 0;
+
+    const hit: SearchHit<TDocument> = {
+      id: String(id),
+      index,
+      score: dist === 0 ? 0 : -dist, // convert distance to similarity (negate so higher = better)
+    };
+
+    // include document fields with id
+    hit.document = { id, ...rest } as unknown as Partial<TDocument>;
+
+    return hit;
+  },
+};
+
+/**
+ * Build rank_by from ranking signals.
+ *
+ * Turbopuffer constraints:
+ * - Sum/Max fusion only works with BM25 (text) signals
+ * - Vector search must be a single ANN query
+ * - Hybrid (text + vector) fusion is not supported in a single query
+ */
+function buildRankBy(signals: RankingSignal[], useMax: boolean): RankBy {
+  const textRankBys: RankBy[] = [];
+  const vectorRankBys: RankBy[] = [];
+
+  for (const signal of signals) {
+    const { weight, ...fields } = signal;
+    for (const [field, value] of Object.entries(fields)) {
+      if (value === undefined) continue;
+
+      if (Array.isArray(value)) {
+        vectorRankBys.push(["vector", "ANN", value as number[]]);
+      } else if (typeof value === "string") {
+        textRankBys.push([field, "BM25", value]);
+      }
+    }
+  }
+
+  const hasVector = vectorRankBys.length > 0;
+  const hasText = textRankBys.length > 0;
+
+  if (!hasVector && !hasText) {
+    throw new Error("No ranking signals provided");
+  }
+
+  // hybrid fusion not supported
+  if (hasVector && hasText) {
+    throw new Error(
+      "Turbopuffer does not support hybrid (vector + text) fusion in a single query. " +
+        "Use separate queries and merge results client-side.",
+    );
+  }
+
+  // multi-vector fusion not supported
+  if (vectorRankBys.length > 1) {
+    throw new Error(
+      "Turbopuffer does not support multi-vector fusion. " +
+        "Use separate queries and merge results client-side.",
+    );
+  }
+
+  // single vector query
+  if (hasVector) {
+    return vectorRankBys[0];
+  }
+
+  // single text query
+  if (textRankBys.length === 1) {
+    return textRankBys[0];
+  }
+
+  // multiple text signals: use Sum or Max fusion
+  const fusion = useMax ? "Max" : "Sum";
+  return [fusion, textRankBys] as RankBy;
+}

package/src/convert/schema.ts

@@ -0,0 +1,163 @@
+/**
+ * Schema conversion codecs.
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type {
+  FieldSchema,
+  VectorFieldSchema,
+  ScalarFieldSchema,
+} from "@kernl-sdk/retrieval";
+import type {
+  AttributeSchema,
+  AttributeSchemaConfig,
+  DistanceMetric,
+} from "@turbopuffer/turbopuffer/resources/namespaces";
+
+type Similarity = VectorFieldSchema["similarity"];
+type ScalarType = ScalarFieldSchema["type"];
+type TpufType = string;
+
+/**
+ * Mapping from kernl scalar types to Turbopuffer attribute types.
+ */
+const SCALAR_TO_TPUF: Record<string, TpufType> = {
+  string: "string",
+  int: "int",
+  bigint: "uint", // Unix epoch timestamps in ms
+  float: "int", // tpuf doesn't have float
+  boolean: "bool",
+  date: "datetime",
+  "string[]": "[]string",
+  "int[]": "[]int",
+  "date[]": "[]datetime",
+};
+
+/**
+ * Mapping from Turbopuffer attribute types to kernl scalar types.
+ */
+const TPUF_TO_SCALAR: Record<string, ScalarType> = {
+  string: "string",
+  int: "int",
+  uint: "bigint",
+  bool: "boolean",
+  datetime: "date",
+  "[]string": "string[]",
+  "[]int": "int[]",
+  "[]datetime": "date[]",
+};
+
+/**
+ * Codec for converting kernl scalar types to Turbopuffer attribute types.
+ */
+export const SCALAR_TYPE: Codec<ScalarType, TpufType> = {
+  encode: (type) => SCALAR_TO_TPUF[type] ?? "string",
+  decode: (type) => TPUF_TO_SCALAR[type] ?? "string",
+};
+
+/**
+ * Codec for converting similarity metric to Turbopuffer distance metric.
+ *
+ * Turbopuffer supports: cosine_distance, euclidean_squared
+ * We support: cosine, euclidean, dot_product
+ */
+export const SIMILARITY: Codec<Similarity, DistanceMetric> = {
+  encode: (similarity) => {
+    switch (similarity) {
+      case "euclidean":
+        return "euclidean_squared";
+      case "cosine":
+      case "dot_product":
+      default:
+        return "cosine_distance";
+    }
+  },
+
+  decode: (metric) => {
+    switch (metric) {
+      case "euclidean_squared":
+        return "euclidean";
+      case "cosine_distance":
+      default:
+        return "cosine";
+    }
+  },
+};
+
+/**
+ * Codec-like converter for FieldSchema to Turbopuffer AttributeSchema.
+ *
+ * Takes the field name as context since Turbopuffer requires `ann: true`
+ * only on the special `vector` attribute.
+ */
+export const FIELD_SCHEMA = {
+  encode: (field: FieldSchema, name: string): AttributeSchema => {
+    // Vector fields
+    if (field.type === "vector" || field.type === "sparse-vector") {
+      const vf = field as VectorFieldSchema;
+      const precision = vf.quantization === "f16" ? "f16" : "f32";
+      return {
+        type: `[${vf.dimensions}]${precision}`,
+        ann: name === "vector",
+      };
+    }
+
+    // Scalar fields
+    const config: AttributeSchemaConfig = {
+      type: SCALAR_TYPE.encode(field.type),
+    };
+
+    if (field.filterable) {
+      config.filterable = true;
+    }
+
+    if (field.fts) {
+      config.full_text_search =
+        typeof field.fts === "object"
+          ? { language: field.fts.language as never }
+          : true;
+    }
+
+    return config;
+  },
+
+  decode: () => {
+    throw new Error("FIELD_SCHEMA.decode: not implemented");
+  },
+};
+
+/**
+ * Codec for converting a full schema record.
+ *
+ * Validates that vector fields are named `vector` since Turbopuffer only
+ * supports ANN indexing on that specific attribute name.
+ */
+export const INDEX_SCHEMA: Codec<
+  Record<string, FieldSchema>,
+  Record<string, AttributeSchema>
+> = {
+  encode: (schema) => {
+    const result: Record<string, AttributeSchema> = {};
+
+    for (const [name, field] of Object.entries(schema)) {
+      const isVector =
+        field.type === "vector" || field.type === "sparse-vector";
+
+      // Enforce vector field naming
+      if (isVector && name !== "vector") {
+        throw new Error(
+          `Turbopuffer requires vector fields to be named "vector", got "${name}". ` +
+            `Rename your field or use a different search provider.`,
+        );
+      }
+
+      result[name] = FIELD_SCHEMA.encode(field, name);
+    }
+
+    return result;
+  },
+
+  decode: () => {
+    throw new Error("INDEX_SCHEMA.decode: not implemented");
+  },
+};

package/src/handle.ts

@@ -0,0 +1,104 @@
+/**
+ * Turbopuffer index handle implementation.
+ */
+
+import type Turbopuffer from "@turbopuffer/turbopuffer";
+import type {
+  IndexHandle,
+  DocumentPatch,
+  UpsertResult,
+  PatchResult,
+  DeleteResult,
+  QueryInput,
+  SearchHit,
+  FieldSchema,
+  UnknownDocument,
+} from "@kernl-sdk/retrieval";
+import { normalizeQuery } from "@kernl-sdk/retrieval";
+
+import { DOCUMENT, PATCH, QUERY, SEARCH_HIT } from "./convert";
+
+/**
+ * Handle for operating on a specific Turbopuffer namespace (index).
+ */
+export class TurbopufferIndexHandle<TDocument = UnknownDocument>
+  implements IndexHandle<TDocument>
+{
+  readonly id: string;
+  private client: Turbopuffer;
+
+  constructor(client: Turbopuffer, id: string) {
+    this.client = client;
+    this.id = id;
+  }
+
+  /**
+   * Query the index.
+   */
+  async query(query: QueryInput): Promise<SearchHit<TDocument>[]> {
+    const q = normalizeQuery(query);
+    const ns = this.client.namespace(this.id);
+
+    const res = await ns.query(QUERY.encode(q));
+    if (!res.rows) {
+      return [];
+    }
+
+    return res.rows.map((row) => SEARCH_HIT.decode<TDocument>(row, this.id));
+  }
+
+  /**
+   * Upsert one or more documents.
+   */
+  async upsert(docs: TDocument | TDocument[]): Promise<UpsertResult> {
+    const arr = Array.isArray(docs) ? docs : [docs];
+    if (arr.length === 0) {
+      return { count: 0 };
+    }
+
+    const ns = this.client.namespace(this.id);
+    const rows = arr.map((doc) => DOCUMENT.encode(doc as UnknownDocument));
+
+    await ns.write({ upsert_rows: rows });
+    return { count: arr.length };
+  }
+
+  /**
+   * Patch one or more documents.
+   */
+  async patch(
+    patches: DocumentPatch<TDocument> | DocumentPatch<TDocument>[],
+  ): Promise<PatchResult> {
+    const arr = Array.isArray(patches) ? patches : [patches];
+    if (arr.length === 0) {
+      return { count: 0 };
+    }
+
+    const ns = this.client.namespace(this.id);
+    const rows = arr.map((p) => PATCH.encode(p as UnknownDocument));
+
+    await ns.write({ patch_rows: rows });
+    return { count: arr.length };
+  }
+
+  /**
+   * Delete one or more documents by ID.
+   */
+  async delete(ids: string | string[]): Promise<DeleteResult> {
+    const arr = Array.isArray(ids) ? ids : [ids];
+    if (arr.length === 0) {
+      return { count: 0 };
+    }
+
+    const ns = this.client.namespace(this.id);
+    await ns.write({ deletes: arr });
+    return { count: arr.length };
+  }
+
+  /**
+   * Add a field to the index schema.
+   */
+  async addField(_field: string, _schema: FieldSchema): Promise<void> {
+    throw new Error("addField not supported by Turbopuffer");
+  }
+}

package/src/index.ts

@@ -0,0 +1,31 @@
+/**
+ * Turbopuffer search index adapter.
+ */
+
+import { TurbopufferSearchIndex } from "./search";
+import type { TurbopufferConfig } from "./types";
+
+export { TurbopufferSearchIndex } from "./search";
+export { TurbopufferIndexHandle } from "./handle";
+export type { TurbopufferConfig } from "./types";
+
+/**
+ * Create a Turbopuffer search index.
+ *
+ * @param config - Turbopuffer API key and region
+ * @returns SearchIndex instance backed by Turbopuffer
+ *
+ * @example
+ * ```ts
+ * const tpuf = turbopuffer({
+ *   apiKey: "your-api-key",
+ *   region: "us-east-1",
+ * });
+ *
+ * const docs = tpuf.index("my-index");
+ * await docs.upsert({ id: "doc-1", fields: { text: "Hello" } });
+ * ```
+ */
+export function turbopuffer(config: TurbopufferConfig) {
+  return new TurbopufferSearchIndex(config);
+}