@kernl-sdk/pg 0.1.10 → 0.1.12

package/src/pgvector/sql/query.ts

@@ -0,0 +1,112 @@
+import type {
+  Filter,
+  OrderBy,
+  SearchQuery,
+  RankingSignal,
+} from "@kernl-sdk/retrieval";
+
+import type { PGIndexConfig } from "../types";
+
+/**
+ * Convert QueryOptions to SQL codec inputs.
+ *
+ * pgvector constraints:
+ * - Only single vector signal supported (no multi-signal fusion)
+ * - No hybrid (vector + text) in same signal
+ * - Filter-only and orderBy-only queries are allowed
+ */
+export function sqlize(query: SearchQuery, config: SqlizeConfig): SqlizedQuery {
+  const signals = query.query ?? query.max ?? [];
+
+  // Validate pgvector constraints
+  if (signals.length > 1) {
+    throw new Error(
+      "pgvector does not support multi-signal fusion. " +
+        "Use a single vector signal, or run multiple queries and fuse client-side.",
+    );
+  }
+
+  if (signals.length === 1) {
+    const { weight, ...fields } = signals[0];
+    let vectorCount = 0;
+    let textCount = 0;
+
+    for (const value of Object.values(fields)) {
+      if (value === undefined) continue;
+      if (Array.isArray(value)) vectorCount++;
+      else if (typeof value === "string") textCount++;
+    }
+
+    if (vectorCount > 1) {
+      throw new Error(
+        "pgvector does not support multi-vector fusion. " +
+          "Use a single vector signal, or run multiple queries and fuse client-side.",
+      );
+    }
+
+    if (vectorCount > 0 && textCount > 0) {
+      throw new Error(
+        "pgvector does not support hybrid (vector + text) fusion. " +
+          "Use a single vector signal, or run multiple queries and fuse client-side.",
+      );
+    }
+  }
+
+  return {
+    select: {
+      pkey: config.pkey,
+      signals,
+      binding: config.binding,
+      include: query.include,
+    },
+    where: { filter: query.filter },
+    order: {
+      signals,
+      orderBy: query.orderBy,
+      binding: config.binding,
+      schema: config.schema,
+      table: config.table,
+    },
+    limit: { topK: query.topK ?? 10, offset: query.offset ?? 0 },
+  };
+}
+
+export interface SqlizedQuery {
+  select: SelectInput;
+  where: Omit<WhereInput, "startIdx">;
+  order: OrderInput;
+  limit: Omit<LimitInput, "startIdx">;
+}
+
+export interface SqlizeConfig {
+  pkey: string;
+  schema: string;
+  table: string;
+  binding?: PGIndexConfig;
+}
+
+export interface SelectInput {
+  pkey: string;
+  signals: RankingSignal[];
+  binding?: PGIndexConfig;
+  include?: string[] | boolean;
+}
+
+export interface WhereInput {
+  filter?: Filter;
+  startIdx: number;
+}
+
+export interface OrderInput {
+  signals: RankingSignal[];
+  orderBy?: OrderBy;
+  binding?: PGIndexConfig;
+  schema?: string;
+  table?: string;
+}
+
+export interface LimitInput {
+  topK: number;
+  offset: number;
+  startIdx: number;
+}

package/src/pgvector/sql/schema.ts

@@ -0,0 +1,61 @@
+/**
+ * Schema conversion codecs for pgvector.
+ */
+
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type {
+  FieldSchema,
+  VectorFieldSchema,
+  ScalarFieldSchema,
+} from "@kernl-sdk/retrieval";
+
+type ScalarType = ScalarFieldSchema["type"];
+type Similarity = VectorFieldSchema["similarity"];
+
+/**
+ * Mapping from kernl scalar types to Postgres column types.
+ */
+const SCALAR_TO_PG: Record<string, string> = {
+  string: "TEXT",
+  int: "INTEGER",
+  bigint: "BIGINT",
+  float: "DOUBLE PRECISION",
+  boolean: "BOOLEAN",
+  date: "TIMESTAMPTZ",
+  object: "JSONB",
+  geopoint: "POINT",
+};
+
+/**
+ * Codec for converting FieldSchema to Postgres column type.
+ */
+export const FIELD_TYPE: Codec<FieldSchema, string> = {
+  encode: (schema) => {
+    if (schema.type === "vector") {
+      return `vector(${schema.dimensions})`;
+    }
+    return SCALAR_TO_PG[schema.type] ?? "TEXT";
+  },
+  decode: () => {
+    throw new Error("FIELD_TYPE.decode not implemented");
+  },
+};
+
+/**
+ * Mapping from similarity metric to pgvector operator class.
+ */
+const SIMILARITY_TO_OPCLASS: Record<string, string> = {
+  cosine: "vector_cosine_ops",
+  euclidean: "vector_l2_ops",
+  dot_product: "vector_ip_ops",
+};
+
+/**
+ * Codec for converting similarity metric to pgvector HNSW operator class.
+ */
+export const SIMILARITY: Codec<Similarity | undefined, string> = {
+  encode: (similarity) => SIMILARITY_TO_OPCLASS[similarity ?? "cosine"],
+  decode: () => {
+    throw new Error("SIMILARITY.decode not implemented");
+  },
+};

package/src/pgvector/sql/select.ts

@@ -0,0 +1,100 @@
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type { SelectInput } from "./query";
+
+export interface SQLClause {
+  sql: string;
+  params: unknown[];
+}
+
+/**
+ * pgvector distance operators by similarity metric.
+ */
+const DISTANCE_OPS = {
+  cosine: "<=>",
+  euclidean: "<->",
+  dot_product: "<#>",
+} as const;
+
+/**
+ * Codec for building SELECT clause with score expression.
+ */
+export const SQL_SELECT: Codec<SelectInput, SQLClause> = {
+  encode({ pkey, signals, binding, include }) {
+    const parts: string[] = [`"${pkey}" as id`];
+    const params: unknown[] = [];
+
+    // find vector signal for scoring
+    const vsig = signals.find((s) => {
+      for (const [key, val] of Object.entries(s)) {
+        if (key !== "weight" && Array.isArray(val)) return true;
+      }
+      return false;
+    });
+
+    if (vsig) {
+      // Extract vector field and value
+      let vecField: string | null = null;
+      let vecValue: number[] | null = null;
+
+      for (const [key, val] of Object.entries(vsig)) {
+        if (key !== "weight" && Array.isArray(val)) {
+          vecField = key;
+          vecValue = val as number[];
+          break;
+        }
+      }
+
+      if (vecField && vecValue) {
+        const col = binding?.fields[vecField]?.column ?? vecField;
+        const similarity = binding?.fields[vecField]?.similarity ?? "cosine";
+        const op = DISTANCE_OPS[similarity];
+
+        params.push(JSON.stringify(vecValue));
+
+        // score expression varies by metric
+        switch (similarity) {
+          case "cosine":
+            parts.push(`1 - ("${col}" ${op} $1::vector) as score`);
+            break;
+          case "euclidean":
+            parts.push(`1 / (1 + ("${col}" ${op} $1::vector)) as score`);
+            break;
+          case "dot_product":
+            parts.push(`-("${col}" ${op} $1::vector) as score`);
+            break;
+        }
+      }
+    } else {
+      parts.push("1 as score");
+    }
+
+    // Select columns based on include
+    // Note: We always select a calculated "score", so skip any document field named "score"
+    // to avoid ambiguity and ensure hit.score is always the similarity score.
+    if (binding && include !== false) {
+      const fields = Object.entries(binding.fields);
+      if (Array.isArray(include)) {
+        // Select only specified fields
+        for (const fieldName of include) {
+          if (fieldName === "score") continue; // Skip to avoid overriding calculated score
+          const field = binding.fields[fieldName];
+          if (field) {
+            parts.push(`"${field.column}"`);
+          }
+        }
+      } else {
+        // include: true or undefined → select all (except score)
+        for (const [fieldName, field] of fields) {
+          if (fieldName === "score") continue; // Skip to avoid overriding calculated score
+          parts.push(`"${field.column}"`);
+        }
+      }
+    }
+
+    return { sql: parts.join(", "), params };
+  },
+
+  decode() {
+    throw new Error("SQL_SELECT.decode not implemented");
+  },
+};

package/src/pgvector/sql/where.ts

@@ -0,0 +1,152 @@
+import type { Codec } from "@kernl-sdk/shared/lib";
+import type { Filter, FieldOps } from "@kernl-sdk/retrieval";
+import type { WhereInput } from "./query";
+import type { SQLClause } from "./select";
+
+/**
+ * Codec for building WHERE clause from MongoDB-style filter.
+ */
+export const SQL_WHERE: Codec<WhereInput, SQLClause> = {
+  encode({ filter, startIdx }) {
+    if (!filter) {
+      return { sql: "", params: [] };
+    }
+    return encodeFilter(filter, startIdx);
+  },
+
+  decode() {
+    throw new Error("SQL_WHERE.decode not implemented");
+  },
+};
+
+/**
+ * Encode a filter to SQL (recursive).
+ */
+function encodeFilter(filter: Filter, startIdx: number): SQLClause {
+  const conditions: string[] = [];
+  const params: unknown[] = [];
+  let idx = startIdx;
+
+  for (const [key, value] of Object.entries(filter)) {
+    if (value === undefined) continue;
+
+    // $and
+    if (key === "$and" && Array.isArray(value)) {
+      const parts: string[] = [];
+      for (const sub of value as Filter[]) {
+        const clause = encodeFilter(sub, idx);
+        if (clause.sql) {
+          parts.push(`(${clause.sql})`);
+          params.push(...clause.params);
+          idx += clause.params.length;
+        }
+      }
+      if (parts.length > 0) {
+        conditions.push(`(${parts.join(" AND ")})`);
+      }
+      continue;
+    }
+
+    // $or
+    if (key === "$or" && Array.isArray(value)) {
+      const parts: string[] = [];
+      for (const sub of value as Filter[]) {
+        const clause = encodeFilter(sub, idx);
+        if (clause.sql) {
+          parts.push(`(${clause.sql})`);
+          params.push(...clause.params);
+          idx += clause.params.length;
+        }
+      }
+      if (parts.length > 0) {
+        conditions.push(`(${parts.join(" OR ")})`);
+      }
+      continue;
+    }
+
+    // $not
+    if (key === "$not" && typeof value === "object" && !Array.isArray(value)) {
+      const clause = encodeFilter(value as Filter, idx);
+      if (clause.sql) {
+        conditions.push(`NOT (${clause.sql})`);
+        params.push(...clause.params);
+        idx += clause.params.length;
+      }
+      continue;
+    }
+
+    // field operators
+    if (isFieldOps(value)) {
+      const ops = value;
+
+      if (ops.$eq !== undefined) {
+        conditions.push(`"${key}" = $${idx++}`);
+        params.push(ops.$eq);
+      }
+      if (ops.$neq !== undefined) {
+        conditions.push(`"${key}" != $${idx++}`);
+        params.push(ops.$neq);
+      }
+      if (ops.$gt !== undefined) {
+        conditions.push(`"${key}" > $${idx++}`);
+        params.push(ops.$gt);
+      }
+      if (ops.$gte !== undefined) {
+        conditions.push(`"${key}" >= $${idx++}`);
+        params.push(ops.$gte);
+      }
+      if (ops.$lt !== undefined) {
+        conditions.push(`"${key}" < $${idx++}`);
+        params.push(ops.$lt);
+      }
+      if (ops.$lte !== undefined) {
+        conditions.push(`"${key}" <= $${idx++}`);
+        params.push(ops.$lte);
+      }
+      if (ops.$in !== undefined) {
+        conditions.push(`"${key}" = ANY($${idx++})`);
+        params.push(ops.$in);
+      }
+      if (ops.$nin !== undefined) {
+        conditions.push(`"${key}" != ALL($${idx++})`);
+        params.push(ops.$nin);
+      }
+      if (ops.$contains !== undefined) {
+        conditions.push(`"${key}" ILIKE $${idx++}`);
+        params.push(`%${ops.$contains}%`);
+      }
+      if (ops.$startsWith !== undefined) {
+        conditions.push(`"${key}" ILIKE $${idx++}`);
+        params.push(`${ops.$startsWith}%`);
+      }
+      if (ops.$endsWith !== undefined) {
+        conditions.push(`"${key}" ILIKE $${idx++}`);
+        params.push(`%${ops.$endsWith}`);
+      }
+      if (ops.$exists !== undefined) {
+        conditions.push(
+          ops.$exists ? `"${key}" IS NOT NULL` : `"${key}" IS NULL`,
+        );
+      }
+      continue;
+    }
+
+    // Equality shorthand: { status: "active" }
+    if (value === null) {
+      conditions.push(`"${key}" IS NULL`);
+    } else {
+      conditions.push(`"${key}" = $${idx++}`);
+      params.push(value);
+    }
+  }
+
+  return { sql: conditions.join(" AND "), params };
+}
+
+/**
+ * Check if value is an operator object (has $-prefixed keys).
+ */
+function isFieldOps(value: unknown): value is FieldOps {
+  if (typeof value !== "object" || value === null) return false;
+  return Object.keys(value).some((k) => k.startsWith("$"));
+}

package/src/pgvector/types.ts

@@ -0,0 +1,21 @@
+import type { FieldSchema } from "@kernl-sdk/retrieval";
+
+/**
+ * Mapping from a logical field name to a Postgres column.
+ */
+export interface PGFieldBinding {
+  column: string;
+  type: FieldSchema["type"];
+  dimensions?: number;
+  similarity?: "cosine" | "euclidean" | "dot_product";
+}
+
+/**
+ * Configuration for binding a Postgres table as a search index.
+ */
+export interface PGIndexConfig {
+  schema: string;
+  table: string;
+  pkey: string;
+  fields: Record<string, PGFieldBinding>;
+}

package/src/pgvector/utils.ts

@@ -0,0 +1,24 @@
+/**
+ * pgvector utility functions.
+ */
+
+/**
+ * Parse index id into schema and table.
+ *
+ * - "docs" → { schema: "public", table: "docs" }
+ * - "analytics.events" → { schema: "analytics", table: "events" }
+ */
+export function parseIndexId(id: string): { schema: string; table: string } {
+  const parts = id.split(".");
+  if (parts.length === 2) {
+    return { schema: parts[0], table: parts[1] };
+  }
+  return { schema: "public", table: id };
+}
+
+/**
+ * Check if a value is a numeric array (vector).
+ */
+export function isVector(val: unknown): val is number[] {
+  return Array.isArray(val) && val.length > 0 && typeof val[0] === "number";
+}

package/src/postgres.ts

@@ -1,12 +1,29 @@
 import { Pool } from "pg";
 import type { KernlStorage } from "kernl";
 
-import { PGStorage } from "./storage";
+import { PGStorage, type PGVectorConfig } from "./storage";
+import { PGSearchIndex } from "./pgvector/search";
 
 /**
- * PostgreSQL connection configuration.
+ * Create a PostgreSQL storage adapter for Kernl.
  */
-export type PostgresConfig =
+export function postgres(config: PostgresConfig): KernlStorage {
+  const p = pool(config);
+  return new PGStorage({ pool: p, vector: config.vector });
+}
+
+/**
+ * Create a pgvector-backed search index.
+ */
+export function pgvector(config: ConnectionConfig) {
+  const p = pool(config);
+  return new PGSearchIndex({ pool: p });
+}
+
+/**
+ * Connection options for PostgreSQL.
+ */
+export type ConnectionConfig =
   | { pool: Pool }
   | { connstr: string }
   | {
@@ -18,39 +35,22 @@ export type PostgresConfig =
     };
 
 /**
- * Create a PostgreSQL storage adapter for Kernl.
- *
- * @param config - Connection configuration (pool, connection string, or credentials)
- * @returns KernlStorage instance backed by PostgreSQL
- *
- * @example
- * ```ts
- * // with connection string
- * const storage = postgres({ connstr: "postgresql://localhost/mydb" });
- *
- * // with connection options
- * const storage = postgres({
- *   host: "localhost",
- *   port: 5432,
- *   database: "mydb",
- *   user: "user",
- *   password: "password"
- * });
- *
- * // existing pool
- * const pool = new Pool({ ... });
- * const storage = postgres({ pool });
- * ```
+ * PostgreSQL storage configuration.
  */
-export function postgres(config: PostgresConfig): KernlStorage {
-  let pool: Pool;
+export type PostgresConfig = ConnectionConfig & {
+  /**
+   * Enable pgvector support for semantic search.
+   */
+  vector?: boolean | PGVectorConfig;
+};
 
+function pool(config: ConnectionConfig): Pool {
   if ("pool" in config) {
-    pool = config.pool;
+    return config.pool;
   } else if ("connstr" in config) {
-    pool = new Pool({ connectionString: config.connstr });
+    return new Pool({ connectionString: config.connstr });
   } else {
-    pool = new Pool({
+    return new Pool({
       host: config.host,
       port: config.port,
       database: config.database,
@@ -58,6 +58,4 @@ export function postgres(config: PostgresConfig): KernlStorage {
       password: config.password,
     });
   }
-
-  return new PGStorage({ pool });
 }