@convex-dev/rag 0.1.7

package/dist/component/convex.config.d.ts

@@ -0,0 +1,3 @@
+declare const component: import("convex/server").ComponentDefinition<any>;
+export default component;
+//# sourceMappingURL=convex.config.d.ts.map

package/dist/component/convex.config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convex.config.d.ts","sourceRoot":"","sources":["../../src/component/convex.config.ts"],"names":[],"mappings":"AAGA,QAAA,MAAM,SAAS,kDAAyB,CAAC;AAGzC,eAAe,SAAS,CAAC"}

package/dist/component/convex.config.js

@@ -0,0 +1,6 @@
+import { defineComponent } from "convex/server";
+import workpool from "@convex-dev/workpool/convex.config";
+const component = defineComponent("rag");
+component.use(workpool);
+export default component;
+//# sourceMappingURL=convex.config.js.map

package/dist/component/convex.config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convex.config.js","sourceRoot":"","sources":["../../src/component/convex.config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,QAAQ,MAAM,oCAAoC,CAAC;AAE1D,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;AACzC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AAExB,eAAe,SAAS,CAAC"}

package/dist/component/embeddings/importance.d.ts

@@ -0,0 +1,21 @@
+/**
+ * This file contains functions for modifying embeddings to include importance.
+ * Terminology is roughly: a "vector" is an "embedding" + importance.
+ * Users pass in embeddings, the tables and vector search deal with vectors.
+ */
+/**
+ * For a search, we need to add a 0 to the end of the embedding so we ignore
+ * the weight value.
+ */
+export declare function searchVector(embedding: number[]): number[];
+/**
+ * For an importance of x (0 to 1):
+ * @param embedding - The vector to modify with an importance weight.
+ * @param importance - 0 - 1, where 0 is no importance and 1 is full importance.
+ * @returns The vector with the importance added.
+ */
+export declare function vectorWithImportance(embedding: number[], importance: number): number[];
+export declare function modifyImportance(vector: number[], importance: number): number[];
+export declare function getImportance(vector: number[]): number;
+export declare function vectorWithImportanceDimension(dimensions: number): number;
+//# sourceMappingURL=importance.d.ts.map

package/dist/component/embeddings/importance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"importance.d.ts","sourceRoot":"","sources":["../../../src/component/embeddings/importance.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;GAGG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,YAK/C;AACD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,YAwB3E;AAcD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,YAKpE;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,UAE7C;AAED,wBAAgB,6BAA6B,CAAC,UAAU,EAAE,MAAM,UAG/D"}

package/dist/component/embeddings/importance.js

@@ -0,0 +1,67 @@
+/**
+ * This file contains functions for modifying embeddings to include importance.
+ * Terminology is roughly: a "vector" is an "embedding" + importance.
+ * Users pass in embeddings, the tables and vector search deal with vectors.
+ */
+/**
+ * For a search, we need to add a 0 to the end of the embedding so we ignore
+ * the weight value.
+ */
+export function searchVector(embedding) {
+    if (embedding.length === 4096) {
+        return [...embedding.slice(0, 4095), 0];
+    }
+    return [...embedding, 0];
+}
+/**
+ * For an importance of x (0 to 1):
+ * @param embedding - The vector to modify with an importance weight.
+ * @param importance - 0 - 1, where 0 is no importance and 1 is full importance.
+ * @returns The vector with the importance added.
+ */
+export function vectorWithImportance(embedding, importance) {
+    /*
+     * Goal: add a weighting that reduces the magnitude of the target vector after
+     * normalization.
+     * 1. Scale the existing vector by importance and add a weight so |v| = 1.
+     * 2. Search with [...embedding, 0].
+     * e.g.:
+     * Say we have an embedding of 2 numbers [.6, .8]
+     * For 50% importance: [.6 * i, .8 * i, sqrt(1 - i^2)] ->  [.3, .4, .866]
+     * For [.6, .8] we used to get 1.0.
+     * Now we get .6*.3 + .8*.4 + 0*.866 = 0.5
+     */
+    // We drop the final dimension if it'd make it larger than 4096.
+    // Unfortunate current limitation of Convex vector search.
+    const vectorToModify = normalizeVector(embedding.length === 4096 ? embedding.slice(0, 4095) : embedding);
+    const scaled = scaleVector(vectorToModify, importance);
+    // |embedding| == 1
+    // weight^2 + importance^2(|embedding|^2) == 1
+    // weight = sqrt(1 - importance^2)
+    const weight = Math.sqrt(1 - importance ** 2);
+    return [...scaled, weight];
+}
+function normalizeVector(vector) {
+    const sumOfSquares = vector.reduce((acc, v) => acc + v * v, 0);
+    const magnitude = Math.sqrt(sumOfSquares);
+    return magnitude === 0
+        ? vector.map(() => 0)
+        : vector.map((v) => v / magnitude);
+}
+function scaleVector(vector, scale) {
+    return vector.map((v) => v * scale);
+}
+export function modifyImportance(vector, importance) {
+    // Note: we don't need to handle 4096 explicitly here
+    // vectorWithImportance will turn it from 4095 to 4096.
+    const vectorToModify = vector.slice(0, vector.length - 1);
+    return vectorWithImportance(vectorToModify, importance);
+}
+export function getImportance(vector) {
+    return Math.sqrt(1 - vector[vector.length - 1] ** 2);
+}
+export function vectorWithImportanceDimension(dimensions) {
+    // +1 for the importance weighting, but respect global limit
+    return dimensions === 4096 ? 4096 : dimensions + 1;
+}
+//# sourceMappingURL=importance.js.map

package/dist/component/embeddings/importance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"importance.js","sourceRoot":"","sources":["../../../src/component/embeddings/importance.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,SAAmB;IAC9C,IAAI,SAAS,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,CAAC,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC;AAC3B,CAAC;AACD;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAmB,EAAE,UAAkB;IAC1E;;;;;;;;;;OAUG;IACH,gEAAgE;IAChE,0DAA0D;IAC1D,MAAM,cAAc,GAAG,eAAe,CACpC,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CACjE,CAAC;IACF,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;IAEvD,mBAAmB;IACnB,8CAA8C;IAC9C,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,UAAU,IAAI,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,eAAe,CAAC,MAAgB;IACvC,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1C,OAAO,SAAS,KAAK,CAAC;QACpB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,WAAW,CAAC,MAAgB,EAAE,KAAa;IAClD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAgB,EAAE,UAAkB;IACnE,qDAAqD;IACrD,uDAAuD;IACvD,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO,oBAAoB,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAgB;IAC5C,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,UAAkB;IAC9D,4DAA4D;IAC5D,OAAO,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC;AACrD,CAAC"}

package/dist/component/embeddings/index.d.ts

@@ -0,0 +1,23 @@
+/**
+ * This file is the interface for interacting with vectors.
+ * It translates from embeddings to the underlying vector storage and search.
+ * It modifies embeddings to include importance.
+ * The outer world deals with filters with user names.
+ * The underlying vector storage has its own names.
+ * This file takes in numbered filters (0-3) to translate without knowing about
+ * user names.
+ */
+import type { Id } from "../_generated/dataModel.js";
+import { type ActionCtx, type MutationCtx } from "../_generated/server.js";
+import { type NumberedFilter } from "../filters.js";
+export declare function insertEmbedding(ctx: MutationCtx, embedding: number[], namespaceId: Id<"namespaces">, importance: number | undefined, filters: NumberedFilter | undefined): Promise<import("convex/values").GenericId<"vectors_4096" | "vectors_128" | "vectors_256" | "vectors_512" | "vectors_768" | "vectors_1024" | "vectors_1408" | "vectors_1536" | "vectors_2048" | "vectors_3072">>;
+export declare function searchEmbeddings(ctx: ActionCtx, { embedding, namespaceId, filters, limit, }: {
+    embedding: number[];
+    namespaceId: Id<"namespaces">;
+    filters: Array<NumberedFilter>;
+    limit: number;
+}): Promise<{
+    _id: import("convex/values").GenericId<"vectors_4096" | "vectors_128" | "vectors_256" | "vectors_512" | "vectors_768" | "vectors_1024" | "vectors_1408" | "vectors_1536" | "vectors_2048" | "vectors_3072">;
+    _score: number;
+}[]>;
+//# sourceMappingURL=index.d.ts.map

package/dist/component/embeddings/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/component/embeddings/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAE3E,OAAO,EAA2B,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAiC7E,wBAAsB,eAAe,CACnC,GAAG,EAAE,WAAW,EAChB,SAAS,EAAE,MAAM,EAAE,EACnB,WAAW,EAAE,EAAE,CAAC,YAAY,CAAC,EAC7B,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,OAAO,EAAE,cAAc,GAAG,SAAS,mNASpC;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,SAAS,EACd,EACE,SAAS,EACT,WAAW,EACX,OAAO,EACP,KAAK,GACN,EAAE;IACD,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC;IAK9B,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;CACf;;;KAqBF"}

package/dist/component/embeddings/index.js

@@ -0,0 +1,54 @@
+import {} from "../_generated/server.js";
+import { getVectorTableName, validateVectorDimension } from "./tables.js";
+import { filterFieldsFromNumbers } from "../filters.js";
+import { searchVector, vectorWithImportance } from "./importance.js";
+// TODO: wait to see if this is needed.
+// export const insertBatch = mutation({
+//   args: {
+//     vectorDimension: vVectorDimension,
+//     vectors: v.array(
+//       v.object({
+//         vector: v.array(v.number()),
+//         namespace: v.id("namespaces"),
+//         importance: v.optional(v.number()),
+//         filters: v.optional(v.any()),
+//       })
+//     ),
+//   },
+//   returns: v.array(vVectorId),
+//   handler: async (ctx, args) => {
+//     return Promise.all(
+//       args.vectors.map(async (vector) =>
+//         insertEmbedding(
+//           ctx,
+//           vector.vector,
+//           vector.namespace,
+//           vector.importance,
+//           vector.filters
+//         )
+//       )
+//     );
+//   },
+// });
+export async function insertEmbedding(ctx, embedding, namespaceId, importance, filters) {
+    const filterFields = filterFieldsFromNumbers(namespaceId, filters);
+    const dimension = validateVectorDimension(embedding.length);
+    return ctx.db.insert(getVectorTableName(dimension), {
+        namespaceId,
+        vector: vectorWithImportance(embedding, importance ?? 1),
+        ...filterFields,
+    });
+}
+export async function searchEmbeddings(ctx, { embedding, namespaceId, filters, limit, }) {
+    const dimension = validateVectorDimension(embedding.length);
+    const tableName = getVectorTableName(dimension);
+    const orFilters = filters.flatMap((filter) => filterFieldsFromNumbers(namespaceId, filter));
+    return ctx.vectorSearch(tableName, "vector", {
+        vector: searchVector(embedding),
+        filter: (q) => orFilters.length === 0
+            ? q.eq("namespaceId", namespaceId)
+            : q.or(...orFilters.flatMap((namedFilter) => Object.entries(namedFilter).map(([filterField, filter]) => q.eq(filterField, filter)))),
+        limit,
+    });
+}
+//# sourceMappingURL=index.js.map

package/dist/component/embeddings/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/component/embeddings/index.ts"],"names":[],"mappings":"AAUA,OAAO,EAAoC,MAAM,yBAAyB,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAuB,MAAM,eAAe,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAErE,uCAAuC;AACvC,wCAAwC;AACxC,YAAY;AACZ,yCAAyC;AACzC,wBAAwB;AACxB,mBAAmB;AACnB,uCAAuC;AACvC,yCAAyC;AACzC,8CAA8C;AAC9C,wCAAwC;AACxC,WAAW;AACX,SAAS;AACT,OAAO;AACP,iCAAiC;AACjC,oCAAoC;AACpC,0BAA0B;AAC1B,2CAA2C;AAC3C,2BAA2B;AAC3B,iBAAiB;AACjB,2BAA2B;AAC3B,8BAA8B;AAC9B,+BAA+B;AAC/B,2BAA2B;AAC3B,YAAY;AACZ,UAAU;AACV,SAAS;AACT,OAAO;AACP,MAAM;AAGN,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAgB,EAChB,SAAmB,EACnB,WAA6B,EAC7B,UAA8B,EAC9B,OAAmC;IAEnC,MAAM,YAAY,GAAG,uBAAuB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,uBAAuB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC5D,OAAO,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,CAAC,EAAE;QAClD,WAAW;QACX,MAAM,EAAE,oBAAoB,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;QACxD,GAAG,YAAY;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAc,EACd,EACE,SAAS,EACT,WAAW,EACX,OAAO,EACP,KAAK,GAUN;IAED,MAAM,SAAS,GAAG,uBAAuB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAC3C,uBAAuB,CAAC,WAAW,EAAE,MAAM,CAAC,CAC7C,CAAC;IACF,OAAO,GAAG,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,EAAE;QAC3C,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC;QAC/B,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,SAAS,CAAC,MAAM,KAAK,CAAC;YACpB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,EAAE,WAAW,CAAC;YAClC,CAAC,CAAC,CAAC,CAAC,EAAE,CACF,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,CACnC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,CACxD,CAAC,CAAC,EAAE,CAAC,WAA+C,EAAE,MAAM,CAAC,CAC9D,CACF,CACF;QACP,KAAK;KACN,CAAC,CAAC;AACL,CAAC"}

package/dist/component/embeddings/tables.d.ts

@@ -0,0 +1,39 @@
+import { type GenericTableSearchIndexes, type SchemaDefinition, type TableDefinition } from "convex/server";
+import { type GenericId, type ObjectType, type VId, type VObject, type VUnion } from "convex/values";
+declare const embeddingsFields: {
+    namespaceId: VId<GenericId<"namespaces">, "required">;
+    filter0: import("convex/values").VArray<[GenericId<"namespaces">, import("convex/values").Value] | undefined, VId<"namespaces", "required"> | import("convex/values").VAny<any, "required", string>, "optional">;
+    filter1: import("convex/values").VArray<[GenericId<"namespaces">, import("convex/values").Value] | undefined, VId<"namespaces", "required"> | import("convex/values").VAny<any, "required", string>, "optional">;
+    filter2: import("convex/values").VArray<[GenericId<"namespaces">, import("convex/values").Value] | undefined, VId<"namespaces", "required"> | import("convex/values").VAny<any, "required", string>, "optional">;
+    filter3: import("convex/values").VArray<[GenericId<"namespaces">, import("convex/values").Value] | undefined, VId<"namespaces", "required"> | import("convex/values").VAny<any, "required", string>, "optional">;
+    vector: import("convex/values").VArray<number[], import("convex/values").VFloat64<number, "required">, "required">;
+};
+type Table = TableDefinition<VObject<ObjectType<typeof embeddingsFields>, typeof embeddingsFields>, {
+    model_table_threadId: ["model", "table", "threadId", "_creationTime"];
+}, GenericTableSearchIndexes, VectorIndex>;
+type VectorIndex = {
+    vector: {
+        vectorField: "vector";
+        dimensions: number;
+        filterFields: string;
+    };
+};
+export type VectorSchema = SchemaDefinition<{
+    [key in VectorTableName]: Table;
+}, true>;
+export declare const VectorDimensions: readonly [128, 256, 512, 768, 1024, 1408, 1536, 2048, 3072, 4096];
+export declare function assertVectorDimension(dimension: number): asserts dimension is VectorDimension;
+export declare function validateVectorDimension(dimension: number): VectorDimension;
+export type VectorDimension = (typeof VectorDimensions)[number];
+export declare const VectorTableNames: `vectors_${(typeof VectorDimensions)[number]}`[];
+export type VectorTableName = (typeof VectorTableNames)[number];
+export type VectorTableId = GenericId<(typeof VectorTableNames)[number]>;
+export declare const vVectorDimension: VUnion<4096 | 128 | 256 | 512 | 768 | 1024 | 1408 | 1536 | 2048 | 3072, any, "required", any>;
+export declare const vVectorTableName: VUnion<"vectors_4096" | "vectors_128" | "vectors_256" | "vectors_512" | "vectors_768" | "vectors_1024" | "vectors_1408" | "vectors_1536" | "vectors_2048" | "vectors_3072", any, "required", any>;
+export declare const vVectorId: VUnion<GenericId<(typeof VectorTableNames)[number]>, VId<(typeof VectorTableNames)[number]>[]>;
+export declare function getVectorTableName(dimension: VectorDimension): VectorTableName;
+declare const tables: {
+    [K in keyof typeof VectorDimensions & number as `vectors_${(typeof VectorDimensions)[K]}`]: Table;
+};
+export default tables;
+//# sourceMappingURL=tables.d.ts.map

package/dist/component/embeddings/tables.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../src/component/embeddings/tables.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,yBAAyB,EAC9B,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,KAAK,SAAS,EACd,KAAK,UAAU,EAEf,KAAK,GAAG,EACR,KAAK,OAAO,EACZ,KAAK,MAAM,EACZ,MAAM,eAAe,CAAC;AAKvB,QAAA,MAAM,gBAAgB;;;;;;;CAGrB,CAAC;AAUF,KAAK,KAAK,GAAG,eAAe,CAC1B,OAAO,CAAC,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAAE,OAAO,gBAAgB,CAAC,EACrE;IAAE,oBAAoB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,eAAe,CAAC,CAAA;CAAE,EACzE,yBAAyB,EACzB,WAAW,CACZ,CAAC;AAEF,KAAK,WAAW,GAAG;IACjB,MAAM,EAAE;QACN,WAAW,EAAE,QAAQ,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG,gBAAgB,CACzC;KAAG,GAAG,IAAI,eAAe,GAAG,KAAK;CAAE,EACnC,IAAI,CACL,CAAC;AAEF,eAAO,MAAM,gBAAgB,mEAEnB,CAAC;AAEX,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,SAAS,IAAI,eAAe,CAMtC;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe,CAO1E;AACD,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AAChE,eAAO,MAAM,gBAAgB,EAExB,WAAW,CAAC,OAAO,gBAAgB,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC;AACtD,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AAChE,MAAM,MAAM,aAAa,GAAG,SAAS,CAAC,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AAEzE,eAAO,MAAM,gBAAgB,+FAAgC,CAAC;AAC9D,eAAO,MAAM,gBAAgB,mMAAgC,CAAC;AAC9D,eAAO,MAAM,SAAS,EAEjB,MAAM,CACT,SAAS,CAAC,CAAC,OAAO,gBAAgB,EAAE,MAAM,CAAC,CAAC,EAC5C,GAAG,CAAC,CAAC,OAAO,gBAAgB,EAAE,MAAM,CAAC,CAAC,EAAE,CACzC,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,eAAe,GAC1B,eAAe,CACjD;AAWD,QAAA,MAAM,MAAM,EAAE;KACX,CAAC,IAAI,MAAM,OAAO,gBAAgB,GACjC,MAAM,IAAI,WAAW,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK;CAMG,CAAC;AAEnE,eAAe,MAAM,CAAC"}

package/dist/component/embeddings/tables.js

@@ -0,0 +1,53 @@
+import { literals } from "convex-helpers/validators";
+import { defineTable, } from "convex/server";
+import { v, } from "convex/values";
+import { vectorWithImportanceDimension } from "./importance.js";
+import { allFilterFieldNames, vAllFilterFields } from "../filters.js";
+// We only generate embeddings for non-tool, non-system messages
+const embeddingsFields = {
+    vector: v.array(v.number()),
+    ...vAllFilterFields,
+};
+function table(dimensions) {
+    return defineTable(embeddingsFields).vectorIndex("vector", {
+        vectorField: "vector",
+        dimensions: vectorWithImportanceDimension(dimensions),
+        filterFields: allFilterFieldNames,
+    });
+}
+export const VectorDimensions = [
+    128, 256, 512, 768, 1024, 1408, 1536, 2048, 3072, 4096,
+];
+export function assertVectorDimension(dimension) {
+    if (!VectorDimensions.includes(dimension)) {
+        throw new Error(`Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`);
+    }
+}
+export function validateVectorDimension(dimension) {
+    if (!VectorDimensions.includes(dimension)) {
+        throw new Error(`Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`);
+    }
+    return dimension;
+}
+export const VectorTableNames = VectorDimensions.map((d) => `vectors_${d}`);
+export const vVectorDimension = literals(...VectorDimensions);
+export const vVectorTableName = literals(...VectorTableNames);
+export const vVectorId = v.union(...VectorTableNames.map((name) => v.id(name)));
+export function getVectorTableName(dimension) {
+    return `vectors_${dimension}`;
+}
+// export function getVectorIdInfo(ctx: QueryCtx, id: VectorTableId) {
+//   for (const dimension of VectorDimensions) {
+//     const tableName = getVectorTableName(dimension);
+//     if (ctx.db.normalizeId(tableName, id)) {
+//       return { tableName, dimension };
+//     }
+//   }
+//   throw new Error(`Unknown vector table id: ${id}`);
+// }
+const tables = Object.fromEntries(VectorDimensions.map((dimensions) => [
+    `vectors_${dimensions}`,
+    table(dimensions),
+]));
+export default tables;
+//# sourceMappingURL=tables.js.map

package/dist/component/embeddings/tables.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tables.js","sourceRoot":"","sources":["../../../src/component/embeddings/tables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EACL,WAAW,GAIZ,MAAM,eAAe,CAAC;AACvB,OAAO,EAGL,CAAC,GAIF,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,6BAA6B,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtE,gEAAgE;AAChE,MAAM,gBAAgB,GAAG;IACvB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC3B,GAAG,gBAAgB;CACpB,CAAC;AAEF,SAAS,KAAK,CAAC,UAA2B;IACxC,OAAO,WAAW,CAAC,gBAAgB,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE;QACzD,WAAW,EAAE,QAAQ;QACrB,UAAU,EAAE,6BAA6B,CAAC,UAAU,CAAC;QACrD,YAAY,EAAE,mBAAmB;KAClC,CAAC,CAAC;AACL,CAAC;AAsBD,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;CAC9C,CAAC;AAEX,MAAM,UAAU,qBAAqB,CACnC,SAAiB;IAEjB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,SAA4B,CAAC,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CACb,+BAA+B,SAAS,gBAAgB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,SAA4B,CAAC,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CACb,+BAA+B,SAAS,gBAAgB,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtF,CAAC;IACJ,CAAC;IACD,OAAO,SAA4B,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,GAAG,CAClD,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,CAC8B,CAAC;AAItD,MAAM,CAAC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,gBAAgB,CAAC,CAAC;AAC9D,MAAM,CAAC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,gBAAgB,CAAC,CAAC;AAC9D,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAC9B,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAI9C,CAAC;AAEF,MAAM,UAAU,kBAAkB,CAAC,SAA0B;IAC3D,OAAO,WAAW,SAAS,EAAqB,CAAC;AACnD,CAAC;AACD,sEAAsE;AACtE,gDAAgD;AAChD,uDAAuD;AACvD,+CAA+C;AAC/C,yCAAyC;AACzC,QAAQ;AACR,MAAM;AACN,uDAAuD;AACvD,IAAI;AAEJ,MAAM,MAAM,GAGR,MAAM,CAAC,WAAW,CACpB,gBAAgB,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC;IACnC,WAAW,UAAU,EAAE;IACvB,KAAK,CAAC,UAAU,CAAC;CAClB,CAAC,CAC8D,CAAC;AAEnE,eAAe,MAAM,CAAC"}

package/dist/component/entries.d.ts

@@ -0,0 +1,167 @@
+import { type Value } from "convex/values";
+import type { EntryFilterValues } from "../shared.js";
+import { type Entry } from "../shared.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import { type QueryCtx } from "./_generated/server.js";
+import { type StatusWithOnComplete } from "./schema.js";
+export declare const addAsync: import("convex/server").RegisteredMutation<"public", {
+    onComplete?: string | undefined;
+    entry: {
+        metadata?: Record<string, any> | undefined;
+        key?: string | undefined;
+        title?: string | undefined;
+        contentHash?: string | undefined;
+        namespaceId: import("convex/values").GenericId<"namespaces">;
+        importance: number;
+        filterValues: {
+            name: string;
+            value: any;
+        }[];
+    };
+    chunker: string;
+}, Promise<{
+    entryId: import("convex/values").GenericId<"entries">;
+    status: "ready";
+    created: boolean;
+} | {
+    entryId: import("convex/values").GenericId<"entries">;
+    status: "pending";
+    created: boolean;
+}>>;
+export declare const addAsyncOnComplete: import("convex/server").RegisteredMutation<"internal", {
+    workId: import("@convex-dev/workpool").WorkId;
+    result: {
+        kind: "success";
+        returnValue: any;
+    } | {
+        kind: "failed";
+        error: string;
+    } | {
+        kind: "canceled";
+    };
+    context: import("convex/values").GenericId<"entries">;
+}, Promise<void>>;
+export declare const add: import("convex/server").RegisteredMutation<"public", {
+    onComplete?: string | undefined;
+    allChunks?: {
+        searchableText?: string | undefined;
+        content: {
+            metadata?: Record<string, any> | undefined;
+            text: string;
+        };
+        embedding: number[];
+    }[] | undefined;
+    entry: {
+        metadata?: Record<string, any> | undefined;
+        key?: string | undefined;
+        title?: string | undefined;
+        contentHash?: string | undefined;
+        namespaceId: import("convex/values").GenericId<"namespaces">;
+        importance: number;
+        filterValues: {
+            name: string;
+            value: any;
+        }[];
+    };
+}, Promise<{
+    entryId: import("convex/values").GenericId<"entries">;
+    status: "ready";
+    created: boolean;
+    replacedVersion: Entry | null;
+} | {
+    entryId: import("convex/values").GenericId<"entries">;
+    status: "pending";
+    created: boolean;
+    replacedVersion: null;
+}>>;
+/**
+ * Lists entries in order of their most recent change
+ */
+export declare const list: import("convex/server").RegisteredQuery<"public", {
+    order?: "asc" | "desc" | undefined;
+    namespaceId: import("convex/values").GenericId<"namespaces">;
+    status: "pending" | "ready" | "replaced";
+    paginationOpts: {
+        id?: number | undefined;
+        endCursor?: string | null | undefined;
+        maximumRowsRead?: number | undefined;
+        maximumBytesRead?: number | undefined;
+        numItems: number;
+        cursor: string | null;
+    };
+}, Promise<{
+    page: Entry[];
+    isDone: boolean;
+    continueCursor: import("convex/server").Cursor;
+    splitCursor?: import("convex/server").Cursor | null;
+    pageStatus?: "SplitRecommended" | "SplitRequired" | null;
+}>>;
+/**
+ * Gets a entry by its id.
+ */
+export declare const get: import("convex/server").RegisteredQuery<"public", {
+    entryId: import("convex/values").GenericId<"entries">;
+}, Promise<Entry | null>>;
+/**
+ * Finds a entry by its key and content hash.
+ */
+export declare const findByContentHash: import("convex/server").RegisteredQuery<"public", {
+    namespace: string;
+    filterNames: string[];
+    dimension: number;
+    modelId: string;
+    key: string;
+    contentHash: string;
+}, Promise<Entry | null>>;
+/**
+ * Promotes a entry to ready, replacing any existing ready entry by key.
+ * It will also call the associated onComplete function if it was pending.
+ * Note: this will not replace the chunks automatically, so you should first
+ * call `replaceChunksPage` on all its chunks.
+ * Edge case: if the entry has already been replaced, it will return the
+ * same entry (replacedVersion.entryId === args.entryId).
+ */
+export declare const promoteToReady: import("convex/server").RegisteredMutation<"public", {
+    entryId: Id<"entries">;
+}, Promise<{
+    replacedVersion: Entry | null;
+}>>;
+export declare function getPreviousEntry(ctx: QueryCtx, entry: Doc<"entries">): Promise<{
+    _id: import("convex/values").GenericId<"entries">;
+    _creationTime: number;
+    metadata?: Record<string, any> | undefined;
+    key?: string | undefined;
+    title?: string | undefined;
+    contentHash?: string | undefined;
+    namespaceId: import("convex/values").GenericId<"namespaces">;
+    status: {
+        onComplete?: string | undefined;
+        kind: "pending";
+    } | {
+        kind: "ready";
+    } | {
+        kind: "replaced";
+        replacedAt: number;
+    };
+    version: number;
+    importance: number;
+    filterValues: {
+        name: string;
+        value: any;
+    }[];
+} | null>;
+export declare function publicEntry(entry: {
+    _id: Id<"entries">;
+    key?: string | undefined;
+    importance: number;
+    filterValues: EntryFilterValues[];
+    contentHash?: string | undefined;
+    title?: string | undefined;
+    metadata?: Record<string, Value> | undefined;
+    status: StatusWithOnComplete;
+}): Entry;
+export declare const deleteAsync: import("convex/server").RegisteredMutation<"public", {
+    entryId: Id<"entries">;
+    startOrder: number;
+}, Promise<void>>;
+//# sourceMappingURL=entries.d.ts.map

package/dist/component/entries.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entries.d.ts","sourceRoot":"","sources":["../../src/component/entries.ts"],"names":[],"mappings":"AAEA,OAAO,EAAK,KAAK,KAAK,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAiB,iBAAiB,EAAW,MAAM,cAAc,CAAC;AAC9E,OAAO,EAOL,KAAK,KAAK,EACX,MAAM,cAAc,CAAC;AAEtB,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAKL,KAAK,QAAQ,EACd,MAAM,wBAAwB,CAAC;AAEhC,OAAe,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AA0BhE,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;GA6DnB,CAAC;AAUH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;iBAkC7B,CAAC;AAgCH,eAAO,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6Dd,CAAC;AA2CH;;GAEG;AACH,eAAO,MAAM,IAAI;;;;;;;;;;;;;;;;;;GAuBf,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,GAAG;;yBAUd,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,iBAAiB;;;;;;;yBA+C5B,CAAC;AAEH;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc;aAYR,EAAE,CAAC,SAAS,CAAC;;;GAJ9B,CAAC;AA4EH,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC;;;;;;;;;;;;;;;;;;;;;;;UAe1E;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE;IACjC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,iBAAiB,EAAE,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC;IAC7C,MAAM,EAAE,oBAAoB,CAAC;CAC9B,GAAG,KAAK,CAaR;AAED,eAAO,MAAM,WAAW;aAWL,EAAE,CAAC,SAAS,CAAC;gBAAc,MAAM;iBAJlD,CAAC"}