@convex-dev/rag 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +371 -0
- package/dist/client/_generated/_ignore.d.ts +1 -0
- package/dist/client/_generated/_ignore.d.ts.map +1 -0
- package/dist/client/_generated/_ignore.js +3 -0
- package/dist/client/_generated/_ignore.js.map +1 -0
- package/dist/client/defaultChunker.d.ts +15 -0
- package/dist/client/defaultChunker.d.ts.map +1 -0
- package/dist/client/defaultChunker.js +148 -0
- package/dist/client/defaultChunker.js.map +1 -0
- package/dist/client/fileUtils.d.ts +24 -0
- package/dist/client/fileUtils.d.ts.map +1 -0
- package/dist/client/fileUtils.js +179 -0
- package/dist/client/fileUtils.js.map +1 -0
- package/dist/client/index.d.ts +442 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +597 -0
- package/dist/client/index.js.map +1 -0
- package/dist/client/types.d.ts +29 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/client/types.js +2 -0
- package/dist/client/types.js.map +1 -0
- package/dist/component/_generated/api.d.ts +439 -0
- package/dist/component/_generated/api.d.ts.map +1 -0
- package/dist/component/_generated/api.js +22 -0
- package/dist/component/_generated/api.js.map +1 -0
- package/dist/component/_generated/dataModel.d.ts +60 -0
- package/dist/component/_generated/server.d.ts +149 -0
- package/dist/component/_generated/server.d.ts.map +1 -0
- package/dist/component/_generated/server.js +74 -0
- package/dist/component/_generated/server.js.map +1 -0
- package/dist/component/chunks.d.ts +139 -0
- package/dist/component/chunks.d.ts.map +1 -0
- package/dist/component/chunks.js +413 -0
- package/dist/component/chunks.js.map +1 -0
- package/dist/component/convex.config.d.ts +3 -0
- package/dist/component/convex.config.d.ts.map +1 -0
- package/dist/component/convex.config.js +6 -0
- package/dist/component/convex.config.js.map +1 -0
- package/dist/component/embeddings/importance.d.ts +21 -0
- package/dist/component/embeddings/importance.d.ts.map +1 -0
- package/dist/component/embeddings/importance.js +67 -0
- package/dist/component/embeddings/importance.js.map +1 -0
- package/dist/component/embeddings/index.d.ts +23 -0
- package/dist/component/embeddings/index.d.ts.map +1 -0
- package/dist/component/embeddings/index.js +54 -0
- package/dist/component/embeddings/index.js.map +1 -0
- package/dist/component/embeddings/tables.d.ts +39 -0
- package/dist/component/embeddings/tables.d.ts.map +1 -0
- package/dist/component/embeddings/tables.js +53 -0
- package/dist/component/embeddings/tables.js.map +1 -0
- package/dist/component/entries.d.ts +167 -0
- package/dist/component/entries.d.ts.map +1 -0
- package/dist/component/entries.js +409 -0
- package/dist/component/entries.js.map +1 -0
- package/dist/component/filters.d.ts +46 -0
- package/dist/component/filters.d.ts.map +1 -0
- package/dist/component/filters.js +72 -0
- package/dist/component/filters.js.map +1 -0
- package/dist/component/namespaces.d.ts +131 -0
- package/dist/component/namespaces.d.ts.map +1 -0
- package/dist/component/namespaces.js +222 -0
- package/dist/component/namespaces.js.map +1 -0
- package/dist/component/schema.d.ts +1697 -0
- package/dist/component/schema.d.ts.map +1 -0
- package/dist/component/schema.js +88 -0
- package/dist/component/schema.js.map +1 -0
- package/dist/component/search.d.ts +20 -0
- package/dist/component/search.d.ts.map +1 -0
- package/dist/component/search.js +69 -0
- package/dist/component/search.js.map +1 -0
- package/dist/package.json +3 -0
- package/dist/react/index.d.ts +2 -0
- package/dist/react/index.d.ts.map +1 -0
- package/dist/react/index.js +6 -0
- package/dist/react/index.js.map +1 -0
- package/dist/shared.d.ts +479 -0
- package/dist/shared.d.ts.map +1 -0
- package/dist/shared.js +98 -0
- package/dist/shared.js.map +1 -0
- package/package.json +97 -0
- package/src/client/_generated/_ignore.ts +1 -0
- package/src/client/defaultChunker.test.ts +243 -0
- package/src/client/defaultChunker.ts +183 -0
- package/src/client/fileUtils.ts +179 -0
- package/src/client/index.test.ts +475 -0
- package/src/client/index.ts +1125 -0
- package/src/client/setup.test.ts +28 -0
- package/src/client/types.ts +69 -0
- package/src/component/_generated/api.d.ts +439 -0
- package/src/component/_generated/api.js +23 -0
- package/src/component/_generated/dataModel.d.ts +60 -0
- package/src/component/_generated/server.d.ts +149 -0
- package/src/component/_generated/server.js +90 -0
- package/src/component/chunks.test.ts +915 -0
- package/src/component/chunks.ts +555 -0
- package/src/component/convex.config.ts +7 -0
- package/src/component/embeddings/importance.test.ts +249 -0
- package/src/component/embeddings/importance.ts +75 -0
- package/src/component/embeddings/index.test.ts +482 -0
- package/src/component/embeddings/index.ts +99 -0
- package/src/component/embeddings/tables.ts +114 -0
- package/src/component/entries.test.ts +341 -0
- package/src/component/entries.ts +546 -0
- package/src/component/filters.ts +119 -0
- package/src/component/namespaces.ts +299 -0
- package/src/component/schema.ts +106 -0
- package/src/component/search.test.ts +445 -0
- package/src/component/search.ts +97 -0
- package/src/component/setup.test.ts +5 -0
- package/src/react/index.ts +7 -0
- package/src/shared.ts +247 -0
- package/src/vitest.config.ts +7 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAK,KAAK,KAAK,EAAE,MAAM,eAAe,CAAC;AAS9C,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;yFAajC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEvE,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyEjB,CAAC;AAEH,eAAO,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;8BAIy9D,eAAe;;uBAA22G,eAAe;yBAAoD,eAAe;0BAA0D,eAAe;yBAAyD,eAAe;wBAAsD,eAAe;0BAAwD,eAAe;;wBAAwG,eAAe;kFAAqH,eAAe;yGAA4G,eAAe;4HAAsH,eAAe,qDAAoD,eAAgB,qEAAqE,CAAC,2HAA0H,aAAc,uLAAuL,eAAe;gMAA0L,eAAe;mFAAyI,GAAG,gDAAwB,WAAW,2BAA0B,WAAY;sBAAuC,eAAe;;CAJxlO,CAAC;AACjC,OAAO,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;AAEnB,eAAe,MAAM,CAAC"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { defineSchema, defineTable } from "convex/server";
|
|
2
|
+
import { v } from "convex/values";
|
|
3
|
+
import embeddingsTables, { vVectorId } from "./embeddings/tables.js";
|
|
4
|
+
import { typedV } from "convex-helpers/validators";
|
|
5
|
+
import { allFilterFieldNames, vAllFilterFields, vNamedFilter, } from "./filters.js";
|
|
6
|
+
export const vStatusWithOnComplete = v.union(v.object({
|
|
7
|
+
kind: v.literal("pending"),
|
|
8
|
+
// Callback function handle for when the namespace/entry is ready/failed.
|
|
9
|
+
onComplete: v.optional(v.string()),
|
|
10
|
+
}), v.object({
|
|
11
|
+
kind: v.literal("ready"),
|
|
12
|
+
}), v.object({
|
|
13
|
+
kind: v.literal("replaced"),
|
|
14
|
+
replacedAt: v.number(),
|
|
15
|
+
}));
|
|
16
|
+
export const schema = defineSchema({
|
|
17
|
+
namespaces: defineTable({
|
|
18
|
+
// user-specified id, eg. userId or "documentation"
|
|
19
|
+
namespace: v.string(),
|
|
20
|
+
version: v.number(),
|
|
21
|
+
modelId: v.string(),
|
|
22
|
+
dimension: v.number(),
|
|
23
|
+
filterNames: v.array(v.string()),
|
|
24
|
+
status: vStatusWithOnComplete,
|
|
25
|
+
}).index("status_namespace_version", ["status.kind", "namespace", "version"]),
|
|
26
|
+
entries: defineTable({
|
|
27
|
+
key: v.optional(v.string()),
|
|
28
|
+
namespaceId: v.id("namespaces"),
|
|
29
|
+
version: v.number(),
|
|
30
|
+
importance: v.number(),
|
|
31
|
+
filterValues: v.array(vNamedFilter),
|
|
32
|
+
// To avoid re-creating/ updating the same entry
|
|
33
|
+
// This is a hash that ideally encompasses the content AND chunking strategy
|
|
34
|
+
// e.g. a hash of the list of chunk content hashes.
|
|
35
|
+
contentHash: v.optional(v.string()),
|
|
36
|
+
// conveneient metadata
|
|
37
|
+
title: v.optional(v.string()),
|
|
38
|
+
metadata: v.optional(v.record(v.string(), v.any())),
|
|
39
|
+
status: vStatusWithOnComplete,
|
|
40
|
+
})
|
|
41
|
+
.index("namespaceId_status_key_version", [
|
|
42
|
+
"namespaceId",
|
|
43
|
+
"status.kind",
|
|
44
|
+
"key",
|
|
45
|
+
"version",
|
|
46
|
+
])
|
|
47
|
+
// To look up most recently changed entries
|
|
48
|
+
.index("status_namespaceId", ["status.kind", "namespaceId"]),
|
|
49
|
+
chunks: defineTable({
|
|
50
|
+
entryId: v.id("entries"),
|
|
51
|
+
order: v.number(),
|
|
52
|
+
state: v.union(v.object({
|
|
53
|
+
kind: v.literal("pending"),
|
|
54
|
+
embedding: v.array(v.number()),
|
|
55
|
+
importance: v.number(),
|
|
56
|
+
pendingSearchableText: v.optional(v.string()),
|
|
57
|
+
}), v.object({
|
|
58
|
+
kind: v.literal("ready"),
|
|
59
|
+
embeddingId: vVectorId,
|
|
60
|
+
// TODO: text search
|
|
61
|
+
searchableText: v.optional(v.string()),
|
|
62
|
+
}), v.object({
|
|
63
|
+
kind: v.literal("replaced"),
|
|
64
|
+
embeddingId: vVectorId,
|
|
65
|
+
vector: v.array(v.number()),
|
|
66
|
+
pendingSearchableText: v.optional(v.string()),
|
|
67
|
+
})),
|
|
68
|
+
// TODO: should content be inline?
|
|
69
|
+
contentId: v.id("content"),
|
|
70
|
+
...vAllFilterFields,
|
|
71
|
+
})
|
|
72
|
+
.index("entryId_order", ["entryId", "order"])
|
|
73
|
+
.index("embeddingId", ["state.embeddingId"])
|
|
74
|
+
.searchIndex("searchableText", {
|
|
75
|
+
searchField: "state.searchableText",
|
|
76
|
+
filterFields: allFilterFieldNames,
|
|
77
|
+
}),
|
|
78
|
+
content: defineTable({
|
|
79
|
+
text: v.string(),
|
|
80
|
+
// convenient metadata
|
|
81
|
+
metadata: v.optional(v.record(v.string(), v.any())),
|
|
82
|
+
}),
|
|
83
|
+
...embeddingsTables,
|
|
84
|
+
});
|
|
85
|
+
export const vv = typedV(schema);
|
|
86
|
+
export { vv as v };
|
|
87
|
+
export default schema;
|
|
88
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAc,MAAM,eAAe,CAAC;AAC9C,OAAO,gBAAgB,EAAE,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAAE,MAAM,EAAE,MAAM,2BAA2B,CAAC;AACnD,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,GACb,MAAM,cAAc,CAAC;AAEtB,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAC1C,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;IAC1B,yEAAyE;IACzE,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CACnC,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;CACzB,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;IACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;CACvB,CAAC,CACH,CAAC;AAIF,MAAM,CAAC,MAAM,MAAM,GAAG,YAAY,CAAC;IACjC,UAAU,EAAE,WAAW,CAAC;QACtB,mDAAmD;QACnD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,EAAE,qBAAqB;KAC9B,CAAC,CAAC,KAAK,CAAC,0BAA0B,EAAE,CAAC,aAAa,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IAC7E,OAAO,EAAE,WAAW,CAAC;QACnB,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3B,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC;QAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;QACnC,gDAAgD;QAChD,4EAA4E;QAC5E,mDAAmD;QACnD,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACnC,uBAAuB;QACvB,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7B,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACnD,MAAM,EAAE,qBAAqB;KAC9B,CAAC;SACC,KAAK,CAAC,gCAAgC,EAAE;QACvC,aAAa;QACb,aAAa;QACb,KAAK;QACL,SAAS;KACV,CAAC;QACF,2CAA2C;SAC1C,KAAK,CAAC,oBAAoB,EAAE,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;IAC9D,MAAM,EAAE,WAAW,CAAC;QAClB,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC;QACxB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,KAAK,EAAE,CAAC,CAAC,KAAK,CACZ,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;YAC1B,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAC9B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;YACtB,qBAAqB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SAC9C,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;YACxB,WAAW,EAAE,SAAS;YACtB,oBAAoB;YACpB,cAAc,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SACvC,CAAC,EACF,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;YAC3B,WAAW,EAAE,SAAS;YACtB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YAC3B,qBAAqB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SAC9C,CAAC,CACH;QACD,kCAAkC;QAClC,SAAS,EAAE,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC;QAC1B,GAAG,gBAAgB;KACpB,CAAC;SACC,KAAK,CAAC,eAAe,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;SAC5C,KAAK,CAAC,aAAa,EAAE,CAAC,mBAAmB,CAAC,CAAC;SAC3C,WAAW,CAAC,gBAAgB,EAAE;QAC7B,WAAW,EAAE,sBAAsB;QACnC,YAAY,EAAE,mBAAmB;KAClC,CAAC;IACJ,OAAO,EAAE,WAAW,CAAC;QACnB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,sBAAsB;QACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;KACpD,CAAC;IAEF,GAAG,gBAAgB;CACpB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;AACjC,OAAO,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;AAEnB,eAAe,MAAM,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { type Entry, type SearchResult } from "../shared.js";
|
|
2
|
+
export declare const search: import("convex/server").RegisteredAction<"public", {
|
|
3
|
+
vectorScoreThreshold?: number | undefined;
|
|
4
|
+
chunkContext?: {
|
|
5
|
+
before: number;
|
|
6
|
+
after: number;
|
|
7
|
+
} | undefined;
|
|
8
|
+
namespace: string;
|
|
9
|
+
modelId: string;
|
|
10
|
+
embedding: number[];
|
|
11
|
+
filters: {
|
|
12
|
+
name: string;
|
|
13
|
+
value: any;
|
|
14
|
+
}[];
|
|
15
|
+
limit: number;
|
|
16
|
+
}, Promise<{
|
|
17
|
+
results: SearchResult[];
|
|
18
|
+
entries: Entry[];
|
|
19
|
+
}>>;
|
|
20
|
+
//# sourceMappingURL=search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/component/search.ts"],"names":[],"mappings":"AAKA,OAAO,EAEL,KAAK,KAAK,EAEV,KAAK,YAAY,EAElB,MAAM,cAAc,CAAC;AAGtB,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;aAqBN,YAAY,EAAE;aACd,KAAK,EAAE;GA8ClB,CAAC"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { v } from "convex/values";
|
|
2
|
+
import { action } from "./_generated/server.js";
|
|
3
|
+
import { searchEmbeddings } from "./embeddings/index.js";
|
|
4
|
+
import { numberedFiltersFromNamedFilters, vNamedFilter } from "./filters.js";
|
|
5
|
+
import { internal } from "./_generated/api.js";
|
|
6
|
+
import { vEntry, vSearchResult, } from "../shared.js";
|
|
7
|
+
export const search = action({
|
|
8
|
+
args: {
|
|
9
|
+
namespace: v.string(),
|
|
10
|
+
embedding: v.array(v.number()),
|
|
11
|
+
modelId: v.string(),
|
|
12
|
+
// These are all OR'd together
|
|
13
|
+
filters: v.array(vNamedFilter),
|
|
14
|
+
limit: v.number(),
|
|
15
|
+
vectorScoreThreshold: v.optional(v.number()),
|
|
16
|
+
chunkContext: v.optional(v.object({ before: v.number(), after: v.number() })),
|
|
17
|
+
},
|
|
18
|
+
returns: v.object({
|
|
19
|
+
results: v.array(vSearchResult),
|
|
20
|
+
entries: v.array(vEntry),
|
|
21
|
+
}),
|
|
22
|
+
handler: async (ctx, args) => {
|
|
23
|
+
const { modelId, embedding, filters, limit } = args;
|
|
24
|
+
const namespace = await ctx.runQuery(internal.namespaces.getCompatibleNamespace, {
|
|
25
|
+
namespace: args.namespace,
|
|
26
|
+
modelId,
|
|
27
|
+
dimension: embedding.length,
|
|
28
|
+
filterNames: filters.map((f) => f.name),
|
|
29
|
+
});
|
|
30
|
+
if (!namespace) {
|
|
31
|
+
console.debug(`No compatible namespace found for ${args.namespace} with model ${args.modelId} and dimension ${embedding.length} and filters ${filters.map((f) => f.name).join(", ")}.`);
|
|
32
|
+
return {
|
|
33
|
+
results: [],
|
|
34
|
+
entries: [],
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
const results = await searchEmbeddings(ctx, {
|
|
38
|
+
embedding,
|
|
39
|
+
namespaceId: namespace._id,
|
|
40
|
+
filters: numberedFiltersFromNamedFilters(filters, namespace.filterNames),
|
|
41
|
+
limit,
|
|
42
|
+
});
|
|
43
|
+
const threshold = args.vectorScoreThreshold ?? -1;
|
|
44
|
+
const aboveThreshold = results.filter((r) => r._score >= threshold);
|
|
45
|
+
const chunkContext = args.chunkContext ?? { before: 0, after: 0 };
|
|
46
|
+
// TODO: break this up if there are too many results
|
|
47
|
+
const { ranges, entries } = await ctx.runQuery(internal.chunks.getRangesOfChunks, {
|
|
48
|
+
embeddingIds: aboveThreshold.map((r) => r._id),
|
|
49
|
+
chunkContext,
|
|
50
|
+
});
|
|
51
|
+
return {
|
|
52
|
+
results: ranges
|
|
53
|
+
.map((r, i) => publicSearchResult(r, aboveThreshold[i]._score))
|
|
54
|
+
.filter((r) => r !== null),
|
|
55
|
+
entries,
|
|
56
|
+
};
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
function publicSearchResult(r, score) {
|
|
60
|
+
if (r === null) {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
...r,
|
|
65
|
+
score,
|
|
66
|
+
entryId: r.entryId,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/component/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAc,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,+BAA+B,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EACL,MAAM,EAEN,aAAa,GAGd,MAAM,cAAc,CAAC;AAGtB,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC;IAC3B,IAAI,EAAE;QACJ,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,8BAA8B;QAC9B,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;QAC9B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,oBAAoB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC5C,YAAY,EAAE,CAAC,CAAC,QAAQ,CACtB,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CACpD;KACF;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;QAC/B,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;KACzB,CAAC;IACF,OAAO,EAAE,KAAK,EACZ,GAAG,EACH,IAAI,EAIH,EAAE;QACH,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,QAAQ,CAClC,QAAQ,CAAC,UAAU,CAAC,sBAAsB,EAC1C;YACE,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO;YACP,SAAS,EAAE,SAAS,CAAC,MAAM;YAC3B,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACxC,CACF,CAAC;QACF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CACX,qCAAqC,IAAI,CAAC,SAAS,eAAe,IAAI,CAAC,OAAO,kBAAkB,SAAS,CAAC,MAAM,gBAAgB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACzK,CAAC;YACF,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,OAAO,EAAE,EAAE;aACZ,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE;YAC1C,SAAS;YACT,WAAW,EAAE,SAAS,CAAC,GAAG;YAC1B,OAAO,EAAE,+BAA+B,CAAC,OAAO,EAAE,SAAS,CAAC,WAAW,CAAC;YACxE,KAAK;SACN,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC,CAAC;QAClD,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;QACpE,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAClE,oDAAoD;QACpD,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,CAC5C,QAAQ,CAAC,MAAM,CAAC,iBAAiB,EACjC;YACE,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAC9C,YAAY;SACb,CACF,CAAC;QACF,OAAO;YACL,OAAO,EAAE,MAAM;iBACZ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;iBAC9D,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;YAC5B,OAAO;SACR,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,kBAAkB,CACzB,CAAoC,EACpC,KAAa;IAEb,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO;QACL,GAAG,CAAC;QACJ,KAAK;QACL,OAAO,EAAE,CAAC,CAAC,OAA6B;KACzC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/react/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,cAAc,UAE1B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/react/index.ts"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,6CAA6C;AAE7C,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,EAAE;IACjC,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC"}
|