@theromans/convex-dobty 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.d.ts +7 -18
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +8 -19
- package/dist/client/index.js.map +1 -1
- package/dist/component/knowledge.d.ts +2 -29
- package/dist/component/knowledge.d.ts.map +1 -1
- package/dist/component/knowledge.js +25 -78
- package/dist/component/knowledge.js.map +1 -1
- package/dist/component/knowledgeSearch.d.ts +23 -25
- package/dist/component/knowledgeSearch.d.ts.map +1 -1
- package/dist/component/knowledgeSearch.js +115 -71
- package/dist/component/knowledgeSearch.js.map +1 -1
- package/package.json +1 -1
- package/src/client/index.ts +8 -32
- package/src/component/knowledge.ts +26 -88
- package/src/component/knowledgeSearch.ts +135 -126
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { action } from "./_generated/server.js";
|
|
1
|
+
import { internalAction, action } from "./_generated/server.js";
|
|
2
2
|
import { internal } from "./_generated/api.js";
|
|
3
3
|
import { v } from "convex/values";
|
|
4
4
|
const EMBEDDING_ENDPOINT = "https://openai.inference.de-txl.ionos.com/v1/embeddings";
|
|
5
5
|
const EMBEDDING_MODEL = "BAAI/bge-m3";
|
|
6
6
|
const EMBEDDING_DIMENSIONS = 1024;
|
|
7
|
-
const EMBEDDING_TIMEOUT_MS =
|
|
7
|
+
const EMBEDDING_TIMEOUT_MS = 15_000;
|
|
8
8
|
const DEFAULT_SEARCH_LIMIT = 5;
|
|
9
9
|
async function getEmbedding(text, apiToken) {
|
|
10
10
|
const controller = new AbortController();
|
|
@@ -25,7 +25,7 @@ async function getEmbedding(text, apiToken) {
|
|
|
25
25
|
});
|
|
26
26
|
if (!response.ok) {
|
|
27
27
|
const body = await response.text();
|
|
28
|
-
throw new Error(`Embedding API
|
|
28
|
+
throw new Error(`Embedding API ${response.status}: ${body}`);
|
|
29
29
|
}
|
|
30
30
|
const json = await response.json();
|
|
31
31
|
return json.data[0].embedding;
|
|
@@ -40,13 +40,118 @@ async function getEmbedding(text, apiToken) {
|
|
|
40
40
|
clearTimeout(timer);
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Embed a single chunk for a single language.
|
|
45
|
+
* Called by the scheduler after a chunk is seeded or updated.
|
|
46
|
+
* Automatically schedules the other language if it is also stale.
|
|
47
|
+
*/
|
|
48
|
+
export const embedChunk = internalAction({
|
|
49
|
+
args: {
|
|
50
|
+
chunkId: v.id("knowledgeChunks"),
|
|
51
|
+
language: v.union(v.literal("en"), v.literal("de")),
|
|
52
|
+
},
|
|
53
|
+
handler: async (ctx, { chunkId, language }) => {
|
|
54
|
+
const apiToken = globalThis.process?.env?.IONOS_API_TOKEN;
|
|
55
|
+
if (!apiToken) {
|
|
56
|
+
console.error("[embedChunk] IONOS_API_TOKEN not set — cannot embed");
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const chunk = await ctx.runQuery(internal.knowledge.getChunk, { chunkId });
|
|
60
|
+
if (!chunk) {
|
|
61
|
+
console.warn(`[embedChunk] chunk ${chunkId} not found, skipping`);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const isStaleEn = !chunk.embedding_en ||
|
|
65
|
+
!chunk.embeddedAt_en ||
|
|
66
|
+
chunk.embeddedAt_en < chunk.updatedAt;
|
|
67
|
+
const isStaleDe = !chunk.embedding_de ||
|
|
68
|
+
!chunk.embeddedAt_de ||
|
|
69
|
+
chunk.embeddedAt_de < chunk.updatedAt;
|
|
70
|
+
const isStale = language === "en" ? isStaleEn : isStaleDe;
|
|
71
|
+
if (!isStale) {
|
|
72
|
+
// Already up to date — schedule the other language if needed
|
|
73
|
+
const otherLanguage = language === "en" ? "de" : "en";
|
|
74
|
+
const otherStale = language === "en" ? isStaleDe : isStaleEn;
|
|
75
|
+
if (otherStale) {
|
|
76
|
+
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
77
|
+
chunkId,
|
|
78
|
+
language: otherLanguage,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const title = language === "en" ? chunk.title_English : chunk.title_Deutsch;
|
|
84
|
+
const content = language === "en" ? chunk.content_English : chunk.content_Deutsch;
|
|
85
|
+
if (!title || !content) {
|
|
86
|
+
console.warn(`[embedChunk] chunk ${chunk.chunkKey} missing ${language} content, skipping`);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
const embedding = await getEmbedding(`${title}\n\n${content}`, apiToken);
|
|
91
|
+
if (language === "en") {
|
|
92
|
+
await ctx.runMutation(internal.knowledge.updateEmbeddingEn, { chunkId, embedding });
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
await ctx.runMutation(internal.knowledge.updateEmbeddingDe, { chunkId, embedding });
|
|
96
|
+
}
|
|
97
|
+
console.log(`[embedChunk] embedded ${chunk.chunkKey} (${language})`);
|
|
98
|
+
// Schedule the other language if it is also stale
|
|
99
|
+
const otherLanguage = language === "en" ? "de" : "en";
|
|
100
|
+
const otherStale = language === "en" ? isStaleDe : isStaleEn;
|
|
101
|
+
if (otherStale) {
|
|
102
|
+
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
103
|
+
chunkId,
|
|
104
|
+
language: otherLanguage,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
catch (e) {
|
|
109
|
+
console.error(`[embedChunk] failed for ${chunk.chunkKey} (${language}): ${e.message}`);
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
});
|
|
113
|
+
/**
|
|
114
|
+
* Schedule embedding jobs for all chunks that are stale
|
|
115
|
+
* (updatedAt > embeddedAt_en or embeddedAt_de).
|
|
116
|
+
* Called automatically after seedKnowledgeChunks, or can be triggered manually.
|
|
117
|
+
*/
|
|
118
|
+
export const scheduleStaleEmbeddings = internalAction({
|
|
119
|
+
args: {},
|
|
120
|
+
handler: async (ctx) => {
|
|
121
|
+
const all = await ctx.runQuery(internal.knowledge.listAllChunks, {});
|
|
122
|
+
let scheduled = 0;
|
|
123
|
+
for (const chunk of all) {
|
|
124
|
+
const staleEn = !chunk.embedding_en ||
|
|
125
|
+
!chunk.embeddedAt_en ||
|
|
126
|
+
chunk.embeddedAt_en < chunk.updatedAt;
|
|
127
|
+
const staleDe = !chunk.embedding_de ||
|
|
128
|
+
!chunk.embeddedAt_de ||
|
|
129
|
+
chunk.embeddedAt_de < chunk.updatedAt;
|
|
130
|
+
if (staleEn) {
|
|
131
|
+
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
132
|
+
chunkId: chunk._id,
|
|
133
|
+
language: "en",
|
|
134
|
+
});
|
|
135
|
+
scheduled++;
|
|
136
|
+
}
|
|
137
|
+
else if (staleDe) {
|
|
138
|
+
// embedChunk for EN will chain to DE when EN is already fresh
|
|
139
|
+
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
140
|
+
chunkId: chunk._id,
|
|
141
|
+
language: "de",
|
|
142
|
+
});
|
|
143
|
+
scheduled++;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
console.log(`[scheduleStaleEmbeddings] scheduled ${scheduled} jobs for ${all.length} chunks`);
|
|
147
|
+
return { total: all.length, scheduled };
|
|
148
|
+
},
|
|
149
|
+
});
|
|
43
150
|
/**
|
|
44
151
|
* Search knowledge chunks by embedding similarity.
|
|
45
|
-
*
|
|
46
|
-
* -
|
|
47
|
-
* -
|
|
48
|
-
* - Falls back to the other language's index if the primary has no vectors.
|
|
49
|
-
* - Returns up to `limit` results (default 5).
|
|
152
|
+
* - language: "en" | "de" — which index to search and which fields to return
|
|
153
|
+
* - Falls back to the other language index if primary has no vectors yet
|
|
154
|
+
* - Returns up to `limit` results (default 5)
|
|
50
155
|
*/
|
|
51
156
|
export const searchKnowledge = action({
|
|
52
157
|
args: {
|
|
@@ -62,11 +167,7 @@ export const searchKnowledge = action({
|
|
|
62
167
|
},
|
|
63
168
|
});
|
|
64
169
|
/**
|
|
65
|
-
* Search
|
|
66
|
-
*
|
|
67
|
-
* - language: the requested language for returned title/content ("en" | "de")
|
|
68
|
-
* - Falls back to the other language's index if primary has no vectors.
|
|
69
|
-
* - Returns up to `limit` results (default 5).
|
|
170
|
+
* Search with a pre-computed embedding vector (no outbound fetch).
|
|
70
171
|
*/
|
|
71
172
|
export const searchByVector = action({
|
|
72
173
|
args: {
|
|
@@ -83,14 +184,12 @@ async function _searchByVector(ctx, { topic, embedding, language, limit, }) {
|
|
|
83
184
|
const primaryIndex = language === "en" ? "by_embedding_en" : "by_embedding_de";
|
|
84
185
|
const fallbackIndex = language === "en" ? "by_embedding_de" : "by_embedding_en";
|
|
85
186
|
const fallbackLanguage = language === "en" ? "de" : "en";
|
|
86
|
-
// Try primary language index first
|
|
87
187
|
let results = await ctx.vectorSearch("knowledgeChunks", primaryIndex, {
|
|
88
188
|
vector: embedding,
|
|
89
189
|
limit,
|
|
90
190
|
filter: (q) => q.eq("topic", topic),
|
|
91
191
|
});
|
|
92
192
|
let embeddingLanguage = language;
|
|
93
|
-
// Fall back to the other language's index if primary returned nothing
|
|
94
193
|
if (results.length === 0) {
|
|
95
194
|
results = await ctx.vectorSearch("knowledgeChunks", fallbackIndex, {
|
|
96
195
|
vector: embedding,
|
|
@@ -102,9 +201,7 @@ async function _searchByVector(ctx, { topic, embedding, language, limit, }) {
|
|
|
102
201
|
if (results.length === 0)
|
|
103
202
|
return [];
|
|
104
203
|
const docs = await Promise.all(results.map(async (r) => {
|
|
105
|
-
const doc = await ctx.runQuery(internal.knowledge.getChunk, {
|
|
106
|
-
chunkId: r._id,
|
|
107
|
-
});
|
|
204
|
+
const doc = await ctx.runQuery(internal.knowledge.getChunk, { chunkId: r._id });
|
|
108
205
|
if (!doc)
|
|
109
206
|
return null;
|
|
110
207
|
const title = language === "en" ? doc.title_English : doc.title_Deutsch;
|
|
@@ -123,57 +220,4 @@ async function _searchByVector(ctx, { topic, embedding, language, limit, }) {
|
|
|
123
220
|
}));
|
|
124
221
|
return docs.filter((d) => d !== null);
|
|
125
222
|
}
|
|
126
|
-
/**
|
|
127
|
-
* Re-embed stale chunks for both languages.
|
|
128
|
-
* Embeds English and Deutsch separately, storing embedding_en and embedding_de.
|
|
129
|
-
*/
|
|
130
|
-
export const refreshEmbeddings = action({
|
|
131
|
-
args: {
|
|
132
|
-
apiToken: v.string(),
|
|
133
|
-
},
|
|
134
|
-
handler: async (ctx, { apiToken }) => {
|
|
135
|
-
const allChunks = await ctx.runQuery(internal.knowledge.listAllChunks, {});
|
|
136
|
-
const staleEn = allChunks.filter((c) => !c.embedding_en || !c.embeddedAt_en || c.embeddedAt_en < c.updatedAt);
|
|
137
|
-
const staleDe = allChunks.filter((c) => !c.embedding_de || !c.embeddedAt_de || c.embeddedAt_de < c.updatedAt);
|
|
138
|
-
let embeddedEn = 0;
|
|
139
|
-
let embeddedDe = 0;
|
|
140
|
-
let errors = 0;
|
|
141
|
-
for (const chunk of staleEn) {
|
|
142
|
-
try {
|
|
143
|
-
const embedding = await getEmbedding(`${chunk.title_English}\n\n${chunk.content_English}`, apiToken);
|
|
144
|
-
await ctx.runMutation(internal.knowledge.updateEmbeddingEn, {
|
|
145
|
-
chunkId: chunk._id,
|
|
146
|
-
embedding,
|
|
147
|
-
});
|
|
148
|
-
embeddedEn++;
|
|
149
|
-
}
|
|
150
|
-
catch (e) {
|
|
151
|
-
console.error(`Failed to embed EN chunk ${chunk.chunkKey}: ${e.message}`);
|
|
152
|
-
errors++;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
for (const chunk of staleDe) {
|
|
156
|
-
try {
|
|
157
|
-
const embedding = await getEmbedding(`${chunk.title_Deutsch}\n\n${chunk.content_Deutsch}`, apiToken);
|
|
158
|
-
await ctx.runMutation(internal.knowledge.updateEmbeddingDe, {
|
|
159
|
-
chunkId: chunk._id,
|
|
160
|
-
embedding,
|
|
161
|
-
});
|
|
162
|
-
embeddedDe++;
|
|
163
|
-
}
|
|
164
|
-
catch (e) {
|
|
165
|
-
console.error(`Failed to embed DE chunk ${chunk.chunkKey}: ${e.message}`);
|
|
166
|
-
errors++;
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
return {
|
|
170
|
-
total: allChunks.length,
|
|
171
|
-
staleEn: staleEn.length,
|
|
172
|
-
staleDe: staleDe.length,
|
|
173
|
-
embeddedEn,
|
|
174
|
-
embeddedDe,
|
|
175
|
-
errors,
|
|
176
|
-
};
|
|
177
|
-
},
|
|
178
|
-
});
|
|
179
223
|
//# sourceMappingURL=knowledgeSearch.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledgeSearch.js","sourceRoot":"","sources":["../../src/component/knowledgeSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledgeSearch.js","sourceRoot":"","sources":["../../src/component/knowledgeSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAoB,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,MAAM,kBAAkB,GACtB,yDAAyD,CAAC;AAC5D,MAAM,eAAe,GAAG,aAAa,CAAC;AACtC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB;IACxD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,oBAAoB,CAAC,CAAC;IACzE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,kBAAkB,EAAE;YAC/C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,QAAQ,EAAE;gBACnC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,eAAe;gBACtB,UAAU,EAAE,oBAAoB;aACjC,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAqB,CAAC;IAC5C,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QAChB,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,iCAAiC,oBAAoB,IAAI,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,cAAc,CAAC;IACvC,IAAI,EAAE;QACJ,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC;QAChC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KACpD;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAI,UAAkB,CAAC,OAAO,EAAE,GAAG,EAAE,eAAqC,CAAC;QACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACrE,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAQ,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,sBAAsB,OAAO,sBAAsB,CAAC,CAAC;YAClE,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GACb,CAAC,KAAK,CAAC,YAAY;YACnB,CAAC,KAAK,CAAC,aAAa;YACpB,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;QACxC,MAAM,SAAS,GACb,CAAC,KAAK,CAAC,YAAY;YACnB,CAAC,KAAK,CAAC,aAAa;YACpB,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;QAExC,MAAM,OAAO,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,6DAA6D;YAC7D,MAAM,aAAa,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACtD,MAAM,UAAU,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7D,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACnE,OAAO;oBACP,QAAQ,EAAE,aAAa;iBACxB,CAAC,CAAC;YACL,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;QAC5E,MAAM,OAAO,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;QAElF,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,sBAAsB,KAAK,CAAC,QAAQ,YAAY,QAAQ,oBAAoB,CAAC,CAAC;YAC3F,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,GAAG,KAAK,OAAO,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;YAEzE,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;gBACtB,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;YACtF,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;YACtF,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,CAAC,QAAQ,KAAK,QAAQ,GAAG,CAAC,CAAC;YAErE,kDAAkD;YAClD,MAAM,aAAa,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACtD,MAAM,UAAU,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7D,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACnE,OAAO;oBACP,QAAQ,EAAE,aAAa;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,QAAQ,KAAK,QAAQ,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,cAAc,CAAC;IACpD,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,GAAG,GAAU,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAE5E,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;YACxB,MAAM,OAAO,GACX,CAAC,KAAK,CAAC,YAAY;gBACnB,CAAC,KAAK,CAAC,aAAa;gBACpB,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;YACxC,MAAM,OAAO,GACX,CAAC,KAAK,CAAC,YAAY;gBACnB,CAAC,KAAK,CAAC,aAAa;gBACpB,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;YAExC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACnE,OAAO,EAAE,KAAK,CAAC,GAAG;oBAClB,QAAQ,EAAE,IAAI;iBACf,CAAC,CAAC;gBACH,SAAS,EAAE,CAAC;YACd,CAAC;iBAAM,IAAI,OAAO,EAAE,CAAC;gBACnB,8DAA8D;gBAC9D,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACnE,OAAO,EAAE,KAAK,CAAC,GAAG;oBAClB,QAAQ,EAAE,IAAI;iBACf,CAAC,CAAC;gBACH,SAAS,EAAE,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,SAAS,aAAa,GAAG,CAAC,MAAM,SAAS,CAAC,CAAC;QAC9F,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAC1C,CAAC;CACF,CAAC,CAAC;AAcH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAC;IACpC,IAAI,EAAE;QACJ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;KACrB;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,GAAG,oBAAoB,EAAE,QAAQ,EAAE,EAAE,EAAE;QACzF,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACtD,OAAO,eAAe,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;CACF,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAC;IACnC,IAAI,EAAE;QACJ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAC/B,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAC9B;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,oBAAoB,EAAE,EAAE,EAAE;QACnF,OAAO,eAAe,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;CACF,CAAC,CAAC;AAEH,KAAK,UAAU,eAAe,CAC5B,GAAQ,EACR,EACE,KAAK,EACL,SAAS,EACT,QAAQ,EACR,KAAK,GAMN;IAED,MAAM,YAAY,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAC/E,MAAM,aAAa,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAChF,MAAM,gBAAgB,GAAgB,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAEtE,IAAI,OAAO,GAAG,MAAM,GAAG,CAAC,YAAY,CAAC,iBAAiB,EAAE,YAAY,EAAE;QACpE,MAAM,EAAE,SAAS;QACjB,KAAK;QACL,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,iBAAiB,GAAgB,QAAQ,CAAC;IAE9C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,GAAG,MAAM,GAAG,CAAC,YAAY,CAAC,iBAAiB,EAAE,aAAa,EAAE;YACjE,MAAM,EAAE,SAAS;YACjB,KAAK;YACL,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC;SACzC,CAAC,CAAC;QACH,iBAAiB,GAAG,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,IAAI,GAA+B,MAAM,OAAO,CAAC,GAAG,CACxD,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAM,EAAmC,EAAE;QAC5D,MAAM,GAAG,GAAQ,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACrF,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,MAAM,KAAK,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC;QACxE,MAAM,OAAO,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC;QAC9E,OAAO;YACL,KAAK;YACL,OAAO;YACP,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,eAAe,EAAE,GAAG,CAAC,eAAe;YACpC,eAAe,EAAE,GAAG,CAAC,eAAe;YACpC,KAAK,EAAE,CAAC,CAAC,MAAM;YACf,QAAQ;YACR,iBAAiB;SAClB,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAwB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AAC9D,CAAC"}
|
package/package.json
CHANGED
package/src/client/index.ts
CHANGED
|
@@ -519,14 +519,6 @@ export class DobtAddon {
|
|
|
519
519
|
return await ctx.runAction(this.component.knowledgeSearch.searchByVector, args);
|
|
520
520
|
}
|
|
521
521
|
|
|
522
|
-
/** Re-embed stale knowledge chunks for both English and Deutsch separately. */
|
|
523
|
-
async refreshEmbeddings(
|
|
524
|
-
ctx: ActionCtx,
|
|
525
|
-
args: { apiToken: string },
|
|
526
|
-
) {
|
|
527
|
-
return await ctx.runAction(this.component.knowledgeSearch.refreshEmbeddings, args);
|
|
528
|
-
}
|
|
529
|
-
|
|
530
522
|
/** List knowledge chunks by topic. */
|
|
531
523
|
async listKnowledgeChunks(
|
|
532
524
|
ctx: QueryCtx,
|
|
@@ -540,29 +532,13 @@ export class DobtAddon {
|
|
|
540
532
|
return await ctx.runQuery(this.component.knowledge.listTopics, {});
|
|
541
533
|
}
|
|
542
534
|
|
|
543
|
-
/**
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
async
|
|
550
|
-
return await ctx.
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
/** Update a single chunk's English embedding by ID. */
|
|
554
|
-
async updateKnowledgeEmbeddingEn(
|
|
555
|
-
ctx: MutationCtx,
|
|
556
|
-
args: { chunkId: string; embedding: number[] },
|
|
557
|
-
) {
|
|
558
|
-
return await ctx.runMutation(this.component.knowledge.updateChunkEmbeddingEn, args);
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
/** Update a single chunk's Deutsch embedding by ID. */
|
|
562
|
-
async updateKnowledgeEmbeddingDe(
|
|
563
|
-
ctx: MutationCtx,
|
|
564
|
-
args: { chunkId: string; embedding: number[] },
|
|
565
|
-
) {
|
|
566
|
-
return await ctx.runMutation(this.component.knowledge.updateChunkEmbeddingDe, args);
|
|
535
|
+
/**
|
|
536
|
+
* Schedule embedding jobs for all chunks where updatedAt > embeddedAt_en/de.
|
|
537
|
+
* Embeddings are processed individually in the background via the scheduler.
|
|
538
|
+
* Called automatically by seedKnowledgeChunks — only needed manually to
|
|
539
|
+
* recover from a stuck state.
|
|
540
|
+
*/
|
|
541
|
+
async scheduleStaleEmbeddings(ctx: ActionCtx) {
|
|
542
|
+
return await ctx.runAction(this.component.knowledgeSearch.scheduleStaleEmbeddings, {});
|
|
567
543
|
}
|
|
568
544
|
}
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
internalMutation,
|
|
5
5
|
internalQuery,
|
|
6
6
|
} from "./_generated/server.js";
|
|
7
|
+
import { internal } from "./_generated/api.js";
|
|
7
8
|
import { v } from "convex/values";
|
|
8
9
|
|
|
9
10
|
/** Upsert knowledge chunks by topic + chunkKey. Only updates if content changed. */
|
|
@@ -25,9 +26,6 @@ export const seedKnowledgeChunks = mutation({
|
|
|
25
26
|
let updated = 0;
|
|
26
27
|
let skipped = 0;
|
|
27
28
|
|
|
28
|
-
// Collect all existing chunkKeys for this topic to detect deletions
|
|
29
|
-
const incomingKeys = new Set(chunks.map((c) => `${c.topic}::${c.chunkKey}`));
|
|
30
|
-
|
|
31
29
|
for (const chunk of chunks) {
|
|
32
30
|
const existing = await ctx.db
|
|
33
31
|
.query("knowledgeChunks")
|
|
@@ -49,17 +47,32 @@ export const seedKnowledgeChunks = mutation({
|
|
|
49
47
|
title_Deutsch: chunk.title_Deutsch,
|
|
50
48
|
content_English: chunk.content_English,
|
|
51
49
|
content_Deutsch: chunk.content_Deutsch,
|
|
52
|
-
//
|
|
53
|
-
embeddedAt_en: undefined,
|
|
54
|
-
embeddedAt_de: undefined,
|
|
50
|
+
// Bump updatedAt — scheduler will detect embeddedAt < updatedAt and re-embed
|
|
55
51
|
updatedAt: Date.now(),
|
|
56
52
|
});
|
|
53
|
+
// Schedule EN embedding; it will chain to DE automatically
|
|
54
|
+
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
55
|
+
chunkId: existing._id,
|
|
56
|
+
language: "en",
|
|
57
|
+
});
|
|
57
58
|
updated++;
|
|
58
59
|
} else {
|
|
60
|
+
// Content unchanged — but schedule embedding if vectors are missing
|
|
61
|
+
const needsEmbed =
|
|
62
|
+
!existing.embedding_en ||
|
|
63
|
+
!existing.embedding_de ||
|
|
64
|
+
!existing.embeddedAt_en ||
|
|
65
|
+
!existing.embeddedAt_de;
|
|
66
|
+
if (needsEmbed) {
|
|
67
|
+
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
68
|
+
chunkId: existing._id,
|
|
69
|
+
language: "en",
|
|
70
|
+
});
|
|
71
|
+
}
|
|
59
72
|
skipped++;
|
|
60
73
|
}
|
|
61
74
|
} else {
|
|
62
|
-
await ctx.db.insert("knowledgeChunks", {
|
|
75
|
+
const id = await ctx.db.insert("knowledgeChunks", {
|
|
63
76
|
topic: chunk.topic,
|
|
64
77
|
chunkKey: chunk.chunkKey,
|
|
65
78
|
title_English: chunk.title_English,
|
|
@@ -68,21 +81,16 @@ export const seedKnowledgeChunks = mutation({
|
|
|
68
81
|
content_Deutsch: chunk.content_Deutsch,
|
|
69
82
|
updatedAt: Date.now(),
|
|
70
83
|
});
|
|
84
|
+
// Schedule EN embedding; it will chain to DE automatically
|
|
85
|
+
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
86
|
+
chunkId: id,
|
|
87
|
+
language: "en",
|
|
88
|
+
});
|
|
71
89
|
inserted++;
|
|
72
90
|
}
|
|
73
91
|
}
|
|
74
92
|
|
|
75
|
-
|
|
76
|
-
const allExisting = await ctx.db.query("knowledgeChunks").collect();
|
|
77
|
-
let deleted = 0;
|
|
78
|
-
for (const existing of allExisting) {
|
|
79
|
-
if (!incomingKeys.has(`${existing.topic}::${existing.chunkKey}`)) {
|
|
80
|
-
await ctx.db.delete(existing._id);
|
|
81
|
-
deleted++;
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
return { inserted, updated, skipped, deleted };
|
|
93
|
+
return { inserted, updated, skipped };
|
|
86
94
|
},
|
|
87
95
|
});
|
|
88
96
|
|
|
@@ -126,77 +134,7 @@ export const listTopics = query({
|
|
|
126
134
|
},
|
|
127
135
|
});
|
|
128
136
|
|
|
129
|
-
/** List chunks that need (re-)embedding for English. */
|
|
130
|
-
export const listStaleChunksEn = query({
|
|
131
|
-
args: {},
|
|
132
|
-
handler: async (ctx) => {
|
|
133
|
-
const all = await ctx.db.query("knowledgeChunks").collect();
|
|
134
|
-
return all
|
|
135
|
-
.filter(
|
|
136
|
-
(c) =>
|
|
137
|
-
!c.embedding_en ||
|
|
138
|
-
!c.embeddedAt_en ||
|
|
139
|
-
c.embeddedAt_en < c.updatedAt,
|
|
140
|
-
)
|
|
141
|
-
.map((c) => ({
|
|
142
|
-
_id: c._id,
|
|
143
|
-
topic: c.topic,
|
|
144
|
-
chunkKey: c.chunkKey,
|
|
145
|
-
title: c.title_English,
|
|
146
|
-
content: c.content_English,
|
|
147
|
-
}));
|
|
148
|
-
},
|
|
149
|
-
});
|
|
150
|
-
|
|
151
|
-
/** List chunks that need (re-)embedding for Deutsch. */
|
|
152
|
-
export const listStaleChunksDe = query({
|
|
153
|
-
args: {},
|
|
154
|
-
handler: async (ctx) => {
|
|
155
|
-
const all = await ctx.db.query("knowledgeChunks").collect();
|
|
156
|
-
return all
|
|
157
|
-
.filter(
|
|
158
|
-
(c) =>
|
|
159
|
-
!c.embedding_de ||
|
|
160
|
-
!c.embeddedAt_de ||
|
|
161
|
-
c.embeddedAt_de < c.updatedAt,
|
|
162
|
-
)
|
|
163
|
-
.map((c) => ({
|
|
164
|
-
_id: c._id,
|
|
165
|
-
topic: c.topic,
|
|
166
|
-
chunkKey: c.chunkKey,
|
|
167
|
-
title: c.title_Deutsch,
|
|
168
|
-
content: c.content_Deutsch,
|
|
169
|
-
}));
|
|
170
|
-
},
|
|
171
|
-
});
|
|
172
|
-
|
|
173
|
-
/** Update a chunk's English embedding by ID. */
|
|
174
|
-
export const updateChunkEmbeddingEn = mutation({
|
|
175
|
-
args: {
|
|
176
|
-
chunkId: v.id("knowledgeChunks"),
|
|
177
|
-
embedding: v.array(v.float64()),
|
|
178
|
-
},
|
|
179
|
-
handler: async (ctx, { chunkId, embedding }) => {
|
|
180
|
-
await ctx.db.patch(chunkId, {
|
|
181
|
-
embedding_en: embedding,
|
|
182
|
-
embeddedAt_en: Date.now(),
|
|
183
|
-
});
|
|
184
|
-
},
|
|
185
|
-
});
|
|
186
137
|
|
|
187
|
-
/** Update a chunk's Deutsch embedding by ID. */
|
|
188
|
-
export const updateChunkEmbeddingDe = mutation({
|
|
189
|
-
args: {
|
|
190
|
-
chunkId: v.id("knowledgeChunks"),
|
|
191
|
-
embedding: v.array(v.float64()),
|
|
192
|
-
},
|
|
193
|
-
handler: async (ctx, { chunkId, embedding }) => {
|
|
194
|
-
await ctx.db.patch(chunkId, {
|
|
195
|
-
embedding_de: embedding,
|
|
196
|
-
embeddedAt_de: Date.now(),
|
|
197
|
-
});
|
|
198
|
-
},
|
|
199
|
-
});
|
|
200
138
|
|
|
201
139
|
// ── Internal functions (called by knowledgeSearch.ts) ──
|
|
202
140
|
|