@theromans/convex-dobty 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component/knowledge.d.ts +0 -1
- package/dist/component/knowledge.d.ts.map +1 -1
- package/dist/component/knowledge.js +11 -17
- package/dist/component/knowledge.js.map +1 -1
- package/dist/component/knowledgeSearch.d.ts +2 -1
- package/dist/component/knowledgeSearch.d.ts.map +1 -1
- package/dist/component/knowledgeSearch.js +26 -10
- package/dist/component/knowledgeSearch.js.map +1 -1
- package/package.json +1 -1
- package/src/component/knowledge.ts +11 -19
- package/src/component/knowledgeSearch.ts +27 -10
|
@@ -12,7 +12,6 @@ export declare const seedKnowledgeChunks: import("convex/server").RegisteredMuta
|
|
|
12
12
|
inserted: number;
|
|
13
13
|
updated: number;
|
|
14
14
|
skipped: number;
|
|
15
|
-
deleted: number;
|
|
16
15
|
}>>;
|
|
17
16
|
/** Delete all knowledge chunks for a given topic (or all topics if omitted). Used before re-seeding. */
|
|
18
17
|
export declare const clearKnowledgeChunks: import("convex/server").RegisteredMutation<"public", {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge.d.ts","sourceRoot":"","sources":["../../src/component/knowledge.ts"],"names":[],"mappings":"AASA,oFAAoF;AACpF,eAAO,MAAM,mBAAmB
|
|
1
|
+
{"version":3,"file":"knowledge.d.ts","sourceRoot":"","sources":["../../src/component/knowledge.ts"],"names":[],"mappings":"AASA,oFAAoF;AACpF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;GAyF9B,CAAC;AAEH,wGAAwG;AACxG,eAAO,MAAM,oBAAoB;;;;GAgB/B,CAAC;AAEH,4BAA4B;AAC5B,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;KAQrB,CAAC;AAEH,qCAAqC;AACrC,eAAO,MAAM,UAAU,0EAOrB,CAAC;AAMH,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;iBAW5B,CAAC;AAEH,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;iBAW5B,CAAC;AAEH,yCAAyC;AACzC,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;KAKxB,CAAC;AAEH,gCAAgC;AAChC,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;UAKnB,CAAC"}
|
|
@@ -17,8 +17,6 @@ export const seedKnowledgeChunks = mutation({
|
|
|
17
17
|
let inserted = 0;
|
|
18
18
|
let updated = 0;
|
|
19
19
|
let skipped = 0;
|
|
20
|
-
// Collect all existing chunkKeys for this topic to detect deletions
|
|
21
|
-
const incomingKeys = new Set(chunks.map((c) => `${c.topic}::${c.chunkKey}`));
|
|
22
20
|
for (const chunk of chunks) {
|
|
23
21
|
const existing = await ctx.db
|
|
24
22
|
.query("knowledgeChunks")
|
|
@@ -38,10 +36,12 @@ export const seedKnowledgeChunks = mutation({
|
|
|
38
36
|
// Bump updatedAt — scheduler will detect embeddedAt < updatedAt and re-embed
|
|
39
37
|
updatedAt: Date.now(),
|
|
40
38
|
});
|
|
41
|
-
// Schedule EN embedding; it will chain to DE automatically
|
|
42
|
-
|
|
39
|
+
// Schedule EN embedding; it will chain to DE automatically.
|
|
40
|
+
// 10s delay lets vector indexes finish bootstrapping after a fresh deploy.
|
|
41
|
+
await ctx.scheduler.runAfter(10_000, internal.knowledgeSearch.embedChunk, {
|
|
43
42
|
chunkId: existing._id,
|
|
44
43
|
language: "en",
|
|
44
|
+
attempt: 0,
|
|
45
45
|
});
|
|
46
46
|
updated++;
|
|
47
47
|
}
|
|
@@ -52,9 +52,10 @@ export const seedKnowledgeChunks = mutation({
|
|
|
52
52
|
!existing.embeddedAt_en ||
|
|
53
53
|
!existing.embeddedAt_de;
|
|
54
54
|
if (needsEmbed) {
|
|
55
|
-
await ctx.scheduler.runAfter(
|
|
55
|
+
await ctx.scheduler.runAfter(10_000, internal.knowledgeSearch.embedChunk, {
|
|
56
56
|
chunkId: existing._id,
|
|
57
57
|
language: "en",
|
|
58
|
+
attempt: 0,
|
|
58
59
|
});
|
|
59
60
|
}
|
|
60
61
|
skipped++;
|
|
@@ -70,24 +71,17 @@ export const seedKnowledgeChunks = mutation({
|
|
|
70
71
|
content_Deutsch: chunk.content_Deutsch,
|
|
71
72
|
updatedAt: Date.now(),
|
|
72
73
|
});
|
|
73
|
-
// Schedule EN embedding; it will chain to DE automatically
|
|
74
|
-
|
|
74
|
+
// Schedule EN embedding; it will chain to DE automatically.
|
|
75
|
+
// 10s delay lets vector indexes finish bootstrapping after a fresh deploy.
|
|
76
|
+
await ctx.scheduler.runAfter(10_000, internal.knowledgeSearch.embedChunk, {
|
|
75
77
|
chunkId: id,
|
|
76
78
|
language: "en",
|
|
79
|
+
attempt: 0,
|
|
77
80
|
});
|
|
78
81
|
inserted++;
|
|
79
82
|
}
|
|
80
83
|
}
|
|
81
|
-
|
|
82
|
-
const allExisting = await ctx.db.query("knowledgeChunks").collect();
|
|
83
|
-
let deleted = 0;
|
|
84
|
-
for (const existing of allExisting) {
|
|
85
|
-
if (!incomingKeys.has(`${existing.topic}::${existing.chunkKey}`)) {
|
|
86
|
-
await ctx.db.delete(existing._id);
|
|
87
|
-
deleted++;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
return { inserted, updated, skipped, deleted };
|
|
84
|
+
return { inserted, updated, skipped };
|
|
91
85
|
},
|
|
92
86
|
});
|
|
93
87
|
/** Delete all knowledge chunks for a given topic (or all topics if omitted). Used before re-seeding. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge.js","sourceRoot":"","sources":["../../src/component/knowledge.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,KAAK,EACL,gBAAgB,EAChB,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,oFAAoF;AACpF,MAAM,CAAC,MAAM,mBAAmB,GAAG,QAAQ,CAAC;IAC1C,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,KAAK,CACb,CAAC,CAAC,MAAM,CAAC;YACP,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;YACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;YACpB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;YACzB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;YACzB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;YAC3B,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;SAC5B,CAAC,CACH;KACF;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;QACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,
|
|
1
|
+
{"version":3,"file":"knowledge.js","sourceRoot":"","sources":["../../src/component/knowledge.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,KAAK,EACL,gBAAgB,EAChB,aAAa,GACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,oFAAoF;AACpF,MAAM,CAAC,MAAM,mBAAmB,GAAG,QAAQ,CAAC;IAC1C,IAAI,EAAE;QACJ,MAAM,EAAE,CAAC,CAAC,KAAK,CACb,CAAC,CAAC,MAAM,CAAC;YACP,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;YACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;YACpB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;YACzB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;YACzB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;YAC3B,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;SAC5B,CAAC,CACH;KACF;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;QACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;iBAC1B,KAAK,CAAC,iBAAiB,CAAC;iBACxB,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE,CACpC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAC1D;iBACA,MAAM,EAAE,CAAC;YAEZ,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,cAAc,GAClB,QAAQ,CAAC,eAAe,KAAK,KAAK,CAAC,eAAe;oBAClD,QAAQ,CAAC,eAAe,KAAK,KAAK,CAAC,eAAe;oBAClD,QAAQ,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa;oBAC9C,QAAQ,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa,CAAC;gBAEjD,IAAI,cAAc,EAAE,CAAC;oBACnB,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE;wBAC/B,aAAa,EAAE,KAAK,CAAC,aAAa;wBAClC,aAAa,EAAE,KAAK,CAAC,aAAa;wBAClC,eAAe,EAAE,KAAK,CAAC,eAAe;wBACtC,eAAe,EAAE,KAAK,CAAC,eAAe;wBACtC,6EAA6E;wBAC7E,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB,CAAC,CAAC;oBACH,4DAA4D;oBAC5D,2EAA2E;oBAC3E,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;wBACxE,OAAO,EAAE,QAAQ,CAAC,GAAG;wBACrB,QAAQ,EAAE,IAAI;wBACd,OAAO,EAAE,CAAC;qBACX,CAAC,CAAC;oBACH,OAAO,EAAE,CAAC;gBACZ,CAAC;qBAAM,CAAC;oBACN,oEAAoE;oBACpE,MAAM,UAAU,GACd,CAAC,QAAQ,CAAC,YAAY;wBACtB,CAAC,QAAQ,CAAC,YAAY;wBACtB,CAAC,QAAQ,CAAC,aAAa;wBACvB,CAAC,QAAQ,CAAC,aAAa,CAAC;oBAC1B,IAAI,UAAU,EAAE,CAAC;wBACf,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;4BACxE,OAAO,EAAE,QAAQ,CAAC,GAAG;4BACrB,QAAQ,EAAE,IAAI;4BACd,OAAO,EAAE,CAAC;yBACX,CAAC,CAAC;oBACL,CAAC;oBACD,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,iBAAiB,EAAE;oBAChD,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,eAAe,EAAE,KAAK,CAAC,eAAe;oBACtC,eAAe,EAAE,KAAK,CAAC,eAAe;oBACtC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC,CAAC;gBACH,4DAA4D;gBAC5D,2EAA2E;gBAC3E,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACxE,OAAO,EAAE,EAAE;oBACX,QAAQ,EAAE,IAAI;oBACd,OAAO,EAAE,CAAC;iBACX,CAAC,CAAC;gBACH,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IACxC,CAAC;CACF,CAAC,CAAC;AAEH,wGAAwG;AACxG,MAAM,CAAC,MAAM,oBAAoB,GAAG,QAAQ,CAAC;IAC3C,IAAI,EAAE;QACJ,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAC9B;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAChC,MAAM,GAAG,GAAG,KAAK;YACf,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;iBACT,KAAK,CAAC,iBAAiB,CAAC;iBACxB,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClD,OAAO,EAAE;YACd,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,OAAO,EAAE,CAAC;QACpD,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YACtB,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;IACjC,CAAC;CACF,CAAC,CAAC;AAEH,4BAA4B;AAC5B,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,CAAC;IAC9B,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE;IAC3B,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAChC,OAAO,MAAM,GAAG,CAAC,EAAE;aAChB,KAAK,CAAC,iBAAiB,CAAC;aACxB,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;aAClD,OAAO,EAAE,CAAC;IACf,CAAC;CACF,CAAC,CAAC;AAEH,qCAAqC;AACrC,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,CAAC;IAC9B,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,OAAO,EAAE,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;IACrB,CAAC;CACF,CAAC,CAAC;AAIH,0DAA0D;AAE1D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAAC;IAChD,IAAI,EAAE;QACJ,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC;QAChC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;KAChC;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE;QAC7C,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE;YAC1B,YAAY,EAAE,SAAS;YACvB,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;SAC1B,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAAC;IAChD,IAAI,EAAE;QACJ,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC;QAChC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;KAChC;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE;QAC7C,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE;YAC1B,YAAY,EAAE,SAAS;YACvB,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;SAC1B,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,yCAAyC;AACzC,MAAM,CAAC,MAAM,aAAa,GAAG,aAAa,CAAC;IACzC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,OAAO,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,OAAO,EAAE,CAAC;IACzD,CAAC;CACF,CAAC,CAAC;AAEH,gCAAgC;AAChC,MAAM,CAAC,MAAM,QAAQ,GAAG,aAAa,CAAC;IACpC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC,EAAE;IAC1C,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;QAClC,OAAO,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;CACF,CAAC,CAAC"}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Embed a single chunk for a single language.
|
|
3
3
|
* Called by the scheduler after a chunk is seeded or updated.
|
|
4
|
-
*
|
|
4
|
+
* Retries on failure with a delay. Chains to the other language when done.
|
|
5
5
|
*/
|
|
6
6
|
export declare const embedChunk: import("convex/server").RegisteredAction<"internal", {
|
|
7
|
+
attempt?: number | undefined;
|
|
7
8
|
chunkId: import("convex/values").GenericId<"knowledgeChunks">;
|
|
8
9
|
language: "en" | "de";
|
|
9
10
|
}, Promise<void>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledgeSearch.d.ts","sourceRoot":"","sources":["../../src/component/knowledgeSearch.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"knowledgeSearch.d.ts","sourceRoot":"","sources":["../../src/component/knowledgeSearch.ts"],"names":[],"mappings":"AA+CA;;;;GAIG;AACH,eAAO,MAAM,UAAU;;;;iBAuFrB,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,uBAAuB;;;GAmClC,CAAC;AAEH,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;IACtB,iBAAiB,EAAE,IAAI,GAAG,IAAI,CAAC;CAChC;AAED;;;;;GAKG;AACH,eAAO,MAAM,eAAe;;;;;;8BAY1B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;8BAUzB,CAAC"}
|
|
@@ -40,17 +40,20 @@ async function getEmbedding(text, apiToken) {
|
|
|
40
40
|
clearTimeout(timer);
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
|
+
const EMBED_RETRY_DELAY_MS = 30_000; // retry after 30s on transient failure
|
|
44
|
+
const EMBED_MAX_RETRIES = 5;
|
|
43
45
|
/**
|
|
44
46
|
* Embed a single chunk for a single language.
|
|
45
47
|
* Called by the scheduler after a chunk is seeded or updated.
|
|
46
|
-
*
|
|
48
|
+
* Retries on failure with a delay. Chains to the other language when done.
|
|
47
49
|
*/
|
|
48
50
|
export const embedChunk = internalAction({
|
|
49
51
|
args: {
|
|
50
52
|
chunkId: v.id("knowledgeChunks"),
|
|
51
53
|
language: v.union(v.literal("en"), v.literal("de")),
|
|
54
|
+
attempt: v.optional(v.number()),
|
|
52
55
|
},
|
|
53
|
-
handler: async (ctx, { chunkId, language }) => {
|
|
56
|
+
handler: async (ctx, { chunkId, language, attempt = 0 }) => {
|
|
54
57
|
const apiToken = globalThis.process?.env?.IONOS_API_TOKEN;
|
|
55
58
|
if (!apiToken) {
|
|
56
59
|
console.error("[embedChunk] IONOS_API_TOKEN not set — cannot embed");
|
|
@@ -68,14 +71,15 @@ export const embedChunk = internalAction({
|
|
|
68
71
|
!chunk.embeddedAt_de ||
|
|
69
72
|
chunk.embeddedAt_de < chunk.updatedAt;
|
|
70
73
|
const isStale = language === "en" ? isStaleEn : isStaleDe;
|
|
74
|
+
const otherLanguage = language === "en" ? "de" : "en";
|
|
75
|
+
const otherStale = language === "en" ? isStaleDe : isStaleEn;
|
|
71
76
|
if (!isStale) {
|
|
72
|
-
// Already up to date —
|
|
73
|
-
const otherLanguage = language === "en" ? "de" : "en";
|
|
74
|
-
const otherStale = language === "en" ? isStaleDe : isStaleEn;
|
|
77
|
+
// Already up to date — chain to other language if needed
|
|
75
78
|
if (otherStale) {
|
|
76
79
|
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
77
80
|
chunkId,
|
|
78
81
|
language: otherLanguage,
|
|
82
|
+
attempt: 0,
|
|
79
83
|
});
|
|
80
84
|
}
|
|
81
85
|
return;
|
|
@@ -94,19 +98,31 @@ export const embedChunk = internalAction({
|
|
|
94
98
|
else {
|
|
95
99
|
await ctx.runMutation(internal.knowledge.updateEmbeddingDe, { chunkId, embedding });
|
|
96
100
|
}
|
|
97
|
-
console.log(`[embedChunk]
|
|
98
|
-
//
|
|
99
|
-
const otherLanguage = language === "en" ? "de" : "en";
|
|
100
|
-
const otherStale = language === "en" ? isStaleDe : isStaleEn;
|
|
101
|
+
console.log(`[embedChunk] ✓ ${chunk.chunkKey} (${language})`);
|
|
102
|
+
// Chain to the other language if also stale
|
|
101
103
|
if (otherStale) {
|
|
102
104
|
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
103
105
|
chunkId,
|
|
104
106
|
language: otherLanguage,
|
|
107
|
+
attempt: 0,
|
|
105
108
|
});
|
|
106
109
|
}
|
|
107
110
|
}
|
|
108
111
|
catch (e) {
|
|
109
|
-
console.error(`[embedChunk] failed
|
|
112
|
+
console.error(`[embedChunk] failed ${chunk.chunkKey} (${language}) attempt ${attempt}: ${e.message}`);
|
|
113
|
+
if (attempt < EMBED_MAX_RETRIES) {
|
|
114
|
+
// Exponential backoff: 30s, 60s, 120s, 240s, 480s
|
|
115
|
+
const delay = EMBED_RETRY_DELAY_MS * Math.pow(2, attempt);
|
|
116
|
+
console.log(`[embedChunk] retrying in ${delay / 1000}s (attempt ${attempt + 1}/${EMBED_MAX_RETRIES})`);
|
|
117
|
+
await ctx.scheduler.runAfter(delay, internal.knowledgeSearch.embedChunk, {
|
|
118
|
+
chunkId,
|
|
119
|
+
language,
|
|
120
|
+
attempt: attempt + 1,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
console.error(`[embedChunk] giving up on ${chunk.chunkKey} (${language}) after ${EMBED_MAX_RETRIES} attempts`);
|
|
125
|
+
}
|
|
110
126
|
}
|
|
111
127
|
},
|
|
112
128
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledgeSearch.js","sourceRoot":"","sources":["../../src/component/knowledgeSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAoB,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,MAAM,kBAAkB,GACtB,yDAAyD,CAAC;AAC5D,MAAM,eAAe,GAAG,aAAa,CAAC;AACtC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB;IACxD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,oBAAoB,CAAC,CAAC;IACzE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,kBAAkB,EAAE;YAC/C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,QAAQ,EAAE;gBACnC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,eAAe;gBACtB,UAAU,EAAE,oBAAoB;aACjC,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAqB,CAAC;IAC5C,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QAChB,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,iCAAiC,oBAAoB,IAAI,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,cAAc,CAAC;IACvC,IAAI,EAAE;QACJ,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC;QAChC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledgeSearch.js","sourceRoot":"","sources":["../../src/component/knowledgeSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAoB,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,MAAM,kBAAkB,GACtB,yDAAyD,CAAC;AAC5D,MAAM,eAAe,GAAG,aAAa,CAAC;AACtC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB;IACxD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,oBAAoB,CAAC,CAAC;IACzE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,kBAAkB,EAAE;YAC/C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,QAAQ,EAAE;gBACnC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,eAAe;gBACtB,UAAU,EAAE,oBAAoB;aACjC,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAqB,CAAC;IAC5C,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QAChB,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,iCAAiC,oBAAoB,IAAI,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,oBAAoB,GAAG,MAAM,CAAC,CAAC,uCAAuC;AAC5E,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,cAAc,CAAC;IACvC,IAAI,EAAE;QACJ,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC;QAChC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAChC;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;QACzD,MAAM,QAAQ,GAAI,UAAkB,CAAC,OAAO,EAAE,GAAG,EAAE,eAAqC,CAAC;QACzF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACrE,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAQ,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,sBAAsB,OAAO,sBAAsB,CAAC,CAAC;YAClE,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GACb,CAAC,KAAK,CAAC,YAAY;YACnB,CAAC,KAAK,CAAC,aAAa;YACpB,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;QACxC,MAAM,SAAS,GACb,CAAC,KAAK,CAAC,YAAY;YACnB,CAAC,KAAK,CAAC,aAAa;YACpB,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;QAExC,MAAM,OAAO,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1D,MAAM,aAAa,GAAgB,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,MAAM,UAAU,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAE7D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,yDAAyD;YACzD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACnE,OAAO;oBACP,QAAQ,EAAE,aAAa;oBACvB,OAAO,EAAE,CAAC;iBACX,CAAC,CAAC;YACL,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;QAC5E,MAAM,OAAO,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;QAElF,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,sBAAsB,KAAK,CAAC,QAAQ,YAAY,QAAQ,oBAAoB,CAAC,CAAC;YAC3F,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,GAAG,KAAK,OAAO,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;YAEzE,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;gBACtB,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;YACtF,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;YACtF,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,QAAQ,KAAK,QAAQ,GAAG,CAAC,CAAC;YAE9D,4CAA4C;YAC5C,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACnE,OAAO;oBACP,QAAQ,EAAE,aAAa;oBACvB,OAAO,EAAE,CAAC;iBACX,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,uBAAuB,KAAK,CAAC,QAAQ,KAAK,QAAQ,aAAa,OAAO,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACtG,IAAI,OAAO,GAAG,iBAAiB,EAAE,CAAC;gBAChC,kDAAkD;gBAClD,MAAM,KAAK,GAAG,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC1D,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,GAAG,IAAI,cAAc,OAAO,GAAG,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC;gBACvG,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACvE,OAAO;oBACP,QAAQ;oBACR,OAAO,EAAE,OAAO,GAAG,CAAC;iBACrB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,6BAA6B,KAAK,CAAC,QAAQ,KAAK,QAAQ,WAAW,iBAAiB,WAAW,CAAC,CAAC;YACjH,CAAC;QACH,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,cAAc,CAAC;IACpD,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,GAAG,GAAU,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAE5E,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;YACxB,MAAM,OAAO,GACX,CAAC,KAAK,CAAC,YAAY;gBACnB,CAAC,KAAK,CAAC,aAAa;gBACpB,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;YACxC,MAAM,OAAO,GACX,CAAC,KAAK,CAAC,YAAY;gBACnB,CAAC,KAAK,CAAC,aAAa;gBACpB,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;YAExC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACnE,OAAO,EAAE,KAAK,CAAC,GAAG;oBAClB,QAAQ,EAAE,IAAI;iBACf,CAAC,CAAC;gBACH,SAAS,EAAE,CAAC;YACd,CAAC;iBAAM,IAAI,OAAO,EAAE,CAAC;gBACnB,8DAA8D;gBAC9D,MAAM,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE;oBACnE,OAAO,EAAE,KAAK,CAAC,GAAG;oBAClB,QAAQ,EAAE,IAAI;iBACf,CAAC,CAAC;gBACH,SAAS,EAAE,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,SAAS,aAAa,GAAG,CAAC,MAAM,SAAS,CAAC,CAAC;QAC9F,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAC1C,CAAC;CACF,CAAC,CAAC;AAcH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAC;IACpC,IAAI,EAAE;QACJ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;KACrB;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,GAAG,oBAAoB,EAAE,QAAQ,EAAE,EAAE,EAAE;QACzF,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACtD,OAAO,eAAe,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;CACF,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAC;IACnC,IAAI,EAAE;QACJ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAC/B,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAC9B;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,GAAG,oBAAoB,EAAE,EAAE,EAAE;QACnF,OAAO,eAAe,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;CACF,CAAC,CAAC;AAEH,KAAK,UAAU,eAAe,CAC5B,GAAQ,EACR,EACE,KAAK,EACL,SAAS,EACT,QAAQ,EACR,KAAK,GAMN;IAED,MAAM,YAAY,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAC/E,MAAM,aAAa,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAChF,MAAM,gBAAgB,GAAgB,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAEtE,IAAI,OAAO,GAAG,MAAM,GAAG,CAAC,YAAY,CAAC,iBAAiB,EAAE,YAAY,EAAE;QACpE,MAAM,EAAE,SAAS;QACjB,KAAK;QACL,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,iBAAiB,GAAgB,QAAQ,CAAC;IAE9C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,GAAG,MAAM,GAAG,CAAC,YAAY,CAAC,iBAAiB,EAAE,aAAa,EAAE;YACjE,MAAM,EAAE,SAAS;YACjB,KAAK;YACL,MAAM,EAAE,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC;SACzC,CAAC,CAAC;QACH,iBAAiB,GAAG,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEpC,MAAM,IAAI,GAA+B,MAAM,OAAO,CAAC,GAAG,CACxD,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAM,EAAmC,EAAE;QAC5D,MAAM,GAAG,GAAQ,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACrF,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,MAAM,KAAK,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC;QACxE,MAAM,OAAO,GAAG,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC;QAC9E,OAAO;YACL,KAAK;YACL,OAAO;YACP,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,eAAe,EAAE,GAAG,CAAC,eAAe;YACpC,eAAe,EAAE,GAAG,CAAC,eAAe;YACpC,KAAK,EAAE,CAAC,CAAC,MAAM;YACf,QAAQ;YACR,iBAAiB;SAClB,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAwB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AAC9D,CAAC"}
|
package/package.json
CHANGED
|
@@ -26,9 +26,6 @@ export const seedKnowledgeChunks = mutation({
|
|
|
26
26
|
let updated = 0;
|
|
27
27
|
let skipped = 0;
|
|
28
28
|
|
|
29
|
-
// Collect all existing chunkKeys for this topic to detect deletions
|
|
30
|
-
const incomingKeys = new Set(chunks.map((c) => `${c.topic}::${c.chunkKey}`));
|
|
31
|
-
|
|
32
29
|
for (const chunk of chunks) {
|
|
33
30
|
const existing = await ctx.db
|
|
34
31
|
.query("knowledgeChunks")
|
|
@@ -53,10 +50,12 @@ export const seedKnowledgeChunks = mutation({
|
|
|
53
50
|
// Bump updatedAt — scheduler will detect embeddedAt < updatedAt and re-embed
|
|
54
51
|
updatedAt: Date.now(),
|
|
55
52
|
});
|
|
56
|
-
// Schedule EN embedding; it will chain to DE automatically
|
|
57
|
-
|
|
53
|
+
// Schedule EN embedding; it will chain to DE automatically.
|
|
54
|
+
// 10s delay lets vector indexes finish bootstrapping after a fresh deploy.
|
|
55
|
+
await ctx.scheduler.runAfter(10_000, internal.knowledgeSearch.embedChunk, {
|
|
58
56
|
chunkId: existing._id,
|
|
59
57
|
language: "en",
|
|
58
|
+
attempt: 0,
|
|
60
59
|
});
|
|
61
60
|
updated++;
|
|
62
61
|
} else {
|
|
@@ -67,9 +66,10 @@ export const seedKnowledgeChunks = mutation({
|
|
|
67
66
|
!existing.embeddedAt_en ||
|
|
68
67
|
!existing.embeddedAt_de;
|
|
69
68
|
if (needsEmbed) {
|
|
70
|
-
await ctx.scheduler.runAfter(
|
|
69
|
+
await ctx.scheduler.runAfter(10_000, internal.knowledgeSearch.embedChunk, {
|
|
71
70
|
chunkId: existing._id,
|
|
72
71
|
language: "en",
|
|
72
|
+
attempt: 0,
|
|
73
73
|
});
|
|
74
74
|
}
|
|
75
75
|
skipped++;
|
|
@@ -84,26 +84,18 @@ export const seedKnowledgeChunks = mutation({
|
|
|
84
84
|
content_Deutsch: chunk.content_Deutsch,
|
|
85
85
|
updatedAt: Date.now(),
|
|
86
86
|
});
|
|
87
|
-
// Schedule EN embedding; it will chain to DE automatically
|
|
88
|
-
|
|
87
|
+
// Schedule EN embedding; it will chain to DE automatically.
|
|
88
|
+
// 10s delay lets vector indexes finish bootstrapping after a fresh deploy.
|
|
89
|
+
await ctx.scheduler.runAfter(10_000, internal.knowledgeSearch.embedChunk, {
|
|
89
90
|
chunkId: id,
|
|
90
91
|
language: "en",
|
|
92
|
+
attempt: 0,
|
|
91
93
|
});
|
|
92
94
|
inserted++;
|
|
93
95
|
}
|
|
94
96
|
}
|
|
95
97
|
|
|
96
|
-
|
|
97
|
-
const allExisting = await ctx.db.query("knowledgeChunks").collect();
|
|
98
|
-
let deleted = 0;
|
|
99
|
-
for (const existing of allExisting) {
|
|
100
|
-
if (!incomingKeys.has(`${existing.topic}::${existing.chunkKey}`)) {
|
|
101
|
-
await ctx.db.delete(existing._id);
|
|
102
|
-
deleted++;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
return { inserted, updated, skipped, deleted };
|
|
98
|
+
return { inserted, updated, skipped };
|
|
107
99
|
},
|
|
108
100
|
});
|
|
109
101
|
|
|
@@ -42,17 +42,21 @@ async function getEmbedding(text: string, apiToken: string): Promise<number[]> {
|
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
const EMBED_RETRY_DELAY_MS = 30_000; // retry after 30s on transient failure
|
|
46
|
+
const EMBED_MAX_RETRIES = 5;
|
|
47
|
+
|
|
45
48
|
/**
|
|
46
49
|
* Embed a single chunk for a single language.
|
|
47
50
|
* Called by the scheduler after a chunk is seeded or updated.
|
|
48
|
-
*
|
|
51
|
+
* Retries on failure with a delay. Chains to the other language when done.
|
|
49
52
|
*/
|
|
50
53
|
export const embedChunk = internalAction({
|
|
51
54
|
args: {
|
|
52
55
|
chunkId: v.id("knowledgeChunks"),
|
|
53
56
|
language: v.union(v.literal("en"), v.literal("de")),
|
|
57
|
+
attempt: v.optional(v.number()),
|
|
54
58
|
},
|
|
55
|
-
handler: async (ctx, { chunkId, language }) => {
|
|
59
|
+
handler: async (ctx, { chunkId, language, attempt = 0 }) => {
|
|
56
60
|
const apiToken = (globalThis as any).process?.env?.IONOS_API_TOKEN as string | undefined;
|
|
57
61
|
if (!apiToken) {
|
|
58
62
|
console.error("[embedChunk] IONOS_API_TOKEN not set — cannot embed");
|
|
@@ -75,14 +79,16 @@ export const embedChunk = internalAction({
|
|
|
75
79
|
chunk.embeddedAt_de < chunk.updatedAt;
|
|
76
80
|
|
|
77
81
|
const isStale = language === "en" ? isStaleEn : isStaleDe;
|
|
82
|
+
const otherLanguage: "en" | "de" = language === "en" ? "de" : "en";
|
|
83
|
+
const otherStale = language === "en" ? isStaleDe : isStaleEn;
|
|
84
|
+
|
|
78
85
|
if (!isStale) {
|
|
79
|
-
// Already up to date —
|
|
80
|
-
const otherLanguage = language === "en" ? "de" : "en";
|
|
81
|
-
const otherStale = language === "en" ? isStaleDe : isStaleEn;
|
|
86
|
+
// Already up to date — chain to other language if needed
|
|
82
87
|
if (otherStale) {
|
|
83
88
|
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
84
89
|
chunkId,
|
|
85
90
|
language: otherLanguage,
|
|
91
|
+
attempt: 0,
|
|
86
92
|
});
|
|
87
93
|
}
|
|
88
94
|
return;
|
|
@@ -105,19 +111,30 @@ export const embedChunk = internalAction({
|
|
|
105
111
|
await ctx.runMutation(internal.knowledge.updateEmbeddingDe, { chunkId, embedding });
|
|
106
112
|
}
|
|
107
113
|
|
|
108
|
-
console.log(`[embedChunk]
|
|
114
|
+
console.log(`[embedChunk] ✓ ${chunk.chunkKey} (${language})`);
|
|
109
115
|
|
|
110
|
-
//
|
|
111
|
-
const otherLanguage = language === "en" ? "de" : "en";
|
|
112
|
-
const otherStale = language === "en" ? isStaleDe : isStaleEn;
|
|
116
|
+
// Chain to the other language if also stale
|
|
113
117
|
if (otherStale) {
|
|
114
118
|
await ctx.scheduler.runAfter(0, internal.knowledgeSearch.embedChunk, {
|
|
115
119
|
chunkId,
|
|
116
120
|
language: otherLanguage,
|
|
121
|
+
attempt: 0,
|
|
117
122
|
});
|
|
118
123
|
}
|
|
119
124
|
} catch (e: any) {
|
|
120
|
-
console.error(`[embedChunk] failed
|
|
125
|
+
console.error(`[embedChunk] failed ${chunk.chunkKey} (${language}) attempt ${attempt}: ${e.message}`);
|
|
126
|
+
if (attempt < EMBED_MAX_RETRIES) {
|
|
127
|
+
// Exponential backoff: 30s, 60s, 120s, 240s, 480s
|
|
128
|
+
const delay = EMBED_RETRY_DELAY_MS * Math.pow(2, attempt);
|
|
129
|
+
console.log(`[embedChunk] retrying in ${delay / 1000}s (attempt ${attempt + 1}/${EMBED_MAX_RETRIES})`);
|
|
130
|
+
await ctx.scheduler.runAfter(delay, internal.knowledgeSearch.embedChunk, {
|
|
131
|
+
chunkId,
|
|
132
|
+
language,
|
|
133
|
+
attempt: attempt + 1,
|
|
134
|
+
});
|
|
135
|
+
} else {
|
|
136
|
+
console.error(`[embedChunk] giving up on ${chunk.chunkKey} (${language}) after ${EMBED_MAX_RETRIES} attempts`);
|
|
137
|
+
}
|
|
121
138
|
}
|
|
122
139
|
},
|
|
123
140
|
});
|