@zakstam/codex-local-component 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -4
- package/dist/app-server/client.d.ts +40 -1
- package/dist/app-server/client.d.ts.map +1 -1
- package/dist/app-server/client.js +56 -3
- package/dist/app-server/client.js.map +1 -1
- package/dist/app-server/index.d.ts +1 -1
- package/dist/app-server/index.d.ts.map +1 -1
- package/dist/app-server/index.js +1 -1
- package/dist/app-server/index.js.map +1 -1
- package/dist/client/index.d.ts +4 -2
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +3 -1
- package/dist/client/index.js.map +1 -1
- package/dist/client/reasoning.d.ts +10 -0
- package/dist/client/reasoning.d.ts.map +1 -0
- package/dist/client/reasoning.js +4 -0
- package/dist/client/reasoning.js.map +1 -0
- package/dist/client/serverRequests.d.ts +14 -0
- package/dist/client/serverRequests.d.ts.map +1 -0
- package/dist/client/serverRequests.js +10 -0
- package/dist/client/serverRequests.js.map +1 -0
- package/dist/client/threads.d.ts +39 -3
- package/dist/client/threads.d.ts.map +1 -1
- package/dist/client/threads.js +18 -0
- package/dist/client/threads.js.map +1 -1
- package/dist/client/types.d.ts +23 -1
- package/dist/client/types.d.ts.map +1 -1
- package/dist/component/approvals.d.ts +1 -1
- package/dist/component/index.d.ts +2 -0
- package/dist/component/index.d.ts.map +1 -1
- package/dist/component/index.js +2 -0
- package/dist/component/index.js.map +1 -1
- package/dist/component/ingest/applyApprovals.d.ts +5 -0
- package/dist/component/ingest/applyApprovals.d.ts.map +1 -0
- package/dist/component/ingest/applyApprovals.js +59 -0
- package/dist/component/ingest/applyApprovals.js.map +1 -0
- package/dist/component/ingest/applyMessages.d.ts +4 -0
- package/dist/component/ingest/applyMessages.d.ts.map +1 -0
- package/dist/component/ingest/applyMessages.js +149 -0
- package/dist/component/ingest/applyMessages.js.map +1 -0
- package/dist/component/ingest/applyStreams.d.ts +7 -0
- package/dist/component/ingest/applyStreams.d.ts.map +1 -0
- package/dist/component/ingest/applyStreams.js +189 -0
- package/dist/component/ingest/applyStreams.js.map +1 -0
- package/dist/component/ingest/applyTurns.d.ts +6 -0
- package/dist/component/ingest/applyTurns.d.ts.map +1 -0
- package/dist/component/ingest/applyTurns.js +94 -0
- package/dist/component/ingest/applyTurns.js.map +1 -0
- package/dist/component/ingest/checkpoints.d.ts +12 -0
- package/dist/component/ingest/checkpoints.d.ts.map +1 -0
- package/dist/component/ingest/checkpoints.js +63 -0
- package/dist/component/ingest/checkpoints.js.map +1 -0
- package/dist/component/ingest/index.d.ts +10 -0
- package/dist/component/ingest/index.d.ts.map +1 -0
- package/dist/component/ingest/index.js +76 -0
- package/dist/component/ingest/index.js.map +1 -0
- package/dist/component/ingest/normalize.d.ts +5 -0
- package/dist/component/ingest/normalize.d.ts.map +1 -0
- package/dist/component/ingest/normalize.js +35 -0
- package/dist/component/ingest/normalize.js.map +1 -0
- package/dist/component/ingest/postIngest.d.ts +4 -0
- package/dist/component/ingest/postIngest.d.ts.map +1 -0
- package/dist/component/ingest/postIngest.js +31 -0
- package/dist/component/ingest/postIngest.js.map +1 -0
- package/dist/component/ingest/sessionGuard.d.ts +9 -0
- package/dist/component/ingest/sessionGuard.d.ts.map +1 -0
- package/dist/component/ingest/sessionGuard.js +102 -0
- package/dist/component/ingest/sessionGuard.js.map +1 -0
- package/dist/component/ingest/stateCache.d.ts +19 -0
- package/dist/component/ingest/stateCache.d.ts.map +1 -0
- package/dist/component/ingest/stateCache.js +121 -0
- package/dist/component/ingest/stateCache.js.map +1 -0
- package/dist/component/ingest/types.d.ts +129 -0
- package/dist/component/ingest/types.d.ts.map +1 -0
- package/dist/component/ingest/types.js +2 -0
- package/dist/component/ingest/types.js.map +1 -0
- package/dist/component/reasoning.d.ts +37 -0
- package/dist/component/reasoning.d.ts.map +1 -0
- package/dist/component/reasoning.js +48 -0
- package/dist/component/reasoning.js.map +1 -0
- package/dist/component/schema.d.ts +87 -11
- package/dist/component/schema.d.ts.map +1 -1
- package/dist/component/schema.js +47 -0
- package/dist/component/schema.js.map +1 -1
- package/dist/component/serverRequests.d.ts +53 -0
- package/dist/component/serverRequests.d.ts.map +1 -0
- package/dist/component/serverRequests.js +187 -0
- package/dist/component/serverRequests.js.map +1 -0
- package/dist/component/streamStats.d.ts +10 -0
- package/dist/component/streamStats.d.ts.map +1 -1
- package/dist/component/streamStats.js +34 -0
- package/dist/component/streamStats.js.map +1 -1
- package/dist/component/sync.d.ts +4 -67
- package/dist/component/sync.d.ts.map +1 -1
- package/dist/component/syncHelpers.d.ts +12 -35
- package/dist/component/syncHelpers.d.ts.map +1 -1
- package/dist/component/syncHelpers.js +11 -228
- package/dist/component/syncHelpers.js.map +1 -1
- package/dist/component/syncIngest.d.ts +2 -72
- package/dist/component/syncIngest.d.ts.map +1 -1
- package/dist/component/syncIngest.js +9 -726
- package/dist/component/syncIngest.js.map +1 -1
- package/dist/component/syncRuntime.d.ts +7 -1
- package/dist/component/syncRuntime.d.ts.map +1 -1
- package/dist/component/syncRuntime.js +8 -2
- package/dist/component/syncRuntime.js.map +1 -1
- package/dist/component/types.d.ts +5 -1
- package/dist/component/types.d.ts.map +1 -1
- package/dist/component/types.js +2 -0
- package/dist/component/types.js.map +1 -1
- package/dist/host/convex-entry.d.ts +3 -0
- package/dist/host/convex-entry.d.ts.map +1 -0
- package/dist/host/convex-entry.js +3 -0
- package/dist/host/convex-entry.js.map +1 -0
- package/dist/host/convex.d.ts +17 -0
- package/dist/host/convex.d.ts.map +1 -1
- package/dist/host/convex.js +9 -0
- package/dist/host/convex.js.map +1 -1
- package/dist/host/convexSlice.d.ts +504 -0
- package/dist/host/convexSlice.d.ts.map +1 -0
- package/dist/host/convexSlice.js +315 -0
- package/dist/host/convexSlice.js.map +1 -0
- package/dist/host/index.d.ts +3 -2
- package/dist/host/index.d.ts.map +1 -1
- package/dist/host/index.js +2 -1
- package/dist/host/index.js.map +1 -1
- package/dist/host/runtime.d.ts +100 -2
- package/dist/host/runtime.d.ts.map +1 -1
- package/dist/host/runtime.js +329 -33
- package/dist/host/runtime.js.map +1 -1
- package/dist/local-adapter/bridge.d.ts +3 -2
- package/dist/local-adapter/bridge.d.ts.map +1 -1
- package/dist/local-adapter/bridge.js.map +1 -1
- package/dist/mapping.d.ts +29 -0
- package/dist/mapping.d.ts.map +1 -1
- package/dist/mapping.js +136 -46
- package/dist/mapping.js.map +1 -1
- package/dist/protocol/classifier.d.ts +2 -12
- package/dist/protocol/classifier.d.ts.map +1 -1
- package/dist/protocol/classifier.js +1 -104
- package/dist/protocol/classifier.js.map +1 -1
- package/dist/protocol/events.d.ts +72 -0
- package/dist/protocol/events.d.ts.map +1 -0
- package/dist/protocol/events.js +533 -0
- package/dist/protocol/events.js.map +1 -0
- package/dist/protocol/generated.d.ts +16 -2
- package/dist/protocol/generated.d.ts.map +1 -1
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.d.ts.map +1 -1
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/index.js.map +1 -1
- package/dist/protocol/outbound.d.ts +13 -0
- package/dist/protocol/outbound.d.ts.map +1 -0
- package/dist/protocol/outbound.js +2 -0
- package/dist/protocol/outbound.js.map +1 -0
- package/dist/protocol/parser.d.ts +3 -2
- package/dist/protocol/parser.d.ts.map +1 -1
- package/dist/protocol/parser.js +99 -3
- package/dist/protocol/parser.js.map +1 -1
- package/dist/protocol/schemas/CommandExecutionRequestApprovalResponse.json +72 -0
- package/dist/protocol/schemas/DynamicToolCallResponse.json +66 -0
- package/dist/protocol/schemas/FileChangeRequestApprovalResponse.json +47 -0
- package/dist/protocol/schemas/ToolRequestUserInputResponse.json +34 -0
- package/dist/react/index.d.ts +3 -1
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +2 -0
- package/dist/react/index.js.map +1 -1
- package/dist/react/types.d.ts +13 -1
- package/dist/react/types.d.ts.map +1 -1
- package/dist/react/useCodexReasoning.d.ts +7 -0
- package/dist/react/useCodexReasoning.d.ts.map +1 -0
- package/dist/react/useCodexReasoning.js +16 -0
- package/dist/react/useCodexReasoning.js.map +1 -0
- package/dist/react/useCodexStreamOverlay.d.ts.map +1 -1
- package/dist/react/useCodexStreamOverlay.js +68 -23
- package/dist/react/useCodexStreamOverlay.js.map +1 -1
- package/dist/react/useCodexStreamingReasoning.d.ts +12 -0
- package/dist/react/useCodexStreamingReasoning.d.ts.map +1 -0
- package/dist/react/useCodexStreamingReasoning.js +21 -0
- package/dist/react/useCodexStreamingReasoning.js.map +1 -0
- package/package.json +5 -1
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { now } from "../utils.js";
|
|
2
|
+
export function collectApprovalEffects(ingest, event) {
|
|
3
|
+
if (!event.turnId) {
|
|
4
|
+
return;
|
|
5
|
+
}
|
|
6
|
+
if (event.approvalRequest) {
|
|
7
|
+
ingest.collected.pendingApprovals.set(`${event.turnId}:${event.approvalRequest.itemId}`, event.approvalRequest);
|
|
8
|
+
}
|
|
9
|
+
if (event.approvalResolution) {
|
|
10
|
+
ingest.collected.resolvedApprovals.set(`${event.turnId}:${event.approvalResolution.itemId}`, event.approvalResolution);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export async function finalizeApprovals(ingest, cache) {
|
|
14
|
+
for (const [key, approval] of ingest.collected.pendingApprovals) {
|
|
15
|
+
const turnId = key.split(":")[0];
|
|
16
|
+
if (!turnId) {
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
const existing = await cache.getApprovalRecord(turnId, approval.itemId);
|
|
20
|
+
if (existing) {
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
const approvalId = await ingest.ctx.db.insert("codex_approvals", {
|
|
24
|
+
tenantId: ingest.args.actor.tenantId,
|
|
25
|
+
userId: ingest.args.actor.userId,
|
|
26
|
+
threadId: ingest.args.threadId,
|
|
27
|
+
turnId,
|
|
28
|
+
itemId: approval.itemId,
|
|
29
|
+
kind: approval.kind,
|
|
30
|
+
status: "pending",
|
|
31
|
+
...(approval.reason ? { reason: approval.reason } : {}),
|
|
32
|
+
createdAt: now(),
|
|
33
|
+
});
|
|
34
|
+
cache.setApprovalRecord(turnId, approval.itemId, {
|
|
35
|
+
_id: approvalId,
|
|
36
|
+
status: "pending",
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
for (const [key, resolution] of ingest.collected.resolvedApprovals) {
|
|
40
|
+
const turnId = key.split(":")[0];
|
|
41
|
+
if (!turnId) {
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
const existing = await cache.getApprovalRecord(turnId, resolution.itemId);
|
|
45
|
+
if (!existing || existing.status !== "pending") {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
await ingest.ctx.db.patch(existing._id, {
|
|
49
|
+
status: resolution.status,
|
|
50
|
+
decidedBy: "runtime",
|
|
51
|
+
decidedAt: now(),
|
|
52
|
+
});
|
|
53
|
+
cache.setApprovalRecord(turnId, resolution.itemId, {
|
|
54
|
+
...existing,
|
|
55
|
+
status: resolution.status,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=applyApprovals.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"applyApprovals.js","sourceRoot":"","sources":["../../../src/component/ingest/applyApprovals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAIlC,MAAM,UAAU,sBAAsB,CAAC,MAAqB,EAAE,KAA6B;IACzF,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO;IACT,CAAC;IAED,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QAC1B,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IAClH,CAAC;IAED,IAAI,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAC7B,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC,GAAG,CACpC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,MAAM,EAAE,EACpD,KAAK,CAAC,kBAAkB,CACzB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAqB,EACrB,KAAuB;IAEvB,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAChE,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxE,IAAI,QAAQ,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,iBAAiB,EAAE;YAC/D,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;YAChC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,MAAM;YACN,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,SAAS;YACjB,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,SAAS,EAAE,GAAG,EAAE;SACjB,CAAC,CAAC;QACH,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;YAC/C,GAAG,EAAE,UAAU;YACf,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,iBAAiB,EAAE,CAAC;QACnE,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/C,SAAS;QACX,CAAC;QAED,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE;YACtC,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS,EAAE,SAAS;YACpB,SAAS,EAAE,GAAG,EAAE;SACjB,CAAC,CAAC;QACH,KAAK,CAAC,iBAAiB,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE;YACjD,GAAG,QAAQ;YACX,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { IngestContext, NormalizedInboundEvent } from "./types.js";
|
|
2
|
+
import type { IngestStateCache } from "./stateCache.js";
|
|
3
|
+
export declare function applyMessageEffectsForEvent(ingest: IngestContext, event: NormalizedInboundEvent, cache: IngestStateCache): Promise<void>;
|
|
4
|
+
//# sourceMappingURL=applyMessages.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"applyMessages.d.ts","sourceRoot":"","sources":["../../../src/component/ingest/applyMessages.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAExD,wBAAsB,2BAA2B,CAC/C,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,sBAAsB,EAC7B,KAAK,EAAE,gBAAgB,GACtB,OAAO,CAAC,IAAI,CAAC,CA+Jf"}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { now } from "../utils.js";
|
|
2
|
+
export async function applyMessageEffectsForEvent(ingest, event, cache) {
|
|
3
|
+
const turnId = event.turnId;
|
|
4
|
+
if (!turnId) {
|
|
5
|
+
return;
|
|
6
|
+
}
|
|
7
|
+
if (event.durableMessage) {
|
|
8
|
+
const existing = await cache.getMessageRecord(turnId, event.durableMessage.messageId);
|
|
9
|
+
if (!existing) {
|
|
10
|
+
const nextOrder = await cache.nextOrderForTurn(turnId);
|
|
11
|
+
const newId = await ingest.ctx.db.insert("codex_messages", {
|
|
12
|
+
tenantId: ingest.args.actor.tenantId,
|
|
13
|
+
userId: ingest.args.actor.userId,
|
|
14
|
+
threadId: ingest.args.threadId,
|
|
15
|
+
turnId,
|
|
16
|
+
messageId: event.durableMessage.messageId,
|
|
17
|
+
role: event.durableMessage.role,
|
|
18
|
+
status: event.durableMessage.status,
|
|
19
|
+
text: event.durableMessage.text,
|
|
20
|
+
sourceItemType: event.durableMessage.sourceItemType,
|
|
21
|
+
orderInTurn: nextOrder,
|
|
22
|
+
payloadJson: event.durableMessage.payloadJson,
|
|
23
|
+
...(event.durableMessage.status === "failed" ? { error: "item failed" } : {}),
|
|
24
|
+
createdAt: event.createdAt,
|
|
25
|
+
updatedAt: now(),
|
|
26
|
+
...(event.durableMessage.status !== "streaming" ? { completedAt: now() } : {}),
|
|
27
|
+
});
|
|
28
|
+
cache.setMessageRecord(turnId, event.durableMessage.messageId, {
|
|
29
|
+
_id: newId,
|
|
30
|
+
status: event.durableMessage.status,
|
|
31
|
+
text: event.durableMessage.text,
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
const nextStatus = (() => {
|
|
36
|
+
if (existing.status === "failed") {
|
|
37
|
+
return "failed";
|
|
38
|
+
}
|
|
39
|
+
if (existing.status === "interrupted" && event.durableMessage?.status !== "failed") {
|
|
40
|
+
return "interrupted";
|
|
41
|
+
}
|
|
42
|
+
if (event.durableMessage?.status === "streaming") {
|
|
43
|
+
return existing.status;
|
|
44
|
+
}
|
|
45
|
+
return event.durableMessage?.status ?? existing.status;
|
|
46
|
+
})();
|
|
47
|
+
cache.queueMessagePatch(existing._id, {
|
|
48
|
+
role: event.durableMessage.role,
|
|
49
|
+
status: nextStatus,
|
|
50
|
+
text: event.durableMessage.text,
|
|
51
|
+
sourceItemType: event.durableMessage.sourceItemType,
|
|
52
|
+
payloadJson: event.durableMessage.payloadJson,
|
|
53
|
+
...(nextStatus === "failed" ? { error: "item failed" } : {}),
|
|
54
|
+
updatedAt: now(),
|
|
55
|
+
...(nextStatus !== "streaming" ? { completedAt: now() } : {}),
|
|
56
|
+
});
|
|
57
|
+
cache.setMessageRecord(turnId, event.durableMessage.messageId, {
|
|
58
|
+
...existing,
|
|
59
|
+
status: nextStatus,
|
|
60
|
+
text: event.durableMessage.text,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (!event.durableDelta) {
|
|
65
|
+
if (!event.reasoningDelta || event.type !== "stream_delta") {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (event.reasoningDelta.channel === "raw" && !ingest.runtime.exposeRawReasoningDeltas) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const existingReasoningEvent = await ingest.ctx.db
|
|
72
|
+
.query("codex_reasoning_segments")
|
|
73
|
+
.withIndex("tenantId_threadId_eventId", (q) => q.eq("tenantId", ingest.args.actor.tenantId).eq("threadId", ingest.args.threadId).eq("eventId", event.eventId))
|
|
74
|
+
.first();
|
|
75
|
+
if (existingReasoningEvent) {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const segmentId = `${event.turnId}:${event.reasoningDelta.itemId}:${event.reasoningDelta.channel}:${event.reasoningDelta.segmentType}:${event.eventId}`;
|
|
79
|
+
await ingest.ctx.db.insert("codex_reasoning_segments", {
|
|
80
|
+
tenantId: ingest.args.actor.tenantId,
|
|
81
|
+
userId: ingest.args.actor.userId,
|
|
82
|
+
threadId: ingest.args.threadId,
|
|
83
|
+
turnId: event.turnId,
|
|
84
|
+
itemId: event.reasoningDelta.itemId,
|
|
85
|
+
segmentId,
|
|
86
|
+
eventId: event.eventId,
|
|
87
|
+
channel: event.reasoningDelta.channel,
|
|
88
|
+
segmentType: event.reasoningDelta.segmentType,
|
|
89
|
+
text: event.reasoningDelta.delta ?? "",
|
|
90
|
+
...(typeof event.reasoningDelta.summaryIndex === "number"
|
|
91
|
+
? { summaryIndex: event.reasoningDelta.summaryIndex }
|
|
92
|
+
: {}),
|
|
93
|
+
...(typeof event.reasoningDelta.contentIndex === "number"
|
|
94
|
+
? { contentIndex: event.reasoningDelta.contentIndex }
|
|
95
|
+
: {}),
|
|
96
|
+
cursorStart: event.cursorStart,
|
|
97
|
+
cursorEnd: event.cursorEnd,
|
|
98
|
+
createdAt: event.createdAt,
|
|
99
|
+
});
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
const existing = await cache.getMessageRecord(turnId, event.durableDelta.messageId);
|
|
103
|
+
if (!existing) {
|
|
104
|
+
const nextOrder = await cache.nextOrderForTurn(turnId);
|
|
105
|
+
const messageId = await ingest.ctx.db.insert("codex_messages", {
|
|
106
|
+
tenantId: ingest.args.actor.tenantId,
|
|
107
|
+
userId: ingest.args.actor.userId,
|
|
108
|
+
threadId: ingest.args.threadId,
|
|
109
|
+
turnId,
|
|
110
|
+
messageId: event.durableDelta.messageId,
|
|
111
|
+
role: "assistant",
|
|
112
|
+
status: "streaming",
|
|
113
|
+
text: event.durableDelta.delta,
|
|
114
|
+
sourceItemType: "agentMessage",
|
|
115
|
+
orderInTurn: nextOrder,
|
|
116
|
+
payloadJson: JSON.stringify({
|
|
117
|
+
type: "agentMessage",
|
|
118
|
+
id: event.durableDelta.messageId,
|
|
119
|
+
text: event.durableDelta.delta,
|
|
120
|
+
}),
|
|
121
|
+
createdAt: event.createdAt,
|
|
122
|
+
updatedAt: now(),
|
|
123
|
+
});
|
|
124
|
+
cache.setMessageRecord(turnId, event.durableDelta.messageId, {
|
|
125
|
+
_id: messageId,
|
|
126
|
+
status: "streaming",
|
|
127
|
+
text: event.durableDelta.delta,
|
|
128
|
+
});
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
if (existing.status !== "streaming") {
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
const nextText = `${existing.text}${event.durableDelta.delta}`;
|
|
135
|
+
cache.queueMessagePatch(existing._id, {
|
|
136
|
+
text: nextText,
|
|
137
|
+
payloadJson: JSON.stringify({
|
|
138
|
+
type: "agentMessage",
|
|
139
|
+
id: event.durableDelta.messageId,
|
|
140
|
+
text: nextText,
|
|
141
|
+
}),
|
|
142
|
+
updatedAt: now(),
|
|
143
|
+
});
|
|
144
|
+
cache.setMessageRecord(turnId, event.durableDelta.messageId, {
|
|
145
|
+
...existing,
|
|
146
|
+
text: nextText,
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
//# sourceMappingURL=applyMessages.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"applyMessages.js","sourceRoot":"","sources":["../../../src/component/ingest/applyMessages.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAIlC,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,MAAqB,EACrB,KAA6B,EAC7B,KAAuB;IAEvB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO;IACT,CAAC;IAED,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAEtF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAEvD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,gBAAgB,EAAE;gBACzD,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;gBACpC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;gBAChC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,MAAM;gBACN,SAAS,EAAE,KAAK,CAAC,cAAc,CAAC,SAAS;gBACzC,IAAI,EAAE,KAAK,CAAC,cAAc,CAAC,IAAI;gBAC/B,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,MAAM;gBACnC,IAAI,EAAE,KAAK,CAAC,cAAc,CAAC,IAAI;gBAC/B,cAAc,EAAE,KAAK,CAAC,cAAc,CAAC,cAAc;gBACnD,WAAW,EAAE,SAAS;gBACtB,WAAW,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;gBAC7C,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,SAAS,EAAE,GAAG,EAAE;gBAChB,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC/E,CAAC,CAAC;YACH,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE;gBAC7D,GAAG,EAAE,KAAK;gBACV,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,MAAM;gBACnC,IAAI,EAAE,KAAK,CAAC,cAAc,CAAC,IAAI;aAChC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE;gBACvB,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;oBACjC,OAAO,QAAiB,CAAC;gBAC3B,CAAC;gBACD,IAAI,QAAQ,CAAC,MAAM,KAAK,aAAa,IAAI,KAAK,CAAC,cAAc,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;oBACnF,OAAO,aAAsB,CAAC;gBAChC,CAAC;gBACD,IAAI,KAAK,CAAC,cAAc,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;oBACjD,OAAO,QAAQ,CAAC,MAAM,CAAC;gBACzB,CAAC;gBACD,OAAO,KAAK,CAAC,cAAc,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC;YACzD,CAAC,CAAC,EAAE,CAAC;YAEL,KAAK,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACpC,IAAI,EAAE,KAAK,CAAC,cAAc,CAAC,IAAI;gBAC/B,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,KAAK,CAAC,cAAc,CAAC,IAAI;gBAC/B,cAAc,EAAE,KAAK,CAAC,cAAc,CAAC,cAAc;gBACnD,WAAW,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;gBAC7C,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,SAAS,EAAE,GAAG,EAAE;gBAChB,GAAG,CAAC,UAAU,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9D,CAAC,CAAC;YACH,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE;gBAC7D,GAAG,QAAQ;gBACX,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,KAAK,CAAC,cAAc,CAAC,IAAI;aAChC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC3D,OAAO;QACT,CAAC;QAED,IAAI,KAAK,CAAC,cAAc,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,wBAAwB,EAAE,CAAC;YACvF,OAAO;QACT,CAAC;QAED,MAAM,sBAAsB,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE;aAC/C,KAAK,CAAC,0BAA0B,CAAC;aACjC,SAAS,CAAC,2BAA2B,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,CAAC,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAC/G;aACA,KAAK,EAAE,CAAC;QACX,IAAI,sBAAsB,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,WAAW,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACxJ,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,0BAA0B,EAAE;YACrD,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;YAChC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,MAAM;YACnC,SAAS;YACT,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,OAAO;YACrC,WAAW,EAAE,KAAK,CAAC,cAAc,CAAC,WAAW;YAC7C,IAAI,EAAE,KAAK,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE;YACtC,GAAG,CAAC,OAAO,KAAK,CAAC,cAAc,CAAC,YAAY,KAAK,QAAQ;gBACvD,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,YAAY,EAAE;gBACrD,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,KAAK,CAAC,cAAc,CAAC,YAAY,KAAK,QAAQ;gBACvD,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,YAAY,EAAE;gBACrD,CAAC,CAAC,EAAE,CAAC;YACP,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAEpF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,gBAAgB,EAAE;YAC7D,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;YAChC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,MAAM;YACN,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS;YACvC,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,WAAW;YACnB,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK;YAC9B,cAAc,EAAE,cAAc;YAC9B,WAAW,EAAE,SAAS;YACtB,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC;gBAC1B,IAAI,EAAE,cAAc;gBACpB,EAAE,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS;gBAChC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK;aAC/B,CAAC;YACF,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,GAAG,EAAE;SACjB,CAAC,CAAC;QACH,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE;YAC3D,GAAG,EAAE,SAAS;YACd,MAAM,EAAE,WAAW;YACnB,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK;SAC/B,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;QACpC,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC/D,KAAK,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,EAAE;QACpC,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC;YAC1B,IAAI,EAAE,cAAc;YACpB,EAAE,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS;YAChC,IAAI,EAAE,QAAQ;SACf,CAAC;QACF,SAAS,EAAE,GAAG,EAAE;KACjB,CAAC,CAAC;IACH,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE;QAC3D,GAAG,QAAQ;QACX,IAAI,EAAE,QAAQ;KACf,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { IngestContext, NormalizedInboundEvent } from "./types.js";
|
|
2
|
+
import type { IngestStateCache } from "./stateCache.js";
|
|
3
|
+
export declare function persistLifecycleEventIfMissing(ingest: IngestContext, event: NormalizedInboundEvent): Promise<void>;
|
|
4
|
+
export declare function applyStreamEvent(ingest: IngestContext, event: NormalizedInboundEvent, cache: IngestStateCache): Promise<void>;
|
|
5
|
+
export declare function finalizeStreamStates(ingest: IngestContext, cache: IngestStateCache): Promise<void>;
|
|
6
|
+
export declare function flushStreamStats(ingest: IngestContext): Promise<void>;
|
|
7
|
+
//# sourceMappingURL=applyStreams.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"applyStreams.d.ts","sourceRoot":"","sources":["../../../src/component/ingest/applyStreams.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAExD,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,sBAAsB,GAC5B,OAAO,CAAC,IAAI,CAAC,CA4Bf;AAED,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,sBAAsB,EAC7B,KAAK,EAAE,gBAAgB,GACtB,OAAO,CAAC,IAAI,CAAC,CA0Hf;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,gBAAgB,GACtB,OAAO,CAAC,IAAI,CAAC,CA4Df;AAED,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAa3E"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { makeFunctionReference } from "convex/server";
|
|
2
|
+
import { DELTA_TTL_MS, DEFAULT_STREAM_DELETE_BATCH_SIZE, LIFECYCLE_EVENT_KINDS, REASONING_RAW_DELTA_EVENT_KINDS, REASONING_SUMMARY_DELTA_EVENT_KINDS, STREAM_TEXT_DELTA_EVENT_KINDS, syncError, } from "../syncRuntime.js";
|
|
3
|
+
import { addStreamDeltaStatsBatch, ensureStreamStat, setStreamStatState, } from "../streamStats.js";
|
|
4
|
+
import { now } from "../utils.js";
|
|
5
|
+
export async function persistLifecycleEventIfMissing(ingest, event) {
|
|
6
|
+
if (event.type !== "lifecycle_event") {
|
|
7
|
+
return;
|
|
8
|
+
}
|
|
9
|
+
const existingLifecycle = await ingest.ctx.db
|
|
10
|
+
.query("codex_lifecycle_events")
|
|
11
|
+
.withIndex("tenantId_threadId_eventId", (q) => q
|
|
12
|
+
.eq("tenantId", ingest.args.actor.tenantId)
|
|
13
|
+
.eq("threadId", ingest.args.threadId)
|
|
14
|
+
.eq("eventId", event.eventId))
|
|
15
|
+
.first();
|
|
16
|
+
if (existingLifecycle) {
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
await ingest.ctx.db.insert("codex_lifecycle_events", {
|
|
20
|
+
tenantId: ingest.args.actor.tenantId,
|
|
21
|
+
threadId: ingest.args.threadId,
|
|
22
|
+
eventId: event.eventId,
|
|
23
|
+
kind: event.kind,
|
|
24
|
+
payloadJson: event.payloadJson,
|
|
25
|
+
createdAt: event.createdAt,
|
|
26
|
+
...(event.turnId ? { turnId: event.turnId } : {}),
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
export async function applyStreamEvent(ingest, event, cache) {
|
|
30
|
+
if (event.type !== "stream_delta") {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
let stream = await cache.getStreamRecord(event.streamId);
|
|
34
|
+
if (!stream) {
|
|
35
|
+
const streamId = await ingest.ctx.db.insert("codex_streams", {
|
|
36
|
+
tenantId: ingest.args.actor.tenantId,
|
|
37
|
+
threadId: ingest.args.threadId,
|
|
38
|
+
turnId: event.turnId,
|
|
39
|
+
streamId: event.streamId,
|
|
40
|
+
state: { kind: "streaming", lastHeartbeatAt: now() },
|
|
41
|
+
startedAt: now(),
|
|
42
|
+
});
|
|
43
|
+
stream = {
|
|
44
|
+
_id: streamId,
|
|
45
|
+
state: { kind: "streaming" },
|
|
46
|
+
turnId: event.turnId,
|
|
47
|
+
};
|
|
48
|
+
cache.setStreamRecord(event.streamId, stream);
|
|
49
|
+
await ensureStreamStat(ingest.ctx, {
|
|
50
|
+
tenantId: ingest.args.actor.tenantId,
|
|
51
|
+
threadId: ingest.args.threadId,
|
|
52
|
+
turnId: event.turnId,
|
|
53
|
+
streamId: event.streamId,
|
|
54
|
+
state: "streaming",
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
if (event.cursorStart >= event.cursorEnd) {
|
|
58
|
+
syncError("E_SYNC_INVALID_CURSOR_RANGE", `Invalid cursor range start=${event.cursorStart} end=${event.cursorEnd} for eventId=${event.eventId}`);
|
|
59
|
+
}
|
|
60
|
+
const streamExpectedCursor = ingest.streamState.expectedCursorByStreamId.get(event.streamId) ?? 0;
|
|
61
|
+
const existingStreamEvent = await ingest.ctx.db
|
|
62
|
+
.query("codex_stream_deltas_ttl")
|
|
63
|
+
.withIndex("tenantId_streamId_eventId", (q) => q
|
|
64
|
+
.eq("tenantId", ingest.args.actor.tenantId)
|
|
65
|
+
.eq("streamId", event.streamId)
|
|
66
|
+
.eq("eventId", event.eventId))
|
|
67
|
+
.first();
|
|
68
|
+
if (existingStreamEvent) {
|
|
69
|
+
ingest.streamState.expectedCursorByStreamId.set(event.streamId, Math.max(streamExpectedCursor, Number(existingStreamEvent.cursorEnd)));
|
|
70
|
+
ingest.streamState.streamCheckpointCursorByStreamId.set(event.streamId, Math.max(ingest.streamState.streamCheckpointCursorByStreamId.get(event.streamId) ?? 0, Number(existingStreamEvent.cursorEnd)));
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (event.cursorStart < streamExpectedCursor) {
|
|
74
|
+
syncError("E_SYNC_OUT_OF_ORDER", `Expected cursorStart>=${streamExpectedCursor} for streamId=${event.streamId} but got ${event.cursorStart} for eventId=${event.eventId}`);
|
|
75
|
+
}
|
|
76
|
+
if (event.cursorStart > streamExpectedCursor) {
|
|
77
|
+
ingest.ingestStatus = "partial";
|
|
78
|
+
}
|
|
79
|
+
if (ingest.collected.inBatchEventIds.has(event.eventId)) {
|
|
80
|
+
syncError("E_SYNC_DUP_EVENT_IN_BATCH", `Duplicate eventId in request batch: ${event.eventId}`);
|
|
81
|
+
}
|
|
82
|
+
ingest.collected.inBatchEventIds.add(event.eventId);
|
|
83
|
+
const shouldPersist = LIFECYCLE_EVENT_KINDS.has(event.kind) ||
|
|
84
|
+
(ingest.runtime.saveStreamDeltas && STREAM_TEXT_DELTA_EVENT_KINDS.has(event.kind)) ||
|
|
85
|
+
(ingest.runtime.saveReasoningDeltas && REASONING_SUMMARY_DELTA_EVENT_KINDS.has(event.kind)) ||
|
|
86
|
+
(ingest.runtime.saveReasoningDeltas &&
|
|
87
|
+
ingest.runtime.exposeRawReasoningDeltas &&
|
|
88
|
+
REASONING_RAW_DELTA_EVENT_KINDS.has(event.kind));
|
|
89
|
+
if (shouldPersist) {
|
|
90
|
+
await ingest.ctx.db.insert("codex_stream_deltas_ttl", {
|
|
91
|
+
tenantId: ingest.args.actor.tenantId,
|
|
92
|
+
streamId: event.streamId,
|
|
93
|
+
turnId: event.turnId,
|
|
94
|
+
eventId: event.eventId,
|
|
95
|
+
cursorStart: event.cursorStart,
|
|
96
|
+
cursorEnd: event.cursorEnd,
|
|
97
|
+
kind: event.kind,
|
|
98
|
+
payloadJson: event.payloadJson,
|
|
99
|
+
createdAt: event.createdAt,
|
|
100
|
+
expiresAt: now() + DELTA_TTL_MS,
|
|
101
|
+
});
|
|
102
|
+
const existing = ingest.streamState.persistedStatsByStreamId.get(event.streamId);
|
|
103
|
+
if (existing) {
|
|
104
|
+
existing.deltaCount += 1;
|
|
105
|
+
existing.latestCursor = Math.max(existing.latestCursor, event.cursorEnd);
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
ingest.streamState.persistedStatsByStreamId.set(event.streamId, {
|
|
109
|
+
threadId: ingest.args.threadId,
|
|
110
|
+
turnId: event.turnId,
|
|
111
|
+
latestCursor: event.cursorEnd,
|
|
112
|
+
deltaCount: 1,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
ingest.lastPersistedCursor = event.cursorEnd;
|
|
116
|
+
ingest.persistedAnyEvent = true;
|
|
117
|
+
}
|
|
118
|
+
ingest.streamState.streamCheckpointCursorByStreamId.set(event.streamId, Math.max(ingest.streamState.streamCheckpointCursorByStreamId.get(event.streamId) ?? 0, event.cursorEnd));
|
|
119
|
+
ingest.streamState.expectedCursorByStreamId.set(event.streamId, event.cursorEnd);
|
|
120
|
+
}
|
|
121
|
+
export async function finalizeStreamStates(ingest, cache) {
|
|
122
|
+
for (const [streamId, terminal] of ingest.collected.terminalByStream) {
|
|
123
|
+
const stream = await cache.getStreamRecord(streamId);
|
|
124
|
+
if (!stream || stream.state.kind !== "streaming") {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
const endedAt = now();
|
|
128
|
+
const cleanupFnId = await ingest.ctx.scheduler.runAfter(ingest.runtime.finishedStreamDeleteDelayMs, makeFunctionReference("streams:cleanupFinishedStream"), {
|
|
129
|
+
tenantId: ingest.args.actor.tenantId,
|
|
130
|
+
streamId,
|
|
131
|
+
batchSize: DEFAULT_STREAM_DELETE_BATCH_SIZE,
|
|
132
|
+
});
|
|
133
|
+
if (terminal.status === "completed") {
|
|
134
|
+
await ingest.ctx.db.patch(stream._id, {
|
|
135
|
+
state: { kind: "finished", endedAt },
|
|
136
|
+
endedAt,
|
|
137
|
+
cleanupScheduledAt: endedAt,
|
|
138
|
+
cleanupFnId,
|
|
139
|
+
});
|
|
140
|
+
await setStreamStatState(ingest.ctx, {
|
|
141
|
+
tenantId: ingest.args.actor.tenantId,
|
|
142
|
+
threadId: ingest.args.threadId,
|
|
143
|
+
turnId: stream.turnId,
|
|
144
|
+
streamId,
|
|
145
|
+
state: "finished",
|
|
146
|
+
});
|
|
147
|
+
cache.setStreamRecord(streamId, {
|
|
148
|
+
...stream,
|
|
149
|
+
state: { kind: "finished" },
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
await ingest.ctx.db.patch(stream._id, {
|
|
154
|
+
state: {
|
|
155
|
+
kind: "aborted",
|
|
156
|
+
reason: terminal.error ?? terminal.status,
|
|
157
|
+
endedAt,
|
|
158
|
+
},
|
|
159
|
+
endedAt,
|
|
160
|
+
cleanupScheduledAt: endedAt,
|
|
161
|
+
cleanupFnId,
|
|
162
|
+
});
|
|
163
|
+
await setStreamStatState(ingest.ctx, {
|
|
164
|
+
tenantId: ingest.args.actor.tenantId,
|
|
165
|
+
threadId: ingest.args.threadId,
|
|
166
|
+
turnId: stream.turnId,
|
|
167
|
+
streamId,
|
|
168
|
+
state: "aborted",
|
|
169
|
+
});
|
|
170
|
+
cache.setStreamRecord(streamId, {
|
|
171
|
+
...stream,
|
|
172
|
+
state: { kind: "aborted" },
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
export async function flushStreamStats(ingest) {
|
|
178
|
+
await addStreamDeltaStatsBatch(ingest.ctx, {
|
|
179
|
+
tenantId: ingest.args.actor.tenantId,
|
|
180
|
+
threadId: ingest.args.threadId,
|
|
181
|
+
updates: Array.from(ingest.streamState.persistedStatsByStreamId.entries()).map(([streamId, stats]) => ({
|
|
182
|
+
streamId,
|
|
183
|
+
turnId: stats.turnId,
|
|
184
|
+
deltaCount: stats.deltaCount,
|
|
185
|
+
latestCursor: stats.latestCursor,
|
|
186
|
+
})),
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=applyStreams.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"applyStreams.js","sourceRoot":"","sources":["../../../src/component/ingest/applyStreams.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EACL,YAAY,EACZ,gCAAgC,EAChC,qBAAqB,EACrB,+BAA+B,EAC/B,mCAAmC,EACnC,6BAA6B,EAC7B,SAAS,GACV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,wBAAwB,EACxB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAIlC,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,MAAqB,EACrB,KAA6B;IAE7B,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACrC,OAAO;IACT,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE;SAC1C,KAAK,CAAC,wBAAwB,CAAC;SAC/B,SAAS,CAAC,2BAA2B,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,CAAC;SACE,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;SAC1C,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;SACpC,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAChC;SACA,KAAK,EAAE,CAAC;IAEX,IAAI,iBAAiB,EAAE,CAAC;QACtB,OAAO;IACT,CAAC;IAED,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,wBAAwB,EAAE;QACnD,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;QACpC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAqB,EACrB,KAA6B,EAC7B,KAAuB;IAEvB,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QAClC,OAAO;IACT,CAAC;IAED,IAAI,MAAM,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,EAAE;YAC3D,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,eAAe,EAAE,GAAG,EAAE,EAAE;YACpD,SAAS,EAAE,GAAG,EAAE;SACjB,CAAC,CAAC;QACH,MAAM,GAAG;YACP,GAAG,EAAE,QAAQ;YACb,KAAK,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;YAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC;QACF,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,gBAAgB,CAAC,MAAM,CAAC,GAAG,EAAE;YACjC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,WAAW;SACnB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACzC,SAAS,CACP,6BAA6B,EAC7B,8BAA8B,KAAK,CAAC,WAAW,QAAQ,KAAK,CAAC,SAAS,gBAAgB,KAAK,CAAC,OAAO,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,MAAM,oBAAoB,GAAG,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClG,MAAM,mBAAmB,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE;SAC5C,KAAK,CAAC,yBAAyB,CAAC;SAChC,SAAS,CAAC,2BAA2B,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,CAAC;SACE,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;SAC1C,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC;SAC9B,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAChC;SACA,KAAK,EAAE,CAAC;IAEX,IAAI,mBAAmB,EAAE,CAAC;QACxB,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAC7C,KAAK,CAAC,QAAQ,EACd,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CACtE,CAAC;QACF,MAAM,CAAC,WAAW,CAAC,gCAAgC,CAAC,GAAG,CACrD,KAAK,CAAC,QAAQ,EACd,IAAI,CAAC,GAAG,CACN,MAAM,CAAC,WAAW,CAAC,gCAAgC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAC5E,MAAM,CAAC,mBAAmB,CAAC,SAAS,CAAC,CACtC,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,KAAK,CAAC,WAAW,GAAG,oBAAoB,EAAE,CAAC;QAC7C,SAAS,CACP,qBAAqB,EACrB,yBAAyB,oBAAoB,iBAAiB,KAAK,CAAC,QAAQ,YAAY,KAAK,CAAC,WAAW,gBAAgB,KAAK,CAAC,OAAO,EAAE,CACzI,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,GAAG,oBAAoB,EAAE,CAAC;QAC7C,MAAM,CAAC,YAAY,GAAG,SAAS,CAAC;IAClC,CAAC;IAED,IAAI,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACxD,SAAS,CAAC,2BAA2B,EAAE,uCAAuC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAEpD,MAAM,aAAa,GACjB,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;QACrC,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,IAAI,6BAA6B,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClF,CAAC,MAAM,CAAC,OAAO,CAAC,mBAAmB,IAAI,mCAAmC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3F,CAAC,MAAM,CAAC,OAAO,CAAC,mBAAmB;YACjC,MAAM,CAAC,OAAO,CAAC,wBAAwB;YACvC,+BAA+B,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAErD,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,yBAAyB,EAAE;YACpD,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,GAAG,EAAE,GAAG,YAAY;SAChC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjF,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;YACzB,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;QAC3E,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE;gBAC9D,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,YAAY,EAAE,KAAK,CAAC,SAAS;gBAC7B,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,mBAAmB,GAAG,KAAK,CAAC,SAAS,CAAC;QAC7C,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,WAAW,CAAC,gCAAgC,CAAC,GAAG,CACrD,KAAK,CAAC,QAAQ,EACd,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,gCAAgC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CACxG,CAAC;IACF,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAqB,EACrB,KAAuB;IAEvB,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC;QACrE,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACjD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;QACtB,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CACrD,MAAM,CAAC,OAAO,CAAC,2BAA2B,EAC1C,qBAAqB,CAAa,+BAA+B,CAAC,EAClE;YACE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,QAAQ;YACR,SAAS,EAAE,gCAAgC;SAC5C,CACF,CAAC;QAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YACpC,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE;gBACpC,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE;gBACpC,OAAO;gBACP,kBAAkB,EAAE,OAAO;gBAC3B,WAAW;aACZ,CAAC,CAAC;YACH,MAAM,kBAAkB,CAAC,MAAM,CAAC,GAAG,EAAE;gBACnC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;gBACpC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ;gBACR,KAAK,EAAE,UAAU;aAClB,CAAC,CAAC;YACH,KAAK,CAAC,eAAe,CAAC,QAAQ,EAAE;gBAC9B,GAAG,MAAM;gBACT,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE;aAC5B,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE;gBACpC,KAAK,EAAE;oBACL,IAAI,EAAE,SAAS;oBACf,MAAM,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM;oBACzC,OAAO;iBACR;gBACD,OAAO;gBACP,kBAAkB,EAAE,OAAO;gBAC3B,WAAW;aACZ,CAAC,CAAC;YACH,MAAM,kBAAkB,CAAC,MAAM,CAAC,GAAG,EAAE;gBACnC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;gBACpC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ;gBACR,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC;YACH,KAAK,CAAC,eAAe,CAAC,QAAQ,EAAE;gBAC9B,GAAG,MAAM;gBACT,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAqB;IAC1D,MAAM,wBAAwB,CAAC,MAAM,CAAC,GAAG,EAAE;QACzC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;QACpC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAC5E,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YACtB,QAAQ;YACR,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;SACjC,CAAC,CACH;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { IngestContext, NormalizedInboundEvent } from "./types.js";
|
|
2
|
+
import type { IngestStateCache } from "./stateCache.js";
|
|
3
|
+
export declare function ensureTurnForEvent(ingest: IngestContext, event: NormalizedInboundEvent): Promise<void>;
|
|
4
|
+
export declare function collectTurnSignals(ingest: IngestContext, event: NormalizedInboundEvent): void;
|
|
5
|
+
export declare function finalizeTurns(ingest: IngestContext, cache: IngestStateCache): Promise<void>;
|
|
6
|
+
//# sourceMappingURL=applyTurns.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"applyTurns.d.ts","sourceRoot":"","sources":["../../../src/component/ingest/applyTurns.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAExD,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,sBAAsB,GAC5B,OAAO,CAAC,IAAI,CAAC,CAuCf;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,sBAAsB,GAAG,IAAI,CAwB7F;AAED,wBAAsB,aAAa,CACjC,MAAM,EAAE,aAAa,EACrB,KAAK,EAAE,gBAAgB,GACtB,OAAO,CAAC,IAAI,CAAC,CAuDf"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { makeFunctionReference } from "convex/server";
|
|
2
|
+
import { pickHigherPriorityTerminalStatus } from "../syncHelpers.js";
|
|
3
|
+
import { authzError, now, requireTurnForActor } from "../utils.js";
|
|
4
|
+
export async function ensureTurnForEvent(ingest, event) {
|
|
5
|
+
const turnId = event.turnId;
|
|
6
|
+
if (!turnId || ingest.collected.knownTurnIds.has(turnId)) {
|
|
7
|
+
return;
|
|
8
|
+
}
|
|
9
|
+
const existingTurn = await ingest.ctx.db
|
|
10
|
+
.query("codex_turns")
|
|
11
|
+
.withIndex("tenantId_threadId_turnId", (q) => q
|
|
12
|
+
.eq("tenantId", ingest.args.actor.tenantId)
|
|
13
|
+
.eq("threadId", ingest.args.threadId)
|
|
14
|
+
.eq("turnId", turnId))
|
|
15
|
+
.first();
|
|
16
|
+
if (!existingTurn) {
|
|
17
|
+
await ingest.ctx.db.insert("codex_turns", {
|
|
18
|
+
tenantId: ingest.args.actor.tenantId,
|
|
19
|
+
userId: ingest.args.actor.userId,
|
|
20
|
+
threadId: ingest.args.threadId,
|
|
21
|
+
turnId,
|
|
22
|
+
status: event.syntheticTurnStatus,
|
|
23
|
+
idempotencyKey: `sync:${ingest.args.threadId}:${turnId}`,
|
|
24
|
+
startedAt: now(),
|
|
25
|
+
...(event.syntheticTurnStatus === "completed" ||
|
|
26
|
+
event.syntheticTurnStatus === "interrupted" ||
|
|
27
|
+
event.syntheticTurnStatus === "failed"
|
|
28
|
+
? { completedAt: now() }
|
|
29
|
+
: {}),
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
else if (existingTurn.userId !== ingest.args.actor.userId) {
|
|
33
|
+
authzError("E_AUTH_TURN_FORBIDDEN", `User ${ingest.args.actor.userId} is not allowed to access turn ${turnId}`);
|
|
34
|
+
}
|
|
35
|
+
ingest.collected.knownTurnIds.add(turnId);
|
|
36
|
+
}
|
|
37
|
+
export function collectTurnSignals(ingest, event) {
|
|
38
|
+
if (!event.turnId) {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
if (event.kind === "turn/started") {
|
|
42
|
+
ingest.collected.startedTurns.add(event.turnId);
|
|
43
|
+
}
|
|
44
|
+
if (event.terminalTurnStatus) {
|
|
45
|
+
const current = ingest.collected.terminalTurns.get(event.turnId);
|
|
46
|
+
ingest.collected.terminalTurns.set(event.turnId, pickHigherPriorityTerminalStatus(current, event.terminalTurnStatus));
|
|
47
|
+
if (event.type === "stream_delta") {
|
|
48
|
+
const currentStream = ingest.collected.terminalByStream.get(event.streamId);
|
|
49
|
+
ingest.collected.terminalByStream.set(event.streamId, pickHigherPriorityTerminalStatus(currentStream, event.terminalTurnStatus));
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
export async function finalizeTurns(ingest, cache) {
|
|
54
|
+
for (const turnId of ingest.collected.startedTurns) {
|
|
55
|
+
const turn = await requireTurnForActor(ingest.ctx, ingest.args.actor, ingest.args.threadId, turnId);
|
|
56
|
+
if (turn.status === "queued") {
|
|
57
|
+
await ingest.ctx.db.patch(turn._id, { status: "inProgress" });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
for (const [turnId, terminal] of ingest.collected.terminalTurns) {
|
|
61
|
+
await ingest.ctx.scheduler.runAfter(0, makeFunctionReference("turnsInternal:finalizeTurnFromStream"), {
|
|
62
|
+
tenantId: ingest.args.actor.tenantId,
|
|
63
|
+
threadId: ingest.args.threadId,
|
|
64
|
+
turnId,
|
|
65
|
+
status: terminal.status,
|
|
66
|
+
...(terminal.error ? { error: terminal.error } : {}),
|
|
67
|
+
});
|
|
68
|
+
if (terminal.status !== "failed" && terminal.status !== "interrupted") {
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
const pendingMessages = await ingest.ctx.db
|
|
72
|
+
.query("codex_messages")
|
|
73
|
+
.withIndex("tenantId_threadId_turnId_status", (q) => q
|
|
74
|
+
.eq("tenantId", ingest.args.actor.tenantId)
|
|
75
|
+
.eq("threadId", ingest.args.threadId)
|
|
76
|
+
.eq("turnId", turnId)
|
|
77
|
+
.eq("status", "streaming"))
|
|
78
|
+
.take(500);
|
|
79
|
+
await Promise.all(pendingMessages.map((message) => ingest.ctx.db.patch(message._id, {
|
|
80
|
+
status: terminal.status,
|
|
81
|
+
...(terminal.error ? { error: terminal.error } : {}),
|
|
82
|
+
updatedAt: now(),
|
|
83
|
+
completedAt: now(),
|
|
84
|
+
})));
|
|
85
|
+
for (const message of pendingMessages) {
|
|
86
|
+
cache.setMessageRecord(turnId, message.messageId, {
|
|
87
|
+
_id: message._id,
|
|
88
|
+
status: terminal.status,
|
|
89
|
+
text: message.text,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=applyTurns.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"applyTurns.js","sourceRoot":"","sources":["../../../src/component/ingest/applyTurns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAInE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAqB,EACrB,KAA6B;IAE7B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACzD,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE;SACrC,KAAK,CAAC,aAAa,CAAC;SACpB,SAAS,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CAC3C,CAAC;SACE,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;SAC1C,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;SACpC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CACxB;SACA,KAAK,EAAE,CAAC;IAEX,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,aAAa,EAAE;YACxC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;YAChC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,MAAM;YACN,MAAM,EAAE,KAAK,CAAC,mBAAmB;YACjC,cAAc,EAAE,QAAQ,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,MAAM,EAAE;YACxD,SAAS,EAAE,GAAG,EAAE;YAChB,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,WAAW;gBAC7C,KAAK,CAAC,mBAAmB,KAAK,aAAa;gBAC3C,KAAK,CAAC,mBAAmB,KAAK,QAAQ;gBACpC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE;gBACxB,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,YAAY,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5D,UAAU,CACR,uBAAuB,EACvB,QAAQ,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,kCAAkC,MAAM,EAAE,CAC3E,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,MAAqB,EAAE,KAA6B;IACrF,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO;IACT,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QAClC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,CAChC,KAAK,CAAC,MAAM,EACZ,gCAAgC,CAAC,OAAO,EAAE,KAAK,CAAC,kBAAkB,CAAC,CACpE,CAAC;QAEF,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC5E,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,GAAG,CACnC,KAAK,CAAC,QAAQ,EACd,gCAAgC,CAAC,aAAa,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAC1E,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAqB,EACrB,KAAuB;IAEvB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACnD,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACpG,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC;QAChE,MAAM,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CACjC,CAAC,EACD,qBAAqB,CAAa,sCAAsC,CAAC,EACzE;YACE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ;YACpC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;YAC9B,MAAM;YACN,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrD,CACF,CAAC;QAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YACtE,SAAS;QACX,CAAC;QAED,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,KAAK,CAAC,gBAAgB,CAAC;aACvB,SAAS,CAAC,iCAAiC,EAAE,CAAC,CAAC,EAAE,EAAE,CAClD,CAAC;aACE,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;aAC1C,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;aACpC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC;aACpB,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC,CAC7B;aACA,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,MAAM,OAAO,CAAC,GAAG,CACf,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAC9B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE;YAC/B,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,SAAS,EAAE,GAAG,EAAE;YAChB,WAAW,EAAE,GAAG,EAAE;SACnB,CAAC,CACH,CACF,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,KAAK,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,EAAE;gBAChD,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,IAAI,EAAE,OAAO,CAAC,IAAI;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { MutationCtx } from "../_generated/server.js";
|
|
2
|
+
import type { ActorContext, IngestContext } from "./types.js";
|
|
3
|
+
export declare function applyStreamCheckpoints(ingest: IngestContext): Promise<void>;
|
|
4
|
+
export declare function upsertCheckpoint(ctx: MutationCtx, args: {
|
|
5
|
+
actor: ActorContext;
|
|
6
|
+
threadId: string;
|
|
7
|
+
streamId: string;
|
|
8
|
+
cursor: number;
|
|
9
|
+
}): Promise<{
|
|
10
|
+
ok: true;
|
|
11
|
+
}>;
|
|
12
|
+
//# sourceMappingURL=checkpoints.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoints.d.ts","sourceRoot":"","sources":["../../../src/component/ingest/checkpoints.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE9D,wBAAsB,sBAAsB,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAqCjF;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,WAAW,EAChB,IAAI,EAAE;IACJ,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACA,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,CAAC,CAkCvB"}
|