@sentry/junior 0.106.0 → 0.107.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-hooks-KR5BCWOK.js → agent-hooks-ICPIJAFY.js} +9 -7
- package/dist/api/conversations/detail.d.ts +1 -1
- package/dist/api/conversations/events.d.ts +14 -0
- package/dist/api/conversations/schema.d.ts +196 -463
- package/dist/api/schema.d.ts +2 -7
- package/dist/api/schema.js +10 -8
- package/dist/api.js +377 -810
- package/dist/app.js +3586 -3614
- package/dist/{catalog-runtime-5CVM2DOF.js → catalog-runtime-FOFNPR3P.js} +3 -2
- package/dist/chat/agent/prompt.d.ts +1 -1
- package/dist/chat/agent/request.d.ts +28 -29
- package/dist/chat/agent/resume.d.ts +7 -7
- package/dist/chat/agent/session.d.ts +5 -6
- package/dist/chat/agent/tools.d.ts +3 -3
- package/dist/chat/agent-dispatch/runner.d.ts +2 -0
- package/dist/chat/config.d.ts +2 -2
- package/dist/chat/conversations/history.d.ts +582 -167
- package/dist/chat/conversations/message-projection.d.ts +6 -0
- package/dist/chat/conversations/message-summaries.d.ts +9 -0
- package/dist/chat/conversations/messages.d.ts +13 -27
- package/dist/chat/conversations/projection.d.ts +30 -53
- package/dist/chat/conversations/provenance.d.ts +42 -0
- package/dist/chat/conversations/search.d.ts +2 -2
- package/dist/chat/conversations/sql/conversation-row.d.ts +1 -1
- package/dist/chat/conversations/sql/event-lock.d.ts +3 -0
- package/dist/chat/conversations/sql/history.d.ts +3 -3
- package/dist/chat/conversations/sql/privacy.d.ts +27 -0
- package/dist/chat/conversations/sql/purge.d.ts +2 -10
- package/dist/chat/conversations/sql/store.d.ts +0 -5
- package/dist/chat/conversations/store.d.ts +8 -13
- package/dist/chat/conversations/turn-lifecycle.d.ts +41 -0
- package/dist/chat/db.d.ts +4 -10
- package/dist/chat/model-profile.d.ts +10 -0
- package/dist/chat/oauth-flow.d.ts +0 -5
- package/dist/chat/pi/conversation-events.d.ts +33 -0
- package/dist/chat/pi/sql-model-usage.d.ts +10 -0
- package/dist/chat/plugins/agent-hooks.d.ts +4 -2
- package/dist/chat/resource-events/ingest.d.ts +0 -13
- package/dist/chat/runtime/delivered-turn-state.d.ts +1 -1
- package/dist/chat/runtime/generated-artifacts.d.ts +1 -1
- package/dist/chat/runtime/reply-executor.d.ts +2 -0
- package/dist/chat/runtime/slack-resume.d.ts +23 -28
- package/dist/chat/runtime/slack-runtime.d.ts +2 -0
- package/dist/chat/runtime/turn.d.ts +14 -1
- package/dist/chat/services/context-compaction-marker.d.ts +0 -1
- package/dist/chat/services/context-compaction.d.ts +1 -0
- package/dist/chat/services/conversation-memory.d.ts +9 -0
- package/dist/chat/services/plugin-auth-orchestration.d.ts +1 -4
- package/dist/chat/services/subscribed-decision.d.ts +1 -1
- package/dist/chat/services/turn-failure-response.d.ts +10 -3
- package/dist/chat/services/turn-result.d.ts +7 -12
- package/dist/chat/services/{turn-reasoning-level.d.ts → turn-router.d.ts} +12 -8
- package/dist/chat/services/turn-session-record.d.ts +15 -16
- package/dist/chat/slack/adapter.d.ts +2 -2
- package/dist/chat/slack/errors.d.ts +2 -0
- package/dist/chat/slack/outbound.d.ts +0 -1
- package/dist/chat/slack/reply.d.ts +2 -8
- package/dist/chat/slack/tools/channel-capabilities.d.ts +2 -2
- package/dist/chat/slack/tools/{send-message.d.ts → send-files.d.ts} +27 -19
- package/dist/chat/state/conversation.d.ts +4 -4
- package/dist/chat/state/turn-id.d.ts +2 -0
- package/dist/chat/state/turn-session-keys.d.ts +6 -0
- package/dist/chat/state/turn-session.d.ts +11 -9
- package/dist/chat/task-execution/state.d.ts +6 -1
- package/dist/chat/task-execution/store.d.ts +1 -1
- package/dist/chat/tool-support/pi-tool-adapter.d.ts +1 -1
- package/dist/chat/tool-support/tool-execution-report.d.ts +1 -0
- package/dist/chat/tools/handoff/tool.d.ts +2 -2
- package/dist/chat/tools/sandbox/file-uploads.d.ts +28 -6
- package/dist/chat/tools/types.d.ts +6 -9
- package/dist/chat/tools/web/image-generate.d.ts +4 -6
- package/dist/{chunk-DDRQ4U7I.js → chunk-4YF7Z6IA.js} +390 -270
- package/dist/{chunk-AZ5OMSN5.js → chunk-AUUOHQAT.js} +2 -2
- package/dist/chunk-B2Z2H66D.js +250 -0
- package/dist/{chunk-4W4O4DCV.js → chunk-CQ7KSO2B.js} +75 -45
- package/dist/{chunk-MJSJAHBW.js → chunk-DY7TRYB5.js} +1 -1
- package/dist/{chunk-GPPUTY5M.js → chunk-EDLNHZH3.js} +861 -891
- package/dist/{chunk-R5BFYK2O.js → chunk-IGHMVDWI.js} +134 -18
- package/dist/chunk-MU6HHZEN.js +99 -0
- package/dist/{chunk-XTKJPSMF.js → chunk-NVOTGWYX.js} +458 -684
- package/dist/{chunk-AEWZ7F65.js → chunk-O24R4DCK.js} +2 -179
- package/dist/{chunk-CCFROYAV.js → chunk-PUAMO4PI.js} +115 -125
- package/dist/{chunk-ZMRLBBD5.js → chunk-RMVOAJRL.js} +30 -13
- package/dist/{chunk-3NYUCBLE.js → chunk-SPUAJVVH.js} +6 -4
- package/dist/{chunk-4UYIRWDW.js → chunk-SS67LUOK.js} +41 -290
- package/dist/{chunk-FXFQNHRU.js → chunk-VFUK3X5B.js} +4 -2
- package/dist/{chunk-3MPPVPGZ.js → chunk-VH6KWKG2.js} +2 -2
- package/dist/chunk-VIXFG2VC.js +182 -0
- package/dist/{chunk-BKM4UXUD.js → chunk-YFQ7CQDE.js} +669 -18
- package/dist/{chunk-77LHNQK4.js → chunk-YIPD3L6L.js} +4 -2
- package/dist/{chunk-3OUSJPR7.js → chunk-YNP2ATQX.js} +6 -4
- package/dist/{chunk-WR7H7VLJ.js → chunk-YT72TOEK.js} +2 -2
- package/dist/cli/chat.js +21 -20
- package/dist/cli/check.js +4 -3
- package/dist/cli/plugins.js +11 -9
- package/dist/cli/snapshot-warmup.js +7 -5
- package/dist/cli/upgrade.js +2041 -105
- package/dist/db/schema/{agent-steps.d.ts → conversation-events.d.ts} +37 -21
- package/dist/db/schema.d.ts +28 -159
- package/dist/{db-S33LFWG2.js → db-DIGO4TGW.js} +6 -8
- package/dist/{runner-HA5XAL6K.js → runner-ACR2HAIC.js} +209 -101
- package/dist/{skills-722AXRV5.js → skills-ITNHBARB.js} +4 -3
- package/dist/{validation-EUI2UFRM.js → validation-AGQ46ZGV.js} +4 -3
- package/migrations/0005_conversation_events.sql +122 -0
- package/migrations/README.md +23 -10
- package/migrations/meta/0005_snapshot.json +948 -0
- package/migrations/meta/_journal.json +7 -0
- package/package.json +5 -5
- package/dist/api/conversations/activity.d.ts +0 -40
- package/dist/api/conversations/detail-projection.d.ts +0 -11
- package/dist/api/conversations/detail.query.d.ts +0 -3
- package/dist/api/conversations/subagent.d.ts +0 -10
- package/dist/api/conversations/transcript.d.ts +0 -28
- package/dist/chat/conversations/legacy-advisor-session.d.ts +0 -7
- package/dist/chat/conversations/legacy-import.d.ts +0 -38
- package/dist/chat/conversations/sql/legacy-history-import.d.ts +0 -55
- package/dist/chat/conversations/sql/messages.d.ts +0 -4
- package/dist/chat/conversations/visible-compactions.d.ts +0 -11
- package/dist/chat/conversations/visible-messages.d.ts +0 -47
- package/dist/chat/services/reply-delivery-plan.d.ts +0 -5
- package/dist/chat/state/session-log.d.ts +0 -384
- package/dist/chunk-P4P7O5WG.js +0 -1072
- package/dist/chunk-XIMUQ4IU.js +0 -147
- package/dist/db/schema/conversation-messages.d.ts +0 -153
- package/dist/handlers/github-webhook/check-suite.d.ts +0 -3
- package/dist/handlers/github-webhook/issue-comment.d.ts +0 -3
- package/dist/handlers/github-webhook/pull-request-review-comment.d.ts +0 -3
- package/dist/handlers/github-webhook/pull-request-review.d.ts +0 -3
- package/dist/handlers/github-webhook/pull-request.d.ts +0 -3
- package/dist/handlers/github-webhook/resource.d.ts +0 -10
- package/dist/handlers/github-webhook.d.ts +0 -15
- package/dist/legacy-import-TMJISRKO.js +0 -18
package/dist/cli/upgrade.js
CHANGED
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
recordConversationActivity,
|
|
5
5
|
recordConversationExecution,
|
|
6
6
|
requestConversationWork
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-YNP2ATQX.js";
|
|
8
8
|
import {
|
|
9
9
|
defineJuniorPlugins,
|
|
10
10
|
pluginCatalogConfigFromEnv,
|
|
@@ -12,16 +12,17 @@ import {
|
|
|
12
12
|
pluginRuntimeRegistrationsFromPluginSet
|
|
13
13
|
} from "../chunk-SG5WAA7H.js";
|
|
14
14
|
import {
|
|
15
|
+
AGENT_TURN_SESSION_PREFIX,
|
|
15
16
|
JUNIOR_THREAD_STATE_TTL_MS,
|
|
16
|
-
|
|
17
|
-
|
|
17
|
+
agentTurnSessionConversationIndexKey,
|
|
18
|
+
agentTurnSessionKey,
|
|
19
|
+
coerceThreadConversationState,
|
|
20
|
+
listAgentTurnSessionSummariesForConversations,
|
|
21
|
+
piMessageSchema
|
|
22
|
+
} from "../chunk-4YF7Z6IA.js";
|
|
18
23
|
import {
|
|
19
|
-
addAgentTurnUsage
|
|
20
|
-
|
|
21
|
-
} from "../chunk-DDRQ4U7I.js";
|
|
22
|
-
import {
|
|
23
|
-
importConversationFromLegacy
|
|
24
|
-
} from "../chunk-P4P7O5WG.js";
|
|
24
|
+
addAgentTurnUsage
|
|
25
|
+
} from "../chunk-MU6HHZEN.js";
|
|
25
26
|
import "../chunk-PDO5BLNM.js";
|
|
26
27
|
import {
|
|
27
28
|
loadAppPluginSet
|
|
@@ -29,19 +30,37 @@ import {
|
|
|
29
30
|
import {
|
|
30
31
|
createPluginLogger,
|
|
31
32
|
createPluginState
|
|
32
|
-
} from "../chunk-
|
|
33
|
+
} from "../chunk-AUUOHQAT.js";
|
|
34
|
+
import {
|
|
35
|
+
disconnectStateAdapter,
|
|
36
|
+
getConnectedStateContext,
|
|
37
|
+
getStateAdapter
|
|
38
|
+
} from "../chunk-B2Z2H66D.js";
|
|
33
39
|
import {
|
|
40
|
+
contextProvenance,
|
|
41
|
+
conversationEventDataSchema,
|
|
42
|
+
conversationMessageProvenanceSchema,
|
|
34
43
|
createJuniorSqlExecutor,
|
|
35
|
-
createSqlConversationMessageStore,
|
|
36
44
|
createSqlStore,
|
|
37
|
-
getDb
|
|
38
|
-
|
|
45
|
+
getDb,
|
|
46
|
+
instructionProvenanceFor,
|
|
47
|
+
juniorConversationEvents,
|
|
48
|
+
juniorConversations,
|
|
49
|
+
newConversationEventSchema,
|
|
50
|
+
sanitizePostgresJson,
|
|
51
|
+
withConversationEventLock
|
|
52
|
+
} from "../chunk-NVOTGWYX.js";
|
|
39
53
|
import {
|
|
40
|
-
|
|
54
|
+
COMPACTION_SUMMARY_PREFIX,
|
|
41
55
|
getChatConfig,
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
56
|
+
isCompactionSummaryText,
|
|
57
|
+
modelIdForProfile,
|
|
58
|
+
modelProfileSchema,
|
|
59
|
+
standardModelId,
|
|
60
|
+
unescapeXml,
|
|
61
|
+
unwrapCurrentInstruction
|
|
62
|
+
} from "../chunk-SS67LUOK.js";
|
|
63
|
+
import "../chunk-VFUK3X5B.js";
|
|
45
64
|
import "../chunk-RT5BUHR5.js";
|
|
46
65
|
import {
|
|
47
66
|
parseDestination,
|
|
@@ -49,8 +68,11 @@ import {
|
|
|
49
68
|
} from "../chunk-B7PUBQQM.js";
|
|
50
69
|
import {
|
|
51
70
|
pluginCatalogRuntime
|
|
52
|
-
} from "../chunk-
|
|
53
|
-
import
|
|
71
|
+
} from "../chunk-YIPD3L6L.js";
|
|
72
|
+
import {
|
|
73
|
+
storedSlackActorSchema
|
|
74
|
+
} from "../chunk-VIXFG2VC.js";
|
|
75
|
+
import "../chunk-O24R4DCK.js";
|
|
54
76
|
import {
|
|
55
77
|
isRecord,
|
|
56
78
|
toOptionalNumber,
|
|
@@ -65,16 +87,13 @@ import "../chunk-MLKGABMK.js";
|
|
|
65
87
|
// src/cli/upgrade.ts
|
|
66
88
|
import { createJiti } from "jiti";
|
|
67
89
|
|
|
68
|
-
// src/
|
|
90
|
+
// src/cli/upgrade/migrations/conversation-history/state-conversation-store.ts
|
|
69
91
|
function createStateConversationStore(state) {
|
|
70
92
|
return {
|
|
71
93
|
get: (args) => getConversation({ ...args, state }),
|
|
72
94
|
// Task-execution state has no destination records, so visibility is never
|
|
73
95
|
// source-confirmed here and cross-context reads fail closed to private.
|
|
74
96
|
getDestinationVisibility: async () => void 0,
|
|
75
|
-
// Child conversations are a SQL-only concept and never use this legacy
|
|
76
|
-
// metadata path.
|
|
77
|
-
ensureChildConversation: async () => void 0,
|
|
78
97
|
recordActivity: (args) => recordConversationActivity({ ...args, state }),
|
|
79
98
|
recordExecution: (args) => recordConversationExecution({ ...args, state }),
|
|
80
99
|
listByActivity: (args) => listConversationsByActivity({ ...args, state })
|
|
@@ -88,10 +107,10 @@ async function migrateConversationsToSql(context, options = {}) {
|
|
|
88
107
|
let target = options.target;
|
|
89
108
|
let closeTarget;
|
|
90
109
|
if (!target) {
|
|
91
|
-
const { sql } = getChatConfig();
|
|
110
|
+
const { sql: sql2 } = getChatConfig();
|
|
92
111
|
const executor = createJuniorSqlExecutor({
|
|
93
|
-
connectionString:
|
|
94
|
-
driver:
|
|
112
|
+
connectionString: sql2.databaseUrl,
|
|
113
|
+
driver: sql2.driver
|
|
95
114
|
});
|
|
96
115
|
target = createSqlStore(executor);
|
|
97
116
|
closeTarget = () => executor.close();
|
|
@@ -172,6 +191,14 @@ var LEGACY_CORE_MIGRATION_IDS = [
|
|
|
172
191
|
"0005_conversation_transcripts"
|
|
173
192
|
];
|
|
174
193
|
var LEGACY_METRICS_MIGRATION_ID = "0006_conversation_metrics";
|
|
194
|
+
var LEGACY_CORE_MIGRATION_CHECKSUMS = {
|
|
195
|
+
"0001_conversation_core": "78fe050d8bec8ba18e2e3192497b3d8ad6b45fbb66ad4859377fb2202ed57651",
|
|
196
|
+
"0002_slack_destination_visibility_backfill": "fb590a09fa51db471a748e3d7abb4137f521ee8df97f6e9ef5563121be98c394",
|
|
197
|
+
"0003_user_identities": "67d9c9c26cbd76213614eb6d7a7cc7e2501fc20e92321eb5176a08ce39cd2efb",
|
|
198
|
+
"0004_actor_cutover": "d41b8bfa66b8a88d69e84af38950025ba4c9be56341565cbe1411f0ca50c1dc2",
|
|
199
|
+
"0005_conversation_transcripts": "add299d1b254e023f89b5993c417dd2248dc009e874efdeaf31ec0732e0d4fb4",
|
|
200
|
+
"0006_conversation_metrics": "7c7ca5c9e11ed4b0e14737fd90d3348ea46e306c88fdf31199b7afb2a11c6a41"
|
|
201
|
+
};
|
|
175
202
|
var MIGRATIONS_TABLE = "__drizzle_junior_core";
|
|
176
203
|
function migrationFolder() {
|
|
177
204
|
const moduleDir = dirname(fileURLToPath(import.meta.url));
|
|
@@ -189,7 +216,7 @@ SELECT
|
|
|
189
216
|
}
|
|
190
217
|
const migrations = readMigrationFiles({ migrationsFolder });
|
|
191
218
|
const [metrics] = await executor.query(`
|
|
192
|
-
SELECT count(*)
|
|
219
|
+
SELECT count(*)::integer AS "columnCount"
|
|
193
220
|
FROM information_schema.columns
|
|
194
221
|
WHERE table_schema = 'public'
|
|
195
222
|
AND table_name = 'junior_conversations'
|
|
@@ -201,17 +228,71 @@ WHERE table_schema = 'public'
|
|
|
201
228
|
)
|
|
202
229
|
`);
|
|
203
230
|
const legacyRecords = await executor.query("SELECT id, checksum FROM junior_schema_migrations");
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
legacyRecords.filter((record) => record.checksum.trim().length > 0).map((record) => record.id)
|
|
231
|
+
const legacyRecordsById = new Map(
|
|
232
|
+
legacyRecords.map((record) => [record.id, record.checksum])
|
|
207
233
|
);
|
|
208
|
-
const
|
|
234
|
+
const metricColumnCount = metrics?.columnCount ?? 0;
|
|
235
|
+
if (metricColumnCount !== 0 && metricColumnCount !== 4) {
|
|
236
|
+
throw new Error(
|
|
237
|
+
`Cannot adopt partial legacy metrics state: found ${metricColumnCount} of 4 required columns`
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
const metricsComplete = metricColumnCount === 4;
|
|
241
|
+
const hasMetricsRecord = legacyRecordsById.has(LEGACY_METRICS_MIGRATION_ID);
|
|
242
|
+
if (metricsComplete !== hasMetricsRecord) {
|
|
243
|
+
throw new Error(
|
|
244
|
+
"Cannot adopt legacy core migration state: legacy metrics migration record does not match physical metric columns"
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
const expectedIds = metricsComplete ? [...LEGACY_CORE_MIGRATION_IDS, LEGACY_METRICS_MIGRATION_ID] : [...LEGACY_CORE_MIGRATION_IDS];
|
|
248
|
+
const missingIds = expectedIds.filter((id) => !legacyRecordsById.has(id));
|
|
209
249
|
if (missingIds.length > 0) {
|
|
210
250
|
throw new Error(
|
|
211
251
|
`Cannot adopt partial legacy core migration state; missing: ${missingIds.join(", ")}`
|
|
212
252
|
);
|
|
213
253
|
}
|
|
214
|
-
const
|
|
254
|
+
const checksumMismatches = expectedIds.filter(
|
|
255
|
+
(id) => legacyRecordsById.get(id) !== LEGACY_CORE_MIGRATION_CHECKSUMS[id]
|
|
256
|
+
);
|
|
257
|
+
if (checksumMismatches.length > 0) {
|
|
258
|
+
throw new Error(
|
|
259
|
+
`Cannot adopt legacy core migration state: checksum mismatch: ${checksumMismatches.join(", ")}`
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
const [baseline] = await executor.query(`
|
|
263
|
+
SELECT
|
|
264
|
+
to_regclass('public.junior_conversation_events')::text AS "conversationEventsTable",
|
|
265
|
+
to_regclass('public.junior_conversation_messages_search_idx')::text AS "searchIndex",
|
|
266
|
+
EXISTS (
|
|
267
|
+
SELECT 1
|
|
268
|
+
FROM information_schema.tables
|
|
269
|
+
WHERE table_schema = 'public'
|
|
270
|
+
AND table_name = 'junior_agent_steps'
|
|
271
|
+
AND table_type = 'BASE TABLE'
|
|
272
|
+
) AS "legacyAgentStepsTable",
|
|
273
|
+
EXISTS (
|
|
274
|
+
SELECT 1
|
|
275
|
+
FROM information_schema.columns
|
|
276
|
+
WHERE table_schema = 'public'
|
|
277
|
+
AND table_name = 'junior_conversations'
|
|
278
|
+
AND column_name = 'metric_run_id'
|
|
279
|
+
) AS "metricRunIdColumn"
|
|
280
|
+
`);
|
|
281
|
+
if (!baseline?.legacyAgentStepsTable || baseline.conversationEventsTable) {
|
|
282
|
+
throw new Error(
|
|
283
|
+
"Cannot adopt legacy core migration state: expected the pre-Drizzle junior_agent_steps table and no junior_conversation_events table"
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
const postBaselineMarkers = [
|
|
287
|
+
baseline.searchIndex ? "junior_conversation_messages_search_idx" : void 0,
|
|
288
|
+
baseline.metricRunIdColumn ? "junior_conversations.metric_run_id" : void 0
|
|
289
|
+
].filter((marker) => marker !== void 0);
|
|
290
|
+
if (postBaselineMarkers.length > 0) {
|
|
291
|
+
throw new Error(
|
|
292
|
+
`Cannot adopt legacy core migration state: post-baseline schema markers are already present: ${postBaselineMarkers.join(", ")}`
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
const migration = metricsComplete ? migrations[1] : migrations[0];
|
|
215
296
|
if (!migration) {
|
|
216
297
|
throw new Error("No core Drizzle migrations were packaged");
|
|
217
298
|
}
|
|
@@ -244,10 +325,10 @@ async function migrateSchema(executor) {
|
|
|
244
325
|
|
|
245
326
|
// src/cli/upgrade/migrations/core-sql.ts
|
|
246
327
|
async function migrateCoreSqlSchema(_context) {
|
|
247
|
-
const { sql } = getChatConfig();
|
|
328
|
+
const { sql: sql2 } = getChatConfig();
|
|
248
329
|
const executor = createJuniorSqlExecutor({
|
|
249
|
-
connectionString:
|
|
250
|
-
driver:
|
|
330
|
+
connectionString: sql2.databaseUrl,
|
|
331
|
+
driver: sql2.driver
|
|
251
332
|
});
|
|
252
333
|
try {
|
|
253
334
|
await migrateSchema(executor);
|
|
@@ -261,56 +342,923 @@ var coreSqlSchemaMigration = {
|
|
|
261
342
|
run: migrateCoreSqlSchema
|
|
262
343
|
};
|
|
263
344
|
|
|
345
|
+
// src/cli/upgrade/migrations/conversation-history/import.ts
|
|
346
|
+
import { z as z3 } from "zod";
|
|
347
|
+
import { eq as eq2 } from "drizzle-orm";
|
|
348
|
+
|
|
349
|
+
// src/cli/upgrade/migrations/conversation-history/session-log.ts
|
|
350
|
+
import { z } from "zod";
|
|
351
|
+
var AGENT_SESSION_LOG_PREFIX = "junior:agent-session-log";
|
|
352
|
+
var AGENT_SESSION_LOG_SCHEMA_VERSION = 2;
|
|
353
|
+
var INITIAL_SESSION_ID = "session_0";
|
|
354
|
+
var schemaVersionSchema = z.union([z.literal(1), z.literal(2)]);
|
|
355
|
+
function legacyActorProvenance(actor) {
|
|
356
|
+
if (actor.teamId && actor.slackUserId && actor.platform) {
|
|
357
|
+
return instructionProvenanceFor({
|
|
358
|
+
platform: "slack",
|
|
359
|
+
teamId: actor.teamId,
|
|
360
|
+
userId: actor.slackUserId,
|
|
361
|
+
...actor.slackUserName ? { userName: actor.slackUserName } : {},
|
|
362
|
+
...actor.fullName ? { fullName: actor.fullName } : {},
|
|
363
|
+
...actor.email ? { email: actor.email } : {}
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
return contextProvenance;
|
|
367
|
+
}
|
|
368
|
+
var piMessageEntrySchema = z.object({
|
|
369
|
+
schemaVersion: schemaVersionSchema,
|
|
370
|
+
type: z.literal("pi_message"),
|
|
371
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
372
|
+
message: piMessageSchema,
|
|
373
|
+
provenance: conversationMessageProvenanceSchema.optional(),
|
|
374
|
+
// Legacy v1 latest-wins actor, decoded into provenance on read.
|
|
375
|
+
actor: storedSlackActorSchema.optional()
|
|
376
|
+
});
|
|
377
|
+
var projectionResetEntrySchema = z.object({
|
|
378
|
+
schemaVersion: schemaVersionSchema,
|
|
379
|
+
type: z.literal("projection_reset"),
|
|
380
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
381
|
+
messages: z.array(piMessageSchema),
|
|
382
|
+
provenance: z.array(conversationMessageProvenanceSchema).optional(),
|
|
383
|
+
// Legacy v1 latest-wins actor; v1 resets carry no per-message provenance.
|
|
384
|
+
actor: storedSlackActorSchema.optional()
|
|
385
|
+
});
|
|
386
|
+
var actorRecordedEntrySchema = z.object({
|
|
387
|
+
schemaVersion: schemaVersionSchema,
|
|
388
|
+
type: z.literal("actor_recorded"),
|
|
389
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
390
|
+
actor: storedSlackActorSchema
|
|
391
|
+
});
|
|
392
|
+
var mcpProviderConnectedEntrySchema = z.object({
|
|
393
|
+
schemaVersion: schemaVersionSchema,
|
|
394
|
+
type: z.literal("mcp_provider_connected"),
|
|
395
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
396
|
+
provider: z.string().min(1)
|
|
397
|
+
});
|
|
398
|
+
var authorizationKindSchema = z.union([
|
|
399
|
+
z.literal("plugin"),
|
|
400
|
+
z.literal("mcp")
|
|
401
|
+
]);
|
|
402
|
+
var authorizationRequestedEntrySchema = z.object({
|
|
403
|
+
schemaVersion: schemaVersionSchema,
|
|
404
|
+
type: z.literal("authorization_requested"),
|
|
405
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
406
|
+
createdAtMs: z.number().int().nonnegative(),
|
|
407
|
+
kind: authorizationKindSchema,
|
|
408
|
+
provider: z.string().min(1),
|
|
409
|
+
actorId: z.string().min(1),
|
|
410
|
+
authorizationId: z.string().min(1),
|
|
411
|
+
delivery: z.union([
|
|
412
|
+
z.literal("private_link_sent"),
|
|
413
|
+
z.literal("private_link_reused")
|
|
414
|
+
])
|
|
415
|
+
});
|
|
416
|
+
var authorizationCompletedEntrySchema = z.object({
|
|
417
|
+
schemaVersion: schemaVersionSchema,
|
|
418
|
+
type: z.literal("authorization_completed"),
|
|
419
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
420
|
+
createdAtMs: z.number().int().nonnegative(),
|
|
421
|
+
kind: authorizationKindSchema,
|
|
422
|
+
provider: z.string().min(1),
|
|
423
|
+
actorId: z.string().min(1),
|
|
424
|
+
authorizationId: z.string().min(1)
|
|
425
|
+
});
|
|
426
|
+
var transcriptRefSchema = z.object({
|
|
427
|
+
type: z.literal("advisor_session"),
|
|
428
|
+
parentConversationId: z.string().min(1),
|
|
429
|
+
key: z.string().min(1)
|
|
430
|
+
});
|
|
431
|
+
var toolExecutionStartedEntrySchema = z.object({
|
|
432
|
+
schemaVersion: schemaVersionSchema,
|
|
433
|
+
type: z.literal("tool_execution_started"),
|
|
434
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
435
|
+
createdAtMs: z.number().int().nonnegative(),
|
|
436
|
+
toolCallId: z.string().min(1),
|
|
437
|
+
toolName: z.string().min(1),
|
|
438
|
+
args: z.unknown().optional()
|
|
439
|
+
});
|
|
440
|
+
var subagentStartedEntrySchema = z.object({
|
|
441
|
+
schemaVersion: schemaVersionSchema,
|
|
442
|
+
type: z.literal("subagent_started"),
|
|
443
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
444
|
+
subagentInvocationId: z.string().min(1),
|
|
445
|
+
subagentKind: z.string().min(1),
|
|
446
|
+
parentToolCallId: z.string().min(1).optional(),
|
|
447
|
+
parentConversationId: z.string().min(1),
|
|
448
|
+
parentSessionId: z.string().min(1).optional(),
|
|
449
|
+
transcriptRef: transcriptRefSchema,
|
|
450
|
+
historyMode: z.literal("shared"),
|
|
451
|
+
modelId: z.string().min(1).optional(),
|
|
452
|
+
reasoningLevel: z.string().min(1).optional(),
|
|
453
|
+
createdAtMs: z.number().int().nonnegative()
|
|
454
|
+
});
|
|
455
|
+
var subagentEndedEntrySchema = z.object({
|
|
456
|
+
schemaVersion: schemaVersionSchema,
|
|
457
|
+
type: z.literal("subagent_ended"),
|
|
458
|
+
sessionId: z.string().min(1).default(INITIAL_SESSION_ID),
|
|
459
|
+
subagentInvocationId: z.string().min(1),
|
|
460
|
+
outcome: z.union([
|
|
461
|
+
z.literal("success"),
|
|
462
|
+
z.literal("error"),
|
|
463
|
+
z.literal("aborted")
|
|
464
|
+
]),
|
|
465
|
+
errorCode: z.string().min(1).optional(),
|
|
466
|
+
transcriptEndMessageIndex: z.number().int().nonnegative().optional(),
|
|
467
|
+
transcriptStartMessageIndex: z.number().int().nonnegative().optional(),
|
|
468
|
+
createdAtMs: z.number().int().nonnegative()
|
|
469
|
+
});
|
|
470
|
+
var sessionLogEntrySchema = z.discriminatedUnion("type", [
|
|
471
|
+
piMessageEntrySchema,
|
|
472
|
+
projectionResetEntrySchema,
|
|
473
|
+
actorRecordedEntrySchema,
|
|
474
|
+
mcpProviderConnectedEntrySchema,
|
|
475
|
+
authorizationRequestedEntrySchema,
|
|
476
|
+
authorizationCompletedEntrySchema,
|
|
477
|
+
toolExecutionStartedEntrySchema,
|
|
478
|
+
subagentStartedEntrySchema,
|
|
479
|
+
subagentEndedEntrySchema
|
|
480
|
+
]);
|
|
481
|
+
function key(scope) {
|
|
482
|
+
const prefix = getChatConfig().state.keyPrefix;
|
|
483
|
+
return [
|
|
484
|
+
...prefix ? [prefix] : [],
|
|
485
|
+
AGENT_SESSION_LOG_PREFIX,
|
|
486
|
+
scope.conversationId
|
|
487
|
+
].join(":");
|
|
488
|
+
}
|
|
489
|
+
function rawKey(scope) {
|
|
490
|
+
return [AGENT_SESSION_LOG_PREFIX, scope.conversationId].join(":");
|
|
491
|
+
}
|
|
492
|
+
function storedRecord(value) {
|
|
493
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
494
|
+
}
|
|
495
|
+
function migrateStoredEntry(value) {
|
|
496
|
+
const record = storedRecord(value);
|
|
497
|
+
if (!record) {
|
|
498
|
+
return value;
|
|
499
|
+
}
|
|
500
|
+
const migrated = { ...record };
|
|
501
|
+
if ("requester" in migrated && !("actor" in migrated)) {
|
|
502
|
+
migrated.actor = migrated.requester;
|
|
503
|
+
}
|
|
504
|
+
delete migrated.requester;
|
|
505
|
+
if (migrated.type === "requester_recorded") {
|
|
506
|
+
migrated.type = "actor_recorded";
|
|
507
|
+
}
|
|
508
|
+
if ("requesterId" in migrated && !("actorId" in migrated)) {
|
|
509
|
+
migrated.actorId = migrated.requesterId;
|
|
510
|
+
}
|
|
511
|
+
delete migrated.requesterId;
|
|
512
|
+
return migrated;
|
|
513
|
+
}
|
|
514
|
+
function decode(value) {
|
|
515
|
+
if (typeof value === "string") {
|
|
516
|
+
return decode(JSON.parse(value));
|
|
517
|
+
}
|
|
518
|
+
const parsed = sessionLogEntrySchema.safeParse(migrateStoredEntry(value));
|
|
519
|
+
if (parsed.success) {
|
|
520
|
+
return parsed.data;
|
|
521
|
+
}
|
|
522
|
+
return {
|
|
523
|
+
schemaVersion: AGENT_SESSION_LOG_SCHEMA_VERSION,
|
|
524
|
+
type: "pi_message",
|
|
525
|
+
sessionId: INITIAL_SESSION_ID,
|
|
526
|
+
message: piMessageSchema.parse(value)
|
|
527
|
+
};
|
|
528
|
+
}
|
|
529
|
+
function redisStore(redisStateAdapter) {
|
|
530
|
+
const client = redisStateAdapter.getClient();
|
|
531
|
+
return {
|
|
532
|
+
async read(scope) {
|
|
533
|
+
const values = await client.lRange(key(scope), 0, -1);
|
|
534
|
+
return values.map(decode);
|
|
535
|
+
}
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
function stateStore() {
|
|
539
|
+
const stateAdapter = getStateAdapter();
|
|
540
|
+
return {
|
|
541
|
+
async read(scope) {
|
|
542
|
+
const listKey = rawKey(scope);
|
|
543
|
+
const value = await stateAdapter.get(listKey);
|
|
544
|
+
if (Array.isArray(value)) {
|
|
545
|
+
return value.map(decode);
|
|
546
|
+
}
|
|
547
|
+
const values = await stateAdapter.getList(listKey);
|
|
548
|
+
return values.map(decode);
|
|
549
|
+
}
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
async function defaultStore() {
|
|
553
|
+
const { redisStateAdapter, stateAdapter } = await getConnectedStateContext();
|
|
554
|
+
if (redisStateAdapter) {
|
|
555
|
+
return redisStore(redisStateAdapter);
|
|
556
|
+
}
|
|
557
|
+
await stateAdapter.connect();
|
|
558
|
+
return stateStore();
|
|
559
|
+
}
|
|
560
|
+
async function loadEntries(args) {
|
|
561
|
+
const store = args.store ?? await defaultStore();
|
|
562
|
+
return (await store.read(args)).map(decode);
|
|
563
|
+
}
|
|
564
|
+
async function readSessionLogEntries(args) {
|
|
565
|
+
return loadEntries(args);
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// src/cli/upgrade/migrations/conversation-history/advisor-session.ts
|
|
569
|
+
import { z as z2 } from "zod";
|
|
570
|
+
var legacyAdvisorMessagesSchema = z2.array(piMessageSchema);
|
|
571
|
+
function key2(conversationId) {
|
|
572
|
+
return `junior:${conversationId}:advisor_session`;
|
|
573
|
+
}
|
|
574
|
+
function createLegacyAdvisorSessionReader() {
|
|
575
|
+
return {
|
|
576
|
+
load: async (conversationId) => {
|
|
577
|
+
const stateAdapter = getStateAdapter();
|
|
578
|
+
await stateAdapter.connect();
|
|
579
|
+
return legacyAdvisorMessagesSchema.parse(
|
|
580
|
+
structuredClone(
|
|
581
|
+
await stateAdapter.get(key2(conversationId)) ?? []
|
|
582
|
+
)
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
// src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts
|
|
589
|
+
import { eq, sql } from "drizzle-orm";
|
|
590
|
+
|
|
591
|
+
// src/cli/upgrade/migrations/conversation-history/legacy-context-message.ts
|
|
592
|
+
var LEGACY_COMPACTION_SUMMARY_PREFIX = "Context handoff summary for future Junior turns:";
|
|
593
|
+
function isLegacyCheckpointText(text) {
|
|
594
|
+
const normalized = (unwrapCurrentInstruction(text) ?? text).trimStart();
|
|
595
|
+
return normalized.startsWith(LEGACY_COMPACTION_SUMMARY_PREFIX);
|
|
596
|
+
}
|
|
597
|
+
function isLegacyOrCurrentCheckpointText(text) {
|
|
598
|
+
return isCompactionSummaryText(text) || isLegacyCheckpointText(text);
|
|
599
|
+
}
|
|
600
|
+
function normalizeText(text) {
|
|
601
|
+
if (!isLegacyCheckpointText(text)) return text;
|
|
602
|
+
return text.replace(
|
|
603
|
+
LEGACY_COMPACTION_SUMMARY_PREFIX,
|
|
604
|
+
COMPACTION_SUMMARY_PREFIX
|
|
605
|
+
);
|
|
606
|
+
}
|
|
607
|
+
function normalizeLegacyContextMessage(message) {
|
|
608
|
+
const record = message;
|
|
609
|
+
if (typeof record.content === "string") {
|
|
610
|
+
const content2 = normalizeText(record.content);
|
|
611
|
+
return content2 === record.content ? message : { ...record, content: content2 };
|
|
612
|
+
}
|
|
613
|
+
if (!Array.isArray(record.content)) return message;
|
|
614
|
+
let changed = false;
|
|
615
|
+
const content = record.content.map((part) => {
|
|
616
|
+
if (!part || typeof part !== "object" || typeof part.text !== "string") {
|
|
617
|
+
return part;
|
|
618
|
+
}
|
|
619
|
+
const text = normalizeText(part.text);
|
|
620
|
+
if (text === part.text) return part;
|
|
621
|
+
changed = true;
|
|
622
|
+
return { ...part, text };
|
|
623
|
+
});
|
|
624
|
+
return changed ? { ...record, content } : message;
|
|
625
|
+
}
|
|
626
|
+
function isLegacyOrCurrentCheckpointMessage(message) {
|
|
627
|
+
const record = message;
|
|
628
|
+
if (typeof record.content === "string") {
|
|
629
|
+
return isLegacyOrCurrentCheckpointText(record.content);
|
|
630
|
+
}
|
|
631
|
+
if (!Array.isArray(record.content)) return false;
|
|
632
|
+
return record.content.some(
|
|
633
|
+
(part) => part !== null && typeof part === "object" && typeof part.text === "string" && isLegacyOrCurrentCheckpointText(part.text)
|
|
634
|
+
);
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// src/cli/upgrade/migrations/conversation-history/legacy-history-import.ts
|
|
638
|
+
var INITIAL_SESSION_ID2 = "session_0";
|
|
639
|
+
var ADVISOR_TASK_OPEN = "<advisor-task>\n";
|
|
640
|
+
var ADVISOR_TASK_CLOSE = "\n</advisor-task>";
|
|
641
|
+
var ADVISOR_CONTEXT_OPEN = "<executor-context>\n";
|
|
642
|
+
var ADVISOR_CONTEXT_CLOSE = "\n</executor-context>";
|
|
643
|
+
function importedAdvisorChildConversationId(parentConversationId) {
|
|
644
|
+
return `advisor:${parentConversationId}`;
|
|
645
|
+
}
|
|
646
|
+
function epochFromSessionId(sessionId) {
|
|
647
|
+
const match = /^session_(\d+)$/.exec(sessionId);
|
|
648
|
+
return match ? Number(match[1]) : 0;
|
|
649
|
+
}
|
|
650
|
+
function messageTimestampMs(message) {
|
|
651
|
+
const timestamp = message.timestamp;
|
|
652
|
+
return typeof timestamp === "number" ? timestamp : void 0;
|
|
653
|
+
}
|
|
654
|
+
function readAdvisorRequest(text) {
|
|
655
|
+
if (!text.startsWith(ADVISOR_TASK_OPEN) || !text.endsWith(ADVISOR_CONTEXT_CLOSE)) {
|
|
656
|
+
return void 0;
|
|
657
|
+
}
|
|
658
|
+
const taskEnd = text.indexOf(ADVISOR_TASK_CLOSE, ADVISOR_TASK_OPEN.length);
|
|
659
|
+
if (taskEnd < 0) {
|
|
660
|
+
return void 0;
|
|
661
|
+
}
|
|
662
|
+
const contextStart = taskEnd + ADVISOR_TASK_CLOSE.length + 2;
|
|
663
|
+
if (!text.startsWith(ADVISOR_CONTEXT_OPEN, contextStart)) {
|
|
664
|
+
return void 0;
|
|
665
|
+
}
|
|
666
|
+
const task = text.slice(ADVISOR_TASK_OPEN.length, taskEnd);
|
|
667
|
+
const context = text.slice(
|
|
668
|
+
contextStart + ADVISOR_CONTEXT_OPEN.length,
|
|
669
|
+
-ADVISOR_CONTEXT_CLOSE.length
|
|
670
|
+
);
|
|
671
|
+
return `${unescapeXml(task)}
|
|
672
|
+
|
|
673
|
+
Executor context:
|
|
674
|
+
${unescapeXml(context)}`;
|
|
675
|
+
}
|
|
676
|
+
function normalizeAdvisorMessage(message) {
|
|
677
|
+
const record = message;
|
|
678
|
+
if (record.role !== "user" || !Array.isArray(record.content)) {
|
|
679
|
+
return message;
|
|
680
|
+
}
|
|
681
|
+
let changed = false;
|
|
682
|
+
const content = record.content.map((part) => {
|
|
683
|
+
if (!part || typeof part !== "object" || part.type !== "text" || typeof part.text !== "string") {
|
|
684
|
+
return part;
|
|
685
|
+
}
|
|
686
|
+
const text = readAdvisorRequest(part.text);
|
|
687
|
+
if (text === void 0) {
|
|
688
|
+
return part;
|
|
689
|
+
}
|
|
690
|
+
changed = true;
|
|
691
|
+
return { ...part, text };
|
|
692
|
+
});
|
|
693
|
+
return changed ? { ...record, content } : message;
|
|
694
|
+
}
|
|
695
|
+
function piEntryProvenance(entry) {
|
|
696
|
+
if (entry.provenance) {
|
|
697
|
+
return entry.provenance;
|
|
698
|
+
}
|
|
699
|
+
if (entry.actor) {
|
|
700
|
+
return legacyActorProvenance(entry.actor);
|
|
701
|
+
}
|
|
702
|
+
return contextProvenance;
|
|
703
|
+
}
|
|
704
|
+
function convertLegacySessionLog(args) {
|
|
705
|
+
const events = [];
|
|
706
|
+
const fallback = args.fallbackCreatedAtMs;
|
|
707
|
+
let advisorChildConversationId;
|
|
708
|
+
let seq = 0;
|
|
709
|
+
const push = (historyVersion, data, createdAtMs) => {
|
|
710
|
+
events.push({ seq, historyVersion, data, createdAtMs });
|
|
711
|
+
seq += 1;
|
|
712
|
+
};
|
|
713
|
+
for (const entry of args.entries) {
|
|
714
|
+
const epoch = epochFromSessionId(entry.sessionId ?? INITIAL_SESSION_ID2);
|
|
715
|
+
switch (entry.type) {
|
|
716
|
+
case "pi_message": {
|
|
717
|
+
const message = entry.message;
|
|
718
|
+
push(
|
|
719
|
+
epoch,
|
|
720
|
+
{
|
|
721
|
+
type: "agent_step",
|
|
722
|
+
message,
|
|
723
|
+
provenance: piEntryProvenance(entry)
|
|
724
|
+
},
|
|
725
|
+
messageTimestampMs(message) ?? fallback
|
|
726
|
+
);
|
|
727
|
+
break;
|
|
728
|
+
}
|
|
729
|
+
case "projection_reset": {
|
|
730
|
+
const provenance = entry.provenance ?? entry.messages.map(() => contextProvenance);
|
|
731
|
+
if (provenance.length !== entry.messages.length) {
|
|
732
|
+
throw new Error(
|
|
733
|
+
"projection_reset provenance must align one-to-one with messages"
|
|
734
|
+
);
|
|
735
|
+
}
|
|
736
|
+
const resetCreatedAtMs = entry.messages.map(messageTimestampMs).find(
|
|
737
|
+
(timestamp) => timestamp !== void 0
|
|
738
|
+
) ?? fallback;
|
|
739
|
+
const checkpointIndex = entry.messages.findIndex(
|
|
740
|
+
isLegacyOrCurrentCheckpointMessage
|
|
741
|
+
);
|
|
742
|
+
push(
|
|
743
|
+
epoch,
|
|
744
|
+
{
|
|
745
|
+
type: "compaction",
|
|
746
|
+
modelProfile: "standard",
|
|
747
|
+
modelId: args.modelId,
|
|
748
|
+
replacementHistory: entry.messages.map((message, index) => ({
|
|
749
|
+
message: index === checkpointIndex ? normalizeLegacyContextMessage(message) : message,
|
|
750
|
+
provenance: provenance[index]
|
|
751
|
+
}))
|
|
752
|
+
},
|
|
753
|
+
resetCreatedAtMs
|
|
754
|
+
);
|
|
755
|
+
break;
|
|
756
|
+
}
|
|
757
|
+
case "mcp_provider_connected": {
|
|
758
|
+
push(
|
|
759
|
+
epoch,
|
|
760
|
+
{ type: "mcp_provider_connected", provider: entry.provider },
|
|
761
|
+
fallback
|
|
762
|
+
);
|
|
763
|
+
break;
|
|
764
|
+
}
|
|
765
|
+
case "authorization_requested": {
|
|
766
|
+
push(
|
|
767
|
+
epoch,
|
|
768
|
+
{
|
|
769
|
+
type: "authorization_requested",
|
|
770
|
+
kind: entry.kind,
|
|
771
|
+
provider: entry.provider,
|
|
772
|
+
actorId: entry.actorId,
|
|
773
|
+
authorizationId: entry.authorizationId,
|
|
774
|
+
delivery: entry.delivery
|
|
775
|
+
},
|
|
776
|
+
entry.createdAtMs
|
|
777
|
+
);
|
|
778
|
+
break;
|
|
779
|
+
}
|
|
780
|
+
case "authorization_completed": {
|
|
781
|
+
push(
|
|
782
|
+
epoch,
|
|
783
|
+
{
|
|
784
|
+
type: "authorization_completed",
|
|
785
|
+
kind: entry.kind,
|
|
786
|
+
provider: entry.provider,
|
|
787
|
+
actorId: entry.actorId,
|
|
788
|
+
authorizationId: entry.authorizationId
|
|
789
|
+
},
|
|
790
|
+
entry.createdAtMs
|
|
791
|
+
);
|
|
792
|
+
break;
|
|
793
|
+
}
|
|
794
|
+
case "tool_execution_started": {
|
|
795
|
+
push(
|
|
796
|
+
epoch,
|
|
797
|
+
{
|
|
798
|
+
type: "tool_execution_started",
|
|
799
|
+
toolCallId: entry.toolCallId,
|
|
800
|
+
toolName: entry.toolName
|
|
801
|
+
},
|
|
802
|
+
entry.createdAtMs
|
|
803
|
+
);
|
|
804
|
+
break;
|
|
805
|
+
}
|
|
806
|
+
case "subagent_started": {
|
|
807
|
+
const childConversationId = importedAdvisorChildConversationId(
|
|
808
|
+
args.conversationId
|
|
809
|
+
);
|
|
810
|
+
advisorChildConversationId = childConversationId;
|
|
811
|
+
push(
|
|
812
|
+
epoch,
|
|
813
|
+
{
|
|
814
|
+
type: "subagent_started",
|
|
815
|
+
subagentInvocationId: entry.subagentInvocationId,
|
|
816
|
+
subagentKind: entry.subagentKind,
|
|
817
|
+
...entry.parentToolCallId ? { parentToolCallId: entry.parentToolCallId } : {},
|
|
818
|
+
childConversationId
|
|
819
|
+
},
|
|
820
|
+
entry.createdAtMs
|
|
821
|
+
);
|
|
822
|
+
break;
|
|
823
|
+
}
|
|
824
|
+
case "subagent_ended": {
|
|
825
|
+
push(
|
|
826
|
+
epoch,
|
|
827
|
+
{
|
|
828
|
+
type: "subagent_ended",
|
|
829
|
+
subagentInvocationId: entry.subagentInvocationId,
|
|
830
|
+
outcome: entry.outcome,
|
|
831
|
+
...entry.errorCode ? { errorCode: entry.errorCode } : {}
|
|
832
|
+
},
|
|
833
|
+
entry.createdAtMs
|
|
834
|
+
);
|
|
835
|
+
break;
|
|
836
|
+
}
|
|
837
|
+
case "actor_recorded": {
|
|
838
|
+
break;
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
return {
|
|
843
|
+
events,
|
|
844
|
+
...advisorChildConversationId ? { advisorChildConversationId } : {}
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
function convertAdvisorMessages(messages, fallbackCreatedAtMs) {
|
|
848
|
+
return messages.map((sourceMessage, seq) => {
|
|
849
|
+
const message = normalizeAdvisorMessage(sourceMessage);
|
|
850
|
+
return {
|
|
851
|
+
seq,
|
|
852
|
+
historyVersion: 0,
|
|
853
|
+
data: { type: "agent_step", message, provenance: contextProvenance },
|
|
854
|
+
createdAtMs: messageTimestampMs(message) ?? fallbackCreatedAtMs
|
|
855
|
+
};
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
function insertRow(conversationId, event) {
|
|
859
|
+
const { type, ...payload } = conversationEventDataSchema.parse(event.data);
|
|
860
|
+
return {
|
|
861
|
+
conversationId,
|
|
862
|
+
seq: event.seq,
|
|
863
|
+
historyVersion: event.historyVersion,
|
|
864
|
+
schemaVersion: 1,
|
|
865
|
+
idempotencyKey: event.idempotencyKey ?? null,
|
|
866
|
+
type,
|
|
867
|
+
payload: sanitizePostgresJson(payload),
|
|
868
|
+
createdAt: new Date(event.createdAtMs)
|
|
869
|
+
};
|
|
870
|
+
}
|
|
871
|
+
function mergeImportedChronology(executionEvents, messages) {
|
|
872
|
+
const visibleEvents = messages.flatMap((message) => {
|
|
873
|
+
const recorded = {
|
|
874
|
+
seq: 0,
|
|
875
|
+
historyVersion: 0,
|
|
876
|
+
idempotencyKey: `message:${message.messageId}`,
|
|
877
|
+
data: {
|
|
878
|
+
type: "message",
|
|
879
|
+
messageId: message.messageId,
|
|
880
|
+
role: message.role,
|
|
881
|
+
text: message.text,
|
|
882
|
+
...message.authorIdentityId ? { authorIdentityId: message.authorIdentityId } : {},
|
|
883
|
+
...message.meta ? { meta: message.meta } : {}
|
|
884
|
+
},
|
|
885
|
+
createdAtMs: message.createdAtMs
|
|
886
|
+
};
|
|
887
|
+
return message.repliedAtMs === void 0 ? [recorded] : [
|
|
888
|
+
recorded,
|
|
889
|
+
{
|
|
890
|
+
seq: 0,
|
|
891
|
+
historyVersion: 0,
|
|
892
|
+
idempotencyKey: `message:${message.messageId}:handled`,
|
|
893
|
+
data: {
|
|
894
|
+
type: "message_handled",
|
|
895
|
+
messageId: message.messageId
|
|
896
|
+
},
|
|
897
|
+
createdAtMs: message.repliedAtMs
|
|
898
|
+
}
|
|
899
|
+
];
|
|
900
|
+
}).map((event, sourceOrder) => ({ event, sourceOrder })).sort(
|
|
901
|
+
(left, right) => left.event.createdAtMs - right.event.createdAtMs || left.sourceOrder - right.sourceOrder
|
|
902
|
+
);
|
|
903
|
+
const merged = [];
|
|
904
|
+
let visibleIndex = 0;
|
|
905
|
+
let currentEpoch = 0;
|
|
906
|
+
const push = (event, historyVersion) => {
|
|
907
|
+
merged.push({ ...event, seq: merged.length, historyVersion });
|
|
908
|
+
};
|
|
909
|
+
for (const executionEvent of [...executionEvents].sort(
|
|
910
|
+
(left, right) => left.seq - right.seq
|
|
911
|
+
)) {
|
|
912
|
+
while (visibleIndex < visibleEvents.length && visibleEvents[visibleIndex].event.createdAtMs <= executionEvent.createdAtMs) {
|
|
913
|
+
push(visibleEvents[visibleIndex].event, currentEpoch);
|
|
914
|
+
visibleIndex += 1;
|
|
915
|
+
}
|
|
916
|
+
push(executionEvent, executionEvent.historyVersion);
|
|
917
|
+
currentEpoch = Math.max(currentEpoch, executionEvent.historyVersion);
|
|
918
|
+
}
|
|
919
|
+
while (visibleIndex < visibleEvents.length) {
|
|
920
|
+
push(visibleEvents[visibleIndex].event, currentEpoch);
|
|
921
|
+
visibleIndex += 1;
|
|
922
|
+
}
|
|
923
|
+
const firstVisibleSeq = merged.find(
|
|
924
|
+
(event) => event.data.type === "message"
|
|
925
|
+
)?.seq;
|
|
926
|
+
return merged.map(
|
|
927
|
+
(event) => event.data.type === "messages_summarized" ? {
|
|
928
|
+
...event,
|
|
929
|
+
data: {
|
|
930
|
+
...event.data,
|
|
931
|
+
historyFromSeq: firstVisibleSeq ?? event.seq + 1
|
|
932
|
+
}
|
|
933
|
+
} : event
|
|
934
|
+
);
|
|
935
|
+
}
|
|
936
|
+
async function writeLegacyImport(executor, args) {
|
|
937
|
+
return withConversationEventLock(executor, args.conversationId, async () => {
|
|
938
|
+
const db = executor.db();
|
|
939
|
+
const conversations = await db.select({
|
|
940
|
+
transcriptPurgedAt: juniorConversations.transcriptPurgedAt
|
|
941
|
+
}).from(juniorConversations).where(eq(juniorConversations.conversationId, args.conversationId)).for("update");
|
|
942
|
+
if (conversations[0]?.transcriptPurgedAt) {
|
|
943
|
+
return false;
|
|
944
|
+
}
|
|
945
|
+
const existing = await db.select({ seq: juniorConversationEvents.seq }).from(juniorConversationEvents).where(eq(juniorConversationEvents.conversationId, args.conversationId)).limit(1);
|
|
946
|
+
if (existing.length > 0) {
|
|
947
|
+
return false;
|
|
948
|
+
}
|
|
949
|
+
const createdAt = new Date(args.fallbackCreatedAtMs);
|
|
950
|
+
await ensureConversationRow(
|
|
951
|
+
executor,
|
|
952
|
+
args.conversationId,
|
|
953
|
+
createdAt,
|
|
954
|
+
new Date(args.lastActivityAtMs)
|
|
955
|
+
);
|
|
956
|
+
const mergedEvents = mergeImportedChronology(
|
|
957
|
+
args.events,
|
|
958
|
+
args.messages ?? []
|
|
959
|
+
);
|
|
960
|
+
if (mergedEvents.length > 0) {
|
|
961
|
+
await db.insert(juniorConversationEvents).values(
|
|
962
|
+
mergedEvents.map((event) => insertRow(args.conversationId, event))
|
|
963
|
+
);
|
|
964
|
+
}
|
|
965
|
+
if (args.child) {
|
|
966
|
+
const childCreatedAtMs = args.child.events.length > 0 ? Math.min(...args.child.events.map((event) => event.createdAtMs)) : args.fallbackCreatedAtMs;
|
|
967
|
+
const childLastActivityAtMs = args.child.events.length > 0 ? Math.max(...args.child.events.map((event) => event.createdAtMs)) : childCreatedAtMs;
|
|
968
|
+
await insertLegacyAdvisorChildRow(
|
|
969
|
+
executor,
|
|
970
|
+
args.child.conversationId,
|
|
971
|
+
args.conversationId,
|
|
972
|
+
new Date(childCreatedAtMs),
|
|
973
|
+
new Date(childLastActivityAtMs)
|
|
974
|
+
);
|
|
975
|
+
if (args.child.events.length > 0) {
|
|
976
|
+
await db.insert(juniorConversationEvents).values(
|
|
977
|
+
args.child.events.map(
|
|
978
|
+
(event) => insertRow(args.child.conversationId, event)
|
|
979
|
+
)
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
return true;
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
async function ensureConversationRow(executor, conversationId, createdAt, lastActivityAt) {
|
|
987
|
+
await executor.db().insert(juniorConversations).values({
|
|
988
|
+
conversationId,
|
|
989
|
+
schemaVersion: 1,
|
|
990
|
+
createdAt,
|
|
991
|
+
lastActivityAt,
|
|
992
|
+
updatedAt: lastActivityAt,
|
|
993
|
+
executionStatus: "idle"
|
|
994
|
+
}).onConflictDoUpdate({
|
|
995
|
+
target: juniorConversations.conversationId,
|
|
996
|
+
set: {
|
|
997
|
+
createdAt: sql`least(${juniorConversations.createdAt}, excluded.created_at)`,
|
|
998
|
+
lastActivityAt: sql`greatest(${juniorConversations.lastActivityAt}, excluded.last_activity_at)`,
|
|
999
|
+
updatedAt: sql`greatest(${juniorConversations.updatedAt}, excluded.updated_at)`
|
|
1000
|
+
}
|
|
1001
|
+
});
|
|
1002
|
+
}
|
|
1003
|
+
async function insertLegacyAdvisorChildRow(executor, childConversationId, parentConversationId, createdAt, lastActivityAt) {
|
|
1004
|
+
await ensureConversationRow(
|
|
1005
|
+
executor,
|
|
1006
|
+
parentConversationId,
|
|
1007
|
+
createdAt,
|
|
1008
|
+
lastActivityAt
|
|
1009
|
+
);
|
|
1010
|
+
await executor.db().insert(juniorConversations).values({
|
|
1011
|
+
conversationId: childConversationId,
|
|
1012
|
+
schemaVersion: 1,
|
|
1013
|
+
parentConversationId,
|
|
1014
|
+
createdAt,
|
|
1015
|
+
lastActivityAt,
|
|
1016
|
+
updatedAt: lastActivityAt,
|
|
1017
|
+
executionStatus: "idle"
|
|
1018
|
+
}).onConflictDoUpdate({
|
|
1019
|
+
target: juniorConversations.conversationId,
|
|
1020
|
+
set: {
|
|
1021
|
+
createdAt: sql`least(${juniorConversations.createdAt}, excluded.created_at)`,
|
|
1022
|
+
lastActivityAt: sql`greatest(${juniorConversations.lastActivityAt}, excluded.last_activity_at)`,
|
|
1023
|
+
updatedAt: sql`greatest(${juniorConversations.updatedAt}, excluded.updated_at)`
|
|
1024
|
+
}
|
|
1025
|
+
});
|
|
1026
|
+
const rows = await executor.db().select({
|
|
1027
|
+
parentConversationId: juniorConversations.parentConversationId
|
|
1028
|
+
}).from(juniorConversations).where(eq(juniorConversations.conversationId, childConversationId));
|
|
1029
|
+
const child = rows[0];
|
|
1030
|
+
if (!child || child.parentConversationId !== parentConversationId) {
|
|
1031
|
+
throw new Error(
|
|
1032
|
+
"Legacy advisor child conflicts with existing conversation lineage"
|
|
1033
|
+
);
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
// src/cli/upgrade/migrations/conversation-history/import.ts
|
|
1038
|
+
var legacyVisibleMessageSchema = z3.object({
|
|
1039
|
+
id: z3.string(),
|
|
1040
|
+
role: z3.enum(["user", "assistant", "system"]),
|
|
1041
|
+
text: z3.string(),
|
|
1042
|
+
createdAtMs: z3.number().finite(),
|
|
1043
|
+
author: z3.object({}).passthrough().optional(),
|
|
1044
|
+
meta: z3.object({}).passthrough().optional()
|
|
1045
|
+
});
|
|
1046
|
+
var legacyCompactionSchema = z3.object({
|
|
1047
|
+
coveredMessageIds: z3.array(z3.string()),
|
|
1048
|
+
createdAtMs: z3.number().finite(),
|
|
1049
|
+
id: z3.string(),
|
|
1050
|
+
summary: z3.string()
|
|
1051
|
+
});
|
|
1052
|
+
function toImportedMessage(message) {
|
|
1053
|
+
const meta = {};
|
|
1054
|
+
if (message.author) meta.author = message.author;
|
|
1055
|
+
const { replied: _replied, ...restMeta } = message.meta ?? {};
|
|
1056
|
+
Object.assign(meta, restMeta);
|
|
1057
|
+
return {
|
|
1058
|
+
messageId: message.id,
|
|
1059
|
+
role: message.role,
|
|
1060
|
+
text: message.text,
|
|
1061
|
+
...Object.keys(meta).length > 0 ? { meta } : {},
|
|
1062
|
+
createdAtMs: message.createdAtMs
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
var legacyThreadStateSnapshotSchema = z3.object({
|
|
1066
|
+
conversation: z3.object({
|
|
1067
|
+
compactions: z3.array(legacyCompactionSchema).optional(),
|
|
1068
|
+
messages: z3.array(legacyVisibleMessageSchema).optional(),
|
|
1069
|
+
stats: z3.object({
|
|
1070
|
+
compactedMessageCount: z3.number().finite().optional(),
|
|
1071
|
+
updatedAtMs: z3.number().finite().optional()
|
|
1072
|
+
}).passthrough().optional()
|
|
1073
|
+
}).passthrough().optional()
|
|
1074
|
+
});
|
|
1075
|
+
async function loadThreadStateSnapshot(conversationId) {
|
|
1076
|
+
const stateAdapter = getStateAdapter();
|
|
1077
|
+
await stateAdapter.connect();
|
|
1078
|
+
const raw = await stateAdapter.get(`thread-state:${conversationId}`);
|
|
1079
|
+
if (!raw) {
|
|
1080
|
+
return { compactions: [], messages: [] };
|
|
1081
|
+
}
|
|
1082
|
+
const conversation = legacyThreadStateSnapshotSchema.parse(raw).conversation;
|
|
1083
|
+
return {
|
|
1084
|
+
compactions: conversation?.compactions ?? [],
|
|
1085
|
+
messages: conversation?.messages ?? [],
|
|
1086
|
+
...conversation?.stats?.compactedMessageCount !== void 0 ? { compactedMessageCount: conversation.stats.compactedMessageCount } : {},
|
|
1087
|
+
...conversation?.stats?.updatedAtMs !== void 0 ? { lastActivityAtMs: conversation.stats.updatedAtMs } : {}
|
|
1088
|
+
};
|
|
1089
|
+
}
|
|
1090
|
+
function intrinsicTimestamps(entries, visible, compactions) {
|
|
1091
|
+
const candidates = [];
|
|
1092
|
+
const pushMessageTs = (message) => {
|
|
1093
|
+
const timestamp = message.timestamp;
|
|
1094
|
+
if (typeof timestamp === "number") {
|
|
1095
|
+
candidates.push(timestamp);
|
|
1096
|
+
}
|
|
1097
|
+
};
|
|
1098
|
+
for (const entry of entries) {
|
|
1099
|
+
if (entry.type === "pi_message") {
|
|
1100
|
+
pushMessageTs(entry.message);
|
|
1101
|
+
} else if (entry.type === "projection_reset") {
|
|
1102
|
+
entry.messages.forEach(pushMessageTs);
|
|
1103
|
+
} else if ("createdAtMs" in entry) {
|
|
1104
|
+
candidates.push(entry.createdAtMs);
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
for (const message of visible) {
|
|
1108
|
+
candidates.push(message.createdAtMs);
|
|
1109
|
+
}
|
|
1110
|
+
for (const compaction of compactions) {
|
|
1111
|
+
candidates.push(compaction.createdAtMs);
|
|
1112
|
+
}
|
|
1113
|
+
return candidates;
|
|
1114
|
+
}
|
|
1115
|
+
async function importConversationFromLegacy(conversationId, deps) {
|
|
1116
|
+
const existing = await deps.executor.db().select({ seq: juniorConversationEvents.seq }).from(juniorConversationEvents).where(eq2(juniorConversationEvents.conversationId, conversationId)).limit(1);
|
|
1117
|
+
if (existing.length > 0) {
|
|
1118
|
+
return { imported: false };
|
|
1119
|
+
}
|
|
1120
|
+
const entries = deps.sessionLogStore ? await deps.sessionLogStore.read({ conversationId }) : await readSessionLogEntries({ conversationId });
|
|
1121
|
+
const snapshot = deps.loadVisibleMessages ? {
|
|
1122
|
+
compactions: deps.legacyCompactions ?? [],
|
|
1123
|
+
messages: await deps.loadVisibleMessages(conversationId)
|
|
1124
|
+
} : await loadThreadStateSnapshot(conversationId);
|
|
1125
|
+
const { compactedMessageCount, compactions, messages: visible } = snapshot;
|
|
1126
|
+
if (entries.length === 0 && visible.length === 0 && compactions.length === 0) {
|
|
1127
|
+
return { imported: false };
|
|
1128
|
+
}
|
|
1129
|
+
const hasAdvisor = entries.some((entry) => entry.type === "subagent_started");
|
|
1130
|
+
const advisorMessages = hasAdvisor ? await (deps.advisorSessionStore ?? createLegacyAdvisorSessionReader()).load(conversationId) : [];
|
|
1131
|
+
const intrinsic = intrinsicTimestamps(entries, visible, compactions);
|
|
1132
|
+
for (const message of advisorMessages) {
|
|
1133
|
+
const timestamp = message.timestamp;
|
|
1134
|
+
if (typeof timestamp === "number") {
|
|
1135
|
+
intrinsic.push(timestamp);
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
const fallbackCreatedAtMs = deps.conversationRecord?.createdAtMs ?? (intrinsic.length > 0 ? Math.min(...intrinsic) : void 0) ?? 0;
|
|
1139
|
+
const lastActivityAtMs = Math.max(
|
|
1140
|
+
fallbackCreatedAtMs,
|
|
1141
|
+
deps.conversationRecord?.lastActivityAtMs ?? 0,
|
|
1142
|
+
deps.legacyLastActivityAtMs ?? 0,
|
|
1143
|
+
intrinsic.length > 0 ? Math.max(...intrinsic) : 0
|
|
1144
|
+
);
|
|
1145
|
+
const converted = convertLegacySessionLog({
|
|
1146
|
+
conversationId,
|
|
1147
|
+
entries,
|
|
1148
|
+
fallbackCreatedAtMs,
|
|
1149
|
+
modelId: deps.modelId
|
|
1150
|
+
});
|
|
1151
|
+
if (compactions.length > 0) {
|
|
1152
|
+
const storedCount = compactions.reduce(
|
|
1153
|
+
(count, compaction) => count + compaction.coveredMessageIds.length,
|
|
1154
|
+
0
|
|
1155
|
+
);
|
|
1156
|
+
const statsCount = typeof compactedMessageCount === "number" && Number.isFinite(compactedMessageCount) && compactedMessageCount > 0 ? Math.floor(compactedMessageCount) : 0;
|
|
1157
|
+
const totalCount = Math.max(storedCount, statsCount);
|
|
1158
|
+
const compacted = compactions.map((compaction, index) => ({
|
|
1159
|
+
id: compaction.id,
|
|
1160
|
+
summary: compaction.summary,
|
|
1161
|
+
createdAtMs: compaction.createdAtMs,
|
|
1162
|
+
coveredMessageCount: compaction.coveredMessageIds.length + (index === 0 ? totalCount - storedCount : 0)
|
|
1163
|
+
}));
|
|
1164
|
+
converted.events.push({
|
|
1165
|
+
seq: converted.events.length,
|
|
1166
|
+
historyVersion: converted.events.at(-1)?.historyVersion ?? 0,
|
|
1167
|
+
data: {
|
|
1168
|
+
type: "messages_summarized",
|
|
1169
|
+
historyFromSeq: 0,
|
|
1170
|
+
compactions: compacted
|
|
1171
|
+
},
|
|
1172
|
+
createdAtMs: compactions.at(-1)?.createdAtMs ?? fallbackCreatedAtMs
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
let child;
|
|
1176
|
+
if (converted.advisorChildConversationId) {
|
|
1177
|
+
child = {
|
|
1178
|
+
conversationId: converted.advisorChildConversationId,
|
|
1179
|
+
events: convertAdvisorMessages(advisorMessages, fallbackCreatedAtMs)
|
|
1180
|
+
};
|
|
1181
|
+
}
|
|
1182
|
+
const messages = visible.map((message) => ({
|
|
1183
|
+
...toImportedMessage(message),
|
|
1184
|
+
...message.meta?.replied ? { repliedAtMs: message.createdAtMs } : {}
|
|
1185
|
+
}));
|
|
1186
|
+
const imported = await writeLegacyImport(deps.executor, {
|
|
1187
|
+
conversationId,
|
|
1188
|
+
fallbackCreatedAtMs,
|
|
1189
|
+
lastActivityAtMs,
|
|
1190
|
+
...messages.length > 0 ? { messages } : {},
|
|
1191
|
+
events: converted.events,
|
|
1192
|
+
...child ? { child } : {}
|
|
1193
|
+
});
|
|
1194
|
+
return { imported };
|
|
1195
|
+
}
|
|
1196
|
+
|
|
264
1197
|
// src/cli/upgrade/migrations/conversations-history-sql.ts
|
|
265
|
-
var
|
|
1198
|
+
var HISTORY_BACKFILL_PAGE_SIZE = 500;
|
|
266
1199
|
async function migrateConversationHistoryToSql(context, options = {}) {
|
|
267
1200
|
const source = createStateConversationStore(context.stateAdapter);
|
|
1201
|
+
const chatConfig = getChatConfig();
|
|
268
1202
|
let executor = options.executor;
|
|
269
1203
|
let closeExecutor;
|
|
270
1204
|
if (!executor) {
|
|
271
|
-
const { sql } =
|
|
1205
|
+
const { sql: sql2 } = chatConfig;
|
|
272
1206
|
executor = createJuniorSqlExecutor({
|
|
273
|
-
connectionString:
|
|
274
|
-
driver:
|
|
1207
|
+
connectionString: sql2.databaseUrl,
|
|
1208
|
+
driver: sql2.driver
|
|
275
1209
|
});
|
|
276
1210
|
closeExecutor = () => executor.close();
|
|
277
1211
|
}
|
|
278
|
-
const
|
|
1212
|
+
const pageSize = Math.max(
|
|
1213
|
+
1,
|
|
1214
|
+
Math.floor(options.batchSize ?? HISTORY_BACKFILL_PAGE_SIZE)
|
|
1215
|
+
);
|
|
279
1216
|
try {
|
|
280
|
-
const messageStore = createSqlConversationMessageStore(executor);
|
|
281
|
-
const conversations = await source.listByActivity({ limit });
|
|
282
1217
|
let migrated = 0;
|
|
283
1218
|
let existing = 0;
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
1219
|
+
let scanned = 0;
|
|
1220
|
+
let offset = 0;
|
|
1221
|
+
while (true) {
|
|
1222
|
+
const conversations = await source.listByActivity({
|
|
1223
|
+
limit: pageSize,
|
|
1224
|
+
offset
|
|
1225
|
+
});
|
|
1226
|
+
if (conversations.length === 0) {
|
|
1227
|
+
break;
|
|
1228
|
+
}
|
|
1229
|
+
for (const conversation of conversations) {
|
|
1230
|
+
const result = await importConversationFromLegacy(
|
|
1231
|
+
conversation.conversationId,
|
|
1232
|
+
{
|
|
1233
|
+
executor,
|
|
1234
|
+
modelId: standardModelId(chatConfig.bot),
|
|
1235
|
+
conversationRecord: conversation,
|
|
1236
|
+
...options.sessionLogStore ? { sessionLogStore: options.sessionLogStore } : {},
|
|
1237
|
+
...options.advisorSessionStore ? { advisorSessionStore: options.advisorSessionStore } : {},
|
|
1238
|
+
...options.loadVisibleMessages ? { loadVisibleMessages: options.loadVisibleMessages } : {}
|
|
1239
|
+
}
|
|
1240
|
+
);
|
|
1241
|
+
if (result.imported) {
|
|
1242
|
+
migrated += 1;
|
|
1243
|
+
} else {
|
|
1244
|
+
existing += 1;
|
|
294
1245
|
}
|
|
295
|
-
);
|
|
296
|
-
if (result.imported) {
|
|
297
|
-
migrated += 1;
|
|
298
|
-
} else {
|
|
299
|
-
existing += 1;
|
|
300
1246
|
}
|
|
1247
|
+
scanned += conversations.length;
|
|
1248
|
+
offset += conversations.length;
|
|
301
1249
|
}
|
|
302
1250
|
return {
|
|
303
1251
|
existing,
|
|
304
1252
|
migrated,
|
|
305
1253
|
missing: 0,
|
|
306
|
-
scanned
|
|
1254
|
+
scanned
|
|
307
1255
|
};
|
|
308
1256
|
} finally {
|
|
309
1257
|
await closeExecutor?.();
|
|
310
1258
|
}
|
|
311
1259
|
}
|
|
312
1260
|
var sqlConversationHistoryMigration = {
|
|
313
|
-
name: "backfill-
|
|
1261
|
+
name: "backfill-conversation-events-sql",
|
|
314
1262
|
run: migrateConversationHistoryToSql
|
|
315
1263
|
};
|
|
316
1264
|
|
|
@@ -407,10 +1355,10 @@ async function runPluginStorageMigrations(context) {
|
|
|
407
1355
|
return emptyResult();
|
|
408
1356
|
}
|
|
409
1357
|
const previousConfig = pluginCatalogRuntime.setConfig(pluginCatalogConfig);
|
|
410
|
-
const
|
|
1358
|
+
const sql2 = getChatConfig().sql;
|
|
411
1359
|
const ownedExecutor = context.db ? void 0 : createJuniorSqlExecutor({
|
|
412
|
-
connectionString:
|
|
413
|
-
driver:
|
|
1360
|
+
connectionString: sql2.databaseUrl,
|
|
1361
|
+
driver: sql2.driver
|
|
414
1362
|
});
|
|
415
1363
|
const sqlUrlDb = ownedExecutor ? ownedExecutor.db() : void 0;
|
|
416
1364
|
try {
|
|
@@ -566,12 +1514,12 @@ async function migratePluginSchemas(executor, roots) {
|
|
|
566
1514
|
|
|
567
1515
|
// src/cli/upgrade/migrations/plugin-sql.ts
|
|
568
1516
|
async function migratePluginsToSql(context) {
|
|
569
|
-
const { sql } = getChatConfig();
|
|
1517
|
+
const { sql: sql2 } = getChatConfig();
|
|
570
1518
|
const { pluginCatalogConfig } = await resolveUpgradePlugins(context);
|
|
571
1519
|
const previousConfig = pluginCatalogRuntime.setConfig(pluginCatalogConfig);
|
|
572
1520
|
const executor = createJuniorSqlExecutor({
|
|
573
|
-
connectionString:
|
|
574
|
-
driver:
|
|
1521
|
+
connectionString: sql2.databaseUrl,
|
|
1522
|
+
driver: sql2.driver
|
|
575
1523
|
});
|
|
576
1524
|
try {
|
|
577
1525
|
const result = await migratePluginSchemas(
|
|
@@ -603,8 +1551,8 @@ var CONVERSATION_ACTIVE_INDEX_KEY = `${CONVERSATION_PREFIX}:active`;
|
|
|
603
1551
|
var LEGACY_CONVERSATION_WORK_PREFIX = "junior:conversation-work";
|
|
604
1552
|
var LEGACY_CONVERSATION_WORK_SCHEMA_VERSION = 1;
|
|
605
1553
|
var LEGACY_CONVERSATION_WORK_INDEX_KEY = `${LEGACY_CONVERSATION_WORK_PREFIX}:index`;
|
|
606
|
-
var
|
|
607
|
-
var AGENT_TURN_SESSION_INDEX_KEY = `${
|
|
1554
|
+
var AGENT_TURN_SESSION_PREFIX2 = "junior:agent_turn_session";
|
|
1555
|
+
var AGENT_TURN_SESSION_INDEX_KEY = `${AGENT_TURN_SESSION_PREFIX2}:index`;
|
|
608
1556
|
var THREAD_STATE_PREFIX = "thread-state";
|
|
609
1557
|
function conversationKey(conversationId) {
|
|
610
1558
|
return `${CONVERSATION_PREFIX}:${conversationId}`;
|
|
@@ -863,9 +1811,9 @@ function uniqueAwaitingContinuationSummaries(values) {
|
|
|
863
1811
|
if (!summary) {
|
|
864
1812
|
continue;
|
|
865
1813
|
}
|
|
866
|
-
const
|
|
867
|
-
if (!summaries.has(
|
|
868
|
-
summaries.set(
|
|
1814
|
+
const key3 = `${summary.conversationId}:${summary.sessionId}`;
|
|
1815
|
+
if (!summaries.has(key3)) {
|
|
1816
|
+
summaries.set(key3, summary);
|
|
869
1817
|
}
|
|
870
1818
|
}
|
|
871
1819
|
return [...summaries.values()];
|
|
@@ -902,7 +1850,7 @@ async function removeEmulatedIndexEntry(args) {
|
|
|
902
1850
|
await args.stateAdapter.set(args.indexKey, next, JUNIOR_THREAD_STATE_TTL_MS);
|
|
903
1851
|
}
|
|
904
1852
|
async function upsertRedisIndexEntry(args) {
|
|
905
|
-
const
|
|
1853
|
+
const key3 = redisIndexKey(args.indexKey);
|
|
906
1854
|
if (args.indexKey === CONVERSATION_BY_ACTIVITY_INDEX_KEY) {
|
|
907
1855
|
const upsertBoundedActivityScript = `
|
|
908
1856
|
redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])
|
|
@@ -917,7 +1865,7 @@ async function upsertRedisIndexEntry(args) {
|
|
|
917
1865
|
"EVAL",
|
|
918
1866
|
upsertBoundedActivityScript,
|
|
919
1867
|
"1",
|
|
920
|
-
|
|
1868
|
+
key3,
|
|
921
1869
|
String(args.score),
|
|
922
1870
|
args.conversationId,
|
|
923
1871
|
String(JUNIOR_THREAD_STATE_TTL_MS),
|
|
@@ -927,13 +1875,13 @@ async function upsertRedisIndexEntry(args) {
|
|
|
927
1875
|
}
|
|
928
1876
|
await args.client.sendCommand([
|
|
929
1877
|
"ZADD",
|
|
930
|
-
|
|
1878
|
+
key3,
|
|
931
1879
|
String(args.score),
|
|
932
1880
|
args.conversationId
|
|
933
1881
|
]);
|
|
934
1882
|
await args.client.sendCommand([
|
|
935
1883
|
"PEXPIRE",
|
|
936
|
-
|
|
1884
|
+
key3,
|
|
937
1885
|
String(JUNIOR_THREAD_STATE_TTL_MS)
|
|
938
1886
|
]);
|
|
939
1887
|
}
|
|
@@ -1128,31 +2076,31 @@ var redisConversationStateMigration = {
|
|
|
1128
2076
|
|
|
1129
2077
|
// src/cli/upgrade/migrations/agent-turn-session-actor.ts
|
|
1130
2078
|
import { THREAD_STATE_TTL_MS } from "chat";
|
|
1131
|
-
var
|
|
1132
|
-
var AGENT_TURN_SESSION_INDEX_KEY2 = `${
|
|
2079
|
+
var AGENT_TURN_SESSION_PREFIX3 = "junior:agent_turn_session";
|
|
2080
|
+
var AGENT_TURN_SESSION_INDEX_KEY2 = `${AGENT_TURN_SESSION_PREFIX3}:index`;
|
|
1133
2081
|
var AGENT_TURN_SESSION_INDEX_MAX_LENGTH = 5e3;
|
|
1134
2082
|
var REDIS_SCAN_COUNT = 500;
|
|
1135
2083
|
function conversationIndexKey(conversationId) {
|
|
1136
|
-
return `${
|
|
2084
|
+
return `${AGENT_TURN_SESSION_PREFIX3}:conversation:${conversationId}:index`;
|
|
1137
2085
|
}
|
|
1138
2086
|
function sessionRecordKey(conversationId, sessionId) {
|
|
1139
|
-
return `${
|
|
2087
|
+
return `${AGENT_TURN_SESSION_PREFIX3}:${conversationId}:${sessionId}`;
|
|
1140
2088
|
}
|
|
1141
2089
|
function logicalConversationIndexPrefix() {
|
|
1142
2090
|
const prefix = getChatConfig().state.keyPrefix;
|
|
1143
2091
|
return [
|
|
1144
2092
|
...prefix ? [prefix] : [],
|
|
1145
|
-
`${
|
|
2093
|
+
`${AGENT_TURN_SESSION_PREFIX3}:conversation:`
|
|
1146
2094
|
].join(":");
|
|
1147
2095
|
}
|
|
1148
|
-
function conversationIdFromRedisListKey(
|
|
2096
|
+
function conversationIdFromRedisListKey(key3) {
|
|
1149
2097
|
const marker = `:list:${logicalConversationIndexPrefix()}`;
|
|
1150
|
-
const markerIndex =
|
|
1151
|
-
if (markerIndex < 0 || !
|
|
2098
|
+
const markerIndex = key3.indexOf(marker);
|
|
2099
|
+
if (markerIndex < 0 || !key3.endsWith(":index")) {
|
|
1152
2100
|
return void 0;
|
|
1153
2101
|
}
|
|
1154
2102
|
return toOptionalString(
|
|
1155
|
-
|
|
2103
|
+
key3.slice(markerIndex + marker.length, -":index".length)
|
|
1156
2104
|
);
|
|
1157
2105
|
}
|
|
1158
2106
|
async function discoverRedisConversationIds(redisStateAdapter) {
|
|
@@ -1178,11 +2126,11 @@ async function discoverRedisConversationIds(redisStateAdapter) {
|
|
|
1178
2126
|
);
|
|
1179
2127
|
}
|
|
1180
2128
|
cursor = String(reply[0]);
|
|
1181
|
-
for (const
|
|
1182
|
-
if (typeof
|
|
2129
|
+
for (const key3 of reply[1]) {
|
|
2130
|
+
if (typeof key3 !== "string") {
|
|
1183
2131
|
continue;
|
|
1184
2132
|
}
|
|
1185
|
-
const conversationId = conversationIdFromRedisListKey(
|
|
2133
|
+
const conversationId = conversationIdFromRedisListKey(key3);
|
|
1186
2134
|
if (conversationId) {
|
|
1187
2135
|
conversationIds.add(conversationId);
|
|
1188
2136
|
}
|
|
@@ -1220,8 +2168,8 @@ async function rewriteList(args) {
|
|
|
1220
2168
|
return { migrated: changed, values: migrated.map((entry) => entry.value) };
|
|
1221
2169
|
}
|
|
1222
2170
|
async function migrateSessionRecord(args) {
|
|
1223
|
-
const
|
|
1224
|
-
const existing = await args.stateAdapter.get(
|
|
2171
|
+
const key3 = sessionRecordKey(args.conversationId, args.sessionId);
|
|
2172
|
+
const existing = await args.stateAdapter.get(key3);
|
|
1225
2173
|
if (existing === void 0) {
|
|
1226
2174
|
return "missing";
|
|
1227
2175
|
}
|
|
@@ -1229,7 +2177,7 @@ async function migrateSessionRecord(args) {
|
|
|
1229
2177
|
if (!migrated.changed) {
|
|
1230
2178
|
return "existing";
|
|
1231
2179
|
}
|
|
1232
|
-
await args.stateAdapter.set(
|
|
2180
|
+
await args.stateAdapter.set(key3, migrated.value, THREAD_STATE_TTL_MS);
|
|
1233
2181
|
return "migrated";
|
|
1234
2182
|
}
|
|
1235
2183
|
async function migrateAgentTurnSessionActor(context) {
|
|
@@ -1327,25 +2275,25 @@ WITH candidates AS MATERIALIZED (
|
|
|
1327
2275
|
ORDER BY conversation_id
|
|
1328
2276
|
LIMIT $2
|
|
1329
2277
|
),
|
|
1330
|
-
-- Occurrence numbers preserve intentional duplicates within one
|
|
1331
|
-
-- collapsing context copies of the same message across
|
|
2278
|
+
-- Occurrence numbers preserve intentional duplicates within one history version
|
|
2279
|
+
-- while collapsing context copies of the same message across replacements.
|
|
1332
2280
|
message_occurrences AS (
|
|
1333
2281
|
SELECT
|
|
1334
|
-
|
|
1335
|
-
|
|
2282
|
+
event.conversation_id,
|
|
2283
|
+
event.payload -> 'message' AS message,
|
|
1336
2284
|
row_number() OVER (
|
|
1337
2285
|
PARTITION BY
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
ORDER BY
|
|
2286
|
+
event.conversation_id,
|
|
2287
|
+
event.history_version,
|
|
2288
|
+
event.payload -> 'message'
|
|
2289
|
+
ORDER BY event.seq
|
|
1342
2290
|
) AS occurrence
|
|
1343
|
-
FROM
|
|
2291
|
+
FROM junior_conversation_events AS event
|
|
1344
2292
|
INNER JOIN candidates AS candidate
|
|
1345
|
-
ON candidate.conversation_id =
|
|
1346
|
-
WHERE
|
|
1347
|
-
AND
|
|
1348
|
-
AND jsonb_typeof(
|
|
2293
|
+
ON candidate.conversation_id = event.conversation_id
|
|
2294
|
+
WHERE event.type = 'agent_step'
|
|
2295
|
+
AND event.payload -> 'message' ->> 'role' = 'assistant'
|
|
2296
|
+
AND jsonb_typeof(event.payload -> 'message' -> 'usage') = 'object'
|
|
1349
2297
|
),
|
|
1350
2298
|
canonical_messages AS (
|
|
1351
2299
|
SELECT DISTINCT conversation_id, message, occurrence
|
|
@@ -1537,10 +2485,10 @@ async function repairConversationUsage(_context, options = {}) {
|
|
|
1537
2485
|
let executor = options.executor;
|
|
1538
2486
|
let closeExecutor;
|
|
1539
2487
|
if (!executor) {
|
|
1540
|
-
const { sql } = getChatConfig();
|
|
2488
|
+
const { sql: sql2 } = getChatConfig();
|
|
1541
2489
|
executor = createJuniorSqlExecutor({
|
|
1542
|
-
connectionString:
|
|
1543
|
-
driver:
|
|
2490
|
+
connectionString: sql2.databaseUrl,
|
|
2491
|
+
driver: sql2.driver
|
|
1544
2492
|
});
|
|
1545
2493
|
closeExecutor = () => executor.close();
|
|
1546
2494
|
}
|
|
@@ -1594,6 +2542,990 @@ var conversationUsageRepairMigration = {
|
|
|
1594
2542
|
run: repairConversationUsage
|
|
1595
2543
|
};
|
|
1596
2544
|
|
|
2545
|
+
// src/cli/upgrade/migrations/conversation-event-cursors.ts
|
|
2546
|
+
var REDIS_SCAN_COUNT2 = 500;
|
|
2547
|
+
function rawTurnSessionKeyPattern() {
|
|
2548
|
+
const statePrefix = getChatConfig().state.keyPrefix;
|
|
2549
|
+
return `*:cache:${statePrefix ? `${statePrefix}:` : ""}${AGENT_TURN_SESSION_PREFIX}:*`;
|
|
2550
|
+
}
|
|
2551
|
+
function logicalStateKey(rawKey2) {
|
|
2552
|
+
const statePrefix = getChatConfig().state.keyPrefix;
|
|
2553
|
+
const cachePrefix = `:cache:${statePrefix ? `${statePrefix}:` : ""}`;
|
|
2554
|
+
const cacheAt = rawKey2.indexOf(cachePrefix);
|
|
2555
|
+
return cacheAt < 0 ? void 0 : rawKey2.slice(cacheAt + cachePrefix.length);
|
|
2556
|
+
}
|
|
2557
|
+
async function discoverRawTurnSessionKeys(redisStateAdapter, conversationIds) {
|
|
2558
|
+
const client = redisStateAdapter?.getClient();
|
|
2559
|
+
const discovered = /* @__PURE__ */ new Map();
|
|
2560
|
+
if (!client) return discovered;
|
|
2561
|
+
const targetKeyPrefixes = [...conversationIds].map(
|
|
2562
|
+
(conversationId) => `${AGENT_TURN_SESSION_PREFIX}:${conversationId}:`
|
|
2563
|
+
);
|
|
2564
|
+
let cursor = "0";
|
|
2565
|
+
do {
|
|
2566
|
+
const reply = await client.sendCommand([
|
|
2567
|
+
"SCAN",
|
|
2568
|
+
cursor,
|
|
2569
|
+
"MATCH",
|
|
2570
|
+
rawTurnSessionKeyPattern(),
|
|
2571
|
+
"COUNT",
|
|
2572
|
+
String(REDIS_SCAN_COUNT2)
|
|
2573
|
+
]);
|
|
2574
|
+
if (!Array.isArray(reply) || reply.length !== 2 || typeof reply[0] !== "string" && typeof reply[0] !== "number" || !Array.isArray(reply[1])) {
|
|
2575
|
+
throw new Error(
|
|
2576
|
+
"Unexpected Redis SCAN response while checking turn-session cursors"
|
|
2577
|
+
);
|
|
2578
|
+
}
|
|
2579
|
+
cursor = String(reply[0]);
|
|
2580
|
+
for (const rawKey2 of reply[1]) {
|
|
2581
|
+
if (typeof rawKey2 !== "string") continue;
|
|
2582
|
+
const logicalKey = logicalStateKey(rawKey2);
|
|
2583
|
+
if (!logicalKey || !targetKeyPrefixes.some((prefix) => logicalKey.startsWith(prefix))) {
|
|
2584
|
+
continue;
|
|
2585
|
+
}
|
|
2586
|
+
const encoded = await client.sendCommand(["GET", rawKey2]);
|
|
2587
|
+
if (encoded === null) continue;
|
|
2588
|
+
if (typeof encoded !== "string") {
|
|
2589
|
+
throw new Error(`Turn-session cursor ${rawKey2} is not JSON text`);
|
|
2590
|
+
}
|
|
2591
|
+
let value;
|
|
2592
|
+
try {
|
|
2593
|
+
value = JSON.parse(encoded);
|
|
2594
|
+
} catch (error) {
|
|
2595
|
+
throw new Error(`Turn-session cursor ${rawKey2} is invalid JSON`, {
|
|
2596
|
+
cause: error
|
|
2597
|
+
});
|
|
2598
|
+
}
|
|
2599
|
+
if (!isRecord(value)) {
|
|
2600
|
+
throw new Error(`Turn-session cursor ${rawKey2} is not an object`);
|
|
2601
|
+
}
|
|
2602
|
+
const conversationId = toOptionalString(value.conversationId);
|
|
2603
|
+
const sessionId = toOptionalString(value.sessionId);
|
|
2604
|
+
if (!conversationId || !sessionId) {
|
|
2605
|
+
throw new Error(`Turn-session cursor ${rawKey2} has no identity`);
|
|
2606
|
+
}
|
|
2607
|
+
const recordKey = agentTurnSessionKey(conversationId, sessionId);
|
|
2608
|
+
if (logicalKey !== recordKey) {
|
|
2609
|
+
throw new Error(
|
|
2610
|
+
`Turn-session cursor ${rawKey2} identity does not match its key`
|
|
2611
|
+
);
|
|
2612
|
+
}
|
|
2613
|
+
if (!conversationIds.has(conversationId)) continue;
|
|
2614
|
+
discovered.set(recordKey, conversationId);
|
|
2615
|
+
}
|
|
2616
|
+
} while (cursor !== "0");
|
|
2617
|
+
return discovered;
|
|
2618
|
+
}
|
|
2619
|
+
async function indexedTurnSessionIds(context, conversationId) {
|
|
2620
|
+
const summaries = await context.stateAdapter.getList(
|
|
2621
|
+
agentTurnSessionConversationIndexKey(conversationId)
|
|
2622
|
+
);
|
|
2623
|
+
return [
|
|
2624
|
+
...new Set(
|
|
2625
|
+
summaries.flatMap((summary) => {
|
|
2626
|
+
if (!isRecord(summary)) return [];
|
|
2627
|
+
const sessionId = toOptionalString(summary.sessionId);
|
|
2628
|
+
return sessionId ? [sessionId] : [];
|
|
2629
|
+
})
|
|
2630
|
+
)
|
|
2631
|
+
];
|
|
2632
|
+
}
|
|
2633
|
+
function hasSeqCursor(record) {
|
|
2634
|
+
return Number.isInteger(record.committedSeq) || Number.isInteger(record.turnStartSeq);
|
|
2635
|
+
}
|
|
2636
|
+
async function prepareConversationEventResequence(context, conversationIds) {
|
|
2637
|
+
if (conversationIds.size === 0) return;
|
|
2638
|
+
await context.stateAdapter.connect();
|
|
2639
|
+
const keys = /* @__PURE__ */ new Set();
|
|
2640
|
+
for (const conversationId of conversationIds) {
|
|
2641
|
+
for (const sessionId of await indexedTurnSessionIds(
|
|
2642
|
+
context,
|
|
2643
|
+
conversationId
|
|
2644
|
+
)) {
|
|
2645
|
+
keys.add(agentTurnSessionKey(conversationId, sessionId));
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
for (const [key3] of await discoverRawTurnSessionKeys(
|
|
2649
|
+
context.redisStateAdapter,
|
|
2650
|
+
conversationIds
|
|
2651
|
+
)) {
|
|
2652
|
+
keys.add(key3);
|
|
2653
|
+
}
|
|
2654
|
+
const terminalKeys = [];
|
|
2655
|
+
for (const key3 of keys) {
|
|
2656
|
+
const value = await context.stateAdapter.get(key3);
|
|
2657
|
+
if (!isRecord(value) || !hasSeqCursor(value)) continue;
|
|
2658
|
+
if (value.state === "running" || value.state === "awaiting_resume") {
|
|
2659
|
+
throw new Error(
|
|
2660
|
+
`Cannot resequence conversation events while unfinished turn session ${key3} retains a seq cursor`
|
|
2661
|
+
);
|
|
2662
|
+
}
|
|
2663
|
+
if (value.state !== "completed" && value.state !== "failed" && value.state !== "abandoned") {
|
|
2664
|
+
throw new Error(
|
|
2665
|
+
`Cannot resequence conversation events with invalid cursor-bearing turn session ${key3}`
|
|
2666
|
+
);
|
|
2667
|
+
}
|
|
2668
|
+
terminalKeys.push(key3);
|
|
2669
|
+
}
|
|
2670
|
+
for (const key3 of terminalKeys) {
|
|
2671
|
+
await context.stateAdapter.delete(key3);
|
|
2672
|
+
}
|
|
2673
|
+
}
|
|
2674
|
+
|
|
2675
|
+
// src/cli/upgrade/migrations/conversation-visible-message-events.ts
|
|
2676
|
+
var VISIBLE_EVENT_BATCH_SIZE = 500;
|
|
2677
|
+
var VISIBLE_EVENT_BACKFILL_LOCK = "junior:upgrade:conversation-visible-message-events";
|
|
2678
|
+
var LOAD_VISIBLE_CONVERSATION_IDS_SQL = `
|
|
2679
|
+
SELECT DISTINCT conversation_id
|
|
2680
|
+
FROM junior_conversation_messages
|
|
2681
|
+
WHERE $2::text IS NULL OR conversation_id > $2::text
|
|
2682
|
+
ORDER BY conversation_id
|
|
2683
|
+
LIMIT $1
|
|
2684
|
+
`;
|
|
2685
|
+
var LOAD_COMPACTED_CONVERSATION_IDS_SQL = `
|
|
2686
|
+
SELECT DISTINCT conversation_id
|
|
2687
|
+
FROM junior_conversation_events
|
|
2688
|
+
WHERE type = 'messages_summarized'
|
|
2689
|
+
AND payload @? '$.compactions[*].coveredMessageIds'
|
|
2690
|
+
AND ($2::text IS NULL OR conversation_id > $2::text)
|
|
2691
|
+
ORDER BY conversation_id
|
|
2692
|
+
LIMIT $1
|
|
2693
|
+
`;
|
|
2694
|
+
var LOAD_MISSING_VISIBLE_EVENTS_SQL = `
|
|
2695
|
+
SELECT
|
|
2696
|
+
message.conversation_id,
|
|
2697
|
+
message.message_id,
|
|
2698
|
+
message.role,
|
|
2699
|
+
message.text,
|
|
2700
|
+
message.author_identity_id,
|
|
2701
|
+
message.meta,
|
|
2702
|
+
message.replied_at,
|
|
2703
|
+
message.created_at
|
|
2704
|
+
FROM junior_conversation_messages message
|
|
2705
|
+
WHERE (
|
|
2706
|
+
NOT EXISTS (
|
|
2707
|
+
SELECT 1
|
|
2708
|
+
FROM junior_conversation_events event
|
|
2709
|
+
WHERE event.conversation_id = message.conversation_id
|
|
2710
|
+
AND event.idempotency_key =
|
|
2711
|
+
'message:' || message.message_id
|
|
2712
|
+
)
|
|
2713
|
+
OR (
|
|
2714
|
+
message.replied_at IS NOT NULL
|
|
2715
|
+
AND NOT EXISTS (
|
|
2716
|
+
SELECT 1
|
|
2717
|
+
FROM junior_conversation_events event
|
|
2718
|
+
WHERE event.conversation_id = message.conversation_id
|
|
2719
|
+
AND event.idempotency_key =
|
|
2720
|
+
'message:' || message.message_id || ':handled'
|
|
2721
|
+
)
|
|
2722
|
+
)
|
|
2723
|
+
)
|
|
2724
|
+
AND (
|
|
2725
|
+
$2::text IS NULL
|
|
2726
|
+
OR (message.conversation_id, message.created_at, message.message_id) >
|
|
2727
|
+
($2::text, $3::timestamptz, $4::text)
|
|
2728
|
+
)
|
|
2729
|
+
ORDER BY message.conversation_id, message.created_at, message.message_id
|
|
2730
|
+
LIMIT $1
|
|
2731
|
+
`;
|
|
2732
|
+
var COUNT_MISSING_VISIBLE_EVENTS_SQL = `
|
|
2733
|
+
SELECT count(*)::integer AS count
|
|
2734
|
+
FROM junior_conversation_messages message
|
|
2735
|
+
WHERE NOT EXISTS (
|
|
2736
|
+
SELECT 1
|
|
2737
|
+
FROM junior_conversation_events event
|
|
2738
|
+
WHERE event.conversation_id = message.conversation_id
|
|
2739
|
+
AND event.idempotency_key =
|
|
2740
|
+
'message:' || message.message_id
|
|
2741
|
+
)
|
|
2742
|
+
OR (
|
|
2743
|
+
message.replied_at IS NOT NULL
|
|
2744
|
+
AND NOT EXISTS (
|
|
2745
|
+
SELECT 1
|
|
2746
|
+
FROM junior_conversation_events event
|
|
2747
|
+
WHERE event.conversation_id = message.conversation_id
|
|
2748
|
+
AND event.idempotency_key =
|
|
2749
|
+
'message:' || message.message_id || ':handled'
|
|
2750
|
+
)
|
|
2751
|
+
)
|
|
2752
|
+
`;
|
|
2753
|
+
function timestampMs(value) {
|
|
2754
|
+
return value instanceof Date ? value.getTime() : new Date(value).getTime();
|
|
2755
|
+
}
|
|
2756
|
+
function eventsForRow(row) {
|
|
2757
|
+
const events = [
|
|
2758
|
+
{
|
|
2759
|
+
idempotencyKey: `message:${row.message_id}`,
|
|
2760
|
+
data: {
|
|
2761
|
+
type: "message",
|
|
2762
|
+
messageId: row.message_id,
|
|
2763
|
+
role: row.role,
|
|
2764
|
+
text: row.text,
|
|
2765
|
+
...row.author_identity_id ? { authorIdentityId: row.author_identity_id } : {},
|
|
2766
|
+
...row.meta ? { meta: row.meta } : {}
|
|
2767
|
+
},
|
|
2768
|
+
createdAtMs: timestampMs(row.created_at)
|
|
2769
|
+
}
|
|
2770
|
+
];
|
|
2771
|
+
if (row.replied_at) {
|
|
2772
|
+
events.push({
|
|
2773
|
+
idempotencyKey: `message:${row.message_id}:handled`,
|
|
2774
|
+
data: { type: "message_handled", messageId: row.message_id },
|
|
2775
|
+
createdAtMs: timestampMs(row.replied_at)
|
|
2776
|
+
});
|
|
2777
|
+
}
|
|
2778
|
+
return events;
|
|
2779
|
+
}
|
|
2780
|
+
async function visitConversationIds(executor, query, batchSize, visit) {
|
|
2781
|
+
let cursor;
|
|
2782
|
+
while (true) {
|
|
2783
|
+
const rows = await executor.query(query, [
|
|
2784
|
+
batchSize,
|
|
2785
|
+
cursor ?? null
|
|
2786
|
+
]);
|
|
2787
|
+
if (rows.length === 0) return;
|
|
2788
|
+
for (const row of rows) {
|
|
2789
|
+
await visit(row.conversation_id);
|
|
2790
|
+
}
|
|
2791
|
+
cursor = rows.at(-1).conversation_id;
|
|
2792
|
+
}
|
|
2793
|
+
}
|
|
2794
|
+
async function visitVisibleConversations(executor, batchSize, visit) {
|
|
2795
|
+
await visitConversationIds(
|
|
2796
|
+
executor,
|
|
2797
|
+
LOAD_VISIBLE_CONVERSATION_IDS_SQL,
|
|
2798
|
+
batchSize,
|
|
2799
|
+
visit
|
|
2800
|
+
);
|
|
2801
|
+
}
|
|
2802
|
+
async function readLegacyCompactionState(context, conversationId) {
|
|
2803
|
+
const raw = await context.stateAdapter.get(
|
|
2804
|
+
`thread-state:${conversationId}`
|
|
2805
|
+
);
|
|
2806
|
+
const conversation = isRecord(raw) && isRecord(raw.conversation) ? raw.conversation : void 0;
|
|
2807
|
+
const stats = conversation && isRecord(conversation.stats) ? conversation.stats : void 0;
|
|
2808
|
+
const rawCount = toOptionalNumber(stats?.compactedMessageCount);
|
|
2809
|
+
const compactedMessageCount = rawCount === void 0 ? void 0 : Math.max(0, Math.floor(rawCount));
|
|
2810
|
+
const messages = conversation && Array.isArray(conversation.messages) ? conversation.messages : [];
|
|
2811
|
+
const liveMessageIds = new Set(
|
|
2812
|
+
messages.flatMap((message) => {
|
|
2813
|
+
if (!isRecord(message)) return [];
|
|
2814
|
+
const id = toOptionalString(message.id);
|
|
2815
|
+
return id ? [id] : [];
|
|
2816
|
+
})
|
|
2817
|
+
);
|
|
2818
|
+
return {
|
|
2819
|
+
...compactedMessageCount === void 0 ? {} : { compactedMessageCount },
|
|
2820
|
+
liveMessageIds
|
|
2821
|
+
};
|
|
2822
|
+
}
|
|
2823
|
+
function sqlVisibleEvents(events) {
|
|
2824
|
+
return events.map((event) => {
|
|
2825
|
+
const parsed = newConversationEventSchema.parse(event);
|
|
2826
|
+
if (!parsed.idempotencyKey) {
|
|
2827
|
+
throw new Error(
|
|
2828
|
+
"Visible-message backfill events require idempotency keys"
|
|
2829
|
+
);
|
|
2830
|
+
}
|
|
2831
|
+
const { type, ...payload } = conversationEventDataSchema.parse(parsed.data);
|
|
2832
|
+
return {
|
|
2833
|
+
createdAtMs: parsed.createdAtMs,
|
|
2834
|
+
idempotencyKey: parsed.idempotencyKey,
|
|
2835
|
+
payload: sanitizePostgresJson(payload),
|
|
2836
|
+
type
|
|
2837
|
+
};
|
|
2838
|
+
});
|
|
2839
|
+
}
|
|
2840
|
+
async function loadVisibleCompactionAnchor(executor, conversationId) {
|
|
2841
|
+
const [row] = await executor.query(
|
|
2842
|
+
`WITH snapshot AS (
|
|
2843
|
+
SELECT payload
|
|
2844
|
+
FROM junior_conversation_events
|
|
2845
|
+
WHERE conversation_id = $1
|
|
2846
|
+
AND type = 'messages_summarized'
|
|
2847
|
+
ORDER BY seq DESC
|
|
2848
|
+
LIMIT 1
|
|
2849
|
+
)
|
|
2850
|
+
SELECT (
|
|
2851
|
+
SELECT event.payload->>'messageId'
|
|
2852
|
+
FROM junior_conversation_events event
|
|
2853
|
+
WHERE event.conversation_id = $1
|
|
2854
|
+
AND event.type = 'message'
|
|
2855
|
+
AND event.seq >= (snapshot.payload->>'historyFromSeq')::integer
|
|
2856
|
+
ORDER BY event.seq
|
|
2857
|
+
LIMIT 1
|
|
2858
|
+
) AS "messageId"
|
|
2859
|
+
FROM snapshot`,
|
|
2860
|
+
[conversationId]
|
|
2861
|
+
);
|
|
2862
|
+
return row;
|
|
2863
|
+
}
|
|
2864
|
+
async function normalizeVisibleCompactionBoundary(executor, conversationId, anchor) {
|
|
2865
|
+
if (!anchor) return;
|
|
2866
|
+
let historyFromSeq;
|
|
2867
|
+
if (anchor.messageId) {
|
|
2868
|
+
const [baseline] = await executor.query(
|
|
2869
|
+
`SELECT seq
|
|
2870
|
+
FROM junior_conversation_events
|
|
2871
|
+
WHERE conversation_id = $1
|
|
2872
|
+
AND type = 'message'
|
|
2873
|
+
AND payload->>'messageId' = $2
|
|
2874
|
+
ORDER BY seq
|
|
2875
|
+
LIMIT 1`,
|
|
2876
|
+
[conversationId, anchor.messageId]
|
|
2877
|
+
);
|
|
2878
|
+
if (!baseline) {
|
|
2879
|
+
throw new Error(
|
|
2880
|
+
`Visible compaction anchor ${anchor.messageId} disappeared during resequence`
|
|
2881
|
+
);
|
|
2882
|
+
}
|
|
2883
|
+
historyFromSeq = baseline.seq;
|
|
2884
|
+
} else {
|
|
2885
|
+
const [end] = await executor.query(
|
|
2886
|
+
`SELECT coalesce(max(seq), -1)::integer + 1 AS seq
|
|
2887
|
+
FROM junior_conversation_events
|
|
2888
|
+
WHERE conversation_id = $1`,
|
|
2889
|
+
[conversationId]
|
|
2890
|
+
);
|
|
2891
|
+
historyFromSeq = end?.seq ?? 0;
|
|
2892
|
+
}
|
|
2893
|
+
await executor.execute(
|
|
2894
|
+
`WITH snapshot AS (
|
|
2895
|
+
SELECT ctid
|
|
2896
|
+
FROM junior_conversation_events
|
|
2897
|
+
WHERE conversation_id = $1
|
|
2898
|
+
AND type = 'messages_summarized'
|
|
2899
|
+
ORDER BY seq DESC
|
|
2900
|
+
LIMIT 1
|
|
2901
|
+
)
|
|
2902
|
+
UPDATE junior_conversation_events event
|
|
2903
|
+
SET payload = jsonb_set(
|
|
2904
|
+
event.payload,
|
|
2905
|
+
'{historyFromSeq}',
|
|
2906
|
+
to_jsonb($2::integer)
|
|
2907
|
+
)
|
|
2908
|
+
FROM snapshot
|
|
2909
|
+
WHERE event.ctid = snapshot.ctid`,
|
|
2910
|
+
[conversationId, historyFromSeq]
|
|
2911
|
+
);
|
|
2912
|
+
}
|
|
2913
|
+
function coveredMessageIds(payload) {
|
|
2914
|
+
if (!Array.isArray(payload.compactions)) return [];
|
|
2915
|
+
return payload.compactions.flatMap((compaction) => {
|
|
2916
|
+
if (!isRecord(compaction) || !Array.isArray(compaction.coveredMessageIds)) {
|
|
2917
|
+
return [];
|
|
2918
|
+
}
|
|
2919
|
+
return compaction.coveredMessageIds.flatMap((id) => {
|
|
2920
|
+
const value = toOptionalString(id);
|
|
2921
|
+
return value ? [value] : [];
|
|
2922
|
+
});
|
|
2923
|
+
});
|
|
2924
|
+
}
|
|
2925
|
+
function retainedCompactionCount(payload) {
|
|
2926
|
+
if (!Array.isArray(payload.compactions)) return 0;
|
|
2927
|
+
return payload.compactions.reduce((total, value) => {
|
|
2928
|
+
if (!isRecord(value)) return total;
|
|
2929
|
+
const stored = toOptionalNumber(value.coveredMessageCount);
|
|
2930
|
+
const legacy = Array.isArray(value.coveredMessageIds) ? value.coveredMessageIds.length : 0;
|
|
2931
|
+
return total + (stored === void 0 ? legacy : Math.max(0, stored));
|
|
2932
|
+
}, 0);
|
|
2933
|
+
}
|
|
2934
|
+
function resolvedHistoryFromSeq(args) {
|
|
2935
|
+
if (args.legacyState.liveMessageIds.size > 0) {
|
|
2936
|
+
const live = args.visibleEvents.find(
|
|
2937
|
+
(event) => args.legacyState.liveMessageIds.has(event.messageId)
|
|
2938
|
+
);
|
|
2939
|
+
if (live) return live.seq;
|
|
2940
|
+
}
|
|
2941
|
+
if (args.legacyState.compactedMessageCount !== void 0) {
|
|
2942
|
+
return args.visibleEvents[args.legacyState.compactedMessageCount]?.seq ?? args.endSeq;
|
|
2943
|
+
}
|
|
2944
|
+
const coveredIds = new Set(args.legacyCoveredMessageIds);
|
|
2945
|
+
if (coveredIds.size > 0) {
|
|
2946
|
+
const lastCoveredSeq = args.visibleEvents.reduce(
|
|
2947
|
+
(last, event) => coveredIds.has(event.messageId) ? Math.max(last, event.seq) : last,
|
|
2948
|
+
-1
|
|
2949
|
+
);
|
|
2950
|
+
return args.visibleEvents.find((event) => event.seq > lastCoveredSeq)?.seq ?? args.endSeq;
|
|
2951
|
+
}
|
|
2952
|
+
return args.currentHistoryFromSeq > 0 ? args.currentHistoryFromSeq : args.visibleEvents[0]?.seq ?? args.endSeq;
|
|
2953
|
+
}
|
|
2954
|
+
async function finalizeVisibleCompactions(context, executor, conversationId) {
|
|
2955
|
+
const legacyState = await readLegacyCompactionState(context, conversationId);
|
|
2956
|
+
await executor.withLock(
|
|
2957
|
+
`junior_conversation:event:${conversationId}`,
|
|
2958
|
+
async () => {
|
|
2959
|
+
const [latest] = await executor.query(
|
|
2960
|
+
`SELECT seq, payload
|
|
2961
|
+
FROM junior_conversation_events
|
|
2962
|
+
WHERE conversation_id = $1
|
|
2963
|
+
AND type = 'messages_summarized'
|
|
2964
|
+
ORDER BY seq DESC
|
|
2965
|
+
LIMIT 1`,
|
|
2966
|
+
[conversationId]
|
|
2967
|
+
);
|
|
2968
|
+
if (!latest) return;
|
|
2969
|
+
const visibleEvents = await executor.query(
|
|
2970
|
+
`SELECT seq, payload->>'messageId' AS "messageId"
|
|
2971
|
+
FROM junior_conversation_events
|
|
2972
|
+
WHERE conversation_id = $1
|
|
2973
|
+
AND type = 'message'
|
|
2974
|
+
ORDER BY seq`,
|
|
2975
|
+
[conversationId]
|
|
2976
|
+
);
|
|
2977
|
+
const [cursor] = await executor.query(
|
|
2978
|
+
`SELECT coalesce(max(seq), -1)::integer + 1 AS "endSeq"
|
|
2979
|
+
FROM junior_conversation_events
|
|
2980
|
+
WHERE conversation_id = $1`,
|
|
2981
|
+
[conversationId]
|
|
2982
|
+
);
|
|
2983
|
+
const currentHistoryFromSeq = toOptionalNumber(latest.payload.historyFromSeq) ?? 0;
|
|
2984
|
+
const historyFromSeq = resolvedHistoryFromSeq({
|
|
2985
|
+
currentHistoryFromSeq: Math.max(0, Math.floor(currentHistoryFromSeq)),
|
|
2986
|
+
endSeq: cursor?.endSeq ?? 0,
|
|
2987
|
+
legacyCoveredMessageIds: coveredMessageIds(latest.payload),
|
|
2988
|
+
legacyState,
|
|
2989
|
+
visibleEvents
|
|
2990
|
+
});
|
|
2991
|
+
const retainedCount = retainedCompactionCount(latest.payload);
|
|
2992
|
+
const countDelta = Math.max(
|
|
2993
|
+
0,
|
|
2994
|
+
(legacyState.compactedMessageCount ?? retainedCount) - retainedCount
|
|
2995
|
+
);
|
|
2996
|
+
await executor.execute(
|
|
2997
|
+
`WITH rewritten AS (
|
|
2998
|
+
SELECT
|
|
2999
|
+
event.ctid,
|
|
3000
|
+
jsonb_agg(
|
|
3001
|
+
(compaction.value - 'coveredMessageIds') ||
|
|
3002
|
+
jsonb_build_object(
|
|
3003
|
+
'coveredMessageCount',
|
|
3004
|
+
coalesce(
|
|
3005
|
+
(compaction.value->>'coveredMessageCount')::integer,
|
|
3006
|
+
jsonb_array_length(
|
|
3007
|
+
coalesce(compaction.value->'coveredMessageIds', '[]'::jsonb)
|
|
3008
|
+
)
|
|
3009
|
+
) + CASE
|
|
3010
|
+
WHEN event.seq = $2 AND compaction.position = 1 THEN $4
|
|
3011
|
+
ELSE 0
|
|
3012
|
+
END
|
|
3013
|
+
)
|
|
3014
|
+
ORDER BY compaction.position
|
|
3015
|
+
) AS compactions
|
|
3016
|
+
FROM junior_conversation_events event
|
|
3017
|
+
CROSS JOIN LATERAL jsonb_array_elements(
|
|
3018
|
+
event.payload->'compactions'
|
|
3019
|
+
) WITH ORDINALITY AS compaction(value, position)
|
|
3020
|
+
WHERE event.conversation_id = $1
|
|
3021
|
+
AND event.type = 'messages_summarized'
|
|
3022
|
+
GROUP BY event.ctid
|
|
3023
|
+
)
|
|
3024
|
+
UPDATE junior_conversation_events event
|
|
3025
|
+
SET payload =
|
|
3026
|
+
(event.payload - 'compactions') ||
|
|
3027
|
+
jsonb_build_object('compactions', rewritten.compactions) ||
|
|
3028
|
+
CASE WHEN event.seq = $2
|
|
3029
|
+
THEN jsonb_build_object('historyFromSeq', $3::integer)
|
|
3030
|
+
ELSE '{}'::jsonb
|
|
3031
|
+
END
|
|
3032
|
+
FROM rewritten
|
|
3033
|
+
WHERE event.ctid = rewritten.ctid`,
|
|
3034
|
+
[conversationId, latest.seq, historyFromSeq, countDelta]
|
|
3035
|
+
);
|
|
3036
|
+
}
|
|
3037
|
+
);
|
|
3038
|
+
}
|
|
3039
|
+
async function mergeVisibleEvents(executor, conversationId, events) {
|
|
3040
|
+
const encoded = sqlVisibleEvents(events);
|
|
3041
|
+
await executor.withLock(
|
|
3042
|
+
`junior_conversation:event:${conversationId}`,
|
|
3043
|
+
async () => {
|
|
3044
|
+
await executor.transaction(async () => {
|
|
3045
|
+
const compactionAnchor = await loadVisibleCompactionAnchor(
|
|
3046
|
+
executor,
|
|
3047
|
+
conversationId
|
|
3048
|
+
);
|
|
3049
|
+
await executor.execute(
|
|
3050
|
+
`WITH input AS (
|
|
3051
|
+
SELECT *
|
|
3052
|
+
FROM jsonb_to_recordset($2::jsonb) AS event(
|
|
3053
|
+
"createdAtMs" bigint,
|
|
3054
|
+
"idempotencyKey" text,
|
|
3055
|
+
payload jsonb,
|
|
3056
|
+
type text
|
|
3057
|
+
)
|
|
3058
|
+
), base AS (
|
|
3059
|
+
SELECT coalesce(max(seq), -1) AS max_seq
|
|
3060
|
+
FROM junior_conversation_events
|
|
3061
|
+
WHERE conversation_id = $1
|
|
3062
|
+
), pending AS (
|
|
3063
|
+
SELECT
|
|
3064
|
+
input.*,
|
|
3065
|
+
row_number() OVER (
|
|
3066
|
+
ORDER BY input."createdAtMs", input."idempotencyKey"
|
|
3067
|
+
) AS offset
|
|
3068
|
+
FROM input
|
|
3069
|
+
WHERE NOT EXISTS (
|
|
3070
|
+
SELECT 1
|
|
3071
|
+
FROM junior_conversation_events existing
|
|
3072
|
+
WHERE existing.conversation_id = $1
|
|
3073
|
+
AND existing.idempotency_key = input."idempotencyKey"
|
|
3074
|
+
)
|
|
3075
|
+
)
|
|
3076
|
+
INSERT INTO junior_conversation_events (
|
|
3077
|
+
conversation_id,
|
|
3078
|
+
seq,
|
|
3079
|
+
history_version,
|
|
3080
|
+
schema_version,
|
|
3081
|
+
idempotency_key,
|
|
3082
|
+
type,
|
|
3083
|
+
payload,
|
|
3084
|
+
created_at
|
|
3085
|
+
)
|
|
3086
|
+
SELECT
|
|
3087
|
+
$1,
|
|
3088
|
+
base.max_seq + pending.offset,
|
|
3089
|
+
coalesce((
|
|
3090
|
+
SELECT prior.history_version
|
|
3091
|
+
FROM junior_conversation_events prior
|
|
3092
|
+
WHERE prior.conversation_id = $1
|
|
3093
|
+
AND prior.created_at < to_timestamp(pending."createdAtMs" / 1000.0)
|
|
3094
|
+
ORDER BY prior.seq DESC
|
|
3095
|
+
LIMIT 1
|
|
3096
|
+
), 0),
|
|
3097
|
+
1,
|
|
3098
|
+
pending."idempotencyKey",
|
|
3099
|
+
pending.type,
|
|
3100
|
+
pending.payload,
|
|
3101
|
+
to_timestamp(pending."createdAtMs" / 1000.0)
|
|
3102
|
+
FROM pending
|
|
3103
|
+
CROSS JOIN base
|
|
3104
|
+
ON CONFLICT (conversation_id, idempotency_key) DO NOTHING`,
|
|
3105
|
+
[conversationId, JSON.stringify(encoded)]
|
|
3106
|
+
);
|
|
3107
|
+
await executor.execute(
|
|
3108
|
+
`UPDATE junior_conversation_events
|
|
3109
|
+
SET seq = -seq - 1
|
|
3110
|
+
WHERE conversation_id = $1`,
|
|
3111
|
+
[conversationId]
|
|
3112
|
+
);
|
|
3113
|
+
await executor.execute(
|
|
3114
|
+
`WITH ranked AS (
|
|
3115
|
+
SELECT
|
|
3116
|
+
event.ctid,
|
|
3117
|
+
row_number() OVER (
|
|
3118
|
+
ORDER BY
|
|
3119
|
+
CASE
|
|
3120
|
+
WHEN event.type IN (
|
|
3121
|
+
'message',
|
|
3122
|
+
'message_handled'
|
|
3123
|
+
) THEN coalesce((
|
|
3124
|
+
SELECT max(-execution.seq - 1)
|
|
3125
|
+
FROM junior_conversation_events execution
|
|
3126
|
+
WHERE execution.conversation_id = event.conversation_id
|
|
3127
|
+
AND execution.type NOT IN (
|
|
3128
|
+
'message',
|
|
3129
|
+
'message_handled'
|
|
3130
|
+
)
|
|
3131
|
+
AND execution.created_at < event.created_at
|
|
3132
|
+
), -1)
|
|
3133
|
+
ELSE -event.seq - 1
|
|
3134
|
+
END,
|
|
3135
|
+
CASE
|
|
3136
|
+
WHEN event.type IN (
|
|
3137
|
+
'message',
|
|
3138
|
+
'message_handled'
|
|
3139
|
+
) THEN 1
|
|
3140
|
+
ELSE 0
|
|
3141
|
+
END,
|
|
3142
|
+
event.created_at,
|
|
3143
|
+
CASE event.type
|
|
3144
|
+
WHEN 'message' THEN 0
|
|
3145
|
+
WHEN 'message_handled' THEN 1
|
|
3146
|
+
ELSE 0
|
|
3147
|
+
END,
|
|
3148
|
+
event.idempotency_key NULLS FIRST,
|
|
3149
|
+
event.type
|
|
3150
|
+
) - 1 AS next_seq
|
|
3151
|
+
FROM junior_conversation_events event
|
|
3152
|
+
WHERE event.conversation_id = $1
|
|
3153
|
+
)
|
|
3154
|
+
UPDATE junior_conversation_events event
|
|
3155
|
+
SET seq = ranked.next_seq
|
|
3156
|
+
FROM ranked
|
|
3157
|
+
WHERE event.ctid = ranked.ctid`,
|
|
3158
|
+
[conversationId]
|
|
3159
|
+
);
|
|
3160
|
+
await normalizeVisibleCompactionBoundary(
|
|
3161
|
+
executor,
|
|
3162
|
+
conversationId,
|
|
3163
|
+
compactionAnchor
|
|
3164
|
+
);
|
|
3165
|
+
});
|
|
3166
|
+
}
|
|
3167
|
+
);
|
|
3168
|
+
}
|
|
3169
|
+
async function migrateConversationVisibleMessageEvents(context, options = {}) {
|
|
3170
|
+
let executor = options.executor;
|
|
3171
|
+
let closeExecutor;
|
|
3172
|
+
if (!executor) {
|
|
3173
|
+
const { sql: sql2 } = getChatConfig();
|
|
3174
|
+
executor = createJuniorSqlExecutor({
|
|
3175
|
+
connectionString: sql2.databaseUrl,
|
|
3176
|
+
driver: sql2.driver
|
|
3177
|
+
});
|
|
3178
|
+
closeExecutor = () => executor.close();
|
|
3179
|
+
}
|
|
3180
|
+
const batchSize = Math.max(
|
|
3181
|
+
1,
|
|
3182
|
+
Math.floor(options.batchSize ?? VISIBLE_EVENT_BATCH_SIZE)
|
|
3183
|
+
);
|
|
3184
|
+
let migrated = 0;
|
|
3185
|
+
let cursor;
|
|
3186
|
+
try {
|
|
3187
|
+
const [legacyTable] = await executor.query(
|
|
3188
|
+
`SELECT to_regclass('public.junior_conversation_messages') IS NOT NULL AS exists`
|
|
3189
|
+
);
|
|
3190
|
+
if (!legacyTable?.exists) {
|
|
3191
|
+
return { existing: 0, migrated: 0, missing: 0, scanned: 0 };
|
|
3192
|
+
}
|
|
3193
|
+
const [before] = await executor.query(
|
|
3194
|
+
COUNT_MISSING_VISIBLE_EVENTS_SQL
|
|
3195
|
+
);
|
|
3196
|
+
if (!before) {
|
|
3197
|
+
throw new Error(
|
|
3198
|
+
"Visible-message event migration could not inspect its projection"
|
|
3199
|
+
);
|
|
3200
|
+
}
|
|
3201
|
+
if (before.count > 0) {
|
|
3202
|
+
const visibleConversationIds = /* @__PURE__ */ new Set();
|
|
3203
|
+
await visitVisibleConversations(
|
|
3204
|
+
executor,
|
|
3205
|
+
batchSize,
|
|
3206
|
+
async (conversationId) => {
|
|
3207
|
+
visibleConversationIds.add(conversationId);
|
|
3208
|
+
}
|
|
3209
|
+
);
|
|
3210
|
+
await prepareConversationEventResequence(context, visibleConversationIds);
|
|
3211
|
+
}
|
|
3212
|
+
while (true) {
|
|
3213
|
+
const rows = await executor.withLock(
|
|
3214
|
+
VISIBLE_EVENT_BACKFILL_LOCK,
|
|
3215
|
+
() => executor.query(LOAD_MISSING_VISIBLE_EVENTS_SQL, [
|
|
3216
|
+
batchSize,
|
|
3217
|
+
cursor?.conversationId ?? null,
|
|
3218
|
+
cursor?.createdAt ?? null,
|
|
3219
|
+
cursor?.messageId ?? null
|
|
3220
|
+
])
|
|
3221
|
+
);
|
|
3222
|
+
if (rows.length === 0) {
|
|
3223
|
+
break;
|
|
3224
|
+
}
|
|
3225
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
3226
|
+
for (const row of rows) {
|
|
3227
|
+
const events = grouped.get(row.conversation_id) ?? [];
|
|
3228
|
+
events.push(...eventsForRow(row));
|
|
3229
|
+
grouped.set(row.conversation_id, events);
|
|
3230
|
+
}
|
|
3231
|
+
for (const [conversationId, events] of grouped) {
|
|
3232
|
+
await mergeVisibleEvents(executor, conversationId, events);
|
|
3233
|
+
}
|
|
3234
|
+
migrated += rows.length;
|
|
3235
|
+
const last = rows.at(-1);
|
|
3236
|
+
cursor = {
|
|
3237
|
+
conversationId: last.conversation_id,
|
|
3238
|
+
createdAt: last.created_at,
|
|
3239
|
+
messageId: last.message_id
|
|
3240
|
+
};
|
|
3241
|
+
}
|
|
3242
|
+
const [remaining] = await executor.query(
|
|
3243
|
+
COUNT_MISSING_VISIBLE_EVENTS_SQL
|
|
3244
|
+
);
|
|
3245
|
+
if (!remaining) {
|
|
3246
|
+
throw new Error(
|
|
3247
|
+
"Visible-message event migration could not verify its projection"
|
|
3248
|
+
);
|
|
3249
|
+
}
|
|
3250
|
+
if (remaining.count > 0) {
|
|
3251
|
+
throw new Error(
|
|
3252
|
+
`Visible-message event migration left ${remaining.count} message row(s) without canonical events`
|
|
3253
|
+
);
|
|
3254
|
+
}
|
|
3255
|
+
await visitConversationIds(
|
|
3256
|
+
executor,
|
|
3257
|
+
LOAD_COMPACTED_CONVERSATION_IDS_SQL,
|
|
3258
|
+
batchSize,
|
|
3259
|
+
async (conversationId) => await finalizeVisibleCompactions(context, executor, conversationId)
|
|
3260
|
+
);
|
|
3261
|
+
return {
|
|
3262
|
+
existing: 0,
|
|
3263
|
+
migrated,
|
|
3264
|
+
missing: 0,
|
|
3265
|
+
scanned: migrated
|
|
3266
|
+
};
|
|
3267
|
+
} finally {
|
|
3268
|
+
await closeExecutor?.();
|
|
3269
|
+
}
|
|
3270
|
+
}
|
|
3271
|
+
var conversationVisibleMessageEventsMigration = {
|
|
3272
|
+
name: "move-conversation-messages-to-events",
|
|
3273
|
+
run: migrateConversationVisibleMessageEvents
|
|
3274
|
+
};
|
|
3275
|
+
|
|
3276
|
+
// src/cli/upgrade/migrations/conversation-context-checkpoints.ts
|
|
3277
|
+
import { isDeepStrictEqual } from "util";
|
|
3278
|
+
var PAGE_SIZE = 250;
|
|
3279
|
+
function messageTextParts(message) {
|
|
3280
|
+
if (typeof message.content === "string") {
|
|
3281
|
+
return [message.content];
|
|
3282
|
+
}
|
|
3283
|
+
if (!Array.isArray(message.content)) {
|
|
3284
|
+
return [];
|
|
3285
|
+
}
|
|
3286
|
+
return message.content.flatMap(
|
|
3287
|
+
(part) => part && typeof part === "object" && part.type === "text" && typeof part.text === "string" ? [part.text] : []
|
|
3288
|
+
);
|
|
3289
|
+
}
|
|
3290
|
+
function checkpointEnd(messages) {
|
|
3291
|
+
return messages.findIndex((row) => {
|
|
3292
|
+
const message = row.payload.message;
|
|
3293
|
+
return message !== null && typeof message === "object" && !Array.isArray(message) && messageTextParts(message).some(
|
|
3294
|
+
isLegacyOrCurrentCheckpointText
|
|
3295
|
+
);
|
|
3296
|
+
});
|
|
3297
|
+
}
|
|
3298
|
+
function matchingMessagePrefix(previous, current) {
|
|
3299
|
+
const limit = Math.min(previous.length, current.length);
|
|
3300
|
+
let count = 0;
|
|
3301
|
+
while (count < limit && isDeepStrictEqual(
|
|
3302
|
+
previous[count].payload.message,
|
|
3303
|
+
current[count].payload.message
|
|
3304
|
+
)) {
|
|
3305
|
+
count += 1;
|
|
3306
|
+
}
|
|
3307
|
+
return count;
|
|
3308
|
+
}
|
|
3309
|
+
function replacementEntry(row, normalizeCheckpoint) {
|
|
3310
|
+
const message = row.payload.message;
|
|
3311
|
+
if (message === null || typeof message !== "object" || Array.isArray(message)) {
|
|
3312
|
+
throw new Error(
|
|
3313
|
+
`Conversation message event ${row.seq} has no message object`
|
|
3314
|
+
);
|
|
3315
|
+
}
|
|
3316
|
+
return {
|
|
3317
|
+
message: normalizeCheckpoint ? normalizeLegacyContextMessage(message) : message,
|
|
3318
|
+
...row.payload.provenance === void 0 ? {} : { provenance: row.payload.provenance }
|
|
3319
|
+
};
|
|
3320
|
+
}
|
|
3321
|
+
function resolvedBinding(payload, bot) {
|
|
3322
|
+
const modelProfile = payload.modelProfile === void 0 ? "standard" : modelProfileSchema.parse(payload.modelProfile);
|
|
3323
|
+
const modelId = payload.modelId;
|
|
3324
|
+
if (modelId !== void 0 && (typeof modelId !== "string" || !modelId)) {
|
|
3325
|
+
throw new Error("Context checkpoint modelId must be a non-empty string");
|
|
3326
|
+
}
|
|
3327
|
+
return {
|
|
3328
|
+
modelProfile,
|
|
3329
|
+
modelId: modelId ?? modelIdForProfile(bot, modelProfile)
|
|
3330
|
+
};
|
|
3331
|
+
}
|
|
3332
|
+
function handoffToolCallId(rows, marker) {
|
|
3333
|
+
if (typeof marker.payload.triggeringToolCallId === "string" && marker.payload.triggeringToolCallId.length > 0) {
|
|
3334
|
+
return marker.payload.triggeringToolCallId;
|
|
3335
|
+
}
|
|
3336
|
+
const tool = [...rows].reverse().find(
|
|
3337
|
+
(row) => row.seq < marker.seq && row.type === "tool_execution_started" && row.payload.toolName === "handoff" && typeof row.payload.toolCallId === "string" && row.payload.toolCallId.length > 0
|
|
3338
|
+
);
|
|
3339
|
+
if (typeof tool?.payload.toolCallId !== "string") {
|
|
3340
|
+
throw new Error(
|
|
3341
|
+
`Cannot correlate handoff checkpoint at event ${marker.seq} during upgrade`
|
|
3342
|
+
);
|
|
3343
|
+
}
|
|
3344
|
+
return tool.payload.toolCallId;
|
|
3345
|
+
}
|
|
3346
|
+
async function normalizeConversation(args) {
|
|
3347
|
+
return await args.executor.withLock(
|
|
3348
|
+
`junior_conversation:event:${args.conversationId}`,
|
|
3349
|
+
async () => await args.executor.transaction(async () => {
|
|
3350
|
+
const rows = await args.executor.query(
|
|
3351
|
+
`SELECT
|
|
3352
|
+
seq,
|
|
3353
|
+
history_version AS "historyVersion",
|
|
3354
|
+
type,
|
|
3355
|
+
payload
|
|
3356
|
+
FROM junior_conversation_events
|
|
3357
|
+
WHERE conversation_id = $1
|
|
3358
|
+
AND type IN ('context_epoch_started', 'agent_step', 'tool_execution_started')
|
|
3359
|
+
ORDER BY seq`,
|
|
3360
|
+
[args.conversationId]
|
|
3361
|
+
);
|
|
3362
|
+
const messagesByEpoch = /* @__PURE__ */ new Map();
|
|
3363
|
+
for (const row of rows) {
|
|
3364
|
+
if (row.type !== "agent_step") continue;
|
|
3365
|
+
const messages = messagesByEpoch.get(row.historyVersion) ?? [];
|
|
3366
|
+
messages.push(row);
|
|
3367
|
+
messagesByEpoch.set(row.historyVersion, messages);
|
|
3368
|
+
}
|
|
3369
|
+
let migrated = 0;
|
|
3370
|
+
const removedSeqs = [];
|
|
3371
|
+
for (const marker of rows) {
|
|
3372
|
+
if (marker.type !== "context_epoch_started") {
|
|
3373
|
+
continue;
|
|
3374
|
+
}
|
|
3375
|
+
const reason = marker.payload.reason;
|
|
3376
|
+
if (reason === "initial") {
|
|
3377
|
+
removedSeqs.push(marker.seq);
|
|
3378
|
+
migrated += 1;
|
|
3379
|
+
continue;
|
|
3380
|
+
}
|
|
3381
|
+
if (reason !== "compaction" && reason !== "handoff" && reason !== "rollback") {
|
|
3382
|
+
throw new Error(
|
|
3383
|
+
`Unknown context checkpoint reason at event ${marker.seq}`
|
|
3384
|
+
);
|
|
3385
|
+
}
|
|
3386
|
+
let replacementHistory = marker.payload.replacementHistory;
|
|
3387
|
+
if (!Array.isArray(replacementHistory)) {
|
|
3388
|
+
const current = messagesByEpoch.get(marker.historyVersion) ?? [];
|
|
3389
|
+
let replacementCount;
|
|
3390
|
+
if (reason === "rollback") {
|
|
3391
|
+
const previous = messagesByEpoch.get(marker.historyVersion - 1) ?? [];
|
|
3392
|
+
replacementCount = matchingMessagePrefix(previous, current);
|
|
3393
|
+
} else {
|
|
3394
|
+
const end = checkpointEnd(current);
|
|
3395
|
+
replacementCount = end < 0 ? current.length : end + 1;
|
|
3396
|
+
}
|
|
3397
|
+
const replacementRows = current.slice(0, replacementCount);
|
|
3398
|
+
replacementHistory = replacementRows.map(
|
|
3399
|
+
(row, index) => replacementEntry(
|
|
3400
|
+
row,
|
|
3401
|
+
reason !== "rollback" && index === replacementRows.length - 1
|
|
3402
|
+
)
|
|
3403
|
+
);
|
|
3404
|
+
removedSeqs.push(...replacementRows.map((row) => row.seq));
|
|
3405
|
+
}
|
|
3406
|
+
const binding = resolvedBinding(marker.payload, args.bot);
|
|
3407
|
+
const parsed = conversationEventDataSchema.parse({
|
|
3408
|
+
type: reason,
|
|
3409
|
+
...binding,
|
|
3410
|
+
...reason === "handoff" ? { triggeringToolCallId: handoffToolCallId(rows, marker) } : {},
|
|
3411
|
+
replacementHistory
|
|
3412
|
+
});
|
|
3413
|
+
const { type, ...payload } = parsed;
|
|
3414
|
+
await args.executor.execute(
|
|
3415
|
+
`UPDATE junior_conversation_events
|
|
3416
|
+
SET type = $3, payload = $4::jsonb
|
|
3417
|
+
WHERE conversation_id = $1 AND seq = $2`,
|
|
3418
|
+
[
|
|
3419
|
+
args.conversationId,
|
|
3420
|
+
marker.seq,
|
|
3421
|
+
type,
|
|
3422
|
+
JSON.stringify(sanitizePostgresJson(payload))
|
|
3423
|
+
]
|
|
3424
|
+
);
|
|
3425
|
+
migrated += 1;
|
|
3426
|
+
}
|
|
3427
|
+
if (removedSeqs.length > 0) {
|
|
3428
|
+
await args.executor.execute(
|
|
3429
|
+
`UPDATE junior_conversation_events
|
|
3430
|
+
SET payload = jsonb_set(
|
|
3431
|
+
payload,
|
|
3432
|
+
'{historyFromSeq}',
|
|
3433
|
+
to_jsonb(
|
|
3434
|
+
(payload->>'historyFromSeq')::integer - (
|
|
3435
|
+
SELECT count(*)::integer
|
|
3436
|
+
FROM unnest($2::integer[]) removed(seq)
|
|
3437
|
+
WHERE removed.seq < (payload->>'historyFromSeq')::integer
|
|
3438
|
+
)
|
|
3439
|
+
)
|
|
3440
|
+
)
|
|
3441
|
+
WHERE conversation_id = $1
|
|
3442
|
+
AND type = 'messages_summarized'
|
|
3443
|
+
AND jsonb_typeof(payload->'historyFromSeq') = 'number'`,
|
|
3444
|
+
[args.conversationId, removedSeqs]
|
|
3445
|
+
);
|
|
3446
|
+
await args.executor.execute(
|
|
3447
|
+
`DELETE FROM junior_conversation_events
|
|
3448
|
+
WHERE conversation_id = $1 AND seq = ANY($2::integer[])`,
|
|
3449
|
+
[args.conversationId, removedSeqs]
|
|
3450
|
+
);
|
|
3451
|
+
await args.executor.execute(
|
|
3452
|
+
`UPDATE junior_conversation_events
|
|
3453
|
+
SET seq = -seq - 1
|
|
3454
|
+
WHERE conversation_id = $1`,
|
|
3455
|
+
[args.conversationId]
|
|
3456
|
+
);
|
|
3457
|
+
await args.executor.execute(
|
|
3458
|
+
`WITH ranked AS (
|
|
3459
|
+
SELECT
|
|
3460
|
+
ctid,
|
|
3461
|
+
row_number() OVER (ORDER BY -seq - 1) - 1 AS next_seq
|
|
3462
|
+
FROM junior_conversation_events
|
|
3463
|
+
WHERE conversation_id = $1
|
|
3464
|
+
)
|
|
3465
|
+
UPDATE junior_conversation_events event
|
|
3466
|
+
SET seq = ranked.next_seq
|
|
3467
|
+
FROM ranked
|
|
3468
|
+
WHERE event.ctid = ranked.ctid`,
|
|
3469
|
+
[args.conversationId]
|
|
3470
|
+
);
|
|
3471
|
+
}
|
|
3472
|
+
return migrated;
|
|
3473
|
+
})
|
|
3474
|
+
);
|
|
3475
|
+
}
|
|
3476
|
+
async function normalizeConversationContextCheckpoints(context, options = {}) {
|
|
3477
|
+
const config = options.executor && options.bot ? void 0 : getChatConfig();
|
|
3478
|
+
const bot = options.bot ?? config?.bot;
|
|
3479
|
+
if (!bot) throw new Error("Context checkpoint upgrade requires bot config");
|
|
3480
|
+
let executor = options.executor;
|
|
3481
|
+
if (!executor) {
|
|
3482
|
+
if (!config)
|
|
3483
|
+
throw new Error("Context checkpoint upgrade requires SQL config");
|
|
3484
|
+
executor = createJuniorSqlExecutor({
|
|
3485
|
+
connectionString: config.sql.databaseUrl,
|
|
3486
|
+
driver: config.sql.driver
|
|
3487
|
+
});
|
|
3488
|
+
}
|
|
3489
|
+
try {
|
|
3490
|
+
const result = {
|
|
3491
|
+
existing: 0,
|
|
3492
|
+
migrated: 0,
|
|
3493
|
+
missing: 0,
|
|
3494
|
+
scanned: 0
|
|
3495
|
+
};
|
|
3496
|
+
while (true) {
|
|
3497
|
+
const conversations = await executor.query(
|
|
3498
|
+
`SELECT DISTINCT conversation_id AS "conversationId"
|
|
3499
|
+
FROM junior_conversation_events
|
|
3500
|
+
WHERE type = 'context_epoch_started'
|
|
3501
|
+
ORDER BY conversation_id
|
|
3502
|
+
LIMIT $1`,
|
|
3503
|
+
[PAGE_SIZE]
|
|
3504
|
+
);
|
|
3505
|
+
if (conversations.length === 0) break;
|
|
3506
|
+
await prepareConversationEventResequence(
|
|
3507
|
+
context,
|
|
3508
|
+
new Set(conversations.map(({ conversationId }) => conversationId))
|
|
3509
|
+
);
|
|
3510
|
+
for (const { conversationId } of conversations) {
|
|
3511
|
+
result.scanned += 1;
|
|
3512
|
+
result.migrated += await normalizeConversation({
|
|
3513
|
+
bot,
|
|
3514
|
+
conversationId,
|
|
3515
|
+
executor
|
|
3516
|
+
});
|
|
3517
|
+
}
|
|
3518
|
+
}
|
|
3519
|
+
return result;
|
|
3520
|
+
} finally {
|
|
3521
|
+
if (!options.executor) await executor.close();
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3524
|
+
var conversationContextCheckpointMigration = {
|
|
3525
|
+
name: "normalize-conversation-context-checkpoints",
|
|
3526
|
+
run: normalizeConversationContextCheckpoints
|
|
3527
|
+
};
|
|
3528
|
+
|
|
1597
3529
|
// src/cli/upgrade.ts
|
|
1598
3530
|
var DEFAULT_IO = {
|
|
1599
3531
|
info: console.log
|
|
@@ -1604,7 +3536,11 @@ var MIGRATIONS = [
|
|
|
1604
3536
|
redisConversationStateMigration,
|
|
1605
3537
|
coreSqlSchemaMigration,
|
|
1606
3538
|
sqlConversationMigration,
|
|
3539
|
+
conversationContextCheckpointMigration,
|
|
3540
|
+
// The external history import must precede the SQL read-model backfill.
|
|
3541
|
+
// Visible events seal a conversation against legacy Redis/session import.
|
|
1607
3542
|
sqlConversationHistoryMigration,
|
|
3543
|
+
conversationVisibleMessageEventsMigration,
|
|
1608
3544
|
conversationUsageRepairMigration,
|
|
1609
3545
|
sqlPluginMigration,
|
|
1610
3546
|
pluginStorageMigration
|