@lota-sdk/core 0.4.13 → 0.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/ai/embedding-cache.ts +17 -11
- package/src/ai-gateway/ai-gateway.ts +164 -94
- package/src/ai-gateway/index.ts +4 -1
- package/src/config/agent-defaults.ts +2 -2
- package/src/config/agent-types.ts +1 -1
- package/src/config/constants.ts +1 -1
- package/src/create-runtime.ts +259 -200
- package/src/db/cursor-pagination.ts +2 -9
- package/src/db/memory-store.ts +194 -175
- package/src/db/memory.ts +125 -71
- package/src/db/schema-fingerprint.ts +5 -4
- package/src/db/service-normalization.ts +4 -3
- package/src/db/service.ts +3 -2
- package/src/db/startup.ts +15 -16
- package/src/effect/errors.ts +161 -21
- package/src/effect/index.ts +0 -1
- package/src/embeddings/provider.ts +15 -7
- package/src/queues/autonomous-job.queue.ts +10 -22
- package/src/queues/delayed-node-promotion.queue.ts +8 -14
- package/src/queues/document-processor.queue.ts +13 -4
- package/src/queues/memory-consolidation.queue.ts +26 -14
- package/src/queues/plan-agent-heartbeat.queue.ts +48 -31
- package/src/queues/plan-scheduler.queue.ts +37 -15
- package/src/queues/queue-factory.ts +59 -35
- package/src/queues/standalone-worker.ts +3 -2
- package/src/redis/connection.ts +10 -3
- package/src/redis/org-memory-lock.ts +1 -1
- package/src/redis/redis-lease-lock.ts +5 -5
- package/src/redis/stream-context.ts +1 -1
- package/src/runtime/chat-message.ts +64 -1
- package/src/runtime/chat-run-orchestration.ts +33 -20
- package/src/runtime/context-compaction/context-compaction-runtime.ts +14 -7
- package/src/runtime/context-compaction/context-compaction.ts +78 -66
- package/src/runtime/domain-layer.ts +19 -13
- package/src/runtime/execution-plan.ts +7 -3
- package/src/runtime/memory/memory-block.ts +3 -9
- package/src/runtime/memory/memory-scope.ts +3 -1
- package/src/runtime/plugin-resolution.ts +2 -1
- package/src/runtime/post-turn-side-effects.ts +6 -5
- package/src/runtime/retrieval-adapters.ts +8 -20
- package/src/runtime/runtime-config.ts +3 -9
- package/src/runtime/runtime-extensions.ts +2 -4
- package/src/runtime/runtime-lifecycle.ts +56 -16
- package/src/runtime/runtime-services.ts +180 -102
- package/src/runtime/runtime-worker-registry.ts +3 -1
- package/src/runtime/social-chat/social-chat-agent-runner.ts +1 -1
- package/src/runtime/social-chat/social-chat-history.ts +21 -18
- package/src/runtime/social-chat/social-chat.ts +356 -223
- package/src/runtime/specialist-runner.ts +3 -1
- package/src/runtime/team-consultation/team-consultation-orchestrator.ts +3 -2
- package/src/runtime/thread-turn-context.ts +142 -102
- package/src/runtime/turn-lifecycle.ts +15 -46
- package/src/services/agent-activity.service.ts +1 -1
- package/src/services/agent-executor.service.ts +107 -77
- package/src/services/autonomous-job.service.ts +354 -293
- package/src/services/background-work.service.ts +3 -3
- package/src/services/context-compaction.service.ts +7 -2
- package/src/services/document-chunk.service.ts +50 -32
- package/src/services/execution-plan/execution-plan-schedule.ts +5 -3
- package/src/services/execution-plan/execution-plan.service.ts +162 -179
- package/src/services/feedback-loop.service.ts +5 -4
- package/src/services/graph-full-routing.ts +37 -36
- package/src/services/institutional-memory.service.ts +28 -30
- package/src/services/learned-skill.service.ts +107 -72
- package/src/services/memory/memory-errors.ts +4 -23
- package/src/services/memory/memory-org-memory.ts +10 -5
- package/src/services/memory/memory-rerank.ts +18 -6
- package/src/services/memory/memory.service.ts +170 -111
- package/src/services/memory/rerank.service.ts +29 -20
- package/src/services/organization-member.service.ts +1 -1
- package/src/services/organization.service.ts +69 -75
- package/src/services/ownership-dispatcher.service.ts +40 -39
- package/src/services/plan/plan-agent-heartbeat.service.ts +22 -24
- package/src/services/plan/plan-agent-query.service.ts +39 -31
- package/src/services/plan/plan-completion-side-effects.ts +13 -17
- package/src/services/plan/plan-coordination.service.ts +2 -1
- package/src/services/plan/plan-cycle.service.ts +6 -5
- package/src/services/plan/plan-deadline.service.ts +57 -54
- package/src/services/plan/plan-event-delivery.service.ts +5 -4
- package/src/services/plan/plan-executor-graph.ts +18 -15
- package/src/services/plan/plan-executor.service.ts +235 -262
- package/src/services/plan/plan-run.service.ts +169 -93
- package/src/services/plan/plan-scheduler.service.ts +192 -202
- package/src/services/plan/plan-template.service.ts +1 -1
- package/src/services/plan/plan-transaction-events.ts +1 -1
- package/src/services/plan/plan-workspace.service.ts +23 -14
- package/src/services/plugin-executor.service.ts +5 -9
- package/src/services/queue-job.service.ts +117 -59
- package/src/services/recent-activity-title.service.ts +13 -12
- package/src/services/recent-activity.service.ts +6 -1
- package/src/services/social-chat-history.service.ts +29 -25
- package/src/services/system-executor.service.ts +5 -9
- package/src/services/thread/thread-active-run.ts +2 -2
- package/src/services/thread/thread-listing.ts +61 -57
- package/src/services/thread/thread-memory-block.ts +73 -48
- package/src/services/thread/thread-message.service.ts +76 -65
- package/src/services/thread/thread-record-store.ts +8 -8
- package/src/services/thread/thread-title.service.ts +10 -4
- package/src/services/thread/thread-turn-execution.ts +43 -45
- package/src/services/thread/thread-turn-preparation.service.ts +257 -135
- package/src/services/thread/thread-turn-streaming.ts +83 -92
- package/src/services/thread/thread-turn.ts +18 -16
- package/src/services/thread/thread.service.ts +135 -100
- package/src/services/user.service.ts +45 -48
- package/src/storage/attachment-parser.ts +6 -2
- package/src/storage/attachment-storage.service.ts +5 -6
- package/src/storage/generated-document-storage.service.ts +1 -1
- package/src/system-agents/context-compaction.agent.ts +10 -9
- package/src/system-agents/delegated-agent-factory.ts +30 -6
- package/src/system-agents/memory-reranker.agent.ts +10 -9
- package/src/system-agents/memory.agent.ts +10 -9
- package/src/system-agents/recent-activity-title-refiner.agent.ts +13 -15
- package/src/system-agents/regular-chat-memory-digest.agent.ts +13 -12
- package/src/system-agents/skill-extractor.agent.ts +13 -12
- package/src/system-agents/skill-manager.agent.ts +13 -12
- package/src/system-agents/thread-router.agent.ts +11 -7
- package/src/system-agents/title-generator.agent.ts +13 -12
- package/src/tools/fetch-webpage.tool.ts +13 -13
- package/src/tools/memory-block.tool.ts +3 -1
- package/src/tools/plan-approval.tool.ts +4 -2
- package/src/tools/read-file-parts.tool.ts +10 -4
- package/src/tools/remember-memory.tool.ts +3 -1
- package/src/tools/research-topic.tool.ts +9 -5
- package/src/tools/search-web.tool.ts +16 -16
- package/src/tools/search.tool.ts +20 -5
- package/src/tools/team-think.tool.ts +61 -38
- package/src/utils/async.ts +5 -5
- package/src/utils/errors.ts +19 -18
- package/src/utils/sse-keepalive.ts +28 -25
- package/src/workers/bootstrap.ts +75 -11
- package/src/workers/memory-consolidation.worker.ts +82 -91
- package/src/workers/organization-learning.worker.ts +14 -4
- package/src/workers/regular-chat-memory-digest.runner.ts +105 -67
- package/src/workers/skill-extraction.runner.ts +97 -61
- package/src/workers/utils/repo-structure-extractor.ts +13 -8
- package/src/workers/utils/thread-message-query.ts +24 -24
- package/src/workers/worker-utils.ts +23 -4
- package/src/effect/helpers.ts +0 -123
|
@@ -8,7 +8,7 @@ import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
|
8
8
|
import type { RecordIdInput } from '../db/record-id'
|
|
9
9
|
import type { SurrealDBService } from '../db/service'
|
|
10
10
|
import { TABLES } from '../db/tables'
|
|
11
|
-
import {
|
|
11
|
+
import { ERROR_TAGS } from '../effect/errors'
|
|
12
12
|
import { DatabaseServiceTag } from '../effect/services'
|
|
13
13
|
import type { MemoryConsolidationJob } from '../queues/memory-consolidation.queue'
|
|
14
14
|
import { QueueJobServiceTag } from '../services/queue-job.service'
|
|
@@ -18,27 +18,30 @@ import { initializeSandboxedWorkerRuntime } from './bootstrap'
|
|
|
18
18
|
import { toSandboxedWorkerError } from './utils/sandbox-error'
|
|
19
19
|
import { createTracedWorkerProcessor } from './worker-utils'
|
|
20
20
|
|
|
21
|
-
class MemoryConsolidationError extends Schema.TaggedErrorClass<MemoryConsolidationError>()(
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
21
|
+
class MemoryConsolidationError extends Schema.TaggedErrorClass<MemoryConsolidationError>()(
|
|
22
|
+
ERROR_TAGS.MemoryConsolidationError,
|
|
23
|
+
{
|
|
24
|
+
stage: Schema.Literals([
|
|
25
|
+
'read-memories',
|
|
26
|
+
'read-neighbors',
|
|
27
|
+
'archive-memory',
|
|
28
|
+
'relate-memories',
|
|
29
|
+
'write-history',
|
|
30
|
+
'load-stale',
|
|
31
|
+
'update-stale',
|
|
32
|
+
'load-middle-nodes',
|
|
33
|
+
'read-existing-relation',
|
|
34
|
+
'write-relation',
|
|
35
|
+
'update-middle-node',
|
|
36
|
+
'decay-standard',
|
|
37
|
+
'decay-ephemeral',
|
|
38
|
+
'cleanup-relations',
|
|
39
|
+
'load-scope-ids',
|
|
40
|
+
]),
|
|
41
|
+
message: Schema.String,
|
|
42
|
+
cause: Schema.Defect,
|
|
43
|
+
},
|
|
44
|
+
) {}
|
|
42
45
|
|
|
43
46
|
function toMemoryConsolidationError(
|
|
44
47
|
stage: MemoryConsolidationError['stage'],
|
|
@@ -47,13 +50,6 @@ function toMemoryConsolidationError(
|
|
|
47
50
|
return new MemoryConsolidationError({ stage, message: getErrorMessage(cause), cause })
|
|
48
51
|
}
|
|
49
52
|
|
|
50
|
-
function tryMemoryConsolidationEffect<A, R = never>(
|
|
51
|
-
stage: MemoryConsolidationError['stage'],
|
|
52
|
-
thunk: () => PromiseLike<A> | Effect.Effect<A, unknown, R>,
|
|
53
|
-
): Effect.Effect<A, MemoryConsolidationError, R> {
|
|
54
|
-
return effectTryPromise(thunk, (cause) => toMemoryConsolidationError(stage, cause))
|
|
55
|
-
}
|
|
56
|
-
|
|
57
53
|
const runtime = await initializeSandboxedWorkerRuntime()
|
|
58
54
|
|
|
59
55
|
const memoryConsolidationDatabaseService: SurrealDBService = await runtime.runPromise(
|
|
@@ -93,8 +89,8 @@ function isContentSubsumed(shorter: string, longer: string): boolean {
|
|
|
93
89
|
|
|
94
90
|
function deduplicateScopeEffect(scopeId: string) {
|
|
95
91
|
return Effect.gen(function* () {
|
|
96
|
-
const memoryRows = yield*
|
|
97
|
-
|
|
92
|
+
const memoryRows = yield* db()
|
|
93
|
+
.query<{
|
|
98
94
|
id: RecordIdInput
|
|
99
95
|
content: string
|
|
100
96
|
importance: number
|
|
@@ -110,8 +106,8 @@ function deduplicateScopeEffect(scopeId: string) {
|
|
|
110
106
|
LIMIT $limit`,
|
|
111
107
|
{ scopeId, limit: MAX_MEMORIES_PER_SCOPE },
|
|
112
108
|
),
|
|
113
|
-
)
|
|
114
|
-
|
|
109
|
+
)
|
|
110
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('read-memories', cause)))
|
|
115
111
|
|
|
116
112
|
const memories = memoryRows.map((row) => ({ ...row, id: toMemoryId(row.id) }))
|
|
117
113
|
if (memories.length < 2) return 0
|
|
@@ -123,8 +119,8 @@ function deduplicateScopeEffect(scopeId: string) {
|
|
|
123
119
|
if (archived.has(memory.id)) continue
|
|
124
120
|
|
|
125
121
|
const candidateLimit = 20
|
|
126
|
-
const neighborStatements = yield*
|
|
127
|
-
|
|
122
|
+
const neighborStatements = yield* db()
|
|
123
|
+
.queryAll<{
|
|
128
124
|
id: RecordIdInput
|
|
129
125
|
content: string
|
|
130
126
|
importance: number
|
|
@@ -159,8 +155,8 @@ function deduplicateScopeEffect(scopeId: string) {
|
|
|
159
155
|
LIMIT ${candidateLimit}`,
|
|
160
156
|
{ scopeId, memoryId: ensureRecordId(memory.id, TABLES.MEMORY), embedding: memory.embedding },
|
|
161
157
|
),
|
|
162
|
-
)
|
|
163
|
-
|
|
158
|
+
)
|
|
159
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('read-neighbors', cause)))
|
|
164
160
|
|
|
165
161
|
const neighbors = (neighborStatements.at(-1) ?? []).map((row) => ({ ...row, id: toMemoryId(row.id) }))
|
|
166
162
|
|
|
@@ -185,31 +181,31 @@ function deduplicateScopeEffect(scopeId: string) {
|
|
|
185
181
|
const winner = keepMemory ? memory : neighbor
|
|
186
182
|
const loser = keepMemory ? neighbor : memory
|
|
187
183
|
|
|
188
|
-
yield*
|
|
189
|
-
|
|
184
|
+
yield* db()
|
|
185
|
+
.query(
|
|
190
186
|
new BoundQuery(
|
|
191
187
|
`UPDATE ${MEMORY_TABLE}
|
|
192
188
|
SET archivedAt = time::now(), validUntil = time::now()
|
|
193
189
|
WHERE id = $loserId AND archivedAt IS NONE`,
|
|
194
190
|
{ loserId: ensureRecordId(loser.id, TABLES.MEMORY) },
|
|
195
191
|
),
|
|
196
|
-
)
|
|
197
|
-
|
|
198
|
-
yield*
|
|
199
|
-
|
|
192
|
+
)
|
|
193
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('archive-memory', cause)))
|
|
194
|
+
yield* db()
|
|
195
|
+
.relate(
|
|
200
196
|
ensureRecordId(winner.id, TABLES.MEMORY),
|
|
201
197
|
MEMORY_RELATION_TABLE,
|
|
202
198
|
ensureRecordId(loser.id, TABLES.MEMORY),
|
|
203
199
|
{ relationType: RELATION_SUPERSEDES, confidence: 1.0 },
|
|
204
|
-
)
|
|
205
|
-
|
|
206
|
-
yield*
|
|
207
|
-
|
|
200
|
+
)
|
|
201
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('relate-memories', cause)))
|
|
202
|
+
yield* db()
|
|
203
|
+
.insert<Record<string, unknown>>(MEMORY_HISTORY_TABLE, {
|
|
208
204
|
memoryId: ensureRecordId(loser.id, TABLES.MEMORY),
|
|
209
205
|
prevValue: loser.content,
|
|
210
206
|
event: 'DELETE',
|
|
211
|
-
})
|
|
212
|
-
|
|
207
|
+
})
|
|
208
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('write-history', cause)))
|
|
213
209
|
|
|
214
210
|
archived.add(loser.id)
|
|
215
211
|
mergeCount++
|
|
@@ -224,8 +220,8 @@ function deduplicateScopeEffect(scopeId: string) {
|
|
|
224
220
|
|
|
225
221
|
function pruneStaleMemoriesEffect() {
|
|
226
222
|
return Effect.gen(function* () {
|
|
227
|
-
const stale = yield*
|
|
228
|
-
|
|
223
|
+
const stale = yield* db()
|
|
224
|
+
.query<{ id: RecordIdInput }>(
|
|
229
225
|
new BoundQuery(
|
|
230
226
|
`SELECT id FROM ${MEMORY_TABLE}
|
|
231
227
|
WHERE accessCount = 0
|
|
@@ -234,15 +230,15 @@ function pruneStaleMemoriesEffect() {
|
|
|
234
230
|
AND importance < 0.5
|
|
235
231
|
LIMIT 200`,
|
|
236
232
|
),
|
|
237
|
-
)
|
|
238
|
-
|
|
233
|
+
)
|
|
234
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('load-stale', cause)))
|
|
239
235
|
|
|
240
236
|
if (stale.length === 0) return 0
|
|
241
237
|
|
|
242
238
|
const staleIds = stale.map((row) => ensureRecordId(row.id, TABLES.MEMORY))
|
|
243
|
-
yield*
|
|
244
|
-
|
|
245
|
-
|
|
239
|
+
yield* db()
|
|
240
|
+
.updateWhere(MEMORY_TABLE, inside('id', staleIds), { archivedAt: nowDate() })
|
|
241
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('update-stale', cause)))
|
|
246
242
|
|
|
247
243
|
return stale.length
|
|
248
244
|
})
|
|
@@ -250,8 +246,8 @@ function pruneStaleMemoriesEffect() {
|
|
|
250
246
|
|
|
251
247
|
function collapseSupersededChainEffect() {
|
|
252
248
|
return Effect.gen(function* () {
|
|
253
|
-
const middleNodes = yield*
|
|
254
|
-
|
|
249
|
+
const middleNodes = yield* db()
|
|
250
|
+
.query<{ middleId: RecordIdInput; predecessors: RecordIdInput[]; successors: RecordIdInput[] }>(
|
|
255
251
|
new BoundQuery(
|
|
256
252
|
`SELECT
|
|
257
253
|
id AS middleId,
|
|
@@ -263,8 +259,8 @@ function collapseSupersededChainEffect() {
|
|
|
263
259
|
AND count(<-${MEMORY_RELATION_TABLE}[WHERE relationType = '${RELATION_SUPERSEDES}']) > 0
|
|
264
260
|
LIMIT ${MEMORY.MAX_KNN_LIMIT}`,
|
|
265
261
|
),
|
|
266
|
-
)
|
|
267
|
-
|
|
262
|
+
)
|
|
263
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('load-middle-nodes', cause)))
|
|
268
264
|
|
|
269
265
|
let collapsed = 0
|
|
270
266
|
|
|
@@ -273,33 +269,28 @@ function collapseSupersededChainEffect() {
|
|
|
273
269
|
for (const succId of node.successors) {
|
|
274
270
|
const predRef = ensureRecordId(predId, TABLES.MEMORY)
|
|
275
271
|
const succRef = ensureRecordId(succId, TABLES.MEMORY)
|
|
276
|
-
const existing = yield*
|
|
277
|
-
|
|
272
|
+
const existing = yield* db()
|
|
273
|
+
.query<{ id: RecordIdInput }>(
|
|
278
274
|
new BoundQuery(
|
|
279
275
|
`SELECT id FROM ${MEMORY_RELATION_TABLE}
|
|
280
276
|
WHERE in = $predId AND out = $succId AND relationType = '${RELATION_SUPERSEDES}'
|
|
281
277
|
LIMIT 1`,
|
|
282
278
|
{ predId: predRef, succId: succRef },
|
|
283
279
|
),
|
|
284
|
-
)
|
|
285
|
-
|
|
280
|
+
)
|
|
281
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('read-existing-relation', cause)))
|
|
286
282
|
|
|
287
283
|
if (existing.length === 0) {
|
|
288
|
-
yield*
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
confidence: 1.0,
|
|
292
|
-
}),
|
|
293
|
-
)
|
|
284
|
+
yield* db()
|
|
285
|
+
.relate(predRef, MEMORY_RELATION_TABLE, succRef, { relationType: RELATION_SUPERSEDES, confidence: 1.0 })
|
|
286
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('write-relation', cause)))
|
|
294
287
|
}
|
|
295
288
|
}
|
|
296
289
|
}
|
|
297
290
|
|
|
298
|
-
yield*
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
}),
|
|
302
|
-
)
|
|
291
|
+
yield* db()
|
|
292
|
+
.updateWhere(MEMORY_TABLE, eq('id', ensureRecordId(node.middleId, TABLES.MEMORY)), { archivedAt: nowDate() })
|
|
293
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('update-middle-node', cause)))
|
|
303
294
|
|
|
304
295
|
collapsed++
|
|
305
296
|
}
|
|
@@ -311,8 +302,8 @@ function collapseSupersededChainEffect() {
|
|
|
311
302
|
function decayImportanceEffect() {
|
|
312
303
|
return Effect.gen(function* () {
|
|
313
304
|
const [standardResult, ephemeralResult] = yield* Effect.all([
|
|
314
|
-
|
|
315
|
-
|
|
305
|
+
db()
|
|
306
|
+
.query<{ count: number }>(
|
|
316
307
|
new BoundQuery(
|
|
317
308
|
`UPDATE ${MEMORY_TABLE}
|
|
318
309
|
SET importance = math::max([0.1, importance * 0.95])
|
|
@@ -323,10 +314,10 @@ function decayImportanceEffect() {
|
|
|
323
314
|
AND (durability = 'standard' OR durability IS NONE)
|
|
324
315
|
RETURN count() AS count`,
|
|
325
316
|
),
|
|
326
|
-
)
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
317
|
+
)
|
|
318
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('decay-standard', cause))),
|
|
319
|
+
db()
|
|
320
|
+
.query<{ count: number }>(
|
|
330
321
|
new BoundQuery(
|
|
331
322
|
`UPDATE ${MEMORY_TABLE}
|
|
332
323
|
SET importance = math::max([0.1, importance * 0.85])
|
|
@@ -337,8 +328,8 @@ function decayImportanceEffect() {
|
|
|
337
328
|
AND durability = 'ephemeral'
|
|
338
329
|
RETURN count() AS count`,
|
|
339
330
|
),
|
|
340
|
-
)
|
|
341
|
-
|
|
331
|
+
)
|
|
332
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('decay-ephemeral', cause))),
|
|
342
333
|
])
|
|
343
334
|
|
|
344
335
|
return (standardResult[0]?.count ?? 0) + (ephemeralResult[0]?.count ?? 0)
|
|
@@ -347,15 +338,15 @@ function decayImportanceEffect() {
|
|
|
347
338
|
|
|
348
339
|
function cleanupOrphanedRelationsEffect() {
|
|
349
340
|
return Effect.gen(function* () {
|
|
350
|
-
const result = yield*
|
|
351
|
-
|
|
341
|
+
const result = yield* db()
|
|
342
|
+
.query<{ count: number }>(
|
|
352
343
|
new BoundQuery(
|
|
353
344
|
`DELETE ${MEMORY_RELATION_TABLE}
|
|
354
345
|
WHERE in.archivedAt IS NOT NONE OR out.archivedAt IS NOT NONE
|
|
355
346
|
RETURN count() AS count`,
|
|
356
347
|
),
|
|
357
|
-
)
|
|
358
|
-
|
|
348
|
+
)
|
|
349
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('cleanup-relations', cause)))
|
|
359
350
|
return result[0]?.count ?? 0
|
|
360
351
|
})
|
|
361
352
|
}
|
|
@@ -369,11 +360,11 @@ const handler = (job: SandboxedJob<MemoryConsolidationJob>) =>
|
|
|
369
360
|
if (targetScope) {
|
|
370
361
|
totalMerged = yield* deduplicateScopeEffect(targetScope)
|
|
371
362
|
} else {
|
|
372
|
-
const scopeIds = yield*
|
|
373
|
-
|
|
363
|
+
const scopeIds = yield* db()
|
|
364
|
+
.query<string>(
|
|
374
365
|
new BoundQuery(`SELECT VALUE scopeId FROM ${MEMORY_TABLE} WHERE archivedAt IS NONE GROUP BY scopeId`),
|
|
375
|
-
)
|
|
376
|
-
|
|
366
|
+
)
|
|
367
|
+
.pipe(Effect.mapError((cause) => toMemoryConsolidationError('load-scope-ids', cause)))
|
|
377
368
|
|
|
378
369
|
const results = yield* Effect.forEach(scopeIds, deduplicateScopeEffect, { concurrency: 2 })
|
|
379
370
|
totalMerged = results.reduce((a, b) => a + b, 0)
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { SandboxedJob } from 'bullmq'
|
|
2
|
-
import { Effect } from 'effect'
|
|
2
|
+
import { Cause, Effect } from 'effect'
|
|
3
3
|
import type { Context } from 'effect'
|
|
4
4
|
|
|
5
|
+
import { AiGatewayModelsTag } from '../ai-gateway/ai-gateway'
|
|
5
6
|
import { serverLogger } from '../config/logger'
|
|
6
|
-
import { effectTryPromise } from '../effect/helpers'
|
|
7
7
|
import {
|
|
8
8
|
AgentConfigServiceTag,
|
|
9
9
|
DatabaseServiceTag,
|
|
@@ -27,9 +27,11 @@ import { createTracedWorkerProcessor } from './worker-utils'
|
|
|
27
27
|
const runtime = await initializeSandboxedWorkerRuntime()
|
|
28
28
|
const resolve = <I, T>(tag: Context.Key<I, T>): Promise<T> => runtime.runPromise(Effect.service(tag))
|
|
29
29
|
const agentConfig = await resolve(AgentConfigServiceTag)
|
|
30
|
+
const aiGatewayModels = await resolve(AiGatewayModelsTag)
|
|
30
31
|
const queues = await resolve(LotaQueuesServiceTag)
|
|
31
32
|
const regularChatDigestServices: RegularChatDigestServices = {
|
|
32
33
|
agentConfig,
|
|
34
|
+
aiGatewayModels,
|
|
33
35
|
databaseService: await resolve(DatabaseServiceTag),
|
|
34
36
|
memoryService: await resolve(MemoryServiceTag),
|
|
35
37
|
socialChatHistoryService: await resolve(SocialChatHistoryServiceTag),
|
|
@@ -39,12 +41,14 @@ const regularChatDigestServices: RegularChatDigestServices = {
|
|
|
39
41
|
const workerRuntimeConfig = await resolve(RuntimeConfigServiceTag)
|
|
40
42
|
const skillExtractionServices: SkillExtractionServices = {
|
|
41
43
|
agentConfig,
|
|
44
|
+
aiGatewayModels,
|
|
42
45
|
databaseService: await resolve(DatabaseServiceTag),
|
|
43
46
|
learnedSkillService: await resolve(LearnedSkillServiceTag),
|
|
44
47
|
socialChatHistoryService: await resolve(SocialChatHistoryServiceTag),
|
|
45
48
|
runtimeAdapters: await resolve(RuntimeAdaptersServiceTag),
|
|
46
49
|
embeddingModel: workerRuntimeConfig.aiGateway.embeddingModel,
|
|
47
50
|
openRouterApiKey: workerRuntimeConfig.aiGateway.openRouterApiKey,
|
|
51
|
+
runPromise: (effect) => runtime.runPromise(effect),
|
|
48
52
|
}
|
|
49
53
|
const organizationLearningQueueJobService = await resolve(QueueJobServiceTag)
|
|
50
54
|
|
|
@@ -55,11 +59,17 @@ const handler = (job: SandboxedJob<OrganizationLearningJob>) =>
|
|
|
55
59
|
Effect.gen(function* () {
|
|
56
60
|
if (job.data.kind === 'regular-chat-memory-digest') {
|
|
57
61
|
const digestJob = job.data
|
|
58
|
-
return yield*
|
|
62
|
+
return yield* Effect.tryPromise({
|
|
63
|
+
try: () => runRegularChatMemoryDigest(digestJob, regularChatDigestServices),
|
|
64
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
65
|
+
})
|
|
59
66
|
}
|
|
60
67
|
|
|
61
68
|
const skillJob = job.data
|
|
62
|
-
return yield*
|
|
69
|
+
return yield* Effect.tryPromise({
|
|
70
|
+
try: () => runSkillExtraction(skillJob, skillExtractionServices),
|
|
71
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
72
|
+
})
|
|
63
73
|
}).pipe(
|
|
64
74
|
Effect.catch((error) =>
|
|
65
75
|
Effect.gen(function* () {
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { Effect, Schema } from 'effect'
|
|
1
|
+
import { Cause, Effect, Schema } from 'effect'
|
|
2
2
|
import type { Context } from 'effect'
|
|
3
3
|
import { BoundQuery } from 'surrealdb'
|
|
4
4
|
import { z } from 'zod'
|
|
5
5
|
|
|
6
|
+
import type { AiGatewayModels } from '../ai-gateway/ai-gateway'
|
|
6
7
|
import type { ResolvedAgentConfig } from '../config/agent-defaults'
|
|
7
8
|
import { isAgentName } from '../config/agent-defaults'
|
|
8
9
|
import { serverLogger } from '../config/logger'
|
|
@@ -10,7 +11,7 @@ import { ensureRecordId, recordIdToString } from '../db/record-id'
|
|
|
10
11
|
import type { RecordIdRef } from '../db/record-id'
|
|
11
12
|
import type { SurrealDBService } from '../db/service'
|
|
12
13
|
import { TABLES } from '../db/tables'
|
|
13
|
-
import {
|
|
14
|
+
import { ERROR_TAGS } from '../effect/errors'
|
|
14
15
|
import type {
|
|
15
16
|
OrganizationLearningQueueRuntime,
|
|
16
17
|
RegularChatMemoryDigestJob,
|
|
@@ -19,7 +20,7 @@ import { createHelperModelRuntime } from '../runtime/helper-model'
|
|
|
19
20
|
import type { LotaRuntimeAdapters, LotaRuntimeBackgroundCursor } from '../runtime/runtime-extensions'
|
|
20
21
|
import type { MemoryServiceTag } from '../services/memory/memory.service'
|
|
21
22
|
import type { SocialChatHistoryServiceTag } from '../services/social-chat-history.service'
|
|
22
|
-
import {
|
|
23
|
+
import { makeRegularChatMemoryDigestAgentFactory } from '../system-agents/regular-chat-memory-digest.agent'
|
|
23
24
|
import { nowIsoDateTimeString } from '../utils/date-time'
|
|
24
25
|
import { compactWhitespace } from '../utils/string'
|
|
25
26
|
import { buildDigestTranscript, resolveWorkspaceBootstrapCutoff } from './regular-chat-memory-digest.helpers'
|
|
@@ -35,8 +36,9 @@ import type { DigestMessage } from './utils/thread-message-query'
|
|
|
35
36
|
// runner handles the regular-chat path after onboarding so longer transcripts
|
|
36
37
|
// can be digested into durable memory and profile projections in the background.
|
|
37
38
|
const StructuredProfilePatchSchema = z.record(z.string(), z.unknown()).default({})
|
|
39
|
+
const encodeJsonString = Schema.encodeSync(Schema.fromJsonString(Schema.Json))
|
|
38
40
|
|
|
39
|
-
class MemoryDigestError extends Schema.TaggedErrorClass<MemoryDigestError>()(
|
|
41
|
+
class MemoryDigestError extends Schema.TaggedErrorClass<MemoryDigestError>()(ERROR_TAGS.MemoryDigestError, {
|
|
40
42
|
message: Schema.String,
|
|
41
43
|
cause: Schema.Defect,
|
|
42
44
|
}) {}
|
|
@@ -45,6 +47,7 @@ const REGULAR_CHAT_MEMORY_DIGEST_TIMEOUT_MS = 10 * 60 * 1000
|
|
|
45
47
|
const WorkspaceMemoryRowSchema = z.object({ content: z.string() })
|
|
46
48
|
export interface RegularChatDigestServices {
|
|
47
49
|
agentConfig: ResolvedAgentConfig
|
|
50
|
+
aiGatewayModels: AiGatewayModels
|
|
48
51
|
databaseService: SurrealDBService
|
|
49
52
|
memoryService: Context.Service.Shape<typeof MemoryServiceTag>
|
|
50
53
|
socialChatHistoryService: Context.Service.Shape<typeof SocialChatHistoryServiceTag>
|
|
@@ -156,7 +159,9 @@ function hasNewEligibleThreadMessages(params: {
|
|
|
156
159
|
: null
|
|
157
160
|
|
|
158
161
|
if (!query) return false
|
|
159
|
-
const rows = yield*
|
|
162
|
+
const rows = yield* params.db
|
|
163
|
+
.query<unknown>(query)
|
|
164
|
+
.pipe(Effect.mapError((cause) => new Cause.UnknownError(cause)))
|
|
160
165
|
return rows.length > 0
|
|
161
166
|
}),
|
|
162
167
|
)
|
|
@@ -164,20 +169,20 @@ function hasNewEligibleThreadMessages(params: {
|
|
|
164
169
|
|
|
165
170
|
function loadExistingOrganizationMemories(db: SurrealDBService, orgId: string): Promise<Array<{ content: string }>> {
|
|
166
171
|
return Effect.runPromise(
|
|
167
|
-
|
|
168
|
-
|
|
172
|
+
db
|
|
173
|
+
.queryMany(
|
|
169
174
|
new BoundQuery(
|
|
170
175
|
`SELECT content, createdAt, id FROM ${TABLES.MEMORY}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
+
WHERE metadata.orgId = $orgId
|
|
177
|
+
AND archivedAt IS NONE
|
|
178
|
+
AND (validUntil IS NONE OR validUntil > time::now())
|
|
179
|
+
ORDER BY createdAt DESC, id DESC
|
|
180
|
+
LIMIT 250`,
|
|
176
181
|
{ orgId },
|
|
177
182
|
),
|
|
178
183
|
WorkspaceMemoryRowSchema,
|
|
179
|
-
)
|
|
180
|
-
|
|
184
|
+
)
|
|
185
|
+
.pipe(Effect.mapError((cause) => new Cause.UnknownError(cause))),
|
|
181
186
|
)
|
|
182
187
|
}
|
|
183
188
|
|
|
@@ -196,33 +201,50 @@ function runRegularChatMemoryDigestEffect(
|
|
|
196
201
|
return { skipped: true, processedThreadMessages: 0, processedSocialMessages: 0, followUpScheduled: false }
|
|
197
202
|
}
|
|
198
203
|
|
|
199
|
-
const workspace = yield*
|
|
204
|
+
const workspace = yield* Effect.tryPromise({
|
|
205
|
+
try: () => workspaceProvider.getWorkspace(orgRef),
|
|
206
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
207
|
+
})
|
|
200
208
|
const lifecycleState = workspaceProvider.getLifecycleState
|
|
201
|
-
? yield*
|
|
209
|
+
? yield* Effect.tryPromise({
|
|
210
|
+
try: () => Promise.resolve(workspaceProvider.getLifecycleState?.(workspace)),
|
|
211
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
212
|
+
})
|
|
202
213
|
: undefined
|
|
203
214
|
if (lifecycleState?.bootstrapActive ?? false) {
|
|
204
215
|
serverLogger.info`Skipping regular chat memory digest for ${orgId}: onboarding is not completed`
|
|
205
216
|
return { skipped: true, processedThreadMessages: 0, processedSocialMessages: 0, followUpScheduled: false }
|
|
206
217
|
}
|
|
207
218
|
const projectionState = workspaceProvider.readProfileProjectionState
|
|
208
|
-
? yield*
|
|
219
|
+
? yield* Effect.tryPromise({
|
|
220
|
+
try: () => Promise.resolve(workspaceProvider.readProfileProjectionState?.(workspace)),
|
|
221
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
222
|
+
})
|
|
209
223
|
: undefined
|
|
210
224
|
|
|
211
|
-
const existingThreadCursor = yield*
|
|
225
|
+
const existingThreadCursor = yield* Effect.tryPromise({
|
|
226
|
+
try: () => getBackgroundCursor('regular-chat-digest', orgRef),
|
|
227
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
228
|
+
})
|
|
212
229
|
const threadOnboardingCutoff = resolveWorkspaceBootstrapCutoff({
|
|
213
230
|
hasExistingCursor: existingThreadCursor !== null,
|
|
214
231
|
bootstrapCompletedAt: lifecycleState?.bootstrapCompletedAt,
|
|
215
232
|
})
|
|
216
233
|
|
|
217
|
-
const threadIds = yield*
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
234
|
+
const threadIds = yield* Effect.tryPromise({
|
|
235
|
+
try: () => listThreadIdsForOrg(services.databaseService, orgRef),
|
|
236
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
237
|
+
})
|
|
238
|
+
const threadMessages = yield* Effect.tryPromise({
|
|
239
|
+
try: () =>
|
|
240
|
+
listEligibleThreadMessages({
|
|
241
|
+
db: services.databaseService,
|
|
242
|
+
threadIds,
|
|
243
|
+
cursor: existingThreadCursor,
|
|
244
|
+
onboardingCutoff: threadOnboardingCutoff,
|
|
245
|
+
}),
|
|
246
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
247
|
+
})
|
|
226
248
|
const existingSocialCursor = yield* services.socialChatHistoryService.getBackgroundCursor(
|
|
227
249
|
'regular-chat-digest',
|
|
228
250
|
orgId,
|
|
@@ -247,30 +269,36 @@ function runRegularChatMemoryDigestEffect(
|
|
|
247
269
|
messages: combinedMessages,
|
|
248
270
|
isKnownAgentName: (value: string) => isAgentName(services.agentConfig, value),
|
|
249
271
|
})
|
|
250
|
-
const existingMemories = yield*
|
|
251
|
-
loadExistingOrganizationMemories(services.databaseService, orgId),
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
272
|
+
const existingMemories = yield* Effect.tryPromise({
|
|
273
|
+
try: () => loadExistingOrganizationMemories(services.databaseService, orgId),
|
|
274
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
275
|
+
})
|
|
276
|
+
const currentStructuredProfile = yield* Schema.decodeUnknownEffect(Schema.Json)(
|
|
277
|
+
projectionState?.structuredProfile ?? {},
|
|
278
|
+
).pipe(Effect.mapError((cause) => new Cause.UnknownError(cause)))
|
|
279
|
+
|
|
280
|
+
const synthesis = yield* Effect.tryPromise({
|
|
281
|
+
try: () =>
|
|
282
|
+
helperModelRuntime.generateHelperStructured({
|
|
283
|
+
tag: 'regular-chat-memory-digest',
|
|
284
|
+
createAgent: makeRegularChatMemoryDigestAgentFactory(services.aiGatewayModels),
|
|
285
|
+
timeoutMs: REGULAR_CHAT_MEMORY_DIGEST_TIMEOUT_MS,
|
|
286
|
+
messages: [
|
|
287
|
+
{
|
|
288
|
+
role: 'user',
|
|
289
|
+
content: buildPrompt({
|
|
290
|
+
workspaceName: projectionState?.workspaceName || 'Workspace',
|
|
291
|
+
currentSummaryBlock: projectionState?.summaryBlock ?? '',
|
|
292
|
+
currentStructuredProfile: encodeJsonString(currentStructuredProfile),
|
|
293
|
+
existingMemories: buildMemoryContext(existingMemories),
|
|
294
|
+
transcript,
|
|
295
|
+
}),
|
|
296
|
+
},
|
|
297
|
+
],
|
|
298
|
+
schema: RegularChatMemoryDigestOutputSchema,
|
|
299
|
+
}),
|
|
300
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
301
|
+
})
|
|
274
302
|
|
|
275
303
|
const summaryBlock = normalizeBlock(synthesis.summaryBlock)
|
|
276
304
|
if (!summaryBlock) {
|
|
@@ -309,25 +337,31 @@ function runRegularChatMemoryDigestEffect(
|
|
|
309
337
|
})
|
|
310
338
|
}
|
|
311
339
|
|
|
312
|
-
yield*
|
|
313
|
-
applyProfileProjection(orgRef, { summaryBlock, structuredPatch: synthesis.structuredProfilePatch }),
|
|
314
|
-
|
|
340
|
+
yield* Effect.tryPromise({
|
|
341
|
+
try: () => applyProfileProjection(orgRef, { summaryBlock, structuredPatch: synthesis.structuredProfilePatch }),
|
|
342
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
343
|
+
})
|
|
315
344
|
if (processedThreadCursor) {
|
|
316
|
-
yield*
|
|
345
|
+
yield* Effect.tryPromise({
|
|
346
|
+
try: () => setBackgroundCursor('regular-chat-digest', orgRef, processedThreadCursor),
|
|
347
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
348
|
+
})
|
|
317
349
|
}
|
|
318
350
|
if (processedSocialCursor) {
|
|
319
351
|
yield* services.socialChatHistoryService.setBackgroundCursor('regular-chat-digest', orgId, processedSocialCursor)
|
|
320
352
|
}
|
|
321
353
|
|
|
322
354
|
const threadBoundaryCursor = processedThreadCursor ?? existingThreadCursor
|
|
323
|
-
const hasMoreThreadMessages = yield*
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
355
|
+
const hasMoreThreadMessages = yield* Effect.tryPromise({
|
|
356
|
+
try: () =>
|
|
357
|
+
hasNewEligibleThreadMessages({
|
|
358
|
+
db: services.databaseService,
|
|
359
|
+
threadIds,
|
|
360
|
+
cursor: threadBoundaryCursor,
|
|
361
|
+
onboardingCutoff: threadBoundaryCursor ? null : threadOnboardingCutoff,
|
|
362
|
+
}),
|
|
363
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
364
|
+
})
|
|
331
365
|
const socialBoundaryCursor = processedSocialCursor ?? existingSocialCursor
|
|
332
366
|
const hasMoreSocialMessages = yield* services.socialChatHistoryService.hasWorkspaceMessages({
|
|
333
367
|
workspaceId: orgId,
|
|
@@ -337,10 +371,14 @@ function runRegularChatMemoryDigestEffect(
|
|
|
337
371
|
|
|
338
372
|
const followUpScheduled = hasMoreThreadMessages || hasMoreSocialMessages
|
|
339
373
|
if (followUpScheduled) {
|
|
340
|
-
yield*
|
|
341
|
-
services.organizationLearningQueue.clearRegularChatMemoryDigestDeduplicationKey(orgId),
|
|
342
|
-
|
|
343
|
-
|
|
374
|
+
yield* Effect.tryPromise({
|
|
375
|
+
try: () => services.organizationLearningQueue.clearRegularChatMemoryDigestDeduplicationKey(orgId),
|
|
376
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
377
|
+
})
|
|
378
|
+
yield* Effect.tryPromise({
|
|
379
|
+
try: () => services.organizationLearningQueue.enqueueRegularChatMemoryDigest({ orgId }),
|
|
380
|
+
catch: (cause) => new Cause.UnknownError(cause),
|
|
381
|
+
})
|
|
344
382
|
}
|
|
345
383
|
|
|
346
384
|
serverLogger.info`Regular chat memory digest completed for ${orgId}: threadMessages=${threadMessages.length}, socialMessages=${socialMessages.length}, facts=${synthesis.facts.length}, followUpScheduled=${followUpScheduled}`
|