@lota-sdk/core 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/infrastructure/schema/00_workstream.surql +55 -0
- package/infrastructure/schema/01_memory.surql +47 -0
- package/infrastructure/schema/02_execution_plan.surql +62 -0
- package/infrastructure/schema/03_learned_skill.surql +32 -0
- package/infrastructure/schema/04_runtime_bootstrap.surql +8 -0
- package/package.json +128 -0
- package/src/ai/definitions.ts +308 -0
- package/src/bifrost/bifrost.ts +256 -0
- package/src/config/agent-defaults.ts +99 -0
- package/src/config/constants.ts +33 -0
- package/src/config/env-shapes.ts +122 -0
- package/src/config/logger.ts +29 -0
- package/src/config/model-constants.ts +31 -0
- package/src/config/search.ts +17 -0
- package/src/config/workstream-defaults.ts +68 -0
- package/src/db/base.service.ts +55 -0
- package/src/db/cursor-pagination.ts +73 -0
- package/src/db/memory-query-builder.ts +207 -0
- package/src/db/memory-store.helpers.ts +118 -0
- package/src/db/memory-store.rows.ts +29 -0
- package/src/db/memory-store.ts +974 -0
- package/src/db/memory-types.ts +193 -0
- package/src/db/memory.ts +505 -0
- package/src/db/record-id.ts +78 -0
- package/src/db/service.ts +932 -0
- package/src/db/startup.ts +152 -0
- package/src/db/tables.ts +20 -0
- package/src/document/org-document-chunking.ts +224 -0
- package/src/document/parsing.ts +40 -0
- package/src/embeddings/provider.ts +76 -0
- package/src/index.ts +302 -0
- package/src/queues/context-compaction.queue.ts +82 -0
- package/src/queues/document-processor.queue.ts +118 -0
- package/src/queues/memory-consolidation.queue.ts +65 -0
- package/src/queues/post-chat-memory.queue.ts +128 -0
- package/src/queues/recent-activity-title-refinement.queue.ts +69 -0
- package/src/queues/regular-chat-memory-digest.config.ts +12 -0
- package/src/queues/regular-chat-memory-digest.queue.ts +73 -0
- package/src/queues/skill-extraction.config.ts +9 -0
- package/src/queues/skill-extraction.queue.ts +62 -0
- package/src/redis/connection.ts +176 -0
- package/src/redis/index.ts +30 -0
- package/src/redis/org-memory-lock.ts +43 -0
- package/src/redis/redis-lease-lock.ts +158 -0
- package/src/runtime/agent-contract.ts +1 -0
- package/src/runtime/agent-prompt-context.ts +119 -0
- package/src/runtime/agent-runtime-policy.ts +192 -0
- package/src/runtime/agent-stream-helpers.ts +117 -0
- package/src/runtime/agent-types.ts +22 -0
- package/src/runtime/approval-continuation.ts +16 -0
- package/src/runtime/chat-attachments.ts +46 -0
- package/src/runtime/chat-message.ts +10 -0
- package/src/runtime/chat-request-routing.ts +21 -0
- package/src/runtime/chat-run-orchestration.ts +25 -0
- package/src/runtime/chat-run-registry.ts +20 -0
- package/src/runtime/chat-types.ts +18 -0
- package/src/runtime/context-compaction-constants.ts +11 -0
- package/src/runtime/context-compaction-runtime.ts +86 -0
- package/src/runtime/context-compaction.ts +909 -0
- package/src/runtime/execution-plan.ts +59 -0
- package/src/runtime/helper-model.ts +405 -0
- package/src/runtime/indexed-repositories-policy.ts +28 -0
- package/src/runtime/instruction-sections.ts +8 -0
- package/src/runtime/llm-content.ts +71 -0
- package/src/runtime/memory-block.ts +264 -0
- package/src/runtime/memory-digest-policy.ts +14 -0
- package/src/runtime/memory-format.ts +8 -0
- package/src/runtime/memory-pipeline.ts +570 -0
- package/src/runtime/memory-prompts-fact.ts +47 -0
- package/src/runtime/memory-prompts-parse.ts +3 -0
- package/src/runtime/memory-prompts-update.ts +37 -0
- package/src/runtime/memory-scope.ts +43 -0
- package/src/runtime/plugin-types.ts +10 -0
- package/src/runtime/retrieval-adapters.ts +25 -0
- package/src/runtime/retrieval-pipeline.ts +3 -0
- package/src/runtime/runtime-extensions.ts +154 -0
- package/src/runtime/skill-extraction-policy.ts +3 -0
- package/src/runtime/team-consultation-orchestrator.ts +245 -0
- package/src/runtime/team-consultation-prompts.ts +32 -0
- package/src/runtime/title-helpers.ts +12 -0
- package/src/runtime/turn-lifecycle.ts +28 -0
- package/src/runtime/workstream-chat-helpers.ts +187 -0
- package/src/runtime/workstream-routing-policy.ts +301 -0
- package/src/runtime/workstream-state.ts +261 -0
- package/src/services/attachment.service.ts +159 -0
- package/src/services/chat-attachments.service.ts +17 -0
- package/src/services/chat-run-registry.service.ts +3 -0
- package/src/services/context-compaction-runtime.ts +13 -0
- package/src/services/context-compaction.service.ts +115 -0
- package/src/services/document-chunk.service.ts +141 -0
- package/src/services/execution-plan.service.ts +890 -0
- package/src/services/learned-skill.service.ts +328 -0
- package/src/services/memory-assessment.service.ts +43 -0
- package/src/services/memory.service.ts +807 -0
- package/src/services/memory.utils.ts +84 -0
- package/src/services/mutating-approval.service.ts +110 -0
- package/src/services/recent-activity-title.service.ts +74 -0
- package/src/services/recent-activity.service.ts +397 -0
- package/src/services/workstream-change-tracker.service.ts +313 -0
- package/src/services/workstream-message.service.ts +283 -0
- package/src/services/workstream-title.service.ts +58 -0
- package/src/services/workstream-turn-preparation.ts +1340 -0
- package/src/services/workstream-turn.ts +37 -0
- package/src/services/workstream.service.ts +854 -0
- package/src/services/workstream.types.ts +118 -0
- package/src/storage/attachment-parser.ts +101 -0
- package/src/storage/attachment-storage.service.ts +391 -0
- package/src/storage/attachments.types.ts +11 -0
- package/src/storage/attachments.utils.ts +58 -0
- package/src/storage/generated-document-storage.service.ts +55 -0
- package/src/system-agents/agent-result.ts +27 -0
- package/src/system-agents/context-compacter.agent.ts +46 -0
- package/src/system-agents/delegated-agent-factory.ts +177 -0
- package/src/system-agents/helper-agent-options.ts +20 -0
- package/src/system-agents/memory-reranker.agent.ts +38 -0
- package/src/system-agents/memory.agent.ts +58 -0
- package/src/system-agents/recent-activity-title-refiner.agent.ts +53 -0
- package/src/system-agents/regular-chat-memory-digest.agent.ts +75 -0
- package/src/system-agents/researcher.agent.ts +34 -0
- package/src/system-agents/skill-extractor.agent.ts +88 -0
- package/src/system-agents/skill-manager.agent.ts +80 -0
- package/src/system-agents/title-generator.agent.ts +42 -0
- package/src/system-agents/workstream-tracker.agent.ts +58 -0
- package/src/tools/execution-plan.tool.ts +163 -0
- package/src/tools/fetch-webpage.tool.ts +132 -0
- package/src/tools/firecrawl-client.ts +12 -0
- package/src/tools/memory-block.tool.ts +55 -0
- package/src/tools/read-file-parts.tool.ts +80 -0
- package/src/tools/remember-memory.tool.ts +85 -0
- package/src/tools/research-topic.tool.ts +15 -0
- package/src/tools/search-tools.ts +55 -0
- package/src/tools/search-web.tool.ts +175 -0
- package/src/tools/team-think.tool.ts +125 -0
- package/src/tools/tool-contract.ts +21 -0
- package/src/tools/user-questions.tool.ts +18 -0
- package/src/utils/async.ts +50 -0
- package/src/utils/date-time.ts +34 -0
- package/src/utils/error.ts +10 -0
- package/src/utils/errors.ts +28 -0
- package/src/utils/hono-error-handler.ts +71 -0
- package/src/utils/string.ts +51 -0
- package/src/workers/bootstrap.ts +44 -0
- package/src/workers/memory-consolidation.worker.ts +318 -0
- package/src/workers/regular-chat-memory-digest.helpers.ts +100 -0
- package/src/workers/regular-chat-memory-digest.runner.ts +363 -0
- package/src/workers/regular-chat-memory-digest.worker.ts +22 -0
- package/src/workers/skill-extraction.runner.ts +331 -0
- package/src/workers/skill-extraction.worker.ts +22 -0
- package/src/workers/utils/repo-indexer-chunker.ts +331 -0
- package/src/workers/utils/repo-structure-extractor.ts +645 -0
- package/src/workers/utils/repomix-process-concurrency.ts +65 -0
- package/src/workers/utils/sandbox-error.ts +5 -0
- package/src/workers/worker-utils.ts +182 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
|
|
3
|
+
export const MemoryTypeSchema = z.enum([
|
|
4
|
+
'fact',
|
|
5
|
+
'preference',
|
|
6
|
+
'interaction',
|
|
7
|
+
'summary',
|
|
8
|
+
'user_request',
|
|
9
|
+
'entity',
|
|
10
|
+
'interest',
|
|
11
|
+
])
|
|
12
|
+
export type MemoryType = z.infer<typeof MemoryTypeSchema>
|
|
13
|
+
|
|
14
|
+
export const DurabilitySchema = z.enum(['core', 'standard', 'ephemeral'])
|
|
15
|
+
export type Durability = z.infer<typeof DurabilitySchema>
|
|
16
|
+
|
|
17
|
+
export const MemoryEventSchema = z.enum(['ADD', 'UPDATE', 'DELETE', 'NONE'])
|
|
18
|
+
export type MemoryEvent = z.infer<typeof MemoryEventSchema>
|
|
19
|
+
|
|
20
|
+
export const RelationTypeSchema = z.enum([
|
|
21
|
+
'contradicts',
|
|
22
|
+
'supports',
|
|
23
|
+
'supersedes',
|
|
24
|
+
'caused_by',
|
|
25
|
+
'depends_on',
|
|
26
|
+
'part_of',
|
|
27
|
+
'implements',
|
|
28
|
+
])
|
|
29
|
+
export type RelationType = z.infer<typeof RelationTypeSchema>
|
|
30
|
+
|
|
31
|
+
export interface MemoryRecord {
|
|
32
|
+
id: string
|
|
33
|
+
content: string
|
|
34
|
+
embedding: number[]
|
|
35
|
+
hash: string
|
|
36
|
+
scopeId: string
|
|
37
|
+
memoryType: MemoryType
|
|
38
|
+
durability: Durability
|
|
39
|
+
metadata: Record<string, unknown>
|
|
40
|
+
importance: number
|
|
41
|
+
accessCount: number
|
|
42
|
+
needsReview: boolean
|
|
43
|
+
lastAccessedAt?: Date
|
|
44
|
+
createdAt: Date
|
|
45
|
+
updatedAt?: Date
|
|
46
|
+
validFrom: Date
|
|
47
|
+
validUntil?: Date
|
|
48
|
+
archivedAt?: Date
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface SearchOptions {
|
|
52
|
+
scopeId: string
|
|
53
|
+
limit?: number
|
|
54
|
+
memoryType?: MemoryType
|
|
55
|
+
pointInTime?: string
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export type LinearNormalization = 'minmax' | 'zscore'
|
|
59
|
+
|
|
60
|
+
export interface WeightedSearchOptions extends SearchOptions {
|
|
61
|
+
weights?: [number, number]
|
|
62
|
+
normalization?: LinearNormalization
|
|
63
|
+
fastMode?: boolean
|
|
64
|
+
includeNeighborContext?: boolean
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface AddOptions {
|
|
68
|
+
scopeId: string
|
|
69
|
+
memoryType: MemoryType
|
|
70
|
+
importance?: number
|
|
71
|
+
metadata?: Record<string, unknown>
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface MemorySearchResult {
|
|
75
|
+
id: string
|
|
76
|
+
content: string
|
|
77
|
+
score: number
|
|
78
|
+
metadata: Record<string, unknown>
|
|
79
|
+
contradictions?: string[]
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const ExtractedFactSchema = z.object({
|
|
83
|
+
content: z.string().describe('The extracted fact or preference.'),
|
|
84
|
+
type: z.enum(['fact', 'preference', 'decision']).describe('Classification of the memory.'),
|
|
85
|
+
confidence: z
|
|
86
|
+
.number()
|
|
87
|
+
.min(0)
|
|
88
|
+
.max(1)
|
|
89
|
+
.describe('1.0 for explicit decisions/confirmed truths, 0.8 for strong implications. Do not extract below 0.6.'),
|
|
90
|
+
durability: z
|
|
91
|
+
.enum(['core', 'standard', 'ephemeral'])
|
|
92
|
+
.describe(
|
|
93
|
+
'core: business decisions, technical architecture, confirmed requirements. standard: general facts, moderate inferences. ephemeral: preferences, one-off interactions, formatting choices.',
|
|
94
|
+
),
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
export type ExtractedFact = z.infer<typeof ExtractedFactSchema>
|
|
98
|
+
|
|
99
|
+
export const FactRetrievalSchema = z.object({
|
|
100
|
+
facts: z.array(ExtractedFactSchema).describe('Extracted durable facts with classification and confidence.'),
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
const MemoryRelationItemSchema = z.object({
|
|
104
|
+
memoryId: z.string().describe('The ID of the related memory from the existing memory list.'),
|
|
105
|
+
relation: RelationTypeSchema.describe('How the new fact relates to this memory.'),
|
|
106
|
+
})
|
|
107
|
+
const MemoryUpdateItemSchema = z.object({
|
|
108
|
+
id: z.string().describe('The unique identifier of the memory item.'),
|
|
109
|
+
text: z.string().describe('The content of the memory item.'),
|
|
110
|
+
event: MemoryEventSchema.describe('The action taken for this memory item (ADD, UPDATE, DELETE, or NONE).'),
|
|
111
|
+
oldMemory: z.string().optional().describe('The previous content of the memory item if the event was UPDATE.'),
|
|
112
|
+
relatesTo: z.array(MemoryRelationItemSchema).optional().describe('Relations to existing memories.'),
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
export const MemoryUpdateSchema = z.object({
|
|
116
|
+
memory: z
|
|
117
|
+
.array(MemoryUpdateItemSchema)
|
|
118
|
+
.describe('An array representing the state of memory items after processing new facts.'),
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
export type MemoryUpdateOutput = z.infer<typeof MemoryUpdateSchema>
|
|
122
|
+
|
|
123
|
+
const MemoryDeltaClassificationSchema = z.enum(['new', 'supersedes', 'contradicts', 'enriches', 'duplicate'])
|
|
124
|
+
|
|
125
|
+
const MemoryDeltaRelationSchema = z
|
|
126
|
+
.object({
|
|
127
|
+
relation: RelationTypeSchema.describe('Relation type to attach from this fact output to a target.'),
|
|
128
|
+
targetMemoryId: z
|
|
129
|
+
.string()
|
|
130
|
+
.min(1)
|
|
131
|
+
.optional()
|
|
132
|
+
.describe('Target existing memory id when relation points to existing memory.'),
|
|
133
|
+
targetFactIndex: z
|
|
134
|
+
.number()
|
|
135
|
+
.int()
|
|
136
|
+
.min(0)
|
|
137
|
+
.optional()
|
|
138
|
+
.describe('Target newFacts index when relation points to another newly provided fact.'),
|
|
139
|
+
})
|
|
140
|
+
.strict()
|
|
141
|
+
.superRefine((value, ctx) => {
|
|
142
|
+
const hasTargetMemoryId = typeof value.targetMemoryId === 'string' && value.targetMemoryId.trim().length > 0
|
|
143
|
+
const hasTargetFactIndex = Number.isInteger(value.targetFactIndex)
|
|
144
|
+
|
|
145
|
+
if ((hasTargetMemoryId && hasTargetFactIndex) || (!hasTargetMemoryId && !hasTargetFactIndex)) {
|
|
146
|
+
ctx.addIssue({
|
|
147
|
+
code: z.ZodIssueCode.custom,
|
|
148
|
+
message: 'Exactly one of targetMemoryId or targetFactIndex must be provided.',
|
|
149
|
+
})
|
|
150
|
+
}
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
const MemoryDeltaItemSchema = z
|
|
154
|
+
.object({
|
|
155
|
+
fact: z.string().min(1).describe('The new fact candidate being classified.'),
|
|
156
|
+
classification: MemoryDeltaClassificationSchema.describe(
|
|
157
|
+
'How this fact relates to existing memories: new, supersedes, contradicts, enriches, duplicate.',
|
|
158
|
+
),
|
|
159
|
+
targetMemoryIds: z
|
|
160
|
+
.array(z.string().min(1))
|
|
161
|
+
.default([])
|
|
162
|
+
.describe('Existing memory IDs that are directly related to this fact.'),
|
|
163
|
+
invalidateTargetIds: z
|
|
164
|
+
.array(z.string().min(1))
|
|
165
|
+
.default([])
|
|
166
|
+
.describe('Subset of targetMemoryIds that should be deleted as obsolete/invalidated.'),
|
|
167
|
+
relations: z
|
|
168
|
+
.array(MemoryDeltaRelationSchema)
|
|
169
|
+
.default([])
|
|
170
|
+
.describe('Explicit semantic relations from this fact to existing memories and/or other new facts by index.'),
|
|
171
|
+
rationale: z.string().min(1).describe('Short rationale for the classification decision.'),
|
|
172
|
+
})
|
|
173
|
+
.strict()
|
|
174
|
+
|
|
175
|
+
export const MemoryDeltaSchema = z
|
|
176
|
+
.object({ deltas: z.array(MemoryDeltaItemSchema).describe('Classification output for each new fact.') })
|
|
177
|
+
.strict()
|
|
178
|
+
export const MemoryImportanceAssessmentSchema = z
|
|
179
|
+
.object({
|
|
180
|
+
importance: z.number().min(0).max(1).describe('Long-term usefulness score from 0 to 1 for storing this memory.'),
|
|
181
|
+
durability: DurabilitySchema.describe('Expected durability for this memory.'),
|
|
182
|
+
classification: z.enum(['durable', 'transient', 'uncertain']).describe('Durability classification for storage.'),
|
|
183
|
+
rationale: z.string().min(1).describe('Concise rationale for the score/classification.'),
|
|
184
|
+
})
|
|
185
|
+
.strict()
|
|
186
|
+
export interface Message {
|
|
187
|
+
role: 'user' | 'agent'
|
|
188
|
+
content: string
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export interface MemoryConfig {
|
|
192
|
+
customPrompt?: string
|
|
193
|
+
}
|
package/src/db/memory.ts
ADDED
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
import { env } from '../config/env-shapes'
|
|
2
|
+
import { aiLogger } from '../config/logger'
|
|
3
|
+
import type { CreateHelperAgentFn } from '../runtime/helper-model'
|
|
4
|
+
import { createHelperModelRuntime } from '../runtime/helper-model'
|
|
5
|
+
import { formatResults } from '../runtime/memory-format'
|
|
6
|
+
import {
|
|
7
|
+
buildMemoryFactMaps,
|
|
8
|
+
compileMemoryUpdatesFromDelta,
|
|
9
|
+
createMemoryActionPlan,
|
|
10
|
+
postProcessMemoryFacts,
|
|
11
|
+
} from '../runtime/memory-pipeline'
|
|
12
|
+
import { getFactRetrievalMessages } from '../runtime/memory-prompts-fact'
|
|
13
|
+
import { parseMessages } from '../runtime/memory-prompts-parse'
|
|
14
|
+
import { getClassifyMemoryDeltaPrompt } from '../runtime/memory-prompts-update'
|
|
15
|
+
import type { SurrealMemoryStore } from './memory-store'
|
|
16
|
+
import { getDefaultMemoryStore } from './memory-store'
|
|
17
|
+
import { hashContent, isUniqueIndexConflict } from './memory-store.helpers'
|
|
18
|
+
import { FactRetrievalSchema, MemoryDeltaSchema, MemoryUpdateSchema } from './memory-types'
|
|
19
|
+
import type {
|
|
20
|
+
AddOptions,
|
|
21
|
+
Durability,
|
|
22
|
+
ExtractedFact,
|
|
23
|
+
MemoryConfig,
|
|
24
|
+
MemorySearchResult,
|
|
25
|
+
MemoryType,
|
|
26
|
+
MemoryUpdateOutput,
|
|
27
|
+
Message,
|
|
28
|
+
MemoryRecord,
|
|
29
|
+
SearchOptions,
|
|
30
|
+
WeightedSearchOptions,
|
|
31
|
+
} from './memory-types'
|
|
32
|
+
|
|
33
|
+
const MEMORY_WORKER_MODEL_TIMEOUT_MS = 10 * 60 * 1000
|
|
34
|
+
const MEMORY_FACT_EXTRACTION_TIMEOUT_MS = MEMORY_WORKER_MODEL_TIMEOUT_MS
|
|
35
|
+
const MEMORY_DELTA_CLASSIFICATION_TIMEOUT_MS = MEMORY_WORKER_MODEL_TIMEOUT_MS
|
|
36
|
+
const MEMORY_DELTA_MAX_CANDIDATE_MEMORIES = 80
|
|
37
|
+
const MEMORY_DELTA_MAX_CANDIDATES_PER_FACT = 10
|
|
38
|
+
const MEMORY_DELTA_MIN_BASELINE_CANDIDATES = 12
|
|
39
|
+
const MEMORY_DELTA_MEMORY_TEXT_MAX_CHARS = 400
|
|
40
|
+
const helperModelRuntime = createHelperModelRuntime()
|
|
41
|
+
|
|
42
|
+
interface PreparedScopeUpdate {
|
|
43
|
+
options: AddOptions
|
|
44
|
+
updates: MemoryUpdateOutput
|
|
45
|
+
factMaps: ReturnType<typeof buildMemoryFactMaps>
|
|
46
|
+
existingMemories: Array<{ id: string; text: string }>
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export class Memory {
|
|
50
|
+
private store: SurrealMemoryStore
|
|
51
|
+
private createAgent: CreateHelperAgentFn
|
|
52
|
+
private maxOutputTokens?: number
|
|
53
|
+
private config: MemoryConfig
|
|
54
|
+
|
|
55
|
+
constructor(agent: { createAgent: CreateHelperAgentFn; maxOutputTokens?: number }, config: MemoryConfig = {}) {
|
|
56
|
+
this.store = getDefaultMemoryStore()
|
|
57
|
+
this.createAgent = agent.createAgent
|
|
58
|
+
this.maxOutputTokens = agent.maxOutputTokens
|
|
59
|
+
|
|
60
|
+
this.config = config
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private buildFactExtractionPrompt(customPrompt?: string): string | undefined {
|
|
64
|
+
const sections = [this.config.customPrompt, customPrompt]
|
|
65
|
+
.map((value) => value?.trim())
|
|
66
|
+
.filter((value): value is string => typeof value === 'string' && value.length > 0)
|
|
67
|
+
if (sections.length === 0) return undefined
|
|
68
|
+
return sections.join('\n\n')
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async insert(
|
|
72
|
+
content: string,
|
|
73
|
+
options: {
|
|
74
|
+
scopeId: string
|
|
75
|
+
memoryType: MemoryType
|
|
76
|
+
importance?: number
|
|
77
|
+
metadata?: Record<string, unknown>
|
|
78
|
+
durability?: Durability
|
|
79
|
+
},
|
|
80
|
+
): Promise<string> {
|
|
81
|
+
return await this.store.insert(
|
|
82
|
+
content,
|
|
83
|
+
options.scopeId,
|
|
84
|
+
options.memoryType,
|
|
85
|
+
options.metadata ?? {},
|
|
86
|
+
options.importance ?? 1,
|
|
87
|
+
options.durability ?? 'standard',
|
|
88
|
+
)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async search(query: string, options: SearchOptions): Promise<string> {
|
|
92
|
+
const results = await this.store.search(
|
|
93
|
+
query,
|
|
94
|
+
options.scopeId,
|
|
95
|
+
options.limit ?? env.MEMORY_SEARCH_K,
|
|
96
|
+
options.memoryType,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
return formatResults(results)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async hybridSearch(query: string, options: SearchOptions): Promise<string> {
|
|
103
|
+
const results = await this.store.hybridSearch(
|
|
104
|
+
query,
|
|
105
|
+
options.scopeId,
|
|
106
|
+
options.limit ?? env.MEMORY_SEARCH_K,
|
|
107
|
+
options.memoryType,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
return formatResults(results)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async hybridSearchWeighted(
|
|
114
|
+
query: string,
|
|
115
|
+
options: SearchOptions & { weights?: [number, number]; normalization?: 'minmax' | 'zscore' },
|
|
116
|
+
): Promise<string> {
|
|
117
|
+
const results = await this.store.hybridSearchWeighted(query, {
|
|
118
|
+
scopeId: options.scopeId,
|
|
119
|
+
limit: options.limit ?? env.MEMORY_SEARCH_K,
|
|
120
|
+
memoryType: options.memoryType,
|
|
121
|
+
weights: options.weights,
|
|
122
|
+
normalization: options.normalization,
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
return formatResults(results)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async searchCandidates(query: string, options: WeightedSearchOptions): Promise<MemorySearchResult[]> {
|
|
129
|
+
const results = await this.store.hybridSearchWeighted(query, {
|
|
130
|
+
scopeId: options.scopeId,
|
|
131
|
+
limit: options.limit ?? env.MEMORY_SEARCH_K,
|
|
132
|
+
memoryType: options.memoryType,
|
|
133
|
+
weights: options.weights,
|
|
134
|
+
normalization: options.normalization,
|
|
135
|
+
fastMode: options.fastMode,
|
|
136
|
+
})
|
|
137
|
+
|
|
138
|
+
if (options.fastMode || options.includeNeighborContext === false) {
|
|
139
|
+
return results
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
return await this.store.enrichWithNeighbors(results)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async listTopMemories(options: {
|
|
146
|
+
scopeId: string
|
|
147
|
+
limit: number
|
|
148
|
+
memoryType?: MemoryType
|
|
149
|
+
durability?: Durability
|
|
150
|
+
minImportance?: number
|
|
151
|
+
}): Promise<MemoryRecord[]> {
|
|
152
|
+
return await this.store.listTopMemories(options)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async updateMemory(id: string, newContent: string): Promise<void> {
|
|
156
|
+
await this.store.update(id, newContent)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async getStaleMemories(scopeId: string, limit?: number): Promise<MemorySearchResult[]> {
|
|
160
|
+
return await this.store.getStaleMemories(scopeId, limit)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async add(
|
|
164
|
+
messages: Message[],
|
|
165
|
+
options: AddOptions,
|
|
166
|
+
extractionOptions?: { customPrompt?: string; maxFacts?: number },
|
|
167
|
+
): Promise<void> {
|
|
168
|
+
const facts = await this.extractFactsFromMessages(messages, extractionOptions)
|
|
169
|
+
if (facts.length === 0) return
|
|
170
|
+
|
|
171
|
+
aiLogger.debug`Extracted ${facts.length} facts from conversation`
|
|
172
|
+
|
|
173
|
+
await this.applyFactsToScope(facts, options)
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async addMultiScope(
|
|
177
|
+
messages: Message[],
|
|
178
|
+
scopes: AddOptions[],
|
|
179
|
+
extractionOptions?: { customPrompt?: string; maxFacts?: number },
|
|
180
|
+
): Promise<void> {
|
|
181
|
+
if (scopes.length === 0) return
|
|
182
|
+
const facts = await this.extractFactsFromMessages(messages, extractionOptions)
|
|
183
|
+
if (facts.length === 0) return
|
|
184
|
+
|
|
185
|
+
aiLogger.debug`Extracted ${facts.length} facts, applying to ${scopes.length} scopes`
|
|
186
|
+
|
|
187
|
+
await this.applyFactsToScopes(facts, scopes)
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async extractFactsFromMessages(
|
|
191
|
+
messages: Message[],
|
|
192
|
+
extractionOptions?: { customPrompt?: string; maxFacts?: number },
|
|
193
|
+
): Promise<ExtractedFact[]> {
|
|
194
|
+
if (messages.length === 0) return []
|
|
195
|
+
|
|
196
|
+
const parsedMessages = parseMessages(messages)
|
|
197
|
+
|
|
198
|
+
const facts = await this.extractFacts(parsedMessages, extractionOptions)
|
|
199
|
+
if (facts.length === 0) {
|
|
200
|
+
aiLogger.debug`No facts extracted from conversation`
|
|
201
|
+
return []
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
return facts
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async applyFactsToScopes(facts: ExtractedFact[], scopes: AddOptions[]): Promise<void> {
|
|
208
|
+
if (facts.length === 0 || scopes.length === 0) return
|
|
209
|
+
|
|
210
|
+
const prepared = await this.prepareFactsToScopes(facts, scopes)
|
|
211
|
+
await this.applyPreparedScopeUpdates(prepared)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
async prepareFactsToScopes(facts: ExtractedFact[], scopes: AddOptions[]): Promise<PreparedScopeUpdate[]> {
|
|
215
|
+
if (facts.length === 0 || scopes.length === 0) return []
|
|
216
|
+
|
|
217
|
+
const prepared: PreparedScopeUpdate[] = []
|
|
218
|
+
for (const scopeOptions of scopes) {
|
|
219
|
+
prepared.push(await this.prepareFactsForScope(facts, scopeOptions))
|
|
220
|
+
}
|
|
221
|
+
return prepared
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async applyPreparedScopeUpdates(prepared: PreparedScopeUpdate[]): Promise<void> {
|
|
225
|
+
if (prepared.length === 0) return
|
|
226
|
+
|
|
227
|
+
for (const item of prepared) {
|
|
228
|
+
await this.applyUpdates(item.updates, item.options, item.factMaps, item.existingMemories)
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
private async applyFactsToScope(facts: ExtractedFact[], options: AddOptions): Promise<void> {
|
|
233
|
+
const prepared = await this.prepareFactsForScope(facts, options)
|
|
234
|
+
await this.applyPreparedScopeUpdates([prepared])
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
private async prepareFactsForScope(facts: ExtractedFact[], options: AddOptions): Promise<PreparedScopeUpdate> {
|
|
238
|
+
const factMaps = buildMemoryFactMaps(facts)
|
|
239
|
+
|
|
240
|
+
const existingMemories = await this.store.list(options.scopeId, options.memoryType)
|
|
241
|
+
const oldMemoryFormat = existingMemories.map((m) => ({ id: m.id, text: m.content }))
|
|
242
|
+
|
|
243
|
+
const factContents = facts.map((f) => f.content)
|
|
244
|
+
const updates = await this.determineUpdates(oldMemoryFormat, factContents)
|
|
245
|
+
|
|
246
|
+
return { options, updates, factMaps, existingMemories: oldMemoryFormat }
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
private async extractFacts(
|
|
250
|
+
parsedMessages: string,
|
|
251
|
+
extractionOptions?: { customPrompt?: string; maxFacts?: number },
|
|
252
|
+
): Promise<ExtractedFact[]> {
|
|
253
|
+
const [systemPrompt, userPrompt] = getFactRetrievalMessages(
|
|
254
|
+
parsedMessages,
|
|
255
|
+
this.buildFactExtractionPrompt(extractionOptions?.customPrompt),
|
|
256
|
+
extractionOptions?.maxFacts,
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
try {
|
|
260
|
+
const result = await helperModelRuntime.generateHelperStructured({
|
|
261
|
+
tag: 'memory-extract-facts',
|
|
262
|
+
createAgent: this.createAgent,
|
|
263
|
+
defaultSystemPrompt: this.config.customPrompt,
|
|
264
|
+
systemPrompt,
|
|
265
|
+
maxOutputTokens: this.maxOutputTokens,
|
|
266
|
+
timeoutMs: MEMORY_FACT_EXTRACTION_TIMEOUT_MS,
|
|
267
|
+
messages: [{ role: 'user', content: userPrompt }],
|
|
268
|
+
schema: FactRetrievalSchema,
|
|
269
|
+
})
|
|
270
|
+
|
|
271
|
+
return postProcessMemoryFacts(
|
|
272
|
+
result.facts,
|
|
273
|
+
typeof extractionOptions?.maxFacts === 'number' ? { maxFacts: extractionOptions.maxFacts } : {},
|
|
274
|
+
)
|
|
275
|
+
} catch (error) {
|
|
276
|
+
aiLogger.error`Failed to extract facts: ${error}`
|
|
277
|
+
return []
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
private async determineUpdates(
|
|
282
|
+
existingMemories: { id: string; text: string }[],
|
|
283
|
+
newFacts: string[],
|
|
284
|
+
): Promise<MemoryUpdateOutput> {
|
|
285
|
+
if (existingMemories.length === 0) {
|
|
286
|
+
return { memory: newFacts.map((fact, index) => ({ id: `new_${index}`, text: fact, event: 'ADD' as const })) }
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
const candidateMemories = this.selectDeltaCandidateMemories(existingMemories, newFacts)
|
|
290
|
+
const { systemPrompt, userPrompt } = getClassifyMemoryDeltaPrompt({ existingMemories: candidateMemories, newFacts })
|
|
291
|
+
aiLogger.debug`Memory delta candidate selection (existing=${existingMemories.length}, selected=${candidateMemories.length}, facts=${newFacts.length})`
|
|
292
|
+
|
|
293
|
+
try {
|
|
294
|
+
const deltas = await helperModelRuntime.generateHelperStructured({
|
|
295
|
+
tag: 'memory-classify-delta',
|
|
296
|
+
createAgent: this.createAgent,
|
|
297
|
+
systemPrompt,
|
|
298
|
+
maxOutputTokens: this.maxOutputTokens,
|
|
299
|
+
timeoutMs: MEMORY_DELTA_CLASSIFICATION_TIMEOUT_MS,
|
|
300
|
+
messages: [{ role: 'user', content: userPrompt }],
|
|
301
|
+
schema: MemoryDeltaSchema,
|
|
302
|
+
})
|
|
303
|
+
const compiled = compileMemoryUpdatesFromDelta({ existingMemories, newFacts, delta: deltas })
|
|
304
|
+
return MemoryUpdateSchema.parse(compiled)
|
|
305
|
+
} catch (error) {
|
|
306
|
+
aiLogger.error`Failed to determine memory updates: ${error}`
|
|
307
|
+
throw error
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
private normalizeMemoryDeltaText(value: string, maxChars?: number): string {
|
|
312
|
+
const normalized = value.replace(/\s+/g, ' ').trim()
|
|
313
|
+
if (!normalized) return ''
|
|
314
|
+
if (typeof maxChars !== 'number' || normalized.length <= maxChars) return normalized
|
|
315
|
+
return `${normalized.slice(0, maxChars - 3)}...`
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
private tokenizeMemoryDeltaText(value: string): string[] {
|
|
319
|
+
return this.normalizeMemoryDeltaText(value)
|
|
320
|
+
.toLowerCase()
|
|
321
|
+
.replace(/[^\w\s$%./:-]/g, ' ')
|
|
322
|
+
.split(/\s+/)
|
|
323
|
+
.map((token) => token.trim())
|
|
324
|
+
.filter((token) => token.length >= 3)
|
|
325
|
+
.slice(0, 24)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
private selectDeltaCandidateMemories(
|
|
329
|
+
existingMemories: { id: string; text: string }[],
|
|
330
|
+
newFacts: string[],
|
|
331
|
+
): { id: string; text: string }[] {
|
|
332
|
+
const normalizedExisting = existingMemories
|
|
333
|
+
.map((memory, index) => ({
|
|
334
|
+
id: memory.id,
|
|
335
|
+
text: this.normalizeMemoryDeltaText(memory.text, MEMORY_DELTA_MEMORY_TEXT_MAX_CHARS),
|
|
336
|
+
index,
|
|
337
|
+
}))
|
|
338
|
+
.filter((memory) => memory.id.trim().length > 0 && memory.text.length > 0)
|
|
339
|
+
|
|
340
|
+
if (normalizedExisting.length <= MEMORY_DELTA_MAX_CANDIDATE_MEMORIES) {
|
|
341
|
+
return normalizedExisting.map((memory) => ({ id: memory.id, text: memory.text }))
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
const scoreById = new Map<string, number>()
|
|
345
|
+
for (const fact of newFacts) {
|
|
346
|
+
const factText = this.normalizeMemoryDeltaText(fact)
|
|
347
|
+
if (!factText) continue
|
|
348
|
+
|
|
349
|
+
const factTokens = this.tokenizeMemoryDeltaText(factText)
|
|
350
|
+
if (factTokens.length === 0) continue
|
|
351
|
+
const factLower = factText.toLowerCase()
|
|
352
|
+
|
|
353
|
+
const scoredForFact = normalizedExisting
|
|
354
|
+
.map((memory) => {
|
|
355
|
+
const memoryLower = memory.text.toLowerCase()
|
|
356
|
+
let overlap = 0
|
|
357
|
+
for (const token of factTokens) {
|
|
358
|
+
if (memoryLower.includes(token)) overlap += 1
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
const phraseBoost = factLower.length >= 10 && memoryLower.includes(factLower) ? 3 : 0
|
|
362
|
+
if (overlap === 0 && phraseBoost === 0) return { id: memory.id, score: 0 }
|
|
363
|
+
|
|
364
|
+
const recencyBoost = Math.max(0, 1 - memory.index / 120)
|
|
365
|
+
const coverageBoost = overlap / factTokens.length
|
|
366
|
+
return { id: memory.id, score: overlap + phraseBoost + coverageBoost + recencyBoost }
|
|
367
|
+
})
|
|
368
|
+
.filter((item) => item.score > 0)
|
|
369
|
+
.sort((left, right) => right.score - left.score)
|
|
370
|
+
.slice(0, MEMORY_DELTA_MAX_CANDIDATES_PER_FACT)
|
|
371
|
+
|
|
372
|
+
for (const candidate of scoredForFact) {
|
|
373
|
+
const current = scoreById.get(candidate.id) ?? 0
|
|
374
|
+
if (candidate.score > current) {
|
|
375
|
+
scoreById.set(candidate.id, candidate.score)
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const targetCandidateCount = Math.min(MEMORY_DELTA_MAX_CANDIDATE_MEMORIES, normalizedExisting.length)
|
|
381
|
+
const baselineCount = Math.min(
|
|
382
|
+
targetCandidateCount,
|
|
383
|
+
Math.max(MEMORY_DELTA_MIN_BASELINE_CANDIDATES, newFacts.length * 4),
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
const selectedIds = new Set<string>(
|
|
387
|
+
[...scoreById.entries()]
|
|
388
|
+
.sort((left, right) => right[1] - left[1])
|
|
389
|
+
.slice(0, targetCandidateCount)
|
|
390
|
+
.map(([id]) => id),
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
if (selectedIds.size < baselineCount) {
|
|
394
|
+
for (const memory of normalizedExisting) {
|
|
395
|
+
selectedIds.add(memory.id)
|
|
396
|
+
if (selectedIds.size >= baselineCount) break
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
if (selectedIds.size === 0) {
|
|
401
|
+
for (const memory of normalizedExisting.slice(0, baselineCount)) {
|
|
402
|
+
selectedIds.add(memory.id)
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
const selected = normalizedExisting
|
|
407
|
+
.filter((memory) => selectedIds.has(memory.id))
|
|
408
|
+
.sort((left, right) => {
|
|
409
|
+
const scoreDiff = (scoreById.get(right.id) ?? 0) - (scoreById.get(left.id) ?? 0)
|
|
410
|
+
if (scoreDiff !== 0) return scoreDiff
|
|
411
|
+
return left.index - right.index
|
|
412
|
+
})
|
|
413
|
+
.slice(0, targetCandidateCount)
|
|
414
|
+
|
|
415
|
+
return selected.map((memory) => ({ id: memory.id, text: memory.text }))
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
private async applyUpdates(
|
|
419
|
+
updates: MemoryUpdateOutput,
|
|
420
|
+
options: AddOptions,
|
|
421
|
+
factMaps: ReturnType<typeof buildMemoryFactMaps>,
|
|
422
|
+
existingMemories: Array<{ id: string; text: string }>,
|
|
423
|
+
): Promise<void> {
|
|
424
|
+
const plan = createMemoryActionPlan({
|
|
425
|
+
updates,
|
|
426
|
+
memoryType: options.memoryType,
|
|
427
|
+
explicitImportance: options.importance,
|
|
428
|
+
confidenceByKey: factMaps.confidenceByKey,
|
|
429
|
+
durabilityByKey: factMaps.durabilityByKey,
|
|
430
|
+
categoryByKey: factMaps.categoryByKey,
|
|
431
|
+
existingMemories,
|
|
432
|
+
})
|
|
433
|
+
|
|
434
|
+
const idMap = new Map<string, string>()
|
|
435
|
+
|
|
436
|
+
for (const action of plan.actions) {
|
|
437
|
+
switch (action.type) {
|
|
438
|
+
case 'add': {
|
|
439
|
+
const truncatedContent = action.text.length > 50 ? `${action.text.slice(0, 50)}...` : action.text
|
|
440
|
+
const metadata = { ...options.metadata, memoryCategory: action.category }
|
|
441
|
+
const hash = hashContent(action.text, options.scopeId, options.memoryType)
|
|
442
|
+
let newId: string
|
|
443
|
+
|
|
444
|
+
try {
|
|
445
|
+
newId = await this.store.insert(
|
|
446
|
+
action.text,
|
|
447
|
+
options.scopeId,
|
|
448
|
+
options.memoryType,
|
|
449
|
+
metadata,
|
|
450
|
+
action.importance,
|
|
451
|
+
action.durability as Durability,
|
|
452
|
+
)
|
|
453
|
+
} catch (error) {
|
|
454
|
+
if (!isUniqueIndexConflict(error, 'memoryHashIdx')) {
|
|
455
|
+
throw error
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
const existing = await this.store.getByHash(hash)
|
|
459
|
+
if (!existing) {
|
|
460
|
+
throw error
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
newId = existing.id
|
|
464
|
+
aiLogger.debug`Skipped duplicate memory insert due to hash conflict: ${newId}`
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
idMap.set(action.refId, newId)
|
|
468
|
+
aiLogger.debug`Added new memory (memoryType: ${options.memoryType}, category: ${action.category}, durability: ${action.durability}, importance: ${action.importance.toFixed(2)}, content: ${truncatedContent})`
|
|
469
|
+
break
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
case 'update': {
|
|
473
|
+
const metadata = { ...options.metadata, ...(action.category ? { memoryCategory: action.category } : {}) }
|
|
474
|
+
await this.store.update(action.refId, action.text, {
|
|
475
|
+
importance: action.importance,
|
|
476
|
+
durability: action.durability as Durability | undefined,
|
|
477
|
+
metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
|
|
478
|
+
})
|
|
479
|
+
idMap.set(action.refId, action.refId)
|
|
480
|
+
aiLogger.debug`Updated memory ${action.refId}: ${action.text.slice(0, 50)}...`
|
|
481
|
+
break
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
case 'delete':
|
|
485
|
+
await this.store.delete(action.refId)
|
|
486
|
+
aiLogger.debug`Deleted memory ${action.refId}`
|
|
487
|
+
break
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
for (const relation of plan.relations) {
|
|
492
|
+
const fromId = idMap.get(relation.fromRefId)
|
|
493
|
+
if (!fromId) continue
|
|
494
|
+
|
|
495
|
+
const toId = idMap.get(relation.toRefId) ?? relation.toRefId
|
|
496
|
+
|
|
497
|
+
try {
|
|
498
|
+
await this.store.addRelation(fromId, toId, relation.relation)
|
|
499
|
+
aiLogger.debug`Created ${relation.relation} relation: ${fromId} -> ${toId}`
|
|
500
|
+
} catch (error) {
|
|
501
|
+
aiLogger.warn`Failed to create relation: ${error}`
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
}
|