ei-tui 0.1.25 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/package.json +2 -1
- package/src/README.md +4 -11
- package/src/cli/README.md +87 -7
- package/src/cli/commands/facts.ts +2 -2
- package/src/cli/commands/people.ts +2 -2
- package/src/cli/commands/quotes.ts +2 -2
- package/src/cli/commands/topics.ts +2 -2
- package/src/cli/mcp.ts +94 -0
- package/src/cli/retrieval.ts +67 -31
- package/src/cli.ts +64 -23
- package/src/core/AGENTS.md +1 -1
- package/src/core/constants/built-in-facts.ts +49 -0
- package/src/core/constants/index.ts +1 -0
- package/src/core/context-utils.ts +0 -1
- package/src/core/embedding-service.ts +8 -0
- package/src/core/handlers/dedup.ts +11 -23
- package/src/core/handlers/heartbeat.ts +2 -3
- package/src/core/handlers/human-extraction.ts +96 -30
- package/src/core/handlers/human-matching.ts +328 -248
- package/src/core/handlers/index.ts +8 -6
- package/src/core/handlers/persona-generation.ts +8 -8
- package/src/core/handlers/rewrite.ts +4 -51
- package/src/core/handlers/utils.ts +23 -1
- package/src/core/heartbeat-manager.ts +2 -4
- package/src/core/human-data-manager.ts +38 -36
- package/src/core/message-manager.ts +10 -10
- package/src/core/orchestrators/ceremony.ts +49 -44
- package/src/core/orchestrators/dedup-phase.ts +2 -4
- package/src/core/orchestrators/human-extraction.ts +351 -207
- package/src/core/orchestrators/index.ts +6 -4
- package/src/core/orchestrators/persona-generation.ts +3 -3
- package/src/core/processor.ts +167 -20
- package/src/core/prompt-context-builder.ts +4 -6
- package/src/core/state/human.ts +1 -26
- package/src/core/state/personas.ts +2 -2
- package/src/core/state-manager.ts +107 -14
- package/src/core/tools/builtin/read-memory.ts +13 -18
- package/src/core/types/data-items.ts +3 -4
- package/src/core/types/entities.ts +7 -4
- package/src/core/types/enums.ts +6 -9
- package/src/core/types/llm.ts +2 -2
- package/src/core/utils/crossFind.ts +2 -5
- package/src/core/utils/event-windows.ts +31 -0
- package/src/integrations/claude-code/importer.ts +14 -5
- package/src/integrations/claude-code/types.ts +3 -0
- package/src/integrations/cursor/importer.ts +282 -0
- package/src/integrations/cursor/index.ts +10 -0
- package/src/integrations/cursor/reader.ts +209 -0
- package/src/integrations/cursor/types.ts +140 -0
- package/src/integrations/opencode/importer.ts +14 -4
- package/src/prompts/AGENTS.md +73 -1
- package/src/prompts/ceremony/dedup.ts +0 -33
- package/src/prompts/ceremony/rewrite.ts +6 -41
- package/src/prompts/ceremony/types.ts +4 -4
- package/src/prompts/generation/descriptions.ts +2 -2
- package/src/prompts/generation/types.ts +2 -2
- package/src/prompts/heartbeat/types.ts +2 -2
- package/src/prompts/human/event-scan.ts +122 -0
- package/src/prompts/human/fact-find.ts +106 -0
- package/src/prompts/human/fact-scan.ts +0 -2
- package/src/prompts/human/index.ts +17 -10
- package/src/prompts/human/person-match.ts +65 -0
- package/src/prompts/human/person-scan.ts +52 -59
- package/src/prompts/human/person-update.ts +241 -0
- package/src/prompts/human/topic-match.ts +65 -0
- package/src/prompts/human/topic-scan.ts +51 -71
- package/src/prompts/human/topic-update.ts +295 -0
- package/src/prompts/human/types.ts +63 -40
- package/src/prompts/index.ts +4 -8
- package/src/prompts/persona/topics-update.ts +2 -2
- package/src/prompts/persona/traits.ts +2 -2
- package/src/prompts/persona/types.ts +3 -3
- package/src/prompts/response/index.ts +1 -1
- package/src/prompts/response/sections.ts +9 -12
- package/src/prompts/response/types.ts +2 -3
- package/src/storage/embeddings.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/indexed.ts +174 -0
- package/src/storage/merge.ts +67 -2
- package/tui/src/commands/me.tsx +5 -14
- package/tui/src/commands/settings.tsx +15 -0
- package/tui/src/context/ei.tsx +5 -14
- package/tui/src/util/yaml-serializers.ts +76 -33
- package/src/cli/commands/traits.ts +0 -25
- package/src/prompts/human/item-match.ts +0 -74
- package/src/prompts/human/item-update.ts +0 -364
- package/src/prompts/human/trait-scan.ts +0 -115
|
@@ -14,8 +14,8 @@ import {
|
|
|
14
14
|
handlePersonaTopicMatch,
|
|
15
15
|
handlePersonaTopicUpdate,
|
|
16
16
|
} from "./persona-topics.js";
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
17
|
+
import { handleFactFind, handleHumanTopicScan, handleHumanPersonScan, handleEventScan } from "./human-extraction.js";
|
|
18
|
+
import { handleTopicMatch, handleTopicUpdate, handlePersonMatch, handlePersonUpdate } from "./human-matching.js";
|
|
19
19
|
import { handleRewriteScan, handleRewriteRewrite } from "./rewrite.js";
|
|
20
20
|
import { handleDedupCurate } from "./dedup.js";
|
|
21
21
|
|
|
@@ -23,12 +23,13 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
|
|
|
23
23
|
handlePersonaResponse,
|
|
24
24
|
handlePersonaGeneration,
|
|
25
25
|
handlePersonaDescriptions,
|
|
26
|
-
|
|
27
|
-
handleHumanTraitScan,
|
|
26
|
+
handleFactFind,
|
|
28
27
|
handleHumanTopicScan,
|
|
29
28
|
handleHumanPersonScan,
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
handleTopicMatch,
|
|
30
|
+
handleTopicUpdate,
|
|
31
|
+
handlePersonMatch,
|
|
32
|
+
handlePersonUpdate,
|
|
32
33
|
handlePersonaTraitExtraction,
|
|
33
34
|
handlePersonaTopicScan,
|
|
34
35
|
handlePersonaTopicMatch,
|
|
@@ -43,4 +44,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
|
|
|
43
44
|
handleRewriteScan,
|
|
44
45
|
handleRewriteRewrite,
|
|
45
46
|
handleDedupCurate,
|
|
47
|
+
handleEventScan,
|
|
46
48
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
type LLMResponse,
|
|
3
|
-
type
|
|
3
|
+
type PersonaTrait,
|
|
4
4
|
type PersonaTopic,
|
|
5
5
|
} from "../types.js";
|
|
6
6
|
import type { StateManager } from "../state-manager.js";
|
|
@@ -27,10 +27,10 @@ export function handlePersonaGeneration(response: LLMResponse, state: StateManag
|
|
|
27
27
|
(existingPartial.traits ?? []).filter(t => t.name?.trim()).map(t => [t.name!.toLowerCase().trim(), t])
|
|
28
28
|
);
|
|
29
29
|
|
|
30
|
-
const mergedLlmTraits:
|
|
30
|
+
const mergedLlmTraits: PersonaTrait[] = (result?.traits || []).map(t => {
|
|
31
31
|
const userTrait = userTraitsByName.get(t.name?.toLowerCase().trim() ?? '');
|
|
32
32
|
return {
|
|
33
|
-
id: (userTrait as
|
|
33
|
+
id: (userTrait as PersonaTrait | undefined)?.id ?? crypto.randomUUID(),
|
|
34
34
|
name: t.name,
|
|
35
35
|
description: userTrait?.description?.trim() || t.description,
|
|
36
36
|
sentiment: userTrait?.sentiment ?? t.sentiment ?? 0,
|
|
@@ -41,10 +41,10 @@ export function handlePersonaGeneration(response: LLMResponse, state: StateManag
|
|
|
41
41
|
|
|
42
42
|
// Keep user-provided traits the LLM didn't return
|
|
43
43
|
const llmTraitNames = new Set(mergedLlmTraits.map(t => t.name?.toLowerCase().trim()));
|
|
44
|
-
const preservedUserTraits:
|
|
44
|
+
const preservedUserTraits: PersonaTrait[] = (existingPartial.traits ?? [])
|
|
45
45
|
.filter(t => t.name?.trim() && !llmTraitNames.has(t.name.toLowerCase().trim()))
|
|
46
46
|
.map(t => ({
|
|
47
|
-
id: (t as
|
|
47
|
+
id: (t as PersonaTrait).id ?? crypto.randomUUID(),
|
|
48
48
|
name: t.name!,
|
|
49
49
|
description: t.description || '',
|
|
50
50
|
sentiment: t.sentiment ?? 0,
|
|
@@ -52,9 +52,9 @@ export function handlePersonaGeneration(response: LLMResponse, state: StateManag
|
|
|
52
52
|
last_updated: now,
|
|
53
53
|
}));
|
|
54
54
|
|
|
55
|
-
const mergedTraits:
|
|
55
|
+
const mergedTraits: PersonaTrait[] = mergedLlmTraits.length > 0
|
|
56
56
|
? [...mergedLlmTraits, ...preservedUserTraits]
|
|
57
|
-
: (existingPartial.traits as
|
|
57
|
+
: (existingPartial.traits as PersonaTrait[] | undefined) ?? [];
|
|
58
58
|
|
|
59
59
|
// Merge LLM topics into user-provided topics by name.
|
|
60
60
|
// User-provided fields win; LLM fills in what the user left blank.
|
|
@@ -151,7 +151,7 @@ export function handlePersonaTraitExtraction(response: LLMResponse, state: State
|
|
|
151
151
|
}
|
|
152
152
|
|
|
153
153
|
const now = new Date().toISOString();
|
|
154
|
-
const traits:
|
|
154
|
+
const traits: PersonaTrait[] = result.map(t => ({
|
|
155
155
|
id: crypto.randomUUID(),
|
|
156
156
|
name: t.name,
|
|
157
157
|
description: t.description,
|
|
@@ -2,10 +2,7 @@ import {
|
|
|
2
2
|
LLMRequestType,
|
|
3
3
|
LLMPriority,
|
|
4
4
|
LLMNextStep,
|
|
5
|
-
ValidationLevel,
|
|
6
5
|
type LLMResponse,
|
|
7
|
-
type Fact,
|
|
8
|
-
type Trait,
|
|
9
6
|
type Topic,
|
|
10
7
|
type Person,
|
|
11
8
|
type DataItemBase,
|
|
@@ -48,7 +45,7 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
|
|
|
48
45
|
// Re-read the item from current state (it may have changed since scan was queued)
|
|
49
46
|
const human = state.getHuman();
|
|
50
47
|
const allItems: DataItemBase[] = [
|
|
51
|
-
...human.
|
|
48
|
+
...human.topics, ...human.people,
|
|
52
49
|
];
|
|
53
50
|
const currentItem = allItems.find(i => i.id === itemId);
|
|
54
51
|
if (!currentItem) {
|
|
@@ -61,11 +58,11 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
|
|
|
61
58
|
for (const searchTerm of subjects) {
|
|
62
59
|
try {
|
|
63
60
|
const results = await searchHumanData(state, searchTerm, {
|
|
64
|
-
types: ["
|
|
61
|
+
types: ["topic", "person"],
|
|
65
62
|
limit: 4, // fetch 4 so we can exclude original and still have 3
|
|
66
63
|
});
|
|
67
64
|
const allMatches: DataItemBase[] = [
|
|
68
|
-
...results.facts, ...results.
|
|
65
|
+
...results.facts, ...results.topics, ...results.people,
|
|
69
66
|
].filter(m => m.id !== itemId); // exclude original
|
|
70
67
|
subjectMatches.push({ searchTerm, matches: allMatches.slice(0, 3) });
|
|
71
68
|
} catch (err) {
|
|
@@ -122,15 +119,13 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
122
119
|
|
|
123
120
|
// Look up the original item to inherit persona_groups
|
|
124
121
|
const allItems: DataItemBase[] = [
|
|
125
|
-
...human.
|
|
122
|
+
...human.topics, ...human.people,
|
|
126
123
|
];
|
|
127
124
|
const originalItem = allItems.find(i => i.id === itemId);
|
|
128
125
|
const inheritedGroups = originalItem?.persona_groups;
|
|
129
126
|
|
|
130
127
|
// Helper: resolve actual type from existing records (don't trust LLM's type field)
|
|
131
128
|
const resolveExistingType = (id: string): RewriteItemType | null => {
|
|
132
|
-
if (human.facts.find(f => f.id === id)) return "fact";
|
|
133
|
-
if (human.traits.find(t => t.id === id)) return "trait";
|
|
134
129
|
if (human.topics.find(t => t.id === id)) return "topic";
|
|
135
130
|
if (human.people.find(p => p.id === id)) return "person";
|
|
136
131
|
return null;
|
|
@@ -163,31 +158,6 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
163
158
|
}
|
|
164
159
|
|
|
165
160
|
switch (resolvedType) {
|
|
166
|
-
case "fact": {
|
|
167
|
-
const existing = human.facts.find(f => f.id === item.id)!;
|
|
168
|
-
state.human_fact_upsert({
|
|
169
|
-
...existing,
|
|
170
|
-
name: item.name,
|
|
171
|
-
description: item.description,
|
|
172
|
-
sentiment: item.sentiment ?? existing.sentiment,
|
|
173
|
-
last_updated: now,
|
|
174
|
-
embedding,
|
|
175
|
-
});
|
|
176
|
-
break;
|
|
177
|
-
}
|
|
178
|
-
case "trait": {
|
|
179
|
-
const existing = human.traits.find(t => t.id === item.id)!;
|
|
180
|
-
state.human_trait_upsert({
|
|
181
|
-
...existing,
|
|
182
|
-
name: item.name,
|
|
183
|
-
description: item.description,
|
|
184
|
-
sentiment: item.sentiment ?? existing.sentiment,
|
|
185
|
-
strength: item.strength ?? existing.strength,
|
|
186
|
-
last_updated: now,
|
|
187
|
-
embedding,
|
|
188
|
-
});
|
|
189
|
-
break;
|
|
190
|
-
}
|
|
191
161
|
case "topic": {
|
|
192
162
|
const existing = human.topics.find(t => t.id === item.id)!;
|
|
193
163
|
state.human_topic_upsert({
|
|
@@ -244,23 +214,6 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
244
214
|
};
|
|
245
215
|
|
|
246
216
|
switch (item.type) {
|
|
247
|
-
case "fact": {
|
|
248
|
-
const fact: Fact = {
|
|
249
|
-
...baseFields,
|
|
250
|
-
validated: ValidationLevel.None,
|
|
251
|
-
validated_date: now,
|
|
252
|
-
};
|
|
253
|
-
state.human_fact_upsert(fact);
|
|
254
|
-
break;
|
|
255
|
-
}
|
|
256
|
-
case "trait": {
|
|
257
|
-
const trait: Trait = {
|
|
258
|
-
...baseFields,
|
|
259
|
-
strength: item.strength ?? 0.5,
|
|
260
|
-
};
|
|
261
|
-
state.human_trait_upsert(trait);
|
|
262
|
-
break;
|
|
263
|
-
}
|
|
264
217
|
case "topic": {
|
|
265
218
|
if (!item.category) {
|
|
266
219
|
console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category — defaulting to "Interest"`);
|
|
@@ -1,7 +1,29 @@
|
|
|
1
1
|
import type { Message, LLMResponse } from "../types.js";
|
|
2
2
|
import type { StateManager } from "../state-manager.js";
|
|
3
3
|
|
|
4
|
-
export
|
|
4
|
+
export function resolveMessageWindow(
|
|
5
|
+
response: LLMResponse,
|
|
6
|
+
state: StateManager
|
|
7
|
+
): { messages_context: Message[]; messages_analyze: Message[] } {
|
|
8
|
+
const personaId = response.request.data.personaId as string;
|
|
9
|
+
const messageIdsToMark = response.request.data.message_ids_to_mark as string[] | undefined;
|
|
10
|
+
const allMessages = state.messages_get(personaId);
|
|
11
|
+
|
|
12
|
+
if (messageIdsToMark && messageIdsToMark.length > 0) {
|
|
13
|
+
const messageIdSet = new Set(messageIdsToMark);
|
|
14
|
+
const messages_analyze = allMessages.filter(m => messageIdSet.has(m.id));
|
|
15
|
+
const analyzeStartTime = messages_analyze[0]?.timestamp ?? '9999';
|
|
16
|
+
const messages_context = allMessages.filter(m =>
|
|
17
|
+
!messageIdSet.has(m.id) && new Date(m.timestamp).getTime() < new Date(analyzeStartTime).getTime()
|
|
18
|
+
);
|
|
19
|
+
return { messages_context, messages_analyze };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const analyzeFrom = response.request.data.analyze_from_timestamp as string | null;
|
|
23
|
+
return splitMessagesByTimestamp(allMessages, analyzeFrom);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type ExtractionFlag = "f" | "t" | "p" | "e";
|
|
5
27
|
|
|
6
28
|
export function splitMessagesByTimestamp(
|
|
7
29
|
messages: Message[],
|
|
@@ -2,7 +2,6 @@ import {
|
|
|
2
2
|
LLMRequestType,
|
|
3
3
|
LLMPriority,
|
|
4
4
|
LLMNextStep,
|
|
5
|
-
ValidationLevel,
|
|
6
5
|
type HumanEntity,
|
|
7
6
|
type Message,
|
|
8
7
|
} from "./types.js";
|
|
@@ -72,9 +71,8 @@ export async function queueEiHeartbeat(
|
|
|
72
71
|
const unverifiedFacts = human.facts
|
|
73
72
|
.filter(
|
|
74
73
|
(f) =>
|
|
75
|
-
f.
|
|
76
|
-
f.
|
|
77
|
-
(f.last_changed_by === undefined || f.last_changed_by !== "ei")
|
|
74
|
+
f.description !== '' &&
|
|
75
|
+
f.validated_date === ''
|
|
78
76
|
)
|
|
79
77
|
.slice(0, 5);
|
|
80
78
|
for (const fact of unverifiedFacts) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { HumanEntity, Fact,
|
|
1
|
+
import type { HumanEntity, Fact, Topic, Person, Quote } from "./types.js";
|
|
2
2
|
import { StateManager } from "./state-manager.js";
|
|
3
3
|
import {
|
|
4
4
|
getEmbeddingService,
|
|
@@ -24,7 +24,7 @@ export async function updateHuman(sm: StateManager, updates: Partial<HumanEntity
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
// =============================================================================
|
|
27
|
-
// FACTS /
|
|
27
|
+
// FACTS / TOPICS / PEOPLE UPSERT
|
|
28
28
|
// =============================================================================
|
|
29
29
|
|
|
30
30
|
export async function upsertFact(sm: StateManager, fact: Fact): Promise<void> {
|
|
@@ -40,18 +40,6 @@ export async function upsertFact(sm: StateManager, fact: Fact): Promise<void> {
|
|
|
40
40
|
sm.human_fact_upsert(fact);
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
-
export async function upsertTrait(sm: StateManager, trait: Trait): Promise<void> {
|
|
44
|
-
const human = sm.getHuman();
|
|
45
|
-
const existing = human.traits.find((t) => t.id === trait.id);
|
|
46
|
-
|
|
47
|
-
if (needsEmbeddingUpdate(existing, trait)) {
|
|
48
|
-
trait.embedding = await computeDataItemEmbedding(trait);
|
|
49
|
-
} else if (existing?.embedding) {
|
|
50
|
-
trait.embedding = existing.embedding;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
sm.human_trait_upsert(trait);
|
|
54
|
-
}
|
|
55
43
|
|
|
56
44
|
export async function upsertTopic(sm: StateManager, topic: Topic): Promise<void> {
|
|
57
45
|
const human = sm.getHuman();
|
|
@@ -81,16 +69,13 @@ export async function upsertPerson(sm: StateManager, person: Person): Promise<vo
|
|
|
81
69
|
|
|
82
70
|
export async function removeDataItem(
|
|
83
71
|
sm: StateManager,
|
|
84
|
-
type: "fact" | "
|
|
72
|
+
type: "fact" | "topic" | "person",
|
|
85
73
|
id: string
|
|
86
74
|
): Promise<void> {
|
|
87
75
|
switch (type) {
|
|
88
76
|
case "fact":
|
|
89
77
|
sm.human_fact_remove(id);
|
|
90
78
|
break;
|
|
91
|
-
case "trait":
|
|
92
|
-
sm.human_trait_remove(id);
|
|
93
|
-
break;
|
|
94
79
|
case "topic":
|
|
95
80
|
sm.human_topic_remove(id);
|
|
96
81
|
break;
|
|
@@ -160,50 +145,72 @@ export async function getQuotesForMessage(sm: StateManager, messageId: string):
|
|
|
160
145
|
export async function searchHumanData(
|
|
161
146
|
sm: StateManager,
|
|
162
147
|
query: string,
|
|
163
|
-
options: { types?: Array<"fact" | "
|
|
148
|
+
options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number; recent?: boolean } = {}
|
|
164
149
|
): Promise<{
|
|
165
150
|
facts: Fact[];
|
|
166
|
-
traits: Trait[];
|
|
167
151
|
topics: Topic[];
|
|
168
152
|
people: Person[];
|
|
169
153
|
quotes: Quote[];
|
|
170
154
|
}> {
|
|
171
|
-
const { types = ["fact", "
|
|
155
|
+
const { types = ["fact", "topic", "person", "quote"], limit = 10, recent } = options;
|
|
172
156
|
const human = sm.getHuman();
|
|
173
157
|
const SIMILARITY_THRESHOLD = 0.3;
|
|
174
158
|
|
|
175
159
|
const result = {
|
|
176
160
|
facts: [] as Fact[],
|
|
177
|
-
traits: [] as Trait[],
|
|
178
161
|
topics: [] as Topic[],
|
|
179
162
|
people: [] as Person[],
|
|
180
163
|
quotes: [] as Quote[],
|
|
181
164
|
};
|
|
182
165
|
|
|
166
|
+
const recentSort = <T extends { last_updated?: string; last_mentioned?: string }>(items: T[]): T[] =>
|
|
167
|
+
[...items].sort((a, b) => {
|
|
168
|
+
const aDate = a.last_mentioned ?? a.last_updated ?? "";
|
|
169
|
+
const bDate = b.last_mentioned ?? b.last_updated ?? "";
|
|
170
|
+
return bDate.localeCompare(aDate);
|
|
171
|
+
});
|
|
172
|
+
|
|
183
173
|
let queryVector: number[] | null = null;
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
174
|
+
if (query) {
|
|
175
|
+
try {
|
|
176
|
+
const embeddingService = getEmbeddingService();
|
|
177
|
+
queryVector = await embeddingService.embed(query);
|
|
178
|
+
} catch (err) {
|
|
179
|
+
console.warn("[searchHumanData] Failed to generate query embedding:", err);
|
|
180
|
+
}
|
|
189
181
|
}
|
|
190
182
|
|
|
191
|
-
const searchItems = <T extends { id: string; embedding?: number[] }>(
|
|
183
|
+
const searchItems = <T extends { id: string; embedding?: number[]; last_updated?: string; last_mentioned?: string }>(
|
|
192
184
|
items: T[],
|
|
193
185
|
textExtractor: (item: T) => string
|
|
194
186
|
): T[] => {
|
|
187
|
+
if (recent && !query) {
|
|
188
|
+
return recentSort(items).slice(0, limit);
|
|
189
|
+
}
|
|
190
|
+
|
|
195
191
|
const withEmbeddings = items.filter((i) => i.embedding?.length);
|
|
196
192
|
|
|
197
193
|
if (queryVector && withEmbeddings.length > 0) {
|
|
198
|
-
|
|
194
|
+
const topK = recent ? Math.max(limit * 5, 50) : limit;
|
|
195
|
+
const found = findTopK(queryVector, withEmbeddings, topK)
|
|
199
196
|
.filter(({ similarity }) => similarity >= SIMILARITY_THRESHOLD)
|
|
200
197
|
.map(({ item }) => item);
|
|
198
|
+
if (recent) {
|
|
199
|
+
return recentSort(found).slice(0, limit);
|
|
200
|
+
}
|
|
201
|
+
return found;
|
|
201
202
|
}
|
|
202
203
|
|
|
204
|
+
if (!query) return [];
|
|
205
|
+
|
|
203
206
|
const lowerQuery = query.toLowerCase();
|
|
204
|
-
|
|
207
|
+
const found = items
|
|
205
208
|
.filter((i) => textExtractor(i).toLowerCase().includes(lowerQuery))
|
|
206
|
-
.slice(0, limit);
|
|
209
|
+
.slice(0, recent ? Math.max(limit * 5, 50) : limit);
|
|
210
|
+
if (recent) {
|
|
211
|
+
return recentSort(found).slice(0, limit);
|
|
212
|
+
}
|
|
213
|
+
return found;
|
|
207
214
|
};
|
|
208
215
|
|
|
209
216
|
if (types.includes("fact")) {
|
|
@@ -211,11 +218,6 @@ export async function searchHumanData(
|
|
|
211
218
|
stripDataItemEmbedding
|
|
212
219
|
);
|
|
213
220
|
}
|
|
214
|
-
if (types.includes("trait")) {
|
|
215
|
-
result.traits = searchItems(human.traits, (t) => `${t.name} ${t.description || ""}`).map(
|
|
216
|
-
stripDataItemEmbedding
|
|
217
|
-
);
|
|
218
|
-
}
|
|
219
221
|
if (types.includes("topic")) {
|
|
220
222
|
result.topics = searchItems(human.topics, (t) => `${t.name} ${t.description || ""}`).map(
|
|
221
223
|
stripDataItemEmbedding
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
} from "../prompts/index.js";
|
|
17
17
|
import { buildResponsePromptData } from "./prompt-context-builder.js";
|
|
18
18
|
import {
|
|
19
|
-
|
|
19
|
+
queueFactFind,
|
|
20
20
|
queueTopicScan,
|
|
21
21
|
queuePersonScan,
|
|
22
22
|
type ExtractionContext,
|
|
@@ -178,7 +178,7 @@ export async function sendMessage(
|
|
|
178
178
|
const traitExtractionData: PersonaTraitExtractionPromptData = {
|
|
179
179
|
persona_name: persona.display_name,
|
|
180
180
|
current_traits: persona.traits,
|
|
181
|
-
messages_context: history.slice(
|
|
181
|
+
messages_context: history.slice(-11, -1),
|
|
182
182
|
messages_analyze: [message],
|
|
183
183
|
};
|
|
184
184
|
const traitPrompt = buildPersonaTraitExtractionPrompt(traitExtractionData);
|
|
@@ -217,7 +217,7 @@ export function checkAndQueueHumanExtraction(
|
|
|
217
217
|
const human = sm.getHuman();
|
|
218
218
|
|
|
219
219
|
const unextractedFacts = sm.messages_getUnextracted(personaId, "f");
|
|
220
|
-
const factsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.facts.length);
|
|
220
|
+
const factsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.facts.filter(f => f.description && f.description !== "").length);
|
|
221
221
|
if (unextractedFacts.length > 0 && unextractedFacts.length >= factsThreshold) {
|
|
222
222
|
const context: ExtractionContext = {
|
|
223
223
|
personaId,
|
|
@@ -226,21 +226,21 @@ export function checkAndQueueHumanExtraction(
|
|
|
226
226
|
messages_analyze: unextractedFacts,
|
|
227
227
|
extraction_flag: "f",
|
|
228
228
|
};
|
|
229
|
-
|
|
229
|
+
queueFactFind(context, sm);
|
|
230
230
|
console.log(
|
|
231
231
|
`[Processor] Human Seed extraction: facts (threshold: ${factsThreshold}, unextracted: ${unextractedFacts.length})`
|
|
232
232
|
);
|
|
233
233
|
}
|
|
234
234
|
|
|
235
|
-
const unextractedTopics = sm.messages_getUnextracted(personaId, "
|
|
235
|
+
const unextractedTopics = sm.messages_getUnextracted(personaId, "t");
|
|
236
236
|
const topicsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.topics.length);
|
|
237
237
|
if (unextractedTopics.length > 0 && unextractedTopics.length >= topicsThreshold) {
|
|
238
238
|
const context: ExtractionContext = {
|
|
239
239
|
personaId,
|
|
240
240
|
personaDisplayName,
|
|
241
|
-
messages_context: history.filter((m) => m.
|
|
241
|
+
messages_context: history.filter((m) => m.t === true),
|
|
242
242
|
messages_analyze: unextractedTopics,
|
|
243
|
-
extraction_flag: "
|
|
243
|
+
extraction_flag: "t",
|
|
244
244
|
};
|
|
245
245
|
queueTopicScan(context, sm);
|
|
246
246
|
console.log(
|
|
@@ -248,15 +248,15 @@ export function checkAndQueueHumanExtraction(
|
|
|
248
248
|
);
|
|
249
249
|
}
|
|
250
250
|
|
|
251
|
-
const unextractedPeople = sm.messages_getUnextracted(personaId, "
|
|
251
|
+
const unextractedPeople = sm.messages_getUnextracted(personaId, "p");
|
|
252
252
|
const peopleThreshold = Math.min(EXTRACTION_TAPER_CAP, human.people.length);
|
|
253
253
|
if (unextractedPeople.length > 0 && unextractedPeople.length >= peopleThreshold) {
|
|
254
254
|
const context: ExtractionContext = {
|
|
255
255
|
personaId,
|
|
256
256
|
personaDisplayName,
|
|
257
|
-
messages_context: history.filter((m) => m.
|
|
257
|
+
messages_context: history.filter((m) => m.p === true),
|
|
258
258
|
messages_analyze: unextractedPeople,
|
|
259
|
-
extraction_flag: "
|
|
259
|
+
extraction_flag: "p",
|
|
260
260
|
};
|
|
261
261
|
queuePersonScan(context, sm);
|
|
262
262
|
console.log(
|
|
@@ -2,10 +2,10 @@ import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MA
|
|
|
2
2
|
import type { StateManager } from "../state-manager.js";
|
|
3
3
|
import { applyDecayToValue } from "../utils/index.js";
|
|
4
4
|
import {
|
|
5
|
-
|
|
6
|
-
queueTraitScan,
|
|
5
|
+
queueFactFind,
|
|
7
6
|
queueTopicScan,
|
|
8
7
|
queuePersonScan,
|
|
8
|
+
queueEventSummary,
|
|
9
9
|
type ExtractionContext,
|
|
10
10
|
type ExtractionOptions,
|
|
11
11
|
} from "./human-extraction.js";
|
|
@@ -109,56 +109,45 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
|
|
|
109
109
|
messages_analyze: unextractedFacts,
|
|
110
110
|
extraction_flag: "f",
|
|
111
111
|
};
|
|
112
|
-
|
|
112
|
+
queueFactFind(context, state, options);
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
-
const unextractedTraits = state.messages_getUnextracted(personaId, "r");
|
|
116
|
-
if (unextractedTraits.length > 0) {
|
|
117
|
-
const context: ExtractionContext = {
|
|
118
|
-
personaId,
|
|
119
|
-
personaDisplayName: persona.display_name,
|
|
120
|
-
messages_context: allMessages.filter(m => m.r === true),
|
|
121
|
-
messages_analyze: unextractedTraits,
|
|
122
|
-
extraction_flag: "r",
|
|
123
|
-
};
|
|
124
|
-
queueTraitScan(context, state, options);
|
|
125
|
-
}
|
|
126
115
|
|
|
127
|
-
const unextractedTopics = state.messages_getUnextracted(personaId, "
|
|
116
|
+
const unextractedTopics = state.messages_getUnextracted(personaId, "t");
|
|
128
117
|
if (unextractedTopics.length > 0) {
|
|
129
118
|
const context: ExtractionContext = {
|
|
130
119
|
personaId,
|
|
131
120
|
personaDisplayName: persona.display_name,
|
|
132
|
-
messages_context: allMessages.filter(m => m.
|
|
121
|
+
messages_context: allMessages.filter(m => m.t === true),
|
|
133
122
|
messages_analyze: unextractedTopics,
|
|
134
|
-
extraction_flag: "
|
|
123
|
+
extraction_flag: "t",
|
|
135
124
|
};
|
|
136
125
|
queueTopicScan(context, state, options);
|
|
137
126
|
}
|
|
138
127
|
|
|
139
|
-
const unextractedPeople = state.messages_getUnextracted(personaId, "
|
|
128
|
+
const unextractedPeople = state.messages_getUnextracted(personaId, "p");
|
|
140
129
|
if (unextractedPeople.length > 0) {
|
|
141
130
|
const context: ExtractionContext = {
|
|
142
131
|
personaId,
|
|
143
132
|
personaDisplayName: persona.display_name,
|
|
144
|
-
messages_context: allMessages.filter(m => m.
|
|
133
|
+
messages_context: allMessages.filter(m => m.p === true),
|
|
145
134
|
messages_analyze: unextractedPeople,
|
|
146
|
-
extraction_flag: "
|
|
135
|
+
extraction_flag: "p",
|
|
147
136
|
};
|
|
148
137
|
queuePersonScan(context, state, options);
|
|
149
138
|
}
|
|
150
139
|
|
|
151
|
-
const totalUnextracted = unextractedFacts.length +
|
|
140
|
+
const totalUnextracted = unextractedFacts.length + unextractedTopics.length + unextractedPeople.length;
|
|
152
141
|
if (totalUnextracted > 0) {
|
|
153
|
-
console.log(`[ceremony:exposure] Queued human extraction scans (f:${unextractedFacts.length},
|
|
142
|
+
console.log(`[ceremony:exposure] Queued human extraction scans (f:${unextractedFacts.length}, t:${unextractedTopics.length}, p:${unextractedPeople.length})`);
|
|
154
143
|
}
|
|
155
144
|
|
|
156
|
-
const unextractedForPersonaTopics = state.messages_getUnextracted(personaId, "
|
|
145
|
+
const unextractedForPersonaTopics = state.messages_getUnextracted(personaId, "t");
|
|
157
146
|
if (unextractedForPersonaTopics.length > 0) {
|
|
158
147
|
const personaTopicContext: PersonaTopicContext = {
|
|
159
148
|
personaId,
|
|
160
149
|
personaDisplayName: persona.display_name,
|
|
161
|
-
messages_context: allMessages.filter(m => m.
|
|
150
|
+
messages_context: allMessages.filter(m => m.t === true),
|
|
162
151
|
messages_analyze: unextractedForPersonaTopics,
|
|
163
152
|
};
|
|
164
153
|
queuePersonaTopicScan(personaTopicContext, state);
|
|
@@ -171,7 +160,7 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
|
|
|
171
160
|
* AND at the end of startCeremony (for the zero-messages edge case).
|
|
172
161
|
*
|
|
173
162
|
* If any ceremony_progress items remain in the queue, does nothing — more work pending.
|
|
174
|
-
*
|
|
163
|
+
* Phase 1: Dedup → Phase 2: Expose → Phase 3: EventSummary → Decay → Expire
|
|
175
164
|
*/
|
|
176
165
|
export function handleCeremonyProgress(state: StateManager, lastPhase: number): void {
|
|
177
166
|
if (state.queue_hasPendingCeremonies()) {
|
|
@@ -193,9 +182,8 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
|
|
|
193
182
|
const personasWithUnprocessed = activePersonas.filter(p => {
|
|
194
183
|
const messages = state.messages_get(p.id);
|
|
195
184
|
return messages.some(msg =>
|
|
185
|
+
!msg.t ||
|
|
196
186
|
!msg.p ||
|
|
197
|
-
!msg.r ||
|
|
198
|
-
!msg.o ||
|
|
199
187
|
!msg.f
|
|
200
188
|
);
|
|
201
189
|
});
|
|
@@ -208,9 +196,22 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
|
|
|
208
196
|
}
|
|
209
197
|
return;
|
|
210
198
|
}
|
|
199
|
+
|
|
200
|
+
if (lastPhase === 2) {
|
|
201
|
+
console.log("[ceremony:progress] Expose complete, starting EventSummary phase");
|
|
202
|
+
const options: ExtractionOptions = { ceremony_progress: 3 };
|
|
203
|
+
queueEventSummaryForAll(state, options);
|
|
204
|
+
|
|
205
|
+
// Zero-work guard: same pattern as DeDupe phase
|
|
206
|
+
if (!state.queue_hasPendingCeremonies()) {
|
|
207
|
+
console.log("[ceremony:progress] No event summary work, advancing to Decay");
|
|
208
|
+
handleCeremonyProgress(state, 3);
|
|
209
|
+
}
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
211
212
|
|
|
212
|
-
// Phase
|
|
213
|
-
console.log("[ceremony:progress]
|
|
213
|
+
// Phase 3 (EventSummary) complete → advance to Decay/Prune/Expire/Explore
|
|
214
|
+
console.log("[ceremony:progress] EventSummary complete, advancing to Decay");
|
|
214
215
|
|
|
215
216
|
const personas = state.persona_getAll();
|
|
216
217
|
const activePersonas = personas.filter(p =>
|
|
@@ -236,7 +237,7 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
|
|
|
236
237
|
// Human ceremony: decay topics + people
|
|
237
238
|
runHumanCeremony(state);
|
|
238
239
|
|
|
239
|
-
// Dedup phase: log near-duplicate human
|
|
240
|
+
// Dedup phase: log near-duplicate human entity candidates for visibility
|
|
240
241
|
runDedupPhase(state);
|
|
241
242
|
|
|
242
243
|
// Rewrite phase: fire-and-forget scans for bloated human data items
|
|
@@ -320,7 +321,7 @@ export function prunePersonaMessages(personaId: string, state: StateManager): vo
|
|
|
320
321
|
const msgMs = new Date(m.timestamp).getTime();
|
|
321
322
|
if (msgMs >= cutoffMs) break; // Sorted by time, no more old ones
|
|
322
323
|
|
|
323
|
-
const fullyExtracted = m.
|
|
324
|
+
const fullyExtracted = m.t && m.p && m.f; // r intentionally excluded — trait extraction deprecated
|
|
324
325
|
if (fullyExtracted) {
|
|
325
326
|
toRemove.push(m.id);
|
|
326
327
|
}
|
|
@@ -518,7 +519,7 @@ export function runHumanCeremony(state: StateManager): void {
|
|
|
518
519
|
}
|
|
519
520
|
|
|
520
521
|
// =============================================================================
|
|
521
|
-
// DEDUP PHASE (synchronous,
|
|
522
|
+
// DEDUP PHASE (synchronous, logging only — candidates are queued for curation by dedup-phase.ts)
|
|
522
523
|
// =============================================================================
|
|
523
524
|
|
|
524
525
|
const DEDUP_DEFAULT_THRESHOLD = 0.85;
|
|
@@ -558,11 +559,10 @@ export function runDedupPhase(state: StateManager): void {
|
|
|
558
559
|
const human = state.getHuman();
|
|
559
560
|
const threshold = human.settings?.ceremony?.dedup_threshold ?? DEDUP_DEFAULT_THRESHOLD;
|
|
560
561
|
|
|
561
|
-
console.log(`[ceremony:dedup]
|
|
562
|
+
console.log(`[ceremony:dedup] Scanning for dedup candidates (threshold: ${threshold})`);
|
|
562
563
|
|
|
563
564
|
const types: Array<{ label: string; items: DedupableItem[] }> = [
|
|
564
565
|
{ label: "facts", items: human.facts },
|
|
565
|
-
{ label: "traits", items: human.traits },
|
|
566
566
|
{ label: "topics", items: human.topics },
|
|
567
567
|
{ label: "people", items: human.people },
|
|
568
568
|
];
|
|
@@ -616,16 +616,6 @@ export function queueRewritePhase(state: StateManager): void {
|
|
|
616
616
|
|
|
617
617
|
const itemsToScan: Array<{ item: DataItemBase; type: RewriteItemType }> = [];
|
|
618
618
|
|
|
619
|
-
for (const fact of human.facts) {
|
|
620
|
-
if ((fact.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
|
|
621
|
-
itemsToScan.push({ item: fact, type: "fact" });
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
for (const trait of human.traits) {
|
|
625
|
-
if ((trait.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
|
|
626
|
-
itemsToScan.push({ item: trait, type: "trait" });
|
|
627
|
-
}
|
|
628
|
-
}
|
|
629
619
|
for (const topic of human.topics) {
|
|
630
620
|
if ((topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
|
|
631
621
|
itemsToScan.push({ item: topic, type: "topic" });
|
|
@@ -664,3 +654,18 @@ export function queueRewritePhase(state: StateManager): void {
|
|
|
664
654
|
|
|
665
655
|
console.log(`[ceremony:rewrite] Queued ${itemsToScan.length} Phase 1 scan(s) at Low priority`);
|
|
666
656
|
}
|
|
657
|
+
|
|
658
|
+
function queueEventSummaryForAll(state: StateManager, options?: ExtractionOptions): void {
|
|
659
|
+
const personas = state.persona_getAll();
|
|
660
|
+
const activePersonas = personas.filter(p =>
|
|
661
|
+
!p.is_paused &&
|
|
662
|
+
!p.is_archived &&
|
|
663
|
+
!p.is_static
|
|
664
|
+
);
|
|
665
|
+
|
|
666
|
+
let totalQueued = 0;
|
|
667
|
+
for (const persona of activePersonas) {
|
|
668
|
+
totalQueued += queueEventSummary(persona.id, state, options);
|
|
669
|
+
}
|
|
670
|
+
console.log(`[ceremony:event] Queued event summary scans for ${activePersonas.length} personas (${totalQueued} total chunks)`);
|
|
671
|
+
}
|