ei-tui 0.1.25 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/package.json +1 -1
- package/src/README.md +4 -11
- package/src/cli/README.md +4 -5
- package/src/cli/retrieval.ts +3 -25
- package/src/cli.ts +3 -7
- package/src/core/AGENTS.md +1 -1
- package/src/core/constants/built-in-facts.ts +49 -0
- package/src/core/constants/index.ts +1 -0
- package/src/core/context-utils.ts +0 -1
- package/src/core/embedding-service.ts +8 -0
- package/src/core/handlers/dedup.ts +10 -16
- package/src/core/handlers/heartbeat.ts +2 -3
- package/src/core/handlers/human-extraction.ts +95 -30
- package/src/core/handlers/human-matching.ts +326 -248
- package/src/core/handlers/index.ts +8 -6
- package/src/core/handlers/persona-generation.ts +8 -8
- package/src/core/handlers/rewrite.ts +4 -29
- package/src/core/handlers/utils.ts +23 -1
- package/src/core/heartbeat-manager.ts +2 -4
- package/src/core/human-data-manager.ts +5 -27
- package/src/core/message-manager.ts +10 -10
- package/src/core/orchestrators/ceremony.ts +50 -39
- package/src/core/orchestrators/dedup-phase.ts +0 -1
- package/src/core/orchestrators/human-extraction.ts +351 -207
- package/src/core/orchestrators/index.ts +6 -4
- package/src/core/orchestrators/persona-generation.ts +3 -3
- package/src/core/processor.ts +99 -17
- package/src/core/prompt-context-builder.ts +4 -6
- package/src/core/state/human.ts +1 -26
- package/src/core/state/personas.ts +2 -2
- package/src/core/state-manager.ts +107 -14
- package/src/core/tools/builtin/read-memory.ts +7 -8
- package/src/core/types/data-items.ts +2 -4
- package/src/core/types/entities.ts +6 -4
- package/src/core/types/enums.ts +6 -9
- package/src/core/types/llm.ts +2 -2
- package/src/core/utils/crossFind.ts +2 -5
- package/src/core/utils/event-windows.ts +31 -0
- package/src/integrations/claude-code/importer.ts +8 -4
- package/src/integrations/claude-code/types.ts +2 -0
- package/src/integrations/opencode/importer.ts +7 -3
- package/src/prompts/AGENTS.md +73 -1
- package/src/prompts/ceremony/rewrite.ts +3 -22
- package/src/prompts/ceremony/types.ts +3 -3
- package/src/prompts/generation/descriptions.ts +2 -2
- package/src/prompts/generation/types.ts +2 -2
- package/src/prompts/heartbeat/types.ts +2 -2
- package/src/prompts/human/event-scan.ts +122 -0
- package/src/prompts/human/fact-find.ts +106 -0
- package/src/prompts/human/fact-scan.ts +0 -2
- package/src/prompts/human/index.ts +17 -10
- package/src/prompts/human/person-match.ts +65 -0
- package/src/prompts/human/person-scan.ts +52 -59
- package/src/prompts/human/person-update.ts +241 -0
- package/src/prompts/human/topic-match.ts +65 -0
- package/src/prompts/human/topic-scan.ts +51 -71
- package/src/prompts/human/topic-update.ts +295 -0
- package/src/prompts/human/types.ts +63 -40
- package/src/prompts/index.ts +4 -8
- package/src/prompts/persona/topics-update.ts +2 -2
- package/src/prompts/persona/traits.ts +2 -2
- package/src/prompts/persona/types.ts +3 -3
- package/src/prompts/response/index.ts +1 -1
- package/src/prompts/response/sections.ts +9 -12
- package/src/prompts/response/types.ts +2 -3
- package/src/storage/embeddings.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/indexed.ts +174 -0
- package/src/storage/merge.ts +67 -2
- package/tui/src/commands/me.tsx +5 -14
- package/tui/src/commands/settings.tsx +15 -0
- package/tui/src/context/ei.tsx +5 -14
- package/tui/src/util/yaml-serializers.ts +48 -33
- package/src/cli/commands/traits.ts +0 -25
- package/src/prompts/human/item-match.ts +0 -74
- package/src/prompts/human/item-update.ts +0 -364
- package/src/prompts/human/trait-scan.ts +0 -115
|
@@ -14,8 +14,8 @@ import {
|
|
|
14
14
|
handlePersonaTopicMatch,
|
|
15
15
|
handlePersonaTopicUpdate,
|
|
16
16
|
} from "./persona-topics.js";
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
17
|
+
import { handleFactFind, handleHumanTopicScan, handleHumanPersonScan, handleEventScan } from "./human-extraction.js";
|
|
18
|
+
import { handleTopicMatch, handleTopicUpdate, handlePersonMatch, handlePersonUpdate } from "./human-matching.js";
|
|
19
19
|
import { handleRewriteScan, handleRewriteRewrite } from "./rewrite.js";
|
|
20
20
|
import { handleDedupCurate } from "./dedup.js";
|
|
21
21
|
|
|
@@ -23,12 +23,13 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
|
|
|
23
23
|
handlePersonaResponse,
|
|
24
24
|
handlePersonaGeneration,
|
|
25
25
|
handlePersonaDescriptions,
|
|
26
|
-
|
|
27
|
-
handleHumanTraitScan,
|
|
26
|
+
handleFactFind,
|
|
28
27
|
handleHumanTopicScan,
|
|
29
28
|
handleHumanPersonScan,
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
handleTopicMatch,
|
|
30
|
+
handleTopicUpdate,
|
|
31
|
+
handlePersonMatch,
|
|
32
|
+
handlePersonUpdate,
|
|
32
33
|
handlePersonaTraitExtraction,
|
|
33
34
|
handlePersonaTopicScan,
|
|
34
35
|
handlePersonaTopicMatch,
|
|
@@ -43,4 +44,5 @@ export const handlers: Record<LLMNextStep, ResponseHandler> = {
|
|
|
43
44
|
handleRewriteScan,
|
|
44
45
|
handleRewriteRewrite,
|
|
45
46
|
handleDedupCurate,
|
|
47
|
+
handleEventScan,
|
|
46
48
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
type LLMResponse,
|
|
3
|
-
type
|
|
3
|
+
type PersonaTrait,
|
|
4
4
|
type PersonaTopic,
|
|
5
5
|
} from "../types.js";
|
|
6
6
|
import type { StateManager } from "../state-manager.js";
|
|
@@ -27,10 +27,10 @@ export function handlePersonaGeneration(response: LLMResponse, state: StateManag
|
|
|
27
27
|
(existingPartial.traits ?? []).filter(t => t.name?.trim()).map(t => [t.name!.toLowerCase().trim(), t])
|
|
28
28
|
);
|
|
29
29
|
|
|
30
|
-
const mergedLlmTraits:
|
|
30
|
+
const mergedLlmTraits: PersonaTrait[] = (result?.traits || []).map(t => {
|
|
31
31
|
const userTrait = userTraitsByName.get(t.name?.toLowerCase().trim() ?? '');
|
|
32
32
|
return {
|
|
33
|
-
id: (userTrait as
|
|
33
|
+
id: (userTrait as PersonaTrait | undefined)?.id ?? crypto.randomUUID(),
|
|
34
34
|
name: t.name,
|
|
35
35
|
description: userTrait?.description?.trim() || t.description,
|
|
36
36
|
sentiment: userTrait?.sentiment ?? t.sentiment ?? 0,
|
|
@@ -41,10 +41,10 @@ export function handlePersonaGeneration(response: LLMResponse, state: StateManag
|
|
|
41
41
|
|
|
42
42
|
// Keep user-provided traits the LLM didn't return
|
|
43
43
|
const llmTraitNames = new Set(mergedLlmTraits.map(t => t.name?.toLowerCase().trim()));
|
|
44
|
-
const preservedUserTraits:
|
|
44
|
+
const preservedUserTraits: PersonaTrait[] = (existingPartial.traits ?? [])
|
|
45
45
|
.filter(t => t.name?.trim() && !llmTraitNames.has(t.name.toLowerCase().trim()))
|
|
46
46
|
.map(t => ({
|
|
47
|
-
id: (t as
|
|
47
|
+
id: (t as PersonaTrait).id ?? crypto.randomUUID(),
|
|
48
48
|
name: t.name!,
|
|
49
49
|
description: t.description || '',
|
|
50
50
|
sentiment: t.sentiment ?? 0,
|
|
@@ -52,9 +52,9 @@ export function handlePersonaGeneration(response: LLMResponse, state: StateManag
|
|
|
52
52
|
last_updated: now,
|
|
53
53
|
}));
|
|
54
54
|
|
|
55
|
-
const mergedTraits:
|
|
55
|
+
const mergedTraits: PersonaTrait[] = mergedLlmTraits.length > 0
|
|
56
56
|
? [...mergedLlmTraits, ...preservedUserTraits]
|
|
57
|
-
: (existingPartial.traits as
|
|
57
|
+
: (existingPartial.traits as PersonaTrait[] | undefined) ?? [];
|
|
58
58
|
|
|
59
59
|
// Merge LLM topics into user-provided topics by name.
|
|
60
60
|
// User-provided fields win; LLM fills in what the user left blank.
|
|
@@ -151,7 +151,7 @@ export function handlePersonaTraitExtraction(response: LLMResponse, state: State
|
|
|
151
151
|
}
|
|
152
152
|
|
|
153
153
|
const now = new Date().toISOString();
|
|
154
|
-
const traits:
|
|
154
|
+
const traits: PersonaTrait[] = result.map(t => ({
|
|
155
155
|
id: crypto.randomUUID(),
|
|
156
156
|
name: t.name,
|
|
157
157
|
description: t.description,
|
|
@@ -2,10 +2,8 @@ import {
|
|
|
2
2
|
LLMRequestType,
|
|
3
3
|
LLMPriority,
|
|
4
4
|
LLMNextStep,
|
|
5
|
-
ValidationLevel,
|
|
6
5
|
type LLMResponse,
|
|
7
6
|
type Fact,
|
|
8
|
-
type Trait,
|
|
9
7
|
type Topic,
|
|
10
8
|
type Person,
|
|
11
9
|
type DataItemBase,
|
|
@@ -48,7 +46,7 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
|
|
|
48
46
|
// Re-read the item from current state (it may have changed since scan was queued)
|
|
49
47
|
const human = state.getHuman();
|
|
50
48
|
const allItems: DataItemBase[] = [
|
|
51
|
-
...human.facts, ...human.
|
|
49
|
+
...human.facts, ...human.topics, ...human.people,
|
|
52
50
|
];
|
|
53
51
|
const currentItem = allItems.find(i => i.id === itemId);
|
|
54
52
|
if (!currentItem) {
|
|
@@ -61,11 +59,11 @@ export async function handleRewriteScan(response: LLMResponse, state: StateManag
|
|
|
61
59
|
for (const searchTerm of subjects) {
|
|
62
60
|
try {
|
|
63
61
|
const results = await searchHumanData(state, searchTerm, {
|
|
64
|
-
types: ["fact", "
|
|
62
|
+
types: ["fact", "topic", "person"],
|
|
65
63
|
limit: 4, // fetch 4 so we can exclude original and still have 3
|
|
66
64
|
});
|
|
67
65
|
const allMatches: DataItemBase[] = [
|
|
68
|
-
...results.facts, ...results.
|
|
66
|
+
...results.facts, ...results.topics, ...results.people,
|
|
69
67
|
].filter(m => m.id !== itemId); // exclude original
|
|
70
68
|
subjectMatches.push({ searchTerm, matches: allMatches.slice(0, 3) });
|
|
71
69
|
} catch (err) {
|
|
@@ -122,7 +120,7 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
122
120
|
|
|
123
121
|
// Look up the original item to inherit persona_groups
|
|
124
122
|
const allItems: DataItemBase[] = [
|
|
125
|
-
...human.facts, ...human.
|
|
123
|
+
...human.facts, ...human.topics, ...human.people,
|
|
126
124
|
];
|
|
127
125
|
const originalItem = allItems.find(i => i.id === itemId);
|
|
128
126
|
const inheritedGroups = originalItem?.persona_groups;
|
|
@@ -130,7 +128,6 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
130
128
|
// Helper: resolve actual type from existing records (don't trust LLM's type field)
|
|
131
129
|
const resolveExistingType = (id: string): RewriteItemType | null => {
|
|
132
130
|
if (human.facts.find(f => f.id === id)) return "fact";
|
|
133
|
-
if (human.traits.find(t => t.id === id)) return "trait";
|
|
134
131
|
if (human.topics.find(t => t.id === id)) return "topic";
|
|
135
132
|
if (human.people.find(p => p.id === id)) return "person";
|
|
136
133
|
return null;
|
|
@@ -175,19 +172,6 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
175
172
|
});
|
|
176
173
|
break;
|
|
177
174
|
}
|
|
178
|
-
case "trait": {
|
|
179
|
-
const existing = human.traits.find(t => t.id === item.id)!;
|
|
180
|
-
state.human_trait_upsert({
|
|
181
|
-
...existing,
|
|
182
|
-
name: item.name,
|
|
183
|
-
description: item.description,
|
|
184
|
-
sentiment: item.sentiment ?? existing.sentiment,
|
|
185
|
-
strength: item.strength ?? existing.strength,
|
|
186
|
-
last_updated: now,
|
|
187
|
-
embedding,
|
|
188
|
-
});
|
|
189
|
-
break;
|
|
190
|
-
}
|
|
191
175
|
case "topic": {
|
|
192
176
|
const existing = human.topics.find(t => t.id === item.id)!;
|
|
193
177
|
state.human_topic_upsert({
|
|
@@ -247,20 +231,11 @@ export async function handleRewriteRewrite(response: LLMResponse, state: StateMa
|
|
|
247
231
|
case "fact": {
|
|
248
232
|
const fact: Fact = {
|
|
249
233
|
...baseFields,
|
|
250
|
-
validated: ValidationLevel.None,
|
|
251
234
|
validated_date: now,
|
|
252
235
|
};
|
|
253
236
|
state.human_fact_upsert(fact);
|
|
254
237
|
break;
|
|
255
238
|
}
|
|
256
|
-
case "trait": {
|
|
257
|
-
const trait: Trait = {
|
|
258
|
-
...baseFields,
|
|
259
|
-
strength: item.strength ?? 0.5,
|
|
260
|
-
};
|
|
261
|
-
state.human_trait_upsert(trait);
|
|
262
|
-
break;
|
|
263
|
-
}
|
|
264
239
|
case "topic": {
|
|
265
240
|
if (!item.category) {
|
|
266
241
|
console.warn(`[handleRewriteRewrite] New topic "${item.name}" missing category — defaulting to "Interest"`);
|
|
@@ -1,7 +1,29 @@
|
|
|
1
1
|
import type { Message, LLMResponse } from "../types.js";
|
|
2
2
|
import type { StateManager } from "../state-manager.js";
|
|
3
3
|
|
|
4
|
-
export
|
|
4
|
+
export function resolveMessageWindow(
|
|
5
|
+
response: LLMResponse,
|
|
6
|
+
state: StateManager
|
|
7
|
+
): { messages_context: Message[]; messages_analyze: Message[] } {
|
|
8
|
+
const personaId = response.request.data.personaId as string;
|
|
9
|
+
const messageIdsToMark = response.request.data.message_ids_to_mark as string[] | undefined;
|
|
10
|
+
const allMessages = state.messages_get(personaId);
|
|
11
|
+
|
|
12
|
+
if (messageIdsToMark && messageIdsToMark.length > 0) {
|
|
13
|
+
const messageIdSet = new Set(messageIdsToMark);
|
|
14
|
+
const messages_analyze = allMessages.filter(m => messageIdSet.has(m.id));
|
|
15
|
+
const analyzeStartTime = messages_analyze[0]?.timestamp ?? '9999';
|
|
16
|
+
const messages_context = allMessages.filter(m =>
|
|
17
|
+
!messageIdSet.has(m.id) && new Date(m.timestamp).getTime() < new Date(analyzeStartTime).getTime()
|
|
18
|
+
);
|
|
19
|
+
return { messages_context, messages_analyze };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const analyzeFrom = response.request.data.analyze_from_timestamp as string | null;
|
|
23
|
+
return splitMessagesByTimestamp(allMessages, analyzeFrom);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type ExtractionFlag = "f" | "t" | "p" | "e";
|
|
5
27
|
|
|
6
28
|
export function splitMessagesByTimestamp(
|
|
7
29
|
messages: Message[],
|
|
@@ -2,7 +2,6 @@ import {
|
|
|
2
2
|
LLMRequestType,
|
|
3
3
|
LLMPriority,
|
|
4
4
|
LLMNextStep,
|
|
5
|
-
ValidationLevel,
|
|
6
5
|
type HumanEntity,
|
|
7
6
|
type Message,
|
|
8
7
|
} from "./types.js";
|
|
@@ -72,9 +71,8 @@ export async function queueEiHeartbeat(
|
|
|
72
71
|
const unverifiedFacts = human.facts
|
|
73
72
|
.filter(
|
|
74
73
|
(f) =>
|
|
75
|
-
f.
|
|
76
|
-
f.
|
|
77
|
-
(f.last_changed_by === undefined || f.last_changed_by !== "ei")
|
|
74
|
+
f.description !== '' &&
|
|
75
|
+
f.validated_date === ''
|
|
78
76
|
)
|
|
79
77
|
.slice(0, 5);
|
|
80
78
|
for (const fact of unverifiedFacts) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { HumanEntity, Fact,
|
|
1
|
+
import type { HumanEntity, Fact, Topic, Person, Quote } from "./types.js";
|
|
2
2
|
import { StateManager } from "./state-manager.js";
|
|
3
3
|
import {
|
|
4
4
|
getEmbeddingService,
|
|
@@ -24,7 +24,7 @@ export async function updateHuman(sm: StateManager, updates: Partial<HumanEntity
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
// =============================================================================
|
|
27
|
-
// FACTS /
|
|
27
|
+
// FACTS / TOPICS / PEOPLE UPSERT
|
|
28
28
|
// =============================================================================
|
|
29
29
|
|
|
30
30
|
export async function upsertFact(sm: StateManager, fact: Fact): Promise<void> {
|
|
@@ -40,18 +40,6 @@ export async function upsertFact(sm: StateManager, fact: Fact): Promise<void> {
|
|
|
40
40
|
sm.human_fact_upsert(fact);
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
-
export async function upsertTrait(sm: StateManager, trait: Trait): Promise<void> {
|
|
44
|
-
const human = sm.getHuman();
|
|
45
|
-
const existing = human.traits.find((t) => t.id === trait.id);
|
|
46
|
-
|
|
47
|
-
if (needsEmbeddingUpdate(existing, trait)) {
|
|
48
|
-
trait.embedding = await computeDataItemEmbedding(trait);
|
|
49
|
-
} else if (existing?.embedding) {
|
|
50
|
-
trait.embedding = existing.embedding;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
sm.human_trait_upsert(trait);
|
|
54
|
-
}
|
|
55
43
|
|
|
56
44
|
export async function upsertTopic(sm: StateManager, topic: Topic): Promise<void> {
|
|
57
45
|
const human = sm.getHuman();
|
|
@@ -81,16 +69,13 @@ export async function upsertPerson(sm: StateManager, person: Person): Promise<vo
|
|
|
81
69
|
|
|
82
70
|
export async function removeDataItem(
|
|
83
71
|
sm: StateManager,
|
|
84
|
-
type: "fact" | "
|
|
72
|
+
type: "fact" | "topic" | "person",
|
|
85
73
|
id: string
|
|
86
74
|
): Promise<void> {
|
|
87
75
|
switch (type) {
|
|
88
76
|
case "fact":
|
|
89
77
|
sm.human_fact_remove(id);
|
|
90
78
|
break;
|
|
91
|
-
case "trait":
|
|
92
|
-
sm.human_trait_remove(id);
|
|
93
|
-
break;
|
|
94
79
|
case "topic":
|
|
95
80
|
sm.human_topic_remove(id);
|
|
96
81
|
break;
|
|
@@ -160,21 +145,19 @@ export async function getQuotesForMessage(sm: StateManager, messageId: string):
|
|
|
160
145
|
export async function searchHumanData(
|
|
161
146
|
sm: StateManager,
|
|
162
147
|
query: string,
|
|
163
|
-
options: { types?: Array<"fact" | "
|
|
148
|
+
options: { types?: Array<"fact" | "topic" | "person" | "quote">; limit?: number } = {}
|
|
164
149
|
): Promise<{
|
|
165
150
|
facts: Fact[];
|
|
166
|
-
traits: Trait[];
|
|
167
151
|
topics: Topic[];
|
|
168
152
|
people: Person[];
|
|
169
153
|
quotes: Quote[];
|
|
170
154
|
}> {
|
|
171
|
-
const { types = ["fact", "
|
|
155
|
+
const { types = ["fact", "topic", "person", "quote"], limit = 10 } = options;
|
|
172
156
|
const human = sm.getHuman();
|
|
173
157
|
const SIMILARITY_THRESHOLD = 0.3;
|
|
174
158
|
|
|
175
159
|
const result = {
|
|
176
160
|
facts: [] as Fact[],
|
|
177
|
-
traits: [] as Trait[],
|
|
178
161
|
topics: [] as Topic[],
|
|
179
162
|
people: [] as Person[],
|
|
180
163
|
quotes: [] as Quote[],
|
|
@@ -211,11 +194,6 @@ export async function searchHumanData(
|
|
|
211
194
|
stripDataItemEmbedding
|
|
212
195
|
);
|
|
213
196
|
}
|
|
214
|
-
if (types.includes("trait")) {
|
|
215
|
-
result.traits = searchItems(human.traits, (t) => `${t.name} ${t.description || ""}`).map(
|
|
216
|
-
stripDataItemEmbedding
|
|
217
|
-
);
|
|
218
|
-
}
|
|
219
197
|
if (types.includes("topic")) {
|
|
220
198
|
result.topics = searchItems(human.topics, (t) => `${t.name} ${t.description || ""}`).map(
|
|
221
199
|
stripDataItemEmbedding
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
} from "../prompts/index.js";
|
|
17
17
|
import { buildResponsePromptData } from "./prompt-context-builder.js";
|
|
18
18
|
import {
|
|
19
|
-
|
|
19
|
+
queueFactFind,
|
|
20
20
|
queueTopicScan,
|
|
21
21
|
queuePersonScan,
|
|
22
22
|
type ExtractionContext,
|
|
@@ -178,7 +178,7 @@ export async function sendMessage(
|
|
|
178
178
|
const traitExtractionData: PersonaTraitExtractionPromptData = {
|
|
179
179
|
persona_name: persona.display_name,
|
|
180
180
|
current_traits: persona.traits,
|
|
181
|
-
messages_context: history.slice(
|
|
181
|
+
messages_context: history.slice(-11, -1),
|
|
182
182
|
messages_analyze: [message],
|
|
183
183
|
};
|
|
184
184
|
const traitPrompt = buildPersonaTraitExtractionPrompt(traitExtractionData);
|
|
@@ -217,7 +217,7 @@ export function checkAndQueueHumanExtraction(
|
|
|
217
217
|
const human = sm.getHuman();
|
|
218
218
|
|
|
219
219
|
const unextractedFacts = sm.messages_getUnextracted(personaId, "f");
|
|
220
|
-
const factsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.facts.length);
|
|
220
|
+
const factsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.facts.filter(f => f.description && f.description !== "").length);
|
|
221
221
|
if (unextractedFacts.length > 0 && unextractedFacts.length >= factsThreshold) {
|
|
222
222
|
const context: ExtractionContext = {
|
|
223
223
|
personaId,
|
|
@@ -226,21 +226,21 @@ export function checkAndQueueHumanExtraction(
|
|
|
226
226
|
messages_analyze: unextractedFacts,
|
|
227
227
|
extraction_flag: "f",
|
|
228
228
|
};
|
|
229
|
-
|
|
229
|
+
queueFactFind(context, sm);
|
|
230
230
|
console.log(
|
|
231
231
|
`[Processor] Human Seed extraction: facts (threshold: ${factsThreshold}, unextracted: ${unextractedFacts.length})`
|
|
232
232
|
);
|
|
233
233
|
}
|
|
234
234
|
|
|
235
|
-
const unextractedTopics = sm.messages_getUnextracted(personaId, "
|
|
235
|
+
const unextractedTopics = sm.messages_getUnextracted(personaId, "t");
|
|
236
236
|
const topicsThreshold = Math.min(EXTRACTION_TAPER_CAP, human.topics.length);
|
|
237
237
|
if (unextractedTopics.length > 0 && unextractedTopics.length >= topicsThreshold) {
|
|
238
238
|
const context: ExtractionContext = {
|
|
239
239
|
personaId,
|
|
240
240
|
personaDisplayName,
|
|
241
|
-
messages_context: history.filter((m) => m.
|
|
241
|
+
messages_context: history.filter((m) => m.t === true),
|
|
242
242
|
messages_analyze: unextractedTopics,
|
|
243
|
-
extraction_flag: "
|
|
243
|
+
extraction_flag: "t",
|
|
244
244
|
};
|
|
245
245
|
queueTopicScan(context, sm);
|
|
246
246
|
console.log(
|
|
@@ -248,15 +248,15 @@ export function checkAndQueueHumanExtraction(
|
|
|
248
248
|
);
|
|
249
249
|
}
|
|
250
250
|
|
|
251
|
-
const unextractedPeople = sm.messages_getUnextracted(personaId, "
|
|
251
|
+
const unextractedPeople = sm.messages_getUnextracted(personaId, "p");
|
|
252
252
|
const peopleThreshold = Math.min(EXTRACTION_TAPER_CAP, human.people.length);
|
|
253
253
|
if (unextractedPeople.length > 0 && unextractedPeople.length >= peopleThreshold) {
|
|
254
254
|
const context: ExtractionContext = {
|
|
255
255
|
personaId,
|
|
256
256
|
personaDisplayName,
|
|
257
|
-
messages_context: history.filter((m) => m.
|
|
257
|
+
messages_context: history.filter((m) => m.p === true),
|
|
258
258
|
messages_analyze: unextractedPeople,
|
|
259
|
-
extraction_flag: "
|
|
259
|
+
extraction_flag: "p",
|
|
260
260
|
};
|
|
261
261
|
queuePersonScan(context, sm);
|
|
262
262
|
console.log(
|
|
@@ -2,10 +2,10 @@ import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MA
|
|
|
2
2
|
import type { StateManager } from "../state-manager.js";
|
|
3
3
|
import { applyDecayToValue } from "../utils/index.js";
|
|
4
4
|
import {
|
|
5
|
-
|
|
6
|
-
queueTraitScan,
|
|
5
|
+
queueFactFind,
|
|
7
6
|
queueTopicScan,
|
|
8
7
|
queuePersonScan,
|
|
8
|
+
queueEventSummary,
|
|
9
9
|
type ExtractionContext,
|
|
10
10
|
type ExtractionOptions,
|
|
11
11
|
} from "./human-extraction.js";
|
|
@@ -109,56 +109,45 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
|
|
|
109
109
|
messages_analyze: unextractedFacts,
|
|
110
110
|
extraction_flag: "f",
|
|
111
111
|
};
|
|
112
|
-
|
|
112
|
+
queueFactFind(context, state, options);
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
-
const unextractedTraits = state.messages_getUnextracted(personaId, "r");
|
|
116
|
-
if (unextractedTraits.length > 0) {
|
|
117
|
-
const context: ExtractionContext = {
|
|
118
|
-
personaId,
|
|
119
|
-
personaDisplayName: persona.display_name,
|
|
120
|
-
messages_context: allMessages.filter(m => m.r === true),
|
|
121
|
-
messages_analyze: unextractedTraits,
|
|
122
|
-
extraction_flag: "r",
|
|
123
|
-
};
|
|
124
|
-
queueTraitScan(context, state, options);
|
|
125
|
-
}
|
|
126
115
|
|
|
127
|
-
const unextractedTopics = state.messages_getUnextracted(personaId, "
|
|
116
|
+
const unextractedTopics = state.messages_getUnextracted(personaId, "t");
|
|
128
117
|
if (unextractedTopics.length > 0) {
|
|
129
118
|
const context: ExtractionContext = {
|
|
130
119
|
personaId,
|
|
131
120
|
personaDisplayName: persona.display_name,
|
|
132
|
-
messages_context: allMessages.filter(m => m.
|
|
121
|
+
messages_context: allMessages.filter(m => m.t === true),
|
|
133
122
|
messages_analyze: unextractedTopics,
|
|
134
|
-
extraction_flag: "
|
|
123
|
+
extraction_flag: "t",
|
|
135
124
|
};
|
|
136
125
|
queueTopicScan(context, state, options);
|
|
137
126
|
}
|
|
138
127
|
|
|
139
|
-
const unextractedPeople = state.messages_getUnextracted(personaId, "
|
|
128
|
+
const unextractedPeople = state.messages_getUnextracted(personaId, "p");
|
|
140
129
|
if (unextractedPeople.length > 0) {
|
|
141
130
|
const context: ExtractionContext = {
|
|
142
131
|
personaId,
|
|
143
132
|
personaDisplayName: persona.display_name,
|
|
144
|
-
messages_context: allMessages.filter(m => m.
|
|
133
|
+
messages_context: allMessages.filter(m => m.p === true),
|
|
145
134
|
messages_analyze: unextractedPeople,
|
|
146
|
-
extraction_flag: "
|
|
135
|
+
extraction_flag: "p",
|
|
147
136
|
};
|
|
148
137
|
queuePersonScan(context, state, options);
|
|
149
138
|
}
|
|
150
139
|
|
|
151
|
-
const totalUnextracted = unextractedFacts.length +
|
|
140
|
+
const totalUnextracted = unextractedFacts.length + unextractedTopics.length + unextractedPeople.length;
|
|
152
141
|
if (totalUnextracted > 0) {
|
|
153
|
-
console.log(`[ceremony:exposure] Queued human extraction scans (f:${unextractedFacts.length},
|
|
142
|
+
console.log(`[ceremony:exposure] Queued human extraction scans (f:${unextractedFacts.length}, t:${unextractedTopics.length}, p:${unextractedPeople.length})`);
|
|
154
143
|
}
|
|
155
144
|
|
|
156
|
-
const unextractedForPersonaTopics = state.messages_getUnextracted(personaId, "
|
|
145
|
+
const unextractedForPersonaTopics = state.messages_getUnextracted(personaId, "t");
|
|
157
146
|
if (unextractedForPersonaTopics.length > 0) {
|
|
158
147
|
const personaTopicContext: PersonaTopicContext = {
|
|
159
148
|
personaId,
|
|
160
149
|
personaDisplayName: persona.display_name,
|
|
161
|
-
messages_context: allMessages.filter(m => m.
|
|
150
|
+
messages_context: allMessages.filter(m => m.t === true),
|
|
162
151
|
messages_analyze: unextractedForPersonaTopics,
|
|
163
152
|
};
|
|
164
153
|
queuePersonaTopicScan(personaTopicContext, state);
|
|
@@ -171,7 +160,7 @@ function queueExposurePhase(personaId: string, state: StateManager, options?: Ex
|
|
|
171
160
|
* AND at the end of startCeremony (for the zero-messages edge case).
|
|
172
161
|
*
|
|
173
162
|
* If any ceremony_progress items remain in the queue, does nothing — more work pending.
|
|
174
|
-
*
|
|
163
|
+
* Phase 1: Dedup → Phase 2: Expose → Phase 3: EventSummary → Decay → Expire
|
|
175
164
|
*/
|
|
176
165
|
export function handleCeremonyProgress(state: StateManager, lastPhase: number): void {
|
|
177
166
|
if (state.queue_hasPendingCeremonies()) {
|
|
@@ -193,9 +182,8 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
|
|
|
193
182
|
const personasWithUnprocessed = activePersonas.filter(p => {
|
|
194
183
|
const messages = state.messages_get(p.id);
|
|
195
184
|
return messages.some(msg =>
|
|
185
|
+
!msg.t ||
|
|
196
186
|
!msg.p ||
|
|
197
|
-
!msg.r ||
|
|
198
|
-
!msg.o ||
|
|
199
187
|
!msg.f
|
|
200
188
|
);
|
|
201
189
|
});
|
|
@@ -208,9 +196,22 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
|
|
|
208
196
|
}
|
|
209
197
|
return;
|
|
210
198
|
}
|
|
199
|
+
|
|
200
|
+
if (lastPhase === 2) {
|
|
201
|
+
console.log("[ceremony:progress] Expose complete, starting EventSummary phase");
|
|
202
|
+
const options: ExtractionOptions = { ceremony_progress: 3 };
|
|
203
|
+
queueEventSummaryForAll(state, options);
|
|
204
|
+
|
|
205
|
+
// Zero-work guard: same pattern as DeDupe phase
|
|
206
|
+
if (!state.queue_hasPendingCeremonies()) {
|
|
207
|
+
console.log("[ceremony:progress] No event summary work, advancing to Decay");
|
|
208
|
+
handleCeremonyProgress(state, 3);
|
|
209
|
+
}
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
211
212
|
|
|
212
|
-
// Phase
|
|
213
|
-
console.log("[ceremony:progress]
|
|
213
|
+
// Phase 3 (EventSummary) complete → advance to Decay/Prune/Expire/Explore
|
|
214
|
+
console.log("[ceremony:progress] EventSummary complete, advancing to Decay");
|
|
214
215
|
|
|
215
216
|
const personas = state.persona_getAll();
|
|
216
217
|
const activePersonas = personas.filter(p =>
|
|
@@ -236,7 +237,7 @@ export function handleCeremonyProgress(state: StateManager, lastPhase: number):
|
|
|
236
237
|
// Human ceremony: decay topics + people
|
|
237
238
|
runHumanCeremony(state);
|
|
238
239
|
|
|
239
|
-
// Dedup phase: log near-duplicate human
|
|
240
|
+
// Dedup phase: log near-duplicate human entity candidates for visibility
|
|
240
241
|
runDedupPhase(state);
|
|
241
242
|
|
|
242
243
|
// Rewrite phase: fire-and-forget scans for bloated human data items
|
|
@@ -320,7 +321,7 @@ export function prunePersonaMessages(personaId: string, state: StateManager): vo
|
|
|
320
321
|
const msgMs = new Date(m.timestamp).getTime();
|
|
321
322
|
if (msgMs >= cutoffMs) break; // Sorted by time, no more old ones
|
|
322
323
|
|
|
323
|
-
const fullyExtracted = m.
|
|
324
|
+
const fullyExtracted = m.t && m.p && m.f; // r intentionally excluded — trait extraction deprecated
|
|
324
325
|
if (fullyExtracted) {
|
|
325
326
|
toRemove.push(m.id);
|
|
326
327
|
}
|
|
@@ -518,7 +519,7 @@ export function runHumanCeremony(state: StateManager): void {
|
|
|
518
519
|
}
|
|
519
520
|
|
|
520
521
|
// =============================================================================
|
|
521
|
-
// DEDUP PHASE (synchronous,
|
|
522
|
+
// DEDUP PHASE (synchronous, logging only — candidates are queued for curation by dedup-phase.ts)
|
|
522
523
|
// =============================================================================
|
|
523
524
|
|
|
524
525
|
const DEDUP_DEFAULT_THRESHOLD = 0.85;
|
|
@@ -558,11 +559,10 @@ export function runDedupPhase(state: StateManager): void {
|
|
|
558
559
|
const human = state.getHuman();
|
|
559
560
|
const threshold = human.settings?.ceremony?.dedup_threshold ?? DEDUP_DEFAULT_THRESHOLD;
|
|
560
561
|
|
|
561
|
-
console.log(`[ceremony:dedup]
|
|
562
|
+
console.log(`[ceremony:dedup] Scanning for dedup candidates (threshold: ${threshold})`);
|
|
562
563
|
|
|
563
564
|
const types: Array<{ label: string; items: DedupableItem[] }> = [
|
|
564
565
|
{ label: "facts", items: human.facts },
|
|
565
|
-
{ label: "traits", items: human.traits },
|
|
566
566
|
{ label: "topics", items: human.topics },
|
|
567
567
|
{ label: "people", items: human.people },
|
|
568
568
|
];
|
|
@@ -621,11 +621,7 @@ export function queueRewritePhase(state: StateManager): void {
|
|
|
621
621
|
itemsToScan.push({ item: fact, type: "fact" });
|
|
622
622
|
}
|
|
623
623
|
}
|
|
624
|
-
|
|
625
|
-
if ((trait.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
|
|
626
|
-
itemsToScan.push({ item: trait, type: "trait" });
|
|
627
|
-
}
|
|
628
|
-
}
|
|
624
|
+
|
|
629
625
|
for (const topic of human.topics) {
|
|
630
626
|
if ((topic.description?.length ?? 0) > REWRITE_DESCRIPTION_THRESHOLD) {
|
|
631
627
|
itemsToScan.push({ item: topic, type: "topic" });
|
|
@@ -664,3 +660,18 @@ export function queueRewritePhase(state: StateManager): void {
|
|
|
664
660
|
|
|
665
661
|
console.log(`[ceremony:rewrite] Queued ${itemsToScan.length} Phase 1 scan(s) at Low priority`);
|
|
666
662
|
}
|
|
663
|
+
|
|
664
|
+
function queueEventSummaryForAll(state: StateManager, options?: ExtractionOptions): void {
|
|
665
|
+
const personas = state.persona_getAll();
|
|
666
|
+
const activePersonas = personas.filter(p =>
|
|
667
|
+
!p.is_paused &&
|
|
668
|
+
!p.is_archived &&
|
|
669
|
+
!p.is_static
|
|
670
|
+
);
|
|
671
|
+
|
|
672
|
+
let totalQueued = 0;
|
|
673
|
+
for (const persona of activePersonas) {
|
|
674
|
+
totalQueued += queueEventSummary(persona.id, state, options);
|
|
675
|
+
}
|
|
676
|
+
console.log(`[ceremony:event] Queued event summary scans for ${activePersonas.length} personas (${totalQueued} total chunks)`);
|
|
677
|
+
}
|
|
@@ -140,7 +140,6 @@ export function queueDedupPhase(state: StateManager): void {
|
|
|
140
140
|
|
|
141
141
|
const entityTypes: Array<{ type: DataItemType; items: DedupableItem[] }> = [
|
|
142
142
|
{ type: "fact", items: human.facts },
|
|
143
|
-
{ type: "trait", items: human.traits },
|
|
144
143
|
{ type: "topic", items: human.topics },
|
|
145
144
|
{ type: "person", items: human.people },
|
|
146
145
|
];
|