ei-tui 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/core/handlers/index.ts +10 -5
- package/src/core/orchestrators/ceremony.ts +77 -1
- package/src/core/processor.ts +9 -3
- package/src/core/state-manager.ts +49 -0
- package/src/core/types.ts +3 -1
- package/src/integrations/opencode/importer.ts +2 -1
- package/src/prompts/human/item-update.ts +32 -2
- package/src/prompts/response/sections.ts +13 -8
- package/src/storage/compress.ts +82 -0
- package/src/storage/index.ts +1 -0
- package/src/storage/local.ts +8 -3
- package/src/storage/remote.ts +6 -2
- package/tui/src/util/yaml-serializers.ts +10 -2
package/package.json
CHANGED
|
@@ -233,6 +233,7 @@ function handleEiHeartbeat(response: LLMResponse, state: StateManager): void {
|
|
|
233
233
|
timestamp: now,
|
|
234
234
|
read: false,
|
|
235
235
|
context_status: ContextStatus.Default,
|
|
236
|
+
f: true, r: true, p: true, o: true,
|
|
236
237
|
});
|
|
237
238
|
|
|
238
239
|
if (found.type === "fact") {
|
|
@@ -672,7 +673,7 @@ async function handleHumanItemUpdate(response: LLMResponse, state: StateManager)
|
|
|
672
673
|
const isEi = personaDisplayName.toLowerCase() === "ei";
|
|
673
674
|
|
|
674
675
|
const human = state.getHuman();
|
|
675
|
-
const getExistingItem = (): { learned_by?: string; persona_groups?: string[] } | undefined => {
|
|
676
|
+
const getExistingItem = (): { learned_by?: string; last_changed_by?: string; persona_groups?: string[] } | undefined => {
|
|
676
677
|
if (isNewItem) return undefined;
|
|
677
678
|
switch (candidateType) {
|
|
678
679
|
case "fact": return human.facts.find(f => f.id === existingItemId);
|
|
@@ -710,7 +711,8 @@ async function handleHumanItemUpdate(response: LLMResponse, state: StateManager)
|
|
|
710
711
|
validated: ValidationLevel.None,
|
|
711
712
|
validated_date: now,
|
|
712
713
|
last_updated: now,
|
|
713
|
-
learned_by: isNewItem ?
|
|
714
|
+
learned_by: isNewItem ? personaId : existingItem?.learned_by,
|
|
715
|
+
last_changed_by: personaId,
|
|
714
716
|
persona_groups: mergeGroups(existingItem?.persona_groups),
|
|
715
717
|
embedding,
|
|
716
718
|
};
|
|
@@ -725,7 +727,8 @@ async function handleHumanItemUpdate(response: LLMResponse, state: StateManager)
|
|
|
725
727
|
sentiment: result.sentiment,
|
|
726
728
|
strength: (result as any).strength ?? 0.5,
|
|
727
729
|
last_updated: now,
|
|
728
|
-
learned_by: isNewItem ?
|
|
730
|
+
learned_by: isNewItem ? personaId : existingItem?.learned_by,
|
|
731
|
+
last_changed_by: personaId,
|
|
729
732
|
persona_groups: mergeGroups(existingItem?.persona_groups),
|
|
730
733
|
embedding,
|
|
731
734
|
};
|
|
@@ -745,7 +748,8 @@ async function handleHumanItemUpdate(response: LLMResponse, state: StateManager)
|
|
|
745
748
|
exposure_current: calculateExposureCurrent(exposureImpact),
|
|
746
749
|
exposure_desired: (result as any).exposure_desired ?? 0.5,
|
|
747
750
|
last_updated: now,
|
|
748
|
-
learned_by: isNewItem ?
|
|
751
|
+
learned_by: isNewItem ? personaId : existingItem?.learned_by,
|
|
752
|
+
last_changed_by: personaId,
|
|
749
753
|
persona_groups: mergeGroups(existingItem?.persona_groups),
|
|
750
754
|
embedding,
|
|
751
755
|
};
|
|
@@ -763,7 +767,8 @@ async function handleHumanItemUpdate(response: LLMResponse, state: StateManager)
|
|
|
763
767
|
exposure_current: calculateExposureCurrent(exposureImpact),
|
|
764
768
|
exposure_desired: (result as any).exposure_desired ?? 0.5,
|
|
765
769
|
last_updated: now,
|
|
766
|
-
learned_by: isNewItem ?
|
|
770
|
+
learned_by: isNewItem ? personaId : existingItem?.learned_by,
|
|
771
|
+
last_changed_by: personaId,
|
|
767
772
|
persona_groups: mergeGroups(existingItem?.persona_groups),
|
|
768
773
|
embedding,
|
|
769
774
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MAX_AGE_DAYS, type CeremonyConfig, type PersonaTopic, type Topic, type Message } from "../types.js";
|
|
1
|
+
import { LLMRequestType, LLMPriority, LLMNextStep, MESSAGE_MIN_COUNT, MESSAGE_MAX_AGE_DAYS, type CeremonyConfig, type PersonaTopic, type Topic, type Message, type DataItemBase } from "../types.js";
|
|
2
2
|
import type { StateManager } from "../state-manager.js";
|
|
3
3
|
import { applyDecayToValue } from "../utils/index.js";
|
|
4
4
|
import {
|
|
@@ -224,6 +224,9 @@ export function handleCeremonyProgress(state: StateManager): void {
|
|
|
224
224
|
|
|
225
225
|
// Human ceremony: decay topics + people
|
|
226
226
|
runHumanCeremony(state);
|
|
227
|
+
|
|
228
|
+
// Dedup phase: log near-duplicate human entities (dry-run only, no mutations)
|
|
229
|
+
runDedupPhase(state);
|
|
227
230
|
|
|
228
231
|
// Expire phase: queue LLM calls for each active persona
|
|
229
232
|
// handlePersonaExpire already chains to Explore → DescriptionCheck
|
|
@@ -498,3 +501,76 @@ export function runHumanCeremony(state: StateManager): void {
|
|
|
498
501
|
console.log(`[ceremony:human] Low exposure items: ${lowExposureTopics.length} topics, ${lowExposurePeople.length} people`);
|
|
499
502
|
}
|
|
500
503
|
}
|
|
504
|
+
|
|
505
|
+
// =============================================================================
|
|
506
|
+
// DEDUP PHASE (synchronous, dry-run — logs candidates, no mutations)
|
|
507
|
+
// =============================================================================
|
|
508
|
+
|
|
509
|
+
const DEDUP_DEFAULT_THRESHOLD = 0.85;
|
|
510
|
+
|
|
511
|
+
type DedupableItem = DataItemBase & { relationship?: string };
|
|
512
|
+
|
|
513
|
+
function findDedupCandidates<T extends DedupableItem>(
|
|
514
|
+
items: T[],
|
|
515
|
+
threshold: number
|
|
516
|
+
): Array<{ a: T; b: T; similarity: number }> {
|
|
517
|
+
const withEmbeddings = items.filter(item =>
|
|
518
|
+
item.embedding && item.embedding.length > 0 &&
|
|
519
|
+
item.relationship !== "Persona"
|
|
520
|
+
);
|
|
521
|
+
|
|
522
|
+
const candidates: Array<{ a: T; b: T; similarity: number }> = [];
|
|
523
|
+
|
|
524
|
+
for (let i = 0; i < withEmbeddings.length; i++) {
|
|
525
|
+
for (let j = i + 1; j < withEmbeddings.length; j++) {
|
|
526
|
+
const a = withEmbeddings[i];
|
|
527
|
+
const b = withEmbeddings[j];
|
|
528
|
+
const dot = a.embedding!.reduce((sum, v, k) => sum + v * b.embedding![k], 0);
|
|
529
|
+
const normA = Math.sqrt(a.embedding!.reduce((sum, v) => sum + v * v, 0));
|
|
530
|
+
const normB = Math.sqrt(b.embedding!.reduce((sum, v) => sum + v * v, 0));
|
|
531
|
+
const similarity = normA && normB ? dot / (normA * normB) : 0;
|
|
532
|
+
|
|
533
|
+
if (similarity >= threshold) {
|
|
534
|
+
candidates.push({ a, b, similarity });
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
return candidates.sort((x, y) => y.similarity - x.similarity);
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
export function runDedupPhase(state: StateManager): void {
|
|
543
|
+
const human = state.getHuman();
|
|
544
|
+
const threshold = human.settings?.ceremony?.dedup_threshold ?? DEDUP_DEFAULT_THRESHOLD;
|
|
545
|
+
|
|
546
|
+
console.log(`[ceremony:dedup] Running dry-run dedup (threshold: ${threshold})`);
|
|
547
|
+
|
|
548
|
+
const types: Array<{ label: string; items: DedupableItem[] }> = [
|
|
549
|
+
{ label: "facts", items: human.facts },
|
|
550
|
+
{ label: "traits", items: human.traits },
|
|
551
|
+
{ label: "topics", items: human.topics },
|
|
552
|
+
{ label: "people", items: human.people },
|
|
553
|
+
];
|
|
554
|
+
|
|
555
|
+
let totalCandidates = 0;
|
|
556
|
+
|
|
557
|
+
for (const { label, items } of types) {
|
|
558
|
+
const candidates = findDedupCandidates(items, threshold);
|
|
559
|
+
if (candidates.length === 0) {
|
|
560
|
+
console.log(`[ceremony:dedup] ${label}: no candidates above ${threshold}`);
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
totalCandidates += candidates.length;
|
|
565
|
+
console.log(`[ceremony:dedup] ${label}: ${candidates.length} candidate pair(s)`);
|
|
566
|
+
for (const { a, b, similarity } of candidates) {
|
|
567
|
+
console.log(
|
|
568
|
+
`[ceremony:dedup] ${(similarity * 100).toFixed(1)}% "${a.name}" ↔ "${b.name}"` +
|
|
569
|
+
(a.description ? `\n[ceremony:dedup] A: ${a.description.slice(0, 80)}` : "") +
|
|
570
|
+
(b.description ? `\n[ceremony:dedup] B: ${b.description.slice(0, 80)}` : "")
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
console.log(`[ceremony:dedup] Done. ${totalCandidates} total candidate pair(s) found.`);
|
|
576
|
+
}
|
package/src/core/processor.ts
CHANGED
|
@@ -565,7 +565,7 @@ export class Processor {
|
|
|
565
565
|
const items: EiHeartbeatItem[] = [];
|
|
566
566
|
|
|
567
567
|
const unverifiedFacts = human.facts
|
|
568
|
-
.filter(f => f.validated === ValidationLevel.None && f.learned_by !== "
|
|
568
|
+
.filter(f => f.validated === ValidationLevel.None && f.learned_by !== "ei" && (f.last_changed_by === undefined || f.last_changed_by !== "ei"))
|
|
569
569
|
.slice(0, 5);
|
|
570
570
|
for (const fact of unverifiedFacts) {
|
|
571
571
|
const quote = human.quotes.find(q => q.data_item_ids.includes(fact.id));
|
|
@@ -1142,8 +1142,14 @@ export class Processor {
|
|
|
1142
1142
|
}
|
|
1143
1143
|
}
|
|
1144
1144
|
|
|
1145
|
-
// Fallback: return
|
|
1146
|
-
return items
|
|
1145
|
+
// Fallback: return top items by recency — never return unbounded list
|
|
1146
|
+
return [...items]
|
|
1147
|
+
.sort((a, b) => {
|
|
1148
|
+
const aTime = (a as { last_updated?: string }).last_updated ?? "";
|
|
1149
|
+
const bTime = (b as { last_updated?: string }).last_updated ?? "";
|
|
1150
|
+
return bTime.localeCompare(aTime);
|
|
1151
|
+
})
|
|
1152
|
+
.slice(0, limit);
|
|
1147
1153
|
};
|
|
1148
1154
|
const selectRelevantQuotes = async (quotes: Quote[]): Promise<Quote[]> => {
|
|
1149
1155
|
if (quotes.length === 0) return [];
|
|
@@ -36,11 +36,60 @@ export class StateManager {
|
|
|
36
36
|
this.humanState.load(state.human);
|
|
37
37
|
this.personaState.load(state.personas);
|
|
38
38
|
this.queueState.load(state.queue);
|
|
39
|
+
this.migrateLearnedByToIds();
|
|
39
40
|
} else {
|
|
40
41
|
this.humanState.load(createDefaultHumanEntity());
|
|
41
42
|
}
|
|
42
43
|
}
|
|
43
44
|
|
|
45
|
+
/**
|
|
46
|
+
* Migration: learned_by used to store display names; now stores persona IDs.
|
|
47
|
+
* On load, attempt to resolve display names -> IDs using current persona map.
|
|
48
|
+
* Unresolvable values (renamed/deleted personas) are cleared to avoid stale display.
|
|
49
|
+
* No-op for already-migrated data (UUIDs or "ei" won't match display names).
|
|
50
|
+
*/
|
|
51
|
+
private migrateLearnedByToIds(): void {
|
|
52
|
+
const personas = this.personaState.getAll();
|
|
53
|
+
const nameToId = new Map<string, string>();
|
|
54
|
+
for (const p of personas) {
|
|
55
|
+
nameToId.set(p.display_name.toLowerCase(), p.id);
|
|
56
|
+
for (const alias of p.aliases ?? []) {
|
|
57
|
+
nameToId.set(alias.toLowerCase(), p.id);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
// "Ei" display name -> "ei" id (hardcoded, always valid)
|
|
61
|
+
nameToId.set("ei", "ei");
|
|
62
|
+
|
|
63
|
+
const human = this.humanState.get();
|
|
64
|
+
let dirty = false;
|
|
65
|
+
const migrateItem = (item: { learned_by?: string; last_changed_by?: string }) => {
|
|
66
|
+
if (item.learned_by && !this.isPersonaId(item.learned_by)) {
|
|
67
|
+
const resolved = nameToId.get(item.learned_by.toLowerCase());
|
|
68
|
+
item.learned_by = resolved ?? undefined; // clear if unresolvable
|
|
69
|
+
dirty = true;
|
|
70
|
+
}
|
|
71
|
+
if (item.last_changed_by && !this.isPersonaId(item.last_changed_by)) {
|
|
72
|
+
const resolved = nameToId.get(item.last_changed_by.toLowerCase());
|
|
73
|
+
item.last_changed_by = resolved ?? undefined;
|
|
74
|
+
dirty = true;
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
[...human.facts, ...human.traits, ...human.topics, ...human.people].forEach(migrateItem);
|
|
78
|
+
if (dirty) {
|
|
79
|
+
this.humanState.set(human);
|
|
80
|
+
console.log("[StateManager] Migrated learned_by fields from display names to persona IDs");
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Returns true if value looks like a persona ID (UUID or the special "ei" id).
|
|
86
|
+
* Display names are free-form strings that won't match UUID format.
|
|
87
|
+
*/
|
|
88
|
+
private isPersonaId(value: string): boolean {
|
|
89
|
+
if (value === "ei") return true;
|
|
90
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value);
|
|
91
|
+
}
|
|
92
|
+
|
|
44
93
|
private buildStorageState(): StorageState {
|
|
45
94
|
return {
|
|
46
95
|
version: 1,
|
package/src/core/types.ts
CHANGED
|
@@ -62,7 +62,8 @@ export interface DataItemBase {
|
|
|
62
62
|
description: string;
|
|
63
63
|
sentiment: number;
|
|
64
64
|
last_updated: string;
|
|
65
|
-
learned_by?: string; // Persona ID that learned this item
|
|
65
|
+
learned_by?: string; // Persona ID that originally learned this item (stable UUID)
|
|
66
|
+
last_changed_by?: string; // Persona ID that most recently updated this item (stable UUID)
|
|
66
67
|
persona_groups?: string[];
|
|
67
68
|
embedding?: number[];
|
|
68
69
|
}
|
|
@@ -188,6 +189,7 @@ export interface CeremonyConfig {
|
|
|
188
189
|
last_ceremony?: string; // ISO timestamp
|
|
189
190
|
decay_rate?: number; // Default: 0.1
|
|
190
191
|
explore_threshold?: number; // Default: 3
|
|
192
|
+
dedup_threshold?: number; // Cosine similarity threshold for dedup candidates. Default: 0.85
|
|
191
193
|
}
|
|
192
194
|
|
|
193
195
|
export interface HumanSettings {
|
|
@@ -272,7 +272,8 @@ async function ensureSessionTopic(
|
|
|
272
272
|
const existingTopic = human.topics.find((t) => t.id === session.id);
|
|
273
273
|
|
|
274
274
|
const firstAgent = await reader.getFirstAgent(session.id);
|
|
275
|
-
const
|
|
275
|
+
const firstPersona = firstAgent ? stateManager.persona_getByName(firstAgent) : null;
|
|
276
|
+
const learnedBy = firstPersona?.id ?? firstAgent ?? "build";
|
|
276
277
|
|
|
277
278
|
if (existingTopic) {
|
|
278
279
|
if (existingTopic.name !== session.title) {
|
|
@@ -2,6 +2,8 @@ import type { ItemUpdatePromptData, PromptOutput } from "./types.js";
|
|
|
2
2
|
import type { DataItemBase } from "../../core/types.js";
|
|
3
3
|
import { formatMessagesAsPlaceholders } from "../message-utils.js";
|
|
4
4
|
|
|
5
|
+
const DESCRIPTION_MAX_CHARS = 500;
|
|
6
|
+
|
|
5
7
|
function formatExistingItem(item: DataItemBase): string {
|
|
6
8
|
return JSON.stringify({
|
|
7
9
|
name: item.name,
|
|
@@ -97,9 +99,28 @@ Examples: "Name Unknown" -> "Robert Jordan", "User was married in the Summer" ->
|
|
|
97
99
|
`;
|
|
98
100
|
|
|
99
101
|
const defaultDescriptionSection = `
|
|
100
|
-
|
|
102
|
+
A concise, evergreen summary of what is currently known about this ${typeLabel}. Personas use this to recall context and make meaningful references.
|
|
103
|
+
|
|
104
|
+
## CRITICAL: Synthesize, don't accumulate
|
|
105
|
+
|
|
106
|
+
Every update must **rewrite** the description as a current-state summary. Never append to it.
|
|
107
|
+
|
|
108
|
+
**Good description**: "Active project to improve test coverage. Settled on Vitest + E2E harness. Currently focused on pipeline integration and extraction logic coverage."
|
|
109
|
+
**Bad description**: "User asked Sisyphus to create a ticket... Later: pruned overengineered framework... Most recent session: added PR checks..."
|
|
110
|
+
|
|
111
|
+
The description should:
|
|
112
|
+
- Capture what is true NOW — the current state, decisions made, where things stand
|
|
113
|
+
- Include details a persona would use to show genuine recall ("Oh right, you were working on the pipeline tests")
|
|
114
|
+
- Be useful to a persona meeting this human for the first time
|
|
115
|
+
- Read as a brief summary paragraph, not a session log
|
|
116
|
+
|
|
117
|
+
The description should NOT:
|
|
118
|
+
- Append "Most recent:", "Latest:", "Current session:", or any temporal marker
|
|
119
|
+
- Accumulate a running history of every conversation that touched this ${typeLabel}
|
|
120
|
+
- Reference specific ticket numbers, commit hashes, or PR numbers unless essential to meaning
|
|
121
|
+
- Exceed 3-4 sentences under any circumstances
|
|
101
122
|
|
|
102
|
-
**ABSOLUTELY VITAL
|
|
123
|
+
**ABSOLUTELY VITAL**: Do **NOT** embellish — personas use their own voice. Capture what is true, not a log of how you got here.
|
|
103
124
|
`;
|
|
104
125
|
|
|
105
126
|
const descriptionSection =
|
|
@@ -332,3 +353,12 @@ If no changes are needed, respond with: \`{}\``;
|
|
|
332
353
|
|
|
333
354
|
return { system, user };
|
|
334
355
|
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Truncate a description to DESCRIPTION_MAX_CHARS for use in prompts.
|
|
359
|
+
* The stored value is unchanged — this only affects what goes into the LLM context.
|
|
360
|
+
*/
|
|
361
|
+
export function truncateDescription(description: string): string {
|
|
362
|
+
if (description.length <= DESCRIPTION_MAX_CHARS) return description;
|
|
363
|
+
return description.slice(0, DESCRIPTION_MAX_CHARS) + "…";
|
|
364
|
+
}
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import type { Trait, Quote, PersonaTopic } from "../../core/types.js";
|
|
7
7
|
import type { ResponsePromptData } from "./types.js";
|
|
8
|
+
import { truncateDescription } from "../human/item-update.js";
|
|
8
9
|
|
|
9
10
|
// =============================================================================
|
|
10
11
|
// IDENTITY SECTION
|
|
@@ -64,10 +65,10 @@ export function buildGuidelinesSection(personaName: string): string {
|
|
|
64
65
|
export function buildTraitsSection(traits: Trait[], header: string): string {
|
|
65
66
|
if (traits.length === 0) return "";
|
|
66
67
|
|
|
67
|
-
const sorted = [...traits].sort((a, b) => (b.strength ?? 0.5) - (a.strength ?? 0.5));
|
|
68
|
+
const sorted = [...traits].sort((a, b) => (b.strength ?? 0.5) - (a.strength ?? 0.5)).slice(0, 15);
|
|
68
69
|
const formatted = sorted.map(t => {
|
|
69
70
|
const strength = t.strength !== undefined ? ` (${Math.round(t.strength * 100)}%)` : "";
|
|
70
|
-
return `- **${t.name}**${strength}: ${t.description}`;
|
|
71
|
+
return `- **${t.name}**${strength}: ${truncateDescription(t.description)}`;
|
|
71
72
|
}).join("\n");
|
|
72
73
|
|
|
73
74
|
return `## ${header}
|
|
@@ -92,6 +93,8 @@ export function buildTopicsSection(topics: PersonaTopic[], header: string): stri
|
|
|
92
93
|
const sorted = [...topics]
|
|
93
94
|
.map(t => ({ topic: t, delta: t.exposure_desired - t.exposure_current }))
|
|
94
95
|
.sort((a, b) => b.delta - a.delta)
|
|
96
|
+
.slice(0, 15)
|
|
97
|
+
.sort((a, b) => b.delta - a.delta)
|
|
95
98
|
.map(x => x.topic);
|
|
96
99
|
|
|
97
100
|
const formatted = sorted.map(t => {
|
|
@@ -137,7 +140,8 @@ export function buildHumanSection(human: ResponsePromptData["human"]): string {
|
|
|
137
140
|
// Facts
|
|
138
141
|
if (human.facts.length > 0) {
|
|
139
142
|
const facts = human.facts
|
|
140
|
-
.
|
|
143
|
+
.slice(0, 15)
|
|
144
|
+
.map(f => `- ${f.name}: ${truncateDescription(f.description)}`)
|
|
141
145
|
.join("\n");
|
|
142
146
|
if (facts) sections.push(`### Key Facts\n${facts}`);
|
|
143
147
|
}
|
|
@@ -145,7 +149,8 @@ export function buildHumanSection(human: ResponsePromptData["human"]): string {
|
|
|
145
149
|
// Traits
|
|
146
150
|
if (human.traits.length > 0) {
|
|
147
151
|
const traits = human.traits
|
|
148
|
-
.
|
|
152
|
+
.slice(0, 15)
|
|
153
|
+
.map(t => `- **${t.name}**: ${truncateDescription(t.description)}`)
|
|
149
154
|
.join("\n");
|
|
150
155
|
sections.push(`### Personality\n${traits}`);
|
|
151
156
|
}
|
|
@@ -155,10 +160,10 @@ export function buildHumanSection(human: ResponsePromptData["human"]): string {
|
|
|
155
160
|
if (activeTopics.length > 0) {
|
|
156
161
|
const topics = activeTopics
|
|
157
162
|
.sort((a, b) => b.exposure_current - a.exposure_current)
|
|
158
|
-
.slice(0,
|
|
163
|
+
.slice(0, 15)
|
|
159
164
|
.map(t => {
|
|
160
165
|
const sentiment = t.sentiment > 0.3 ? "(enjoys)" : t.sentiment < -0.3 ? "(dislikes)" : "";
|
|
161
|
-
return `- **${t.name}** ${sentiment}: ${t.description}`;
|
|
166
|
+
return `- **${t.name}** ${sentiment}: ${truncateDescription(t.description)}`;
|
|
162
167
|
})
|
|
163
168
|
.join("\n");
|
|
164
169
|
sections.push(`### Current Interests\n${topics}`);
|
|
@@ -168,8 +173,8 @@ export function buildHumanSection(human: ResponsePromptData["human"]): string {
|
|
|
168
173
|
if (human.people.length > 0) {
|
|
169
174
|
const people = human.people
|
|
170
175
|
.sort((a, b) => b.exposure_current - a.exposure_current)
|
|
171
|
-
.slice(0,
|
|
172
|
-
.map(p => `- **${p.name}** (${p.relationship}): ${p.description}`)
|
|
176
|
+
.slice(0, 15)
|
|
177
|
+
.map(p => `- **${p.name}** (${p.relationship}): ${truncateDescription(p.description)}`)
|
|
173
178
|
.join("\n");
|
|
174
179
|
sections.push(`### People in Their Life\n${people}`);
|
|
175
180
|
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gzip compression utilities for storage.
|
|
3
|
+
*
|
|
4
|
+
* Uses the native CompressionStream/DecompressionStream API, available in
|
|
5
|
+
* both modern browsers and Bun (no external dependencies).
|
|
6
|
+
*
|
|
7
|
+
* Compressed output is base64-encoded so it can be stored as a plain string
|
|
8
|
+
* (LocalStorage, remote API body, etc.).
|
|
9
|
+
*
|
|
10
|
+
* FileStorage deliberately does NOT use these — uncompressed JSON on disk
|
|
11
|
+
* stays human-readable and debuggable.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export async function compress(json: string): Promise<string> {
|
|
15
|
+
const encoder = new TextEncoder();
|
|
16
|
+
const input = encoder.encode(json);
|
|
17
|
+
|
|
18
|
+
const cs = new CompressionStream("gzip");
|
|
19
|
+
const writer = cs.writable.getWriter();
|
|
20
|
+
writer.write(input);
|
|
21
|
+
writer.close();
|
|
22
|
+
|
|
23
|
+
const chunks: Uint8Array[] = [];
|
|
24
|
+
const reader = cs.readable.getReader();
|
|
25
|
+
while (true) {
|
|
26
|
+
const { done, value } = await reader.read();
|
|
27
|
+
if (done) break;
|
|
28
|
+
chunks.push(value);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
32
|
+
const merged = new Uint8Array(totalLength);
|
|
33
|
+
let offset = 0;
|
|
34
|
+
for (const chunk of chunks) {
|
|
35
|
+
merged.set(chunk, offset);
|
|
36
|
+
offset += chunk.length;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// btoa needs a binary string — convert byte-by-byte
|
|
40
|
+
let binary = "";
|
|
41
|
+
for (let i = 0; i < merged.length; i++) {
|
|
42
|
+
binary += String.fromCharCode(merged[i]);
|
|
43
|
+
}
|
|
44
|
+
return btoa(binary);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function decompress(b64: string): Promise<string> {
|
|
48
|
+
const binary = atob(b64);
|
|
49
|
+
const bytes = new Uint8Array(binary.length);
|
|
50
|
+
for (let i = 0; i < binary.length; i++) {
|
|
51
|
+
bytes[i] = binary.charCodeAt(i);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const ds = new DecompressionStream("gzip");
|
|
55
|
+
const writer = ds.writable.getWriter();
|
|
56
|
+
writer.write(bytes);
|
|
57
|
+
writer.close();
|
|
58
|
+
|
|
59
|
+
const chunks: Uint8Array[] = [];
|
|
60
|
+
const reader = ds.readable.getReader();
|
|
61
|
+
while (true) {
|
|
62
|
+
const { done, value } = await reader.read();
|
|
63
|
+
if (done) break;
|
|
64
|
+
chunks.push(value);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
68
|
+
const merged = new Uint8Array(totalLength);
|
|
69
|
+
let offset = 0;
|
|
70
|
+
for (const chunk of chunks) {
|
|
71
|
+
merged.set(chunk, offset);
|
|
72
|
+
offset += chunk.length;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return new TextDecoder().decode(merged);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Returns true if the string looks like a base64-encoded gzip payload. */
|
|
79
|
+
export function isCompressed(value: string): boolean {
|
|
80
|
+
// gzip magic bytes are 0x1f 0x8b — base64-encoded that starts with "H4sI"
|
|
81
|
+
return value.startsWith("H4sI");
|
|
82
|
+
}
|
package/src/storage/index.ts
CHANGED
|
@@ -3,3 +3,4 @@ export { LocalStorage } from "./local.js";
|
|
|
3
3
|
export { remoteSync, RemoteSync, type RemoteSyncCredentials, type RemoteTimestamp, type SyncResult, type FetchResult } from "./remote.js";
|
|
4
4
|
export { encrypt, decrypt, generateUserId, type CryptoCredentials, type EncryptedPayload } from "./crypto.js";
|
|
5
5
|
export { yoloMerge } from "./merge.js";
|
|
6
|
+
export { compress, decompress, isCompressed } from "./compress.js";
|
package/src/storage/local.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { StorageState } from "../core/types.js";
|
|
2
2
|
import type { Storage } from "./interface.js";
|
|
3
|
+
import { compress, decompress, isCompressed } from "./compress.js";
|
|
3
4
|
|
|
4
5
|
const STATE_KEY = "ei_state";
|
|
5
6
|
const BACKUP_KEY = "ei_state_backup";
|
|
@@ -19,7 +20,9 @@ export class LocalStorage implements Storage {
|
|
|
19
20
|
async save(state: StorageState): Promise<void> {
|
|
20
21
|
state.timestamp = new Date().toISOString();
|
|
21
22
|
try {
|
|
22
|
-
|
|
23
|
+
const json = JSON.stringify(state);
|
|
24
|
+
const payload = await compress(json);
|
|
25
|
+
globalThis.localStorage.setItem(STATE_KEY, payload);
|
|
23
26
|
} catch (e) {
|
|
24
27
|
if (this.isQuotaError(e)) {
|
|
25
28
|
throw new Error("STORAGE_SAVE_FAILED: localStorage quota exceeded");
|
|
@@ -32,7 +35,8 @@ export class LocalStorage implements Storage {
|
|
|
32
35
|
const current = globalThis.localStorage?.getItem(STATE_KEY);
|
|
33
36
|
if (current) {
|
|
34
37
|
try {
|
|
35
|
-
|
|
38
|
+
const json = isCompressed(current) ? await decompress(current) : current;
|
|
39
|
+
return JSON.parse(json) as StorageState;
|
|
36
40
|
} catch {
|
|
37
41
|
return null;
|
|
38
42
|
}
|
|
@@ -62,7 +66,8 @@ export class LocalStorage implements Storage {
|
|
|
62
66
|
const backup = globalThis.localStorage?.getItem(BACKUP_KEY);
|
|
63
67
|
if (backup) {
|
|
64
68
|
try {
|
|
65
|
-
|
|
69
|
+
const json = isCompressed(backup) ? await decompress(backup) : backup;
|
|
70
|
+
return JSON.parse(json) as StorageState;
|
|
66
71
|
} catch {
|
|
67
72
|
return null;
|
|
68
73
|
}
|
package/src/storage/remote.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { StorageState } from "../core/types.js";
|
|
2
2
|
import { encrypt, decrypt, generateUserId, type CryptoCredentials, type EncryptedPayload } from "./crypto.js";
|
|
3
|
+
import { compress, decompress, isCompressed } from "./compress.js";
|
|
3
4
|
|
|
4
5
|
const API_BASE = "https://flare576.com/ei/api";
|
|
5
6
|
|
|
@@ -71,7 +72,8 @@ export class RemoteSync {
|
|
|
71
72
|
|
|
72
73
|
try {
|
|
73
74
|
const stateJson = JSON.stringify(state);
|
|
74
|
-
const
|
|
75
|
+
const compressed = await compress(stateJson);
|
|
76
|
+
const encrypted = await encrypt(compressed, this.credentials);
|
|
75
77
|
const encryptedJson = JSON.stringify(encrypted);
|
|
76
78
|
|
|
77
79
|
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
|
@@ -124,7 +126,9 @@ export class RemoteSync {
|
|
|
124
126
|
|
|
125
127
|
const body = await response.json();
|
|
126
128
|
const encrypted: EncryptedPayload = JSON.parse(body.data);
|
|
127
|
-
const
|
|
129
|
+
const decryptedPayload = await decrypt(encrypted, this.credentials);
|
|
130
|
+
// Support both compressed (new) and uncompressed (legacy) remote state
|
|
131
|
+
const decryptedJson = isCompressed(decryptedPayload) ? await decompress(decryptedPayload) : decryptedPayload;
|
|
128
132
|
const state = JSON.parse(decryptedJson) as StorageState;
|
|
129
133
|
// Capture etag for concurrency protection
|
|
130
134
|
this.lastEtag = response.headers.get("ETag");
|
|
@@ -350,7 +350,10 @@ export function humanToYAML(human: HumanEntity): string {
|
|
|
350
350
|
|
|
351
351
|
return YAML.stringify(data, {
|
|
352
352
|
lineWidth: 0,
|
|
353
|
-
})
|
|
353
|
+
})
|
|
354
|
+
.replace(/^(\s+validated:\s+\S+)$/mg, '$1 # none | ei | human')
|
|
355
|
+
.replace(/^(\s+)(learned_by: .+)$/mg, '$1# [read-only] $2')
|
|
356
|
+
.replace(/^(\s+)(last_changed_by: .+)$/mg, '$1# [read-only] $2');
|
|
354
357
|
}
|
|
355
358
|
|
|
356
359
|
export interface HumanYAMLResult {
|
|
@@ -365,7 +368,12 @@ export interface HumanYAMLResult {
|
|
|
365
368
|
}
|
|
366
369
|
|
|
367
370
|
export function humanFromYAML(yamlContent: string): HumanYAMLResult {
|
|
368
|
-
|
|
371
|
+
// Strip read-only comment lines before parsing so users can't accidentally corrupt them
|
|
372
|
+
const stripped = yamlContent
|
|
373
|
+
.split('\n')
|
|
374
|
+
.filter(line => !/^\s*#\s*\[read-only\]/.test(line))
|
|
375
|
+
.join('\n');
|
|
376
|
+
const data = YAML.parse(stripped) as EditableHumanData;
|
|
369
377
|
|
|
370
378
|
const deletedFactIds: string[] = [];
|
|
371
379
|
const deletedTraitIds: string[] = [];
|