@qearlyao/familiar 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/config.example.toml +1 -1
- package/dist/added-models.js +6 -15
- package/dist/agent-events.js +1 -3
- package/dist/agent.js +3 -4
- package/dist/browser-tools.js +12 -11
- package/dist/chat-log.js +3 -2
- package/dist/cli.js +2 -2
- package/dist/config-overrides.js +5 -14
- package/dist/config-registry.js +1 -4
- package/dist/config.js +45 -113
- package/dist/contact-note.js +2 -12
- package/dist/data-retention.js +1 -3
- package/dist/discord.js +2 -2
- package/dist/generated-media.js +3 -2
- package/dist/hot-reload.js +1 -3
- package/dist/image-gen.js +12 -51
- package/dist/inbound-attachments.js +53 -22
- package/dist/memory/diary/ambient-injector.js +1 -3
- package/dist/memory/diary/ambient.js +1 -3
- package/dist/memory/diary/chunks.js +1 -3
- package/dist/memory/diary/indexer.js +1 -3
- package/dist/memory/doctor.js +3 -8
- package/dist/memory/index/chunk-indexer.js +6 -2
- package/dist/memory/index/retrieval.js +1 -3
- package/dist/memory/index/store.js +47 -19
- package/dist/memory/lcm/backfill.js +19 -16
- package/dist/memory/lcm/context-transformer.js +12 -24
- package/dist/memory/lcm/context.js +10 -4
- package/dist/memory/lcm/eviction-score.js +25 -13
- package/dist/memory/lcm/indexer.js +1 -5
- package/dist/memory/lcm/normalize.js +22 -1
- package/dist/memory/lcm/store.js +27 -24
- package/dist/memory/operator.js +2 -4
- package/dist/memory/service.js +1 -3
- package/dist/memory/tools.js +0 -4
- package/dist/memory/util.js +6 -0
- package/dist/models.js +3 -0
- package/dist/persona.js +2 -14
- package/dist/runtime.js +2 -23
- package/dist/scheduler.js +15 -49
- package/dist/service.js +24 -14
- package/dist/settings.js +7 -32
- package/dist/tts.js +0 -6
- package/dist/util/fs.js +41 -0
- package/dist/util/guards.js +8 -0
- package/dist/util/image-mime.js +31 -0
- package/dist/util/time.js +29 -0
- package/dist/web-auth.js +4 -1
- package/dist/web-tools.js +8 -5
- package/dist/web.js +188 -62
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/web/dist/assets/index-B23WT77N.js +63 -0
- package/web/dist/assets/index-D3MotFzN.css +2 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-C-w9fjBf.js +0 -61
- package/web/dist/assets/index-CcQ13VAY.css +0 -2
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { condense } from "./condense.js";
|
|
2
|
-
import { createRawContextItems, estimateAgentMessageTokens,
|
|
2
|
+
import { createRawContextItems, estimateAgentMessageTokens, renderLcmRecordPartsForSummary, resolveFreshTailStartIndex, selectLcmCompactionCandidatePromptAware, } from "./context.js";
|
|
3
3
|
import { indexLcmSummaries } from "./indexer.js";
|
|
4
4
|
import { createSyntheticLcmSummaryMessage } from "./summarizer.js";
|
|
5
5
|
const LCM_SUMMARY_OPEN_TAG = "<from_earlier>";
|
|
@@ -48,6 +48,7 @@ export class LcmContextTransformer {
|
|
|
48
48
|
signal,
|
|
49
49
|
model: options.model,
|
|
50
50
|
promptText,
|
|
51
|
+
initialPressure: pressure,
|
|
51
52
|
});
|
|
52
53
|
}
|
|
53
54
|
}
|
|
@@ -76,7 +77,9 @@ export class LcmContextTransformer {
|
|
|
76
77
|
}
|
|
77
78
|
async serviceCompactionDebtForState(input) {
|
|
78
79
|
for (let round = 0; input.state.compactionDebt > 0 && round < this.settings.maxRounds; round += 1) {
|
|
79
|
-
const pressure =
|
|
80
|
+
const pressure = round === 0 && input.initialPressure
|
|
81
|
+
? input.initialPressure
|
|
82
|
+
: this.evaluateCompactionPressure(input.state, input.model, input.promptText ?? "");
|
|
80
83
|
if (!pressure.candidate.shouldCompact) {
|
|
81
84
|
if (pressure.thresholdOverflowTokens > 0) {
|
|
82
85
|
const condensed = await this.condenseRuntimeSummaries({
|
|
@@ -262,8 +265,9 @@ export class LcmContextTransformer {
|
|
|
262
265
|
this.lcmStore.db
|
|
263
266
|
.transaction(() => {
|
|
264
267
|
for (const insert of inserts) {
|
|
265
|
-
|
|
266
|
-
insert.item.
|
|
268
|
+
const result = this.lcmStore.insertRecordReturningStored(insert.input);
|
|
269
|
+
insert.item.recordId = result.record.id;
|
|
270
|
+
insert.item.record = result.record;
|
|
267
271
|
}
|
|
268
272
|
})
|
|
269
273
|
.immediate();
|
|
@@ -275,20 +279,7 @@ export class LcmContextTransformer {
|
|
|
275
279
|
const items = [];
|
|
276
280
|
for (const row of rows) {
|
|
277
281
|
if (row.type === "raw") {
|
|
278
|
-
|
|
279
|
-
if (!record) {
|
|
280
|
-
console.error(`memory LCM context item dropped because record ${row.recordId} is missing`);
|
|
281
|
-
continue;
|
|
282
|
-
}
|
|
283
|
-
const message = lcmRecordToAgentMessage(record);
|
|
284
|
-
items.push({
|
|
285
|
-
type: "raw",
|
|
286
|
-
id: row.fingerprint,
|
|
287
|
-
recordId: record.id,
|
|
288
|
-
record,
|
|
289
|
-
message,
|
|
290
|
-
tokens: estimateAgentMessageTokens(message),
|
|
291
|
-
});
|
|
282
|
+
// Raw history is owned by transcripts; legacy raw context rows must not replay it.
|
|
292
283
|
continue;
|
|
293
284
|
}
|
|
294
285
|
const summary = this.lcmStore.getSummary(row.summaryId);
|
|
@@ -359,8 +350,6 @@ function syncContextState(state, messages) {
|
|
|
359
350
|
const replacement = rawById.get(item.id);
|
|
360
351
|
if (replacement && !covered.has(item.id))
|
|
361
352
|
next.push(replacement);
|
|
362
|
-
else if (item.recordId !== null && !covered.has(item.id))
|
|
363
|
-
next.push(item);
|
|
364
353
|
}
|
|
365
354
|
for (const item of rawItems) {
|
|
366
355
|
if (!covered.has(item.id) && !next.some((existing) => existing.type === "raw" && existing.id === item.id)) {
|
|
@@ -390,8 +379,6 @@ function contextItemsForStorage(items) {
|
|
|
390
379
|
const timestamp = item.message.timestamp;
|
|
391
380
|
const happenedAt = typeof timestamp === "number" && Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
|
|
392
381
|
if (item.type === "raw") {
|
|
393
|
-
if (item.recordId !== null)
|
|
394
|
-
stored.push({ type: "raw", recordId: item.recordId, fingerprint: item.id, happenedAt });
|
|
395
382
|
continue;
|
|
396
383
|
}
|
|
397
384
|
if (item.persistedSummaryId !== undefined) {
|
|
@@ -522,9 +509,10 @@ function assembleWithinBudget(state, settings, model) {
|
|
|
522
509
|
const freshTail = state.items.slice(resolveFreshTailStartIndexForState(state.items, settings));
|
|
523
510
|
const selected = new Set(freshTail);
|
|
524
511
|
let tokens = sumItemTokens(freshTail);
|
|
512
|
+
const originalIndexes = new Map(state.items.map((item, index) => [item, index]));
|
|
525
513
|
const summaries = state.items
|
|
526
514
|
.filter((item) => item.type === "summary" && !selected.has(item))
|
|
527
|
-
.sort((a, b) => b.depth - a.depth ||
|
|
515
|
+
.sort((a, b) => b.depth - a.depth || (originalIndexes.get(b) ?? 0) - (originalIndexes.get(a) ?? 0));
|
|
528
516
|
for (const item of summaries) {
|
|
529
517
|
if (tokens + item.tokens > budget && selected.size > 0)
|
|
530
518
|
continue;
|
|
@@ -587,7 +575,7 @@ function lcmRecordPartsFromAgentMessage(message) {
|
|
|
587
575
|
kind: "tool_result",
|
|
588
576
|
toolCallId: toolResult.toolCallId ?? "",
|
|
589
577
|
toolName: toolResult.toolName ?? "tool",
|
|
590
|
-
output:
|
|
578
|
+
output: textFromContent(toolResult.content),
|
|
591
579
|
...(toolResult.isError ? { isError: true } : {}),
|
|
592
580
|
},
|
|
593
581
|
];
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { scoreEvictable, tokenBag } from "./eviction-score.js";
|
|
2
|
+
import { buildEvictionScoreContext, scoreEvictable, tokenBag } from "./eviction-score.js";
|
|
3
3
|
const MESSAGE_OVERHEAD_TOKENS = 6;
|
|
4
4
|
const RECORD_OVERHEAD_TOKENS = 4;
|
|
5
5
|
const IMAGE_TOKEN_ESTIMATE = 1200;
|
|
@@ -167,9 +167,12 @@ function selectLeafChunk(items, leafChunkTokens, promptText, config) {
|
|
|
167
167
|
const records = items.map((item) => item.record).filter((record) => !!record);
|
|
168
168
|
if (records.length === 0)
|
|
169
169
|
return selectOldestLeafChunk(items, leafChunkTokens);
|
|
170
|
+
const scoreContext = buildEvictionScoreContext(promptText, records);
|
|
171
|
+
if (!scoreContext)
|
|
172
|
+
return selectOldestLeafChunk(items, leafChunkTokens);
|
|
170
173
|
const scored = targetRanges.map((range) => ({
|
|
171
174
|
...range,
|
|
172
|
-
score: range.items.reduce((total, item) => total + (item.record ? scoreEvictable(item.record, promptText, records) : 0), 0),
|
|
175
|
+
score: range.items.reduce((total, item) => total + (item.record ? scoreEvictable(item.record, promptText, records, scoreContext) : 0), 0),
|
|
173
176
|
}));
|
|
174
177
|
scored.sort((a, b) => a.score - b.score || a.startIndex - b.startIndex);
|
|
175
178
|
return scored[0]?.items ?? [];
|
|
@@ -179,7 +182,7 @@ function createValidLeafRanges(items, leafChunkTokens) {
|
|
|
179
182
|
for (let startIndex = 0; startIndex < items.length; startIndex += 1) {
|
|
180
183
|
if (isToolResultContinuingPreviousToolCall(items, startIndex))
|
|
181
184
|
continue;
|
|
182
|
-
const chunk =
|
|
185
|
+
const chunk = selectOldestLeafChunkFromIndex(items, startIndex, leafChunkTokens);
|
|
183
186
|
if (chunk.length === 0)
|
|
184
187
|
continue;
|
|
185
188
|
const chunkTokens = chunk.reduce((total, item) => total + item.tokens, 0);
|
|
@@ -194,9 +197,12 @@ function createValidLeafRanges(items, leafChunkTokens) {
|
|
|
194
197
|
return ranges;
|
|
195
198
|
}
|
|
196
199
|
function selectOldestLeafChunk(items, leafChunkTokens) {
|
|
200
|
+
return selectOldestLeafChunkFromIndex(items, 0, leafChunkTokens);
|
|
201
|
+
}
|
|
202
|
+
function selectOldestLeafChunkFromIndex(items, startIndex, leafChunkTokens) {
|
|
197
203
|
const chunk = [];
|
|
198
204
|
let tokens = 0;
|
|
199
|
-
for (let index =
|
|
205
|
+
for (let index = startIndex; index < items.length; index += 1) {
|
|
200
206
|
const item = items[index];
|
|
201
207
|
if (!item)
|
|
202
208
|
continue;
|
|
@@ -4,34 +4,46 @@ export function tokenBag(text) {
|
|
|
4
4
|
.split(/[^a-z0-9]+/)
|
|
5
5
|
.filter((token) => token.length >= 2);
|
|
6
6
|
}
|
|
7
|
-
export function
|
|
7
|
+
export function buildEvictionScoreContext(prompt, allRecords) {
|
|
8
8
|
const promptTerms = tokenBag(prompt);
|
|
9
9
|
if (promptTerms.length === 0)
|
|
10
|
-
return
|
|
11
|
-
const recordTerms = tokenBag(record.text);
|
|
12
|
-
if (recordTerms.length === 0)
|
|
13
|
-
return 0;
|
|
14
|
-
const recordFreq = new Map();
|
|
15
|
-
for (const term of recordTerms)
|
|
16
|
-
recordFreq.set(term, (recordFreq.get(term) ?? 0) + 1);
|
|
10
|
+
return null;
|
|
17
11
|
const promptUniqueTerms = new Set(promptTerms);
|
|
18
12
|
const documentFrequencies = new Map();
|
|
13
|
+
const recordTerms = new Map();
|
|
19
14
|
for (const candidate of allRecords) {
|
|
20
|
-
const
|
|
15
|
+
const terms = tokenBag(candidate.text);
|
|
16
|
+
recordTerms.set(candidate, terms);
|
|
17
|
+
const candidateTerms = new Set(terms);
|
|
21
18
|
for (const term of promptUniqueTerms) {
|
|
22
19
|
if (candidateTerms.has(term))
|
|
23
20
|
documentFrequencies.set(term, (documentFrequencies.get(term) ?? 0) + 1);
|
|
24
21
|
}
|
|
25
22
|
}
|
|
26
|
-
|
|
23
|
+
return {
|
|
24
|
+
promptUniqueTerms,
|
|
25
|
+
documentFrequencies,
|
|
26
|
+
recordTerms,
|
|
27
|
+
candidateCount: Math.max(0, allRecords.length),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export function scoreEvictable(record, prompt, allRecords, context = buildEvictionScoreContext(prompt, allRecords)) {
|
|
31
|
+
if (!context)
|
|
32
|
+
return 0;
|
|
33
|
+
const recordTerms = context.recordTerms.get(record) ?? tokenBag(record.text);
|
|
34
|
+
if (recordTerms.length === 0)
|
|
35
|
+
return 0;
|
|
36
|
+
const recordFreq = new Map();
|
|
37
|
+
for (const term of recordTerms)
|
|
38
|
+
recordFreq.set(term, (recordFreq.get(term) ?? 0) + 1);
|
|
27
39
|
let score = 0;
|
|
28
|
-
for (const term of promptUniqueTerms) {
|
|
40
|
+
for (const term of context.promptUniqueTerms) {
|
|
29
41
|
const tf = recordFreq.get(term) ?? 0;
|
|
30
42
|
if (tf <= 0)
|
|
31
43
|
continue;
|
|
32
44
|
const normalizedTf = tf / recordTerms.length;
|
|
33
|
-
const df = documentFrequencies.get(term) ?? 0;
|
|
34
|
-
const idf = Math.log((candidateCount + 1) / (df + 1) + 1);
|
|
45
|
+
const df = context.documentFrequencies.get(term) ?? 0;
|
|
46
|
+
const idf = Math.log((context.candidateCount + 1) / (df + 1) + 1);
|
|
35
47
|
score += normalizedTf * idf;
|
|
36
48
|
}
|
|
37
49
|
return score;
|
|
@@ -8,11 +8,7 @@ export async function projectNormalizedLcmBatch(options) {
|
|
|
8
8
|
}
|
|
9
9
|
const storedRecords = [];
|
|
10
10
|
for (const record of options.batch.records) {
|
|
11
|
-
|
|
12
|
-
const stored = options.lcmStore.getRecord(id);
|
|
13
|
-
if (!stored)
|
|
14
|
-
throw new Error(`Failed to read projected LCM record: ${id}`);
|
|
15
|
-
storedRecords.push(stored);
|
|
11
|
+
storedRecords.push(options.lcmStore.insertRecordReturningStored(record).record);
|
|
16
12
|
}
|
|
17
13
|
return {
|
|
18
14
|
segmentIds,
|
|
@@ -59,7 +59,7 @@ export function normalizeChatRecords(chatRecords, options) {
|
|
|
59
59
|
kind: "tool_result",
|
|
60
60
|
toolCallId: record.event.toolCallId,
|
|
61
61
|
toolName: record.event.toolName,
|
|
62
|
-
output: record.event.result,
|
|
62
|
+
output: toolResultVisibleOutput(record.event.result),
|
|
63
63
|
...(record.event.isError ? { isError: true } : {}),
|
|
64
64
|
},
|
|
65
65
|
];
|
|
@@ -191,6 +191,27 @@ function briefJson(value, maxLength = 160) {
|
|
|
191
191
|
const compact = text.replace(/\s+/g, " ").trim();
|
|
192
192
|
return compact.length > maxLength ? `${compact.slice(0, maxLength - 1)}...` : compact;
|
|
193
193
|
}
|
|
194
|
+
function toolResultVisibleOutput(result) {
|
|
195
|
+
if (!result || typeof result !== "object" || !("content" in result))
|
|
196
|
+
return result;
|
|
197
|
+
const content = result.content;
|
|
198
|
+
if (!Array.isArray(content))
|
|
199
|
+
return result;
|
|
200
|
+
return textFromToolResultContent(content);
|
|
201
|
+
}
|
|
202
|
+
function textFromToolResultContent(content) {
|
|
203
|
+
if (!Array.isArray(content))
|
|
204
|
+
return content;
|
|
205
|
+
return content
|
|
206
|
+
.map((item) => {
|
|
207
|
+
if (item && typeof item === "object" && item.type === "text") {
|
|
208
|
+
return item.text;
|
|
209
|
+
}
|
|
210
|
+
return "";
|
|
211
|
+
})
|
|
212
|
+
.filter((item) => typeof item === "string" && item.length > 0)
|
|
213
|
+
.join("\n");
|
|
214
|
+
}
|
|
194
215
|
function chatRecordSource(record, sourcePath) {
|
|
195
216
|
const messageId = "messageId" in record && typeof record.messageId === "string"
|
|
196
217
|
? record.messageId
|
package/dist/memory/lcm/store.js
CHANGED
|
@@ -3,6 +3,7 @@ import { mkdirSync } from "node:fs";
|
|
|
3
3
|
import { dirname, resolve } from "node:path";
|
|
4
4
|
import Database from "better-sqlite3";
|
|
5
5
|
import { normalizeFtsMatchQuery } from "../index/fts-query.js";
|
|
6
|
+
import { runInTransaction } from "../util.js";
|
|
6
7
|
import { readMeta, runLcmMigrations } from "./schema.js";
|
|
7
8
|
export class LcmStore {
|
|
8
9
|
db;
|
|
@@ -34,6 +35,9 @@ export class LcmStore {
|
|
|
34
35
|
const raw = readMeta(this.db, "schema_version");
|
|
35
36
|
return raw ? Number(raw) : null;
|
|
36
37
|
}
|
|
38
|
+
computeRecordKey(input) {
|
|
39
|
+
return computeLcmRecordKey(input);
|
|
40
|
+
}
|
|
37
41
|
ensureSegment(input) {
|
|
38
42
|
const startedAt = input.startedAt ?? new Date().toISOString();
|
|
39
43
|
this.db
|
|
@@ -68,6 +72,9 @@ export class LcmStore {
|
|
|
68
72
|
return rows.map(segmentFromRow);
|
|
69
73
|
}
|
|
70
74
|
insertRecord(input) {
|
|
75
|
+
return this.insertRecordReturningStored(input).record.id;
|
|
76
|
+
}
|
|
77
|
+
insertRecordReturningStored(input) {
|
|
71
78
|
const normalized = normalizeRecordInput(input);
|
|
72
79
|
const runInsert = () => {
|
|
73
80
|
this.ensureSegment({
|
|
@@ -76,14 +83,13 @@ export class LcmStore {
|
|
|
76
83
|
channelKey: normalized.channelKey,
|
|
77
84
|
startedAt: normalized.happenedAt,
|
|
78
85
|
});
|
|
79
|
-
const existing = this.db
|
|
80
|
-
.prepare("SELECT id FROM lcm_records WHERE record_key = ?")
|
|
81
|
-
.get(normalized.recordKey);
|
|
86
|
+
const existing = this.db.prepare("SELECT * FROM lcm_records WHERE record_key = ?").get(normalized.recordKey);
|
|
82
87
|
if (existing)
|
|
83
|
-
return existing
|
|
84
|
-
|
|
88
|
+
return { record: recordFromRow(existing), inserted: false };
|
|
89
|
+
const inserted = insertRecordPrepared(this.db, normalized);
|
|
90
|
+
return { record: recordFromRow(inserted), inserted: true };
|
|
85
91
|
};
|
|
86
|
-
return this.db
|
|
92
|
+
return runInTransaction(this.db, runInsert);
|
|
87
93
|
}
|
|
88
94
|
getRecord(id) {
|
|
89
95
|
const row = this.db.prepare("SELECT * FROM lcm_records WHERE id = ?").get(id);
|
|
@@ -132,7 +138,7 @@ export class LcmStore {
|
|
|
132
138
|
this.insertSummarySources(id, sources);
|
|
133
139
|
this.insertSummaryParents(id, parents);
|
|
134
140
|
});
|
|
135
|
-
const result = this.db
|
|
141
|
+
const result = runInTransaction(this.db, runInsert);
|
|
136
142
|
return result;
|
|
137
143
|
}
|
|
138
144
|
getSummary(id) {
|
|
@@ -246,8 +252,14 @@ export class LcmStore {
|
|
|
246
252
|
.all(segmentId, retainDepth);
|
|
247
253
|
for (const summary of summaries) {
|
|
248
254
|
report.indexDeletes.push({ corpus: "lcm_summary", sourceId: lcmSummaryIndexSourceId(summary.id) });
|
|
249
|
-
report.summaryFtsRowsDeleted += this.deleteSummaryFtsRow(summary.id);
|
|
250
255
|
}
|
|
256
|
+
report.summaryFtsRowsDeleted += this.db
|
|
257
|
+
.prepare(`DELETE FROM lcm_summaries_fts
|
|
258
|
+
WHERE rowid IN (
|
|
259
|
+
SELECT id FROM lcm_summaries
|
|
260
|
+
WHERE segment_id = ? AND pinned = 0 AND depth < ?
|
|
261
|
+
)`)
|
|
262
|
+
.run(segmentId, retainDepth).changes;
|
|
251
263
|
report.summariesDeleted += this.db
|
|
252
264
|
.prepare("DELETE FROM lcm_summaries WHERE segment_id = ? AND pinned = 0 AND depth < ?")
|
|
253
265
|
.run(segmentId, retainDepth).changes;
|
|
@@ -257,8 +269,10 @@ export class LcmStore {
|
|
|
257
269
|
const records = this.db.prepare("SELECT id FROM lcm_records WHERE segment_id = ?").all(segmentId);
|
|
258
270
|
for (const record of records) {
|
|
259
271
|
report.indexDeletes.push({ corpus: "lcm_record", sourceId: lcmRecordIndexSourceId(record.id) });
|
|
260
|
-
report.recordFtsRowsDeleted += this.deleteRecordFtsRow(record.id);
|
|
261
272
|
}
|
|
273
|
+
report.recordFtsRowsDeleted += this.db
|
|
274
|
+
.prepare("DELETE FROM lcm_records_fts WHERE rowid IN (SELECT id FROM lcm_records WHERE segment_id = ?)")
|
|
275
|
+
.run(segmentId).changes;
|
|
262
276
|
report.rawRecordsDeleted += this.db
|
|
263
277
|
.prepare("DELETE FROM lcm_records WHERE segment_id = ?")
|
|
264
278
|
.run(segmentId).changes;
|
|
@@ -318,20 +332,6 @@ export class LcmStore {
|
|
|
318
332
|
}
|
|
319
333
|
return map;
|
|
320
334
|
}
|
|
321
|
-
deleteRecordFtsRow(id) {
|
|
322
|
-
const row = this.db.prepare("SELECT text_full FROM lcm_records WHERE id = ?").get(id);
|
|
323
|
-
if (!row)
|
|
324
|
-
return 0;
|
|
325
|
-
this.db.prepare("DELETE FROM lcm_records_fts WHERE rowid = ?").run(id);
|
|
326
|
-
return 1;
|
|
327
|
-
}
|
|
328
|
-
deleteSummaryFtsRow(id) {
|
|
329
|
-
const row = this.db.prepare("SELECT text_full FROM lcm_summaries WHERE id = ?").get(id);
|
|
330
|
-
if (!row)
|
|
331
|
-
return 0;
|
|
332
|
-
this.db.prepare("DELETE FROM lcm_summaries_fts WHERE rowid = ?").run(id);
|
|
333
|
-
return 1;
|
|
334
|
-
}
|
|
335
335
|
snapshotSummariesForPrunedRecords(segmentId) {
|
|
336
336
|
const summaries = this.db
|
|
337
337
|
.prepare(`SELECT * FROM lcm_summaries
|
|
@@ -424,7 +424,10 @@ function insertRecordPrepared(db, normalized) {
|
|
|
424
424
|
if (normalized.kind !== "boundary") {
|
|
425
425
|
db.prepare("INSERT INTO lcm_records_fts(rowid, text_full) VALUES (?, ?)").run(id, normalized.text);
|
|
426
426
|
}
|
|
427
|
-
|
|
427
|
+
const row = db.prepare("SELECT * FROM lcm_records WHERE id = ?").get(id);
|
|
428
|
+
if (!row)
|
|
429
|
+
throw new Error(`Failed to read inserted LCM record: ${id}`);
|
|
430
|
+
return row;
|
|
428
431
|
}
|
|
429
432
|
function normalizeSummaryInput(input) {
|
|
430
433
|
const text = (input.text ?? "").trim();
|
package/dist/memory/operator.js
CHANGED
|
@@ -11,6 +11,7 @@ import { backfillFromChatLogs } from "./lcm/backfill.js";
|
|
|
11
11
|
import { indexLcmRecords, indexLcmSummaries } from "./lcm/indexer.js";
|
|
12
12
|
import { lcmRecordIndexSourceId, lcmSummaryIndexSourceId } from "./lcm/store.js";
|
|
13
13
|
import { MemoryService } from "./service.js";
|
|
14
|
+
import { runInTransaction } from "./util.js";
|
|
14
15
|
export async function runMemoryOperator(config, argv) {
|
|
15
16
|
const [command, ...args] = argv;
|
|
16
17
|
if (!command || command === "--help" || command === "help") {
|
|
@@ -151,10 +152,7 @@ async function reindex(config, service, options, embeddingProvider, signal) {
|
|
|
151
152
|
for (const corpus of corpora)
|
|
152
153
|
deleteCorpus(service.memoryStore, corpus);
|
|
153
154
|
};
|
|
154
|
-
|
|
155
|
-
runDelete();
|
|
156
|
-
else
|
|
157
|
-
service.memoryStore.db.transaction(runDelete).immediate();
|
|
155
|
+
runInTransaction(service.memoryStore.db, runDelete);
|
|
158
156
|
const indexer = options.force
|
|
159
157
|
? new ChunkIndexer({
|
|
160
158
|
store: service.memoryStore,
|
package/dist/memory/service.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { watch } from "node:fs";
|
|
2
2
|
import { basename, resolve } from "node:path";
|
|
3
|
+
import { isEnoent } from "../util/fs.js";
|
|
3
4
|
import { __ambientDiaryInjectorTest, AmbientDiaryInjector } from "./diary/ambient-injector.js";
|
|
4
5
|
import { DIARY_INDEX_FILE_RE, indexAllDiaryFiles, indexDiaryFile, removeDiaryFileIndex } from "./diary/indexer.js";
|
|
5
6
|
import { ChunkIndexer } from "./index/chunk-indexer.js";
|
|
@@ -196,7 +197,4 @@ function parseIndexSourceId(value, prefix) {
|
|
|
196
197
|
const id = Number(value.slice(prefix.length + 1));
|
|
197
198
|
return Number.isInteger(id) && id > 0 ? id : null;
|
|
198
199
|
}
|
|
199
|
-
function isEnoent(error) {
|
|
200
|
-
return !!error && typeof error === "object" && "code" in error && error.code === "ENOENT";
|
|
201
|
-
}
|
|
202
200
|
export const __memoryServiceTest = __ambientDiaryInjectorTest;
|
package/dist/memory/tools.js
CHANGED
package/dist/models.js
CHANGED
|
@@ -95,6 +95,9 @@ export function createConfiguredModel(config) {
|
|
|
95
95
|
}
|
|
96
96
|
return resolveModel(ref, config);
|
|
97
97
|
}
|
|
98
|
+
export function resolveProviderSetting(records, provider, modelId) {
|
|
99
|
+
return records[`${provider}/${modelId}`] ?? records[provider];
|
|
100
|
+
}
|
|
98
101
|
export function resolveModelApiKey(config, model) {
|
|
99
102
|
const configuredEnv = config.models.apiKeyEnvs[`${model.provider}/${model.id}`] ??
|
|
100
103
|
config.models.apiKeyEnvs[model.provider] ??
|
package/dist/persona.js
CHANGED
|
@@ -1,23 +1,11 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
|
-
|
|
3
|
-
return error instanceof Error && "code" in error && error.code === "ENOENT";
|
|
4
|
-
}
|
|
5
|
-
async function readOptionalPersonaFile(path) {
|
|
6
|
-
try {
|
|
7
|
-
return await readFile(path, "utf8");
|
|
8
|
-
}
|
|
9
|
-
catch (error) {
|
|
10
|
-
if (isMissingFile(error))
|
|
11
|
-
return null;
|
|
12
|
-
throw error;
|
|
13
|
-
}
|
|
14
|
-
}
|
|
2
|
+
import { readFileOrNull } from "./util/fs.js";
|
|
15
3
|
export async function loadPersona(config) {
|
|
16
4
|
const [soul, user, memory, inner] = await Promise.all([
|
|
17
5
|
readFile(config.persona.soul, "utf8"),
|
|
18
6
|
readFile(config.persona.user, "utf8"),
|
|
19
7
|
readFile(config.persona.memory, "utf8"),
|
|
20
|
-
|
|
8
|
+
readFileOrNull(config.persona.inner, "utf8"),
|
|
21
9
|
]);
|
|
22
10
|
return { soul, user, memory, inner };
|
|
23
11
|
}
|
package/dist/runtime.js
CHANGED
|
@@ -1,31 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { buildRecordBase, } from "./chat-log.js";
|
|
3
3
|
import { promptAttachmentNotes } from "./inbound-attachments.js";
|
|
4
|
+
import { formatLocalTimestamp } from "./util/time.js";
|
|
4
5
|
function formatAuthor(authorName, authorId) {
|
|
5
6
|
return authorName ? `${authorName} (uid:${authorId})` : `uid:${authorId}`;
|
|
6
7
|
}
|
|
7
|
-
function formatLocalTimestamp(ts) {
|
|
8
|
-
const date = new Date(ts);
|
|
9
|
-
if (Number.isNaN(date.getTime()))
|
|
10
|
-
return ts;
|
|
11
|
-
const offsetMinutes = -date.getTimezoneOffset();
|
|
12
|
-
const sign = offsetMinutes >= 0 ? "+" : "-";
|
|
13
|
-
const absolute = Math.abs(offsetMinutes);
|
|
14
|
-
const hours = Math.floor(absolute / 60);
|
|
15
|
-
const minutes = absolute % 60;
|
|
16
|
-
const offset = minutes === 0 ? `GMT${sign}${hours}` : `GMT${sign}${hours}:${String(minutes).padStart(2, "0")}`;
|
|
17
|
-
const local = [
|
|
18
|
-
date.getFullYear(),
|
|
19
|
-
String(date.getMonth() + 1).padStart(2, "0"),
|
|
20
|
-
String(date.getDate()).padStart(2, "0"),
|
|
21
|
-
].join("-");
|
|
22
|
-
const time = [
|
|
23
|
-
String(date.getHours()).padStart(2, "0"),
|
|
24
|
-
String(date.getMinutes()).padStart(2, "0"),
|
|
25
|
-
String(date.getSeconds()).padStart(2, "0"),
|
|
26
|
-
].join(":");
|
|
27
|
-
return `${local} ${time} ${offset}`;
|
|
28
|
-
}
|
|
29
8
|
function formatPromptRecord(record) {
|
|
30
9
|
const text = record.text.trim() || "(no text)";
|
|
31
10
|
const author = record.authorName?.trim()
|
|
@@ -365,7 +344,7 @@ export class ConversationRuntime {
|
|
|
365
344
|
}
|
|
366
345
|
async noteOutbound(options) {
|
|
367
346
|
const text = options.text.trim();
|
|
368
|
-
if (!text && options.messageIds.length === 0)
|
|
347
|
+
if (!text && options.messageIds.length === 0 && !options.silent)
|
|
369
348
|
return undefined;
|
|
370
349
|
const record = {
|
|
371
350
|
type: "outbound",
|
package/dist/scheduler.js
CHANGED
|
@@ -1,35 +1,9 @@
|
|
|
1
|
-
import { appendFile, mkdir,
|
|
1
|
+
import { appendFile, mkdir, rename, writeFile } from "node:fs/promises";
|
|
2
2
|
import { dirname, resolve } from "node:path";
|
|
3
|
+
import { readFileOrNull } from "./util/fs.js";
|
|
4
|
+
import { formatLocalTimestamp, toDate } from "./util/time.js";
|
|
5
|
+
export { formatLocalTimestamp } from "./util/time.js";
|
|
3
6
|
const stateWriteQueues = new Map();
|
|
4
|
-
function toDate(value) {
|
|
5
|
-
if (value instanceof Date)
|
|
6
|
-
return value;
|
|
7
|
-
return new Date(value);
|
|
8
|
-
}
|
|
9
|
-
function formatOffset(date) {
|
|
10
|
-
const offsetMinutes = -date.getTimezoneOffset();
|
|
11
|
-
const sign = offsetMinutes >= 0 ? "+" : "-";
|
|
12
|
-
const absolute = Math.abs(offsetMinutes);
|
|
13
|
-
const hours = Math.floor(absolute / 60);
|
|
14
|
-
const minutes = absolute % 60;
|
|
15
|
-
return minutes === 0 ? `GMT${sign}${hours}` : `GMT${sign}${hours}:${String(minutes).padStart(2, "0")}`;
|
|
16
|
-
}
|
|
17
|
-
export function formatLocalTimestamp(value) {
|
|
18
|
-
const date = toDate(value);
|
|
19
|
-
if (Number.isNaN(date.getTime()))
|
|
20
|
-
return String(value);
|
|
21
|
-
const localDate = [
|
|
22
|
-
date.getFullYear(),
|
|
23
|
-
String(date.getMonth() + 1).padStart(2, "0"),
|
|
24
|
-
String(date.getDate()).padStart(2, "0"),
|
|
25
|
-
].join("-");
|
|
26
|
-
const localTime = [
|
|
27
|
-
String(date.getHours()).padStart(2, "0"),
|
|
28
|
-
String(date.getMinutes()).padStart(2, "0"),
|
|
29
|
-
String(date.getSeconds()).padStart(2, "0"),
|
|
30
|
-
].join(":");
|
|
31
|
-
return `${localDate} ${localTime} ${formatOffset(date)}`;
|
|
32
|
-
}
|
|
33
7
|
export function formatIdleDuration(ms) {
|
|
34
8
|
if (!Number.isFinite(ms) || ms <= 0)
|
|
35
9
|
return "0m";
|
|
@@ -53,9 +27,7 @@ export function buildHeartbeatInjectionText(options) {
|
|
|
53
27
|
const body = options.body ??
|
|
54
28
|
`hey~ been quiet for a bit. this is your time now.
|
|
55
29
|
|
|
56
|
-
what you do with it is up to you — HEARTBEAT.md has the menu if you don't remember it. once you know the shape of it you don't have to re-read every fire
|
|
57
|
-
|
|
58
|
-
it's okay to sit one out, but only when that's actually the real answer — not when it's the easy one.`;
|
|
30
|
+
what you do with it is up to you — HEARTBEAT.md has the menu if you don't remember it. once you know the shape of it you don't have to re-read every fire.`;
|
|
59
31
|
return `<heartbeat local_time="${formatLocalTimestamp(nowDate)}" idle_duration="${formatIdleDuration(idleDurationMs)}" idle_minutes="${idleMinutes}">\n${body}\n</heartbeat>`;
|
|
60
32
|
}
|
|
61
33
|
function pad2(value) {
|
|
@@ -146,22 +118,16 @@ export function schedulerLogPath(dataDir, now = new Date()) {
|
|
|
146
118
|
return resolve(dataDir, "scheduler", `${now.toISOString().slice(0, 10)}.jsonl`);
|
|
147
119
|
}
|
|
148
120
|
export async function loadSchedulerState(dataDir) {
|
|
149
|
-
const
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
}
|
|
160
|
-
catch (error) {
|
|
161
|
-
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
|
|
162
|
-
return { cron: {} };
|
|
163
|
-
throw error;
|
|
164
|
-
}
|
|
121
|
+
const raw = await readFileOrNull(schedulerStatePath(dataDir), "utf8");
|
|
122
|
+
if (raw === null)
|
|
123
|
+
return { cron: {} };
|
|
124
|
+
const parsed = JSON.parse(raw);
|
|
125
|
+
return {
|
|
126
|
+
heartbeat: parsed.heartbeat && typeof parsed.heartbeat === "object" && !Array.isArray(parsed.heartbeat)
|
|
127
|
+
? parsed.heartbeat
|
|
128
|
+
: undefined,
|
|
129
|
+
cron: parsed.cron && typeof parsed.cron === "object" && !Array.isArray(parsed.cron) ? parsed.cron : {},
|
|
130
|
+
};
|
|
165
131
|
}
|
|
166
132
|
export async function saveSchedulerState(dataDir, state) {
|
|
167
133
|
const path = schedulerStatePath(dataDir);
|
package/dist/service.js
CHANGED
|
@@ -141,6 +141,26 @@ async function run(command, args, options) {
|
|
|
141
141
|
}
|
|
142
142
|
await execFileAsync(command, args);
|
|
143
143
|
}
|
|
144
|
+
async function runInteractive(command, args, options, errorPrefix) {
|
|
145
|
+
const currentPlatform = options.platform ?? platform();
|
|
146
|
+
if (options.runCommand) {
|
|
147
|
+
await options.runCommand(command, args);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
await new Promise((resolveRun, rejectRun) => {
|
|
151
|
+
const child = spawn(command, args, {
|
|
152
|
+
shell: currentPlatform === "win32",
|
|
153
|
+
stdio: "inherit",
|
|
154
|
+
});
|
|
155
|
+
child.on("exit", (code) => {
|
|
156
|
+
if (code === 0)
|
|
157
|
+
resolveRun();
|
|
158
|
+
else
|
|
159
|
+
rejectRun(new Error(`${errorPrefix} failed with exit code ${code ?? "unknown"}`));
|
|
160
|
+
});
|
|
161
|
+
child.on("error", rejectRun);
|
|
162
|
+
});
|
|
163
|
+
}
|
|
144
164
|
async function capture(command, args, options) {
|
|
145
165
|
if (options.captureCommand)
|
|
146
166
|
return options.captureCommand(command, args);
|
|
@@ -254,22 +274,12 @@ async function supervisorState(spec, options) {
|
|
|
254
274
|
return "not-loaded";
|
|
255
275
|
}
|
|
256
276
|
}
|
|
257
|
-
export async function upgradeFamiliar(options = {}) {
|
|
277
|
+
export async function upgradeFamiliar(workspacePath, options = {}) {
|
|
258
278
|
const currentPlatform = options.platform ?? platform();
|
|
259
279
|
const npmCommand = currentPlatform === "win32" ? "npm.cmd" : "npm";
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
stdio: "inherit",
|
|
264
|
-
});
|
|
265
|
-
child.on("exit", (code) => {
|
|
266
|
-
if (code === 0)
|
|
267
|
-
resolveUpgrade();
|
|
268
|
-
else
|
|
269
|
-
rejectUpgrade(new Error(`npm upgrade failed with exit code ${code ?? "unknown"}`));
|
|
270
|
-
});
|
|
271
|
-
child.on("error", rejectUpgrade);
|
|
272
|
-
});
|
|
280
|
+
const familiarCommand = currentPlatform === "win32" ? "familiar.cmd" : "familiar";
|
|
281
|
+
await runInteractive(npmCommand, ["install", "-g", "@qearlyao/familiar@latest"], options, "npm upgrade");
|
|
282
|
+
await runInteractive(familiarCommand, ["init", workspacePath], options, "workspace default refresh");
|
|
273
283
|
}
|
|
274
284
|
export function formatServiceResult(result) {
|
|
275
285
|
return [result.title, ...result.details].join("\n");
|