@qearlyao/familiar 0.2.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HEARTBEAT.md +1 -1
- package/README.md +33 -0
- package/config.example.toml +4 -2
- package/dist/{agent.js → agent/factory.js} +97 -328
- package/dist/agent/payload-normalizers.js +52 -0
- package/dist/agent/session-helpers.js +86 -0
- package/dist/agent/tool-descriptions.js +4 -0
- package/dist/agent/tools.js +30 -0
- package/dist/agent/transcript-log.js +93 -0
- package/dist/cli.js +45 -15
- package/dist/config/enums.js +35 -0
- package/dist/{config.js → config/index.js} +9 -272
- package/dist/config/interpolate.js +15 -0
- package/dist/config/model-refs.js +11 -0
- package/dist/{config-overrides.js → config/overrides.js} +1 -1
- package/dist/config/readers.js +116 -0
- package/dist/{config-registry.js → config/registry.js} +27 -8
- package/dist/config/sections.js +113 -0
- package/dist/{settings.js → config/settings.js} +5 -2
- package/dist/config/types.js +1 -0
- package/dist/{chat-log.js → conversation/chat-log.js} +16 -4
- package/dist/{contact-note.js → conversation/contact-note.js} +1 -1
- package/dist/conversation/ids.js +11 -0
- package/dist/conversation/owner-identity.js +29 -0
- package/dist/discord/channel.js +32 -0
- package/dist/discord/chunking.js +163 -0
- package/dist/discord/client.js +44 -0
- package/dist/discord/commands.js +181 -0
- package/dist/discord/daemon.js +379 -0
- package/dist/discord/inbound.js +44 -0
- package/dist/discord/send.js +115 -0
- package/dist/discord/turn.js +55 -0
- package/dist/index.js +12 -11
- package/dist/lifecycle/control.js +1 -0
- package/dist/{data-retention.js → lifecycle/data-retention.js} +1 -1
- package/dist/{hot-reload.js → lifecycle/hot-reload.js} +2 -2
- package/dist/{service.js → lifecycle/service.js} +1 -0
- package/dist/media/attachment-limits.js +3 -0
- package/dist/{generated-media.js → media/generated-media.js} +1 -1
- package/dist/{image-gen.js → media/image-gen.js} +2 -2
- package/dist/{inbound-attachments.js → media/inbound-attachments.js} +47 -43
- package/dist/media/media-understanding.js +215 -0
- package/dist/memory/index/store.js +21 -17
- package/dist/memory/index/vector-codec.js +2 -2
- package/dist/memory/lcm/context-transformer.js +6 -2
- package/dist/memory/lcm/segment-manager.js +6 -2
- package/dist/memory/lcm/store/index-ids.js +6 -0
- package/dist/memory/lcm/store/inserts.js +31 -0
- package/dist/memory/lcm/store/normalizers.js +91 -0
- package/dist/memory/lcm/store/row-mappers.js +114 -0
- package/dist/memory/lcm/store/row-types.js +1 -0
- package/dist/memory/lcm/store/serialization.js +37 -0
- package/dist/memory/lcm/store/snapshots.js +73 -0
- package/dist/memory/lcm/store.js +20 -360
- package/dist/memory/lcm/summarizer.js +1 -1
- package/dist/{added-models.js → models/added-models.js} +1 -1
- package/dist/{persona.js → prompting/persona.js} +1 -1
- package/dist/runtime/agent-core.js +82 -0
- package/dist/{agent-events.js → runtime/agent-events.js} +1 -1
- package/dist/runtime/agent-work-queue.js +55 -0
- package/dist/{runtime.js → runtime/conversation-runtime.js} +91 -43
- package/dist/runtime/runtime-manager.js +51 -0
- package/dist/runtime/scheduler-runner.js +243 -0
- package/dist/{scheduler.js → runtime/scheduler.js} +4 -4
- package/dist/{browser-tools.js → tools/browser-tools.js} +24 -34
- package/dist/util/fs.js +2 -1
- package/dist/web/agent-routes.js +104 -0
- package/dist/web/auth-routes.js +39 -0
- package/dist/web/auth.js +205 -0
- package/dist/web/config-routes.js +55 -0
- package/dist/web/conversation-routes.js +122 -0
- package/dist/web/daemon.js +108 -0
- package/dist/web/diary-routes.js +88 -0
- package/dist/web/errors.js +3 -0
- package/dist/web/event-hub.js +246 -0
- package/dist/{web-http.js → web/http.js} +19 -5
- package/dist/web/memes.js +25 -0
- package/dist/web/messages.js +348 -0
- package/dist/web/multipart.js +86 -0
- package/dist/web/payloads.js +34 -0
- package/dist/web/request-context.js +25 -0
- package/dist/web/route-helpers.js +9 -0
- package/dist/web/routes.js +37 -0
- package/dist/web/runtime-actions.js +231 -0
- package/dist/web/session-store.js +161 -0
- package/dist/{web-static.js → web/static.js} +19 -14
- package/dist/web/stream.js +78 -0
- package/dist/web-tools/cache.js +42 -0
- package/dist/web-tools/config.js +16 -0
- package/dist/web-tools/fetch-providers.js +119 -0
- package/dist/web-tools/format.js +88 -0
- package/dist/web-tools/http.js +81 -0
- package/dist/web-tools/index.js +152 -0
- package/dist/web-tools/routing.js +29 -0
- package/dist/web-tools/safety.js +73 -0
- package/dist/web-tools/search-providers.js +277 -0
- package/dist/web-tools/types.js +54 -0
- package/dist/web-tools/util.js +23 -0
- package/npm-shrinkwrap.json +319 -201
- package/package.json +6 -4
- package/web/dist/assets/index-C-k4O5Dz.js +6 -0
- package/web/dist/assets/index-Dj-L9nX4.css +2 -0
- package/web/dist/assets/markdown-kaIeGxdv.js +14 -0
- package/web/dist/assets/react-Bi_azaFt.js +9 -0
- package/web/dist/assets/rolldown-runtime-S-ySWqyJ.js +1 -0
- package/web/dist/assets/ui-C12-nN_X.js +51 -0
- package/web/dist/assets/vendor-D1QXMhXm.js +16 -0
- package/web/dist/index.html +11 -3
- package/dist/discord.js +0 -1299
- package/dist/media-understanding.js +0 -120
- package/dist/web-auth.js +0 -111
- package/dist/web-tools.js +0 -941
- package/dist/web.js +0 -1209
- package/web/dist/assets/index-B23WT77N.js +0 -63
- package/web/dist/assets/index-D3MotFzN.css +0 -2
- /package/dist/{control.js → agent/types.js} +0 -0
- /package/dist/{image-derivatives.js → media/image-derivatives.js} +0 -0
- /package/dist/{tts.js → media/tts.js} +0 -0
- /package/dist/{models.js → models/index.js} +0 -0
- /package/dist/{skills.js → prompting/skills.js} +0 -0
- /package/dist/{silent-marker.js → runtime/silent-marker.js} +0 -0
- /package/dist/{web-events.js → web/events.js} +0 -0
- /package/dist/{web-types.js → web/types.js} +0 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { parseJsonArray, parseJsonObject } from "./serialization.js";
|
|
2
|
+
function sourceFromRow(row) {
|
|
3
|
+
return {
|
|
4
|
+
sourceType: row.source_type,
|
|
5
|
+
sourcePath: row.source_path,
|
|
6
|
+
sourceLine: row.source_line,
|
|
7
|
+
sourceRecordId: row.source_record_id,
|
|
8
|
+
sourceMessageId: row.source_message_id,
|
|
9
|
+
sourceRef: row.source_ref,
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export function segmentFromRow(row) {
|
|
13
|
+
return {
|
|
14
|
+
id: row.id,
|
|
15
|
+
status: row.status,
|
|
16
|
+
sessionId: row.session_id,
|
|
17
|
+
channelKey: row.channel_key,
|
|
18
|
+
startedAt: row.started_at,
|
|
19
|
+
closedAt: row.closed_at,
|
|
20
|
+
rawPrunedAt: row.raw_pruned_at,
|
|
21
|
+
boundarySource: parseJsonObject(row.boundary_source_json),
|
|
22
|
+
metadata: parseJsonObject(row.metadata_json),
|
|
23
|
+
createdAt: row.created_at,
|
|
24
|
+
updatedAt: row.updated_at,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
export function recordFromRow(row) {
|
|
28
|
+
return {
|
|
29
|
+
id: row.id,
|
|
30
|
+
recordKey: row.record_key,
|
|
31
|
+
segmentId: row.segment_id,
|
|
32
|
+
kind: row.kind,
|
|
33
|
+
text: row.text_full,
|
|
34
|
+
parts: parseJsonArray(row.parts_json),
|
|
35
|
+
happenedAt: row.happened_at,
|
|
36
|
+
sessionId: row.session_id,
|
|
37
|
+
channelKey: row.channel_key,
|
|
38
|
+
channelId: row.channel_id,
|
|
39
|
+
jobId: row.job_id,
|
|
40
|
+
source: sourceFromRow(row),
|
|
41
|
+
attachments: parseJsonArray(row.attachments_json),
|
|
42
|
+
metadata: parseJsonObject(row.metadata_json),
|
|
43
|
+
createdAt: row.created_at,
|
|
44
|
+
updatedAt: row.updated_at,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export function summaryFromRow(row, parents = []) {
|
|
48
|
+
return {
|
|
49
|
+
id: row.id,
|
|
50
|
+
summaryKey: row.summary_key,
|
|
51
|
+
segmentId: row.segment_id,
|
|
52
|
+
depth: row.depth,
|
|
53
|
+
status: row.status,
|
|
54
|
+
text: row.text_full,
|
|
55
|
+
pinned: row.pinned === 1,
|
|
56
|
+
coversFromRecordId: row.covers_from_record_id,
|
|
57
|
+
coversToRecordId: row.covers_to_record_id,
|
|
58
|
+
source: sourceFromRow(row),
|
|
59
|
+
metadata: parseJsonObject(row.metadata_json),
|
|
60
|
+
snapshot: parseJsonArray(row.snapshot_json),
|
|
61
|
+
parents,
|
|
62
|
+
createdAt: row.created_at,
|
|
63
|
+
updatedAt: row.updated_at,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export function summarySourceFromRow(row) {
|
|
67
|
+
return {
|
|
68
|
+
summaryId: row.summary_id,
|
|
69
|
+
ord: row.ord,
|
|
70
|
+
recordId: row.record_id,
|
|
71
|
+
sourceSummaryId: row.source_summary_id,
|
|
72
|
+
sourceRef: row.source_ref,
|
|
73
|
+
snapshot: parseJsonObject(row.snapshot_json),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
export function contextItemFromRow(row) {
|
|
77
|
+
if (row.item_type === "raw") {
|
|
78
|
+
if (row.record_id === null)
|
|
79
|
+
throw new Error(`Invalid raw LCM context item at ordinal ${row.ordinal}`);
|
|
80
|
+
return {
|
|
81
|
+
sessionKey: row.session_key,
|
|
82
|
+
ordinal: row.ordinal,
|
|
83
|
+
type: "raw",
|
|
84
|
+
recordId: row.record_id,
|
|
85
|
+
summaryId: null,
|
|
86
|
+
fingerprint: row.fingerprint,
|
|
87
|
+
happenedAt: row.happened_at,
|
|
88
|
+
updatedAt: row.updated_at,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
if (row.item_type === "summary") {
|
|
92
|
+
if (row.summary_id === null)
|
|
93
|
+
throw new Error(`Invalid summary LCM context item at ordinal ${row.ordinal}`);
|
|
94
|
+
return {
|
|
95
|
+
sessionKey: row.session_key,
|
|
96
|
+
ordinal: row.ordinal,
|
|
97
|
+
type: "summary",
|
|
98
|
+
recordId: null,
|
|
99
|
+
summaryId: row.summary_id,
|
|
100
|
+
fingerprint: row.fingerprint,
|
|
101
|
+
happenedAt: row.happened_at,
|
|
102
|
+
updatedAt: row.updated_at,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
throw new Error(`Unknown LCM context item type: ${row.item_type}`);
|
|
106
|
+
}
|
|
107
|
+
export function sessionStateFromRow(row) {
|
|
108
|
+
return {
|
|
109
|
+
sessionKey: row.session_key,
|
|
110
|
+
compactionDebt: row.compaction_debt,
|
|
111
|
+
cacheTouchedAt: row.cache_touched_at,
|
|
112
|
+
updatedAt: row.updated_at,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { isRecord } from "../../../util/guards.js";
|
|
3
|
+
export function jsonOrNull(value) {
|
|
4
|
+
if (value === null || value === undefined)
|
|
5
|
+
return null;
|
|
6
|
+
return JSON.stringify(value);
|
|
7
|
+
}
|
|
8
|
+
export function parseJsonObject(value) {
|
|
9
|
+
if (!value)
|
|
10
|
+
return null;
|
|
11
|
+
try {
|
|
12
|
+
const parsed = JSON.parse(value);
|
|
13
|
+
return isRecord(parsed) ? parsed : null;
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function parseJsonArray(value) {
|
|
20
|
+
if (!value)
|
|
21
|
+
return null;
|
|
22
|
+
try {
|
|
23
|
+
const parsed = JSON.parse(value);
|
|
24
|
+
return Array.isArray(parsed) ? parsed : null;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function stableHash(value) {
|
|
31
|
+
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
32
|
+
}
|
|
33
|
+
export function sourceRecordIdToString(value) {
|
|
34
|
+
if (value === null || value === undefined)
|
|
35
|
+
return null;
|
|
36
|
+
return String(value);
|
|
37
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { parseJsonArray, parseJsonObject } from "./serialization.js";
|
|
2
|
+
const SUMMARY_SNAPSHOT_TEXT_LIMIT = 4 * 1024;
|
|
3
|
+
const SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX = "…[truncated]";
|
|
4
|
+
export function buildSummarySnapshot(db, summary) {
|
|
5
|
+
const rows = db
|
|
6
|
+
.prepare(`SELECT * FROM lcm_records
|
|
7
|
+
WHERE segment_id = ?
|
|
8
|
+
AND id BETWEEN ? AND ?
|
|
9
|
+
ORDER BY happened_at, id`)
|
|
10
|
+
.all(summary.segment_id, summary.covers_from_record_id, summary.covers_to_record_id);
|
|
11
|
+
return rows.map(snapshotRecordFromRow);
|
|
12
|
+
}
|
|
13
|
+
export function buildSummaryParentSnapshot(db, summaryId, visiting) {
|
|
14
|
+
if (visiting.has(summaryId))
|
|
15
|
+
throw new Error(`Cycle detected in LCM summary parents at ${summaryId}`);
|
|
16
|
+
visiting.add(summaryId);
|
|
17
|
+
const row = db.prepare("SELECT * FROM lcm_summaries WHERE id = ?").get(summaryId);
|
|
18
|
+
if (!row)
|
|
19
|
+
throw new Error(`LCM summary does not exist: ${summaryId}`);
|
|
20
|
+
let snapshot = parseJsonArray(row.snapshot_json);
|
|
21
|
+
if (!snapshot &&
|
|
22
|
+
row.covers_from_record_id !== null &&
|
|
23
|
+
row.covers_to_record_id !== null &&
|
|
24
|
+
db
|
|
25
|
+
.prepare("SELECT 1 FROM lcm_records WHERE segment_id = ? AND id BETWEEN ? AND ? LIMIT 1")
|
|
26
|
+
.get(row.segment_id, row.covers_from_record_id, row.covers_to_record_id)) {
|
|
27
|
+
snapshot = buildSummarySnapshot(db, row);
|
|
28
|
+
}
|
|
29
|
+
const parentRows = db
|
|
30
|
+
.prepare(`SELECT parent_summary_id
|
|
31
|
+
FROM lcm_summary_parents
|
|
32
|
+
WHERE summary_id = ?
|
|
33
|
+
ORDER BY ord, parent_summary_id`)
|
|
34
|
+
.all(summaryId);
|
|
35
|
+
const parents = parentRows.map((parent) => buildSummaryParentSnapshot(db, parent.parent_summary_id, visiting));
|
|
36
|
+
visiting.delete(summaryId);
|
|
37
|
+
return {
|
|
38
|
+
summaryId: row.id,
|
|
39
|
+
depth: row.depth,
|
|
40
|
+
text: row.text_full,
|
|
41
|
+
coversFromRecordId: row.covers_from_record_id,
|
|
42
|
+
coversToRecordId: row.covers_to_record_id,
|
|
43
|
+
snapshot,
|
|
44
|
+
parents,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
export function snapshotRecordFromRow(row) {
|
|
48
|
+
const metadata = parseJsonObject(row.metadata_json);
|
|
49
|
+
return {
|
|
50
|
+
id: row.id,
|
|
51
|
+
kind: row.kind,
|
|
52
|
+
happened_at: row.happened_at,
|
|
53
|
+
role: snapshotRole(row.kind, metadata),
|
|
54
|
+
text: truncateSummarySnapshotText(row.text_full),
|
|
55
|
+
parts: parseJsonArray(row.parts_json),
|
|
56
|
+
attachments: parseJsonArray(row.attachments_json),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
export function snapshotRole(kind, metadata) {
|
|
60
|
+
if (typeof metadata?.role === "string" && metadata.role.trim())
|
|
61
|
+
return metadata.role;
|
|
62
|
+
if (kind === "user" || kind === "assistant")
|
|
63
|
+
return kind;
|
|
64
|
+
if (kind === "tool")
|
|
65
|
+
return "tool";
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
export function truncateSummarySnapshotText(text) {
|
|
69
|
+
if (text.length <= SUMMARY_SNAPSHOT_TEXT_LIMIT)
|
|
70
|
+
return text;
|
|
71
|
+
const retainedLength = Math.max(0, SUMMARY_SNAPSHOT_TEXT_LIMIT - SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX.length);
|
|
72
|
+
return `${text.slice(0, retainedLength)}${SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX}`;
|
|
73
|
+
}
|
package/dist/memory/lcm/store.js
CHANGED
|
@@ -1,10 +1,17 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
1
|
import { mkdirSync } from "node:fs";
|
|
3
2
|
import { dirname, resolve } from "node:path";
|
|
4
3
|
import Database from "better-sqlite3";
|
|
5
4
|
import { normalizeFtsMatchQuery } from "../index/fts-query.js";
|
|
6
5
|
import { runInTransaction } from "../util.js";
|
|
7
6
|
import { readMeta, runLcmMigrations } from "./schema.js";
|
|
7
|
+
import { lcmRecordIndexSourceId, lcmSummaryIndexSourceId } from "./store/index-ids.js";
|
|
8
|
+
import { insertRecordPrepared, insertSummaryPrepared } from "./store/inserts.js";
|
|
9
|
+
import { computeLcmRecordKey, dedupeSummaryParentIds, normalizeRecordInput, normalizeSummaryInput, } from "./store/normalizers.js";
|
|
10
|
+
import { contextItemFromRow, recordFromRow, segmentFromRow, sessionStateFromRow, summaryFromRow, summarySourceFromRow, } from "./store/row-mappers.js";
|
|
11
|
+
import { jsonOrNull } from "./store/serialization.js";
|
|
12
|
+
import { buildSummaryParentSnapshot, buildSummarySnapshot } from "./store/snapshots.js";
|
|
13
|
+
export { lcmRecordIndexSourceId, lcmSummaryIndexSourceId } from "./store/index-ids.js";
|
|
14
|
+
export { computeLcmRecordKey } from "./store/normalizers.js";
|
|
8
15
|
export class LcmStore {
|
|
9
16
|
db;
|
|
10
17
|
ownsDb;
|
|
@@ -134,7 +141,7 @@ export class LcmStore {
|
|
|
134
141
|
throw new Error("LCM summary depth must be an integer >= 0");
|
|
135
142
|
}
|
|
136
143
|
const normalized = normalizeSummaryInput(input);
|
|
137
|
-
const runInsert = () => runSummaryInsertTransaction(
|
|
144
|
+
const runInsert = () => this.runSummaryInsertTransaction(normalized, (id, sources, parents) => {
|
|
138
145
|
this.insertSummarySources(id, sources);
|
|
139
146
|
this.insertSummaryParents(id, parents);
|
|
140
147
|
});
|
|
@@ -294,15 +301,14 @@ export class LcmStore {
|
|
|
294
301
|
summary_id, ord, record_id, source_summary_id, source_ref, snapshot_json
|
|
295
302
|
) VALUES (?, ?, ?, ?, ?, ?)`);
|
|
296
303
|
for (const [index, source] of sources.entries()) {
|
|
297
|
-
// source_summary_id is
|
|
298
|
-
// parent edge is lcm_summary_parents, and this column is scheduled for removal.
|
|
304
|
+
// source_summary_id is advisory only; the canonical parent edge is lcm_summary_parents.
|
|
299
305
|
insert.run(summaryId, index, source.recordId ?? null, null, source.sourceRef ?? null, jsonOrNull(source.snapshot ?? null));
|
|
300
306
|
}
|
|
301
307
|
}
|
|
302
308
|
insertSummaryParents(summaryId, parents) {
|
|
303
309
|
if (parents.length === 0)
|
|
304
310
|
return;
|
|
305
|
-
const uniqueParents =
|
|
311
|
+
const uniqueParents = dedupeSummaryParentIds(parents);
|
|
306
312
|
const existingRows = this.db
|
|
307
313
|
.prepare(`SELECT id FROM lcm_summaries WHERE id IN (${uniqueParents.map(() => "?").join(",")})`)
|
|
308
314
|
.all(...uniqueParents);
|
|
@@ -315,6 +321,15 @@ export class LcmStore {
|
|
|
315
321
|
for (const [index, parentId] of uniqueParents.entries())
|
|
316
322
|
insert.run(summaryId, parentId, index);
|
|
317
323
|
}
|
|
324
|
+
runSummaryInsertTransaction(normalized, insertEdges) {
|
|
325
|
+
this.ensureSegment({ id: normalized.segmentId });
|
|
326
|
+
const existing = this.db
|
|
327
|
+
.prepare("SELECT id FROM lcm_summaries WHERE summary_key = ?")
|
|
328
|
+
.get(normalized.summaryKey);
|
|
329
|
+
if (existing)
|
|
330
|
+
return existing.id;
|
|
331
|
+
return insertSummaryPrepared(this.db, normalized, insertEdges);
|
|
332
|
+
}
|
|
318
333
|
summaryParentMap(summaryIds) {
|
|
319
334
|
const map = new Map();
|
|
320
335
|
if (summaryIds.length === 0)
|
|
@@ -368,358 +383,3 @@ export class LcmStore {
|
|
|
368
383
|
}
|
|
369
384
|
}
|
|
370
385
|
}
|
|
371
|
-
export function lcmRecordIndexSourceId(id) {
|
|
372
|
-
return `lcm_record:${id}`;
|
|
373
|
-
}
|
|
374
|
-
export function lcmSummaryIndexSourceId(id) {
|
|
375
|
-
return `lcm_summary:${id}`;
|
|
376
|
-
}
|
|
377
|
-
export function computeLcmRecordKey(input) {
|
|
378
|
-
const text = (input.text ?? "").trim();
|
|
379
|
-
if (!text && input.kind !== "boundary")
|
|
380
|
-
throw new Error("LCM record text must not be empty");
|
|
381
|
-
const parts = input.parts?.length ? input.parts : null;
|
|
382
|
-
return stableHash({
|
|
383
|
-
segmentId: input.segmentId,
|
|
384
|
-
kind: input.kind,
|
|
385
|
-
text: text || "Session boundary",
|
|
386
|
-
parts,
|
|
387
|
-
happenedAt: input.happenedAt ?? new Date().toISOString(),
|
|
388
|
-
source: normalizeSource(input.source),
|
|
389
|
-
});
|
|
390
|
-
}
|
|
391
|
-
function normalizeRecordInput(input) {
|
|
392
|
-
const text = (input.text ?? "").trim();
|
|
393
|
-
if (!text && input.kind !== "boundary")
|
|
394
|
-
throw new Error("LCM record text must not be empty");
|
|
395
|
-
const source = normalizeSource(input.source);
|
|
396
|
-
const happenedAt = input.happenedAt ?? new Date().toISOString();
|
|
397
|
-
const parts = input.parts?.length ? input.parts : null;
|
|
398
|
-
const normalizedText = text || "Session boundary";
|
|
399
|
-
return {
|
|
400
|
-
segmentId: input.segmentId,
|
|
401
|
-
kind: input.kind,
|
|
402
|
-
text: normalizedText,
|
|
403
|
-
parts,
|
|
404
|
-
happenedAt,
|
|
405
|
-
sessionId: input.sessionId ?? null,
|
|
406
|
-
channelKey: input.channelKey ?? null,
|
|
407
|
-
channelId: input.channelId ?? null,
|
|
408
|
-
jobId: input.jobId ?? null,
|
|
409
|
-
source,
|
|
410
|
-
attachments: input.attachments?.length ? input.attachments : null,
|
|
411
|
-
metadata: input.metadata ?? null,
|
|
412
|
-
recordKey: computeLcmRecordKey({ ...input, happenedAt }),
|
|
413
|
-
};
|
|
414
|
-
}
|
|
415
|
-
function insertRecordPrepared(db, normalized) {
|
|
416
|
-
const inserted = db
|
|
417
|
-
.prepare(`INSERT INTO lcm_records (
|
|
418
|
-
record_key, segment_id, kind, text_full, happened_at, session_id, channel_key,
|
|
419
|
-
channel_id, job_id, source_type, source_path, source_line, source_record_id,
|
|
420
|
-
source_message_id, source_ref, attachments_json, metadata_json, parts_json
|
|
421
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
422
|
-
.run(normalized.recordKey, normalized.segmentId, normalized.kind, normalized.text, normalized.happenedAt, normalized.sessionId, normalized.channelKey, normalized.channelId, normalized.jobId, normalized.source.sourceType, normalized.source.sourcePath ?? null, normalized.source.sourceLine ?? null, sourceRecordIdToString(normalized.source.sourceRecordId), normalized.source.sourceMessageId ?? null, normalized.source.sourceRef ?? null, jsonOrNull(normalized.attachments), jsonOrNull(normalized.metadata), jsonOrNull(normalized.parts));
|
|
423
|
-
const id = Number(inserted.lastInsertRowid);
|
|
424
|
-
if (normalized.kind !== "boundary") {
|
|
425
|
-
db.prepare("INSERT INTO lcm_records_fts(rowid, text_full) VALUES (?, ?)").run(id, normalized.text);
|
|
426
|
-
}
|
|
427
|
-
const row = db.prepare("SELECT * FROM lcm_records WHERE id = ?").get(id);
|
|
428
|
-
if (!row)
|
|
429
|
-
throw new Error(`Failed to read inserted LCM record: ${id}`);
|
|
430
|
-
return row;
|
|
431
|
-
}
|
|
432
|
-
function normalizeSummaryInput(input) {
|
|
433
|
-
const text = (input.text ?? "").trim();
|
|
434
|
-
const source = normalizeSource(input.source);
|
|
435
|
-
const status = input.status ?? (text ? "ready" : "placeholder");
|
|
436
|
-
const normalizedText = text || "";
|
|
437
|
-
return {
|
|
438
|
-
segmentId: input.segmentId,
|
|
439
|
-
depth: input.depth,
|
|
440
|
-
status,
|
|
441
|
-
text: normalizedText,
|
|
442
|
-
pinned: input.pinned ?? false,
|
|
443
|
-
coversFromRecordId: input.coversFromRecordId ?? null,
|
|
444
|
-
coversToRecordId: input.coversToRecordId ?? null,
|
|
445
|
-
source,
|
|
446
|
-
sourceItems: input.sourceItems ?? [],
|
|
447
|
-
parents: input.parents ?? [],
|
|
448
|
-
metadata: input.metadata ?? null,
|
|
449
|
-
summaryKey: stableHash({
|
|
450
|
-
segmentId: input.segmentId,
|
|
451
|
-
depth: input.depth,
|
|
452
|
-
status,
|
|
453
|
-
text: normalizedText,
|
|
454
|
-
coversFromRecordId: input.coversFromRecordId ?? null,
|
|
455
|
-
coversToRecordId: input.coversToRecordId ?? null,
|
|
456
|
-
source,
|
|
457
|
-
parents: input.parents ?? [],
|
|
458
|
-
}),
|
|
459
|
-
};
|
|
460
|
-
}
|
|
461
|
-
function insertSummaryPrepared(db, normalized, insertEdges) {
|
|
462
|
-
const inserted = db
|
|
463
|
-
.prepare(`INSERT INTO lcm_summaries (
|
|
464
|
-
summary_key, segment_id, depth, status, text_full, pinned,
|
|
465
|
-
covers_from_record_id, covers_to_record_id, snapshot_json, source_type, source_path,
|
|
466
|
-
source_line, source_record_id, source_message_id, source_ref, metadata_json
|
|
467
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
468
|
-
.run(normalized.summaryKey, normalized.segmentId, normalized.depth, normalized.status, normalized.text, normalized.pinned ? 1 : 0, normalized.coversFromRecordId, normalized.coversToRecordId, null, normalized.source.sourceType, normalized.source.sourcePath ?? null, normalized.source.sourceLine ?? null, sourceRecordIdToString(normalized.source.sourceRecordId), normalized.source.sourceMessageId ?? null, normalized.source.sourceRef ?? null, jsonOrNull(normalized.metadata));
|
|
469
|
-
const id = Number(inserted.lastInsertRowid);
|
|
470
|
-
db.prepare("INSERT INTO lcm_summaries_fts(rowid, text_full) VALUES (?, ?)").run(id, normalized.text);
|
|
471
|
-
insertEdges(id, normalized.sourceItems, normalized.parents);
|
|
472
|
-
return id;
|
|
473
|
-
}
|
|
474
|
-
function runSummaryInsertTransaction(store, normalized, insertEdges) {
|
|
475
|
-
store.ensureSegment({ id: normalized.segmentId });
|
|
476
|
-
const existing = store.db.prepare("SELECT id FROM lcm_summaries WHERE summary_key = ?").get(normalized.summaryKey);
|
|
477
|
-
if (existing)
|
|
478
|
-
return existing.id;
|
|
479
|
-
return insertSummaryPrepared(store.db, normalized, insertEdges);
|
|
480
|
-
}
|
|
481
|
-
function dedupeNumbers(values) {
|
|
482
|
-
const seen = new Set();
|
|
483
|
-
const result = [];
|
|
484
|
-
for (const value of values) {
|
|
485
|
-
if (!Number.isInteger(value) || value <= 0)
|
|
486
|
-
throw new Error("LCM summary parents must be positive integer ids");
|
|
487
|
-
if (seen.has(value))
|
|
488
|
-
continue;
|
|
489
|
-
seen.add(value);
|
|
490
|
-
result.push(value);
|
|
491
|
-
}
|
|
492
|
-
return result;
|
|
493
|
-
}
|
|
494
|
-
function normalizeSource(source) {
|
|
495
|
-
return {
|
|
496
|
-
sourceType: source.sourceType,
|
|
497
|
-
sourcePath: source.sourcePath ?? null,
|
|
498
|
-
sourceLine: source.sourceLine ?? null,
|
|
499
|
-
sourceRecordId: source.sourceRecordId ?? null,
|
|
500
|
-
sourceMessageId: source.sourceMessageId ?? null,
|
|
501
|
-
sourceRef: source.sourceRef ?? null,
|
|
502
|
-
};
|
|
503
|
-
}
|
|
504
|
-
function stableHash(value) {
|
|
505
|
-
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
506
|
-
}
|
|
507
|
-
function sourceRecordIdToString(value) {
|
|
508
|
-
if (value === null || value === undefined)
|
|
509
|
-
return null;
|
|
510
|
-
return String(value);
|
|
511
|
-
}
|
|
512
|
-
function jsonOrNull(value) {
|
|
513
|
-
if (value === null || value === undefined)
|
|
514
|
-
return null;
|
|
515
|
-
return JSON.stringify(value);
|
|
516
|
-
}
|
|
517
|
-
function parseJsonObject(value) {
|
|
518
|
-
if (!value)
|
|
519
|
-
return null;
|
|
520
|
-
try {
|
|
521
|
-
const parsed = JSON.parse(value);
|
|
522
|
-
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
|
523
|
-
? parsed
|
|
524
|
-
: null;
|
|
525
|
-
}
|
|
526
|
-
catch {
|
|
527
|
-
return null;
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
function parseJsonArray(value) {
|
|
531
|
-
if (!value)
|
|
532
|
-
return null;
|
|
533
|
-
try {
|
|
534
|
-
const parsed = JSON.parse(value);
|
|
535
|
-
return Array.isArray(parsed) ? parsed : null;
|
|
536
|
-
}
|
|
537
|
-
catch {
|
|
538
|
-
return null;
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
function sourceFromRow(row) {
|
|
542
|
-
return {
|
|
543
|
-
sourceType: row.source_type,
|
|
544
|
-
sourcePath: row.source_path,
|
|
545
|
-
sourceLine: row.source_line,
|
|
546
|
-
sourceRecordId: row.source_record_id,
|
|
547
|
-
sourceMessageId: row.source_message_id,
|
|
548
|
-
sourceRef: row.source_ref,
|
|
549
|
-
};
|
|
550
|
-
}
|
|
551
|
-
function segmentFromRow(row) {
|
|
552
|
-
return {
|
|
553
|
-
id: row.id,
|
|
554
|
-
status: row.status,
|
|
555
|
-
sessionId: row.session_id,
|
|
556
|
-
channelKey: row.channel_key,
|
|
557
|
-
startedAt: row.started_at,
|
|
558
|
-
closedAt: row.closed_at,
|
|
559
|
-
rawPrunedAt: row.raw_pruned_at,
|
|
560
|
-
boundarySource: parseJsonObject(row.boundary_source_json),
|
|
561
|
-
metadata: parseJsonObject(row.metadata_json),
|
|
562
|
-
createdAt: row.created_at,
|
|
563
|
-
updatedAt: row.updated_at,
|
|
564
|
-
};
|
|
565
|
-
}
|
|
566
|
-
function recordFromRow(row) {
|
|
567
|
-
return {
|
|
568
|
-
id: row.id,
|
|
569
|
-
recordKey: row.record_key,
|
|
570
|
-
segmentId: row.segment_id,
|
|
571
|
-
kind: row.kind,
|
|
572
|
-
text: row.text_full,
|
|
573
|
-
parts: parseJsonArray(row.parts_json),
|
|
574
|
-
happenedAt: row.happened_at,
|
|
575
|
-
sessionId: row.session_id,
|
|
576
|
-
channelKey: row.channel_key,
|
|
577
|
-
channelId: row.channel_id,
|
|
578
|
-
jobId: row.job_id,
|
|
579
|
-
source: sourceFromRow(row),
|
|
580
|
-
attachments: parseJsonArray(row.attachments_json),
|
|
581
|
-
metadata: parseJsonObject(row.metadata_json),
|
|
582
|
-
createdAt: row.created_at,
|
|
583
|
-
updatedAt: row.updated_at,
|
|
584
|
-
};
|
|
585
|
-
}
|
|
586
|
-
function summaryFromRow(row, parents = []) {
|
|
587
|
-
return {
|
|
588
|
-
id: row.id,
|
|
589
|
-
summaryKey: row.summary_key,
|
|
590
|
-
segmentId: row.segment_id,
|
|
591
|
-
depth: row.depth,
|
|
592
|
-
status: row.status,
|
|
593
|
-
text: row.text_full,
|
|
594
|
-
pinned: row.pinned === 1,
|
|
595
|
-
coversFromRecordId: row.covers_from_record_id,
|
|
596
|
-
coversToRecordId: row.covers_to_record_id,
|
|
597
|
-
source: sourceFromRow(row),
|
|
598
|
-
metadata: parseJsonObject(row.metadata_json),
|
|
599
|
-
snapshot: parseJsonArray(row.snapshot_json),
|
|
600
|
-
parents,
|
|
601
|
-
createdAt: row.created_at,
|
|
602
|
-
updatedAt: row.updated_at,
|
|
603
|
-
};
|
|
604
|
-
}
|
|
605
|
-
function summarySourceFromRow(row) {
|
|
606
|
-
return {
|
|
607
|
-
summaryId: row.summary_id,
|
|
608
|
-
ord: row.ord,
|
|
609
|
-
recordId: row.record_id,
|
|
610
|
-
sourceSummaryId: row.source_summary_id,
|
|
611
|
-
sourceRef: row.source_ref,
|
|
612
|
-
snapshot: parseJsonObject(row.snapshot_json),
|
|
613
|
-
};
|
|
614
|
-
}
|
|
615
|
-
function contextItemFromRow(row) {
|
|
616
|
-
if (row.item_type === "raw") {
|
|
617
|
-
if (row.record_id === null)
|
|
618
|
-
throw new Error(`Invalid raw LCM context item at ordinal ${row.ordinal}`);
|
|
619
|
-
return {
|
|
620
|
-
sessionKey: row.session_key,
|
|
621
|
-
ordinal: row.ordinal,
|
|
622
|
-
type: "raw",
|
|
623
|
-
recordId: row.record_id,
|
|
624
|
-
summaryId: null,
|
|
625
|
-
fingerprint: row.fingerprint,
|
|
626
|
-
happenedAt: row.happened_at,
|
|
627
|
-
updatedAt: row.updated_at,
|
|
628
|
-
};
|
|
629
|
-
}
|
|
630
|
-
if (row.item_type === "summary") {
|
|
631
|
-
if (row.summary_id === null)
|
|
632
|
-
throw new Error(`Invalid summary LCM context item at ordinal ${row.ordinal}`);
|
|
633
|
-
return {
|
|
634
|
-
sessionKey: row.session_key,
|
|
635
|
-
ordinal: row.ordinal,
|
|
636
|
-
type: "summary",
|
|
637
|
-
recordId: null,
|
|
638
|
-
summaryId: row.summary_id,
|
|
639
|
-
fingerprint: row.fingerprint,
|
|
640
|
-
happenedAt: row.happened_at,
|
|
641
|
-
updatedAt: row.updated_at,
|
|
642
|
-
};
|
|
643
|
-
}
|
|
644
|
-
throw new Error(`Unknown LCM context item type: ${row.item_type}`);
|
|
645
|
-
}
|
|
646
|
-
function sessionStateFromRow(row) {
|
|
647
|
-
return {
|
|
648
|
-
sessionKey: row.session_key,
|
|
649
|
-
compactionDebt: row.compaction_debt,
|
|
650
|
-
cacheTouchedAt: row.cache_touched_at,
|
|
651
|
-
updatedAt: row.updated_at,
|
|
652
|
-
};
|
|
653
|
-
}
|
|
654
|
-
const SUMMARY_SNAPSHOT_TEXT_LIMIT = 4 * 1024;
|
|
655
|
-
const SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX = "…[truncated]";
|
|
656
|
-
function buildSummarySnapshot(db, summary) {
|
|
657
|
-
const rows = db
|
|
658
|
-
.prepare(`SELECT * FROM lcm_records
|
|
659
|
-
WHERE segment_id = ?
|
|
660
|
-
AND id BETWEEN ? AND ?
|
|
661
|
-
ORDER BY happened_at, id`)
|
|
662
|
-
.all(summary.segment_id, summary.covers_from_record_id, summary.covers_to_record_id);
|
|
663
|
-
return rows.map(snapshotRecordFromRow);
|
|
664
|
-
}
|
|
665
|
-
function buildSummaryParentSnapshot(db, summaryId, visiting) {
|
|
666
|
-
if (visiting.has(summaryId))
|
|
667
|
-
throw new Error(`Cycle detected in LCM summary parents at ${summaryId}`);
|
|
668
|
-
visiting.add(summaryId);
|
|
669
|
-
const row = db.prepare("SELECT * FROM lcm_summaries WHERE id = ?").get(summaryId);
|
|
670
|
-
if (!row)
|
|
671
|
-
throw new Error(`LCM summary does not exist: ${summaryId}`);
|
|
672
|
-
let snapshot = parseJsonArray(row.snapshot_json);
|
|
673
|
-
if (!snapshot &&
|
|
674
|
-
row.covers_from_record_id !== null &&
|
|
675
|
-
row.covers_to_record_id !== null &&
|
|
676
|
-
db
|
|
677
|
-
.prepare("SELECT 1 FROM lcm_records WHERE segment_id = ? AND id BETWEEN ? AND ? LIMIT 1")
|
|
678
|
-
.get(row.segment_id, row.covers_from_record_id, row.covers_to_record_id)) {
|
|
679
|
-
snapshot = buildSummarySnapshot(db, row);
|
|
680
|
-
}
|
|
681
|
-
const parentRows = db
|
|
682
|
-
.prepare(`SELECT parent_summary_id
|
|
683
|
-
FROM lcm_summary_parents
|
|
684
|
-
WHERE summary_id = ?
|
|
685
|
-
ORDER BY ord, parent_summary_id`)
|
|
686
|
-
.all(summaryId);
|
|
687
|
-
const parents = parentRows.map((parent) => buildSummaryParentSnapshot(db, parent.parent_summary_id, visiting));
|
|
688
|
-
visiting.delete(summaryId);
|
|
689
|
-
return {
|
|
690
|
-
summaryId: row.id,
|
|
691
|
-
depth: row.depth,
|
|
692
|
-
text: row.text_full,
|
|
693
|
-
coversFromRecordId: row.covers_from_record_id,
|
|
694
|
-
coversToRecordId: row.covers_to_record_id,
|
|
695
|
-
snapshot,
|
|
696
|
-
parents,
|
|
697
|
-
};
|
|
698
|
-
}
|
|
699
|
-
function snapshotRecordFromRow(row) {
|
|
700
|
-
const metadata = parseJsonObject(row.metadata_json);
|
|
701
|
-
return {
|
|
702
|
-
id: row.id,
|
|
703
|
-
kind: row.kind,
|
|
704
|
-
happened_at: row.happened_at,
|
|
705
|
-
role: snapshotRole(row.kind, metadata),
|
|
706
|
-
text: truncateSummarySnapshotText(row.text_full),
|
|
707
|
-
parts: parseJsonArray(row.parts_json),
|
|
708
|
-
attachments: parseJsonArray(row.attachments_json),
|
|
709
|
-
};
|
|
710
|
-
}
|
|
711
|
-
function snapshotRole(kind, metadata) {
|
|
712
|
-
if (typeof metadata?.role === "string" && metadata.role.trim())
|
|
713
|
-
return metadata.role;
|
|
714
|
-
if (kind === "user" || kind === "assistant")
|
|
715
|
-
return kind;
|
|
716
|
-
if (kind === "tool")
|
|
717
|
-
return "tool";
|
|
718
|
-
return null;
|
|
719
|
-
}
|
|
720
|
-
function truncateSummarySnapshotText(text) {
|
|
721
|
-
if (text.length <= SUMMARY_SNAPSHOT_TEXT_LIMIT)
|
|
722
|
-
return text;
|
|
723
|
-
const retainedLength = Math.max(0, SUMMARY_SNAPSHOT_TEXT_LIMIT - SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX.length);
|
|
724
|
-
return `${text.slice(0, retainedLength)}${SUMMARY_SNAPSHOT_TRUNCATED_SUFFIX}`;
|
|
725
|
-
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
2
|
import { completeSimple, } from "@earendil-works/pi-ai";
|
|
3
|
-
import { assertModelCanAuthenticate, parseModelRef, resolveModel, resolveModelApiKey } from "../../models.js";
|
|
3
|
+
import { assertModelCanAuthenticate, parseModelRef, resolveModel, resolveModelApiKey } from "../../models/index.js";
|
|
4
4
|
export const LCM_SUMMARIZER_SYSTEM_PROMPT = "You write continuity memory for a companion agent — notes it reads back later to stay close to a real person it talks with. Raw conversation history is preserved separately; the agent can search it on demand. Summaries aren't the last copy of anything — they're the index that lets the agent know what to look up. Preserve emotional shape and retrieval scent. Keep the moments that mattered emotionally over the ones that were lexically rich. Accurate, specific, understated. Don't dramatize, don't flatten. Plain text only.";
|
|
5
5
|
export class DefaultLcmSummarizer {
|
|
6
6
|
config;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
|
-
import { atomicWriteJson, createWriteQueue, isEnoent } from "
|
|
3
|
+
import { atomicWriteJson, createWriteQueue, isEnoent } from "../util/fs.js";
|
|
4
4
|
let addedModelsPath = resolve(process.cwd(), "data", "settings", "added-models.json");
|
|
5
5
|
let loaded = false;
|
|
6
6
|
let modelsCache = [];
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
|
-
import { readFileOrNull } from "
|
|
2
|
+
import { readFileOrNull } from "../util/fs.js";
|
|
3
3
|
export async function loadPersona(config) {
|
|
4
4
|
const [soul, user, memory, inner] = await Promise.all([
|
|
5
5
|
readFile(config.persona.soul, "utf8"),
|