@modusensus/dsh-mneme 0.6.1 → 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/lib/api.js +77 -1
- package/lib/client.js +234 -5
- package/lib/config.js +18 -0
- package/lib/dream/tag-extractor.js +156 -0
- package/lib/dream.js +20 -0
- package/lib/mirror.js +9 -0
- package/lib/parser/tag.js +59 -0
- package/lib/search/tag-boost.js +61 -0
- package/lib/service.js +134 -4
- package/lib/store.js +162 -0
- package/package.json +1 -1
- package/src/api.js +77 -1
- package/src/config.js +18 -0
- package/src/dream/tag-extractor.js +156 -0
- package/src/dream.js +20 -0
- package/src/mirror.js +9 -0
- package/src/parser/tag.js +59 -0
- package/src/search/tag-boost.js +61 -0
- package/src/service.js +134 -4
- package/src/store.js +162 -0
- package/test/api.test.js +80 -0
- package/test/boundary-v0625.test.js +82 -0
- package/test/client.test.js +98 -0
- package/test/directory.test.js +134 -0
- package/test/tag-boost.test.js +125 -0
- package/test/tag.test.js +312 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
// v0.6.2 auto-tag: a lightweight LLM pass that runs after an autoDream
|
|
2
|
+
// consolidation and extracts 1-3 tags per retained memory. Opt-in via
|
|
3
|
+
// config.autoTagEnabled (default false); autoTagMaxPerRun (default 10) bounds
|
|
4
|
+
// how many memories are tagged per run.
|
|
5
|
+
//
|
|
6
|
+
// Design notes:
|
|
7
|
+
// - ONE batched LLM call per run (the consolidation route is reused). Each
|
|
8
|
+
// memory is a prompt line; the model replies with a JSON array of
|
|
9
|
+
// {"id","tags"} entries — the per-memory contract is `{"tags":[...]}`.
|
|
10
|
+
// - Fail-safe everywhere: a missing route / aborted stream / unparseable
|
|
11
|
+
// JSON / unknown id / illegal tag are all skipped, never thrown. Tagging
|
|
12
|
+
// must never degrade the consolidation run it rides on.
|
|
13
|
+
// - The actual writes go through service.applyMemoryTags inside a
|
|
14
|
+
// service.transaction, so the mirror re-renders exactly once per pass and
|
|
15
|
+
// the write hooks fire once (never per memory).
|
|
16
|
+
import { sanitizeTags, MAX_TAG_LENGTH } from "../parser/tag.js";
|
|
17
|
+
|
|
18
|
+
const TAG_PROMPT = `你是记忆库标签助手。下面是保留的记忆条目(id、标题、内容)。
|
|
19
|
+
对每条记忆提取 1-3 个中文或英文标签,用于检索分类。
|
|
20
|
+
标签规则:
|
|
21
|
+
- 只允许字符:字母、数字、下划线、中文、连字符(如:linux、考研、deepseek-r1)
|
|
22
|
+
- 标签长度 ≤ ${MAX_TAG_LENGTH} 字符
|
|
23
|
+
- 宁缺毋滥:提取最核心的 1-3 个,不要凑数
|
|
24
|
+
- 不要输出内容里没有依据的标签
|
|
25
|
+
只输出一个 JSON 数组,每项形如 { "id": "<记忆id>", "tags": ["标签1", "标签2"] }。
|
|
26
|
+
不要输出其他文字。`;
|
|
27
|
+
|
|
28
|
+
/** Same stream consumption contract as dream.js. Returns accumulated text or
|
|
29
|
+
* undefined when the stream aborted/errored. */
|
|
30
|
+
async function streamText(ctx, options) {
|
|
31
|
+
if (!ctx?.llm?.stream) return undefined;
|
|
32
|
+
let text = "";
|
|
33
|
+
for await (const chunk of ctx.llm.stream(options)) {
|
|
34
|
+
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
35
|
+
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return text;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Pull the outermost JSON array out of a model reply (same tolerant contract
|
|
43
|
+
* as sleep.js parseJsonArray): find the first `[` … last `]` and parse. */
|
|
44
|
+
function parseJsonArray(text) {
|
|
45
|
+
if (typeof text !== "string") return undefined;
|
|
46
|
+
const start = text.indexOf("[");
|
|
47
|
+
const end = text.lastIndexOf("]");
|
|
48
|
+
if (start === -1 || end <= start) return undefined;
|
|
49
|
+
try {
|
|
50
|
+
const parsed = JSON.parse(text.slice(start, end + 1));
|
|
51
|
+
return Array.isArray(parsed) ? parsed : undefined;
|
|
52
|
+
} catch {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Resolve the LLM route for the tag pass: the caller-provided consolidation
|
|
58
|
+
* route first, then dreamProvider/dreamModel. Falls through to undefined. */
|
|
59
|
+
function resolveTagRoute(route, config, logger) {
|
|
60
|
+
if (route?.provider && route?.model) return route;
|
|
61
|
+
if (config?.dreamProvider && config?.dreamModel) return { provider: config.dreamProvider, model: config.dreamModel };
|
|
62
|
+
logger?.warn?.("dsh-mneme auto-tag: no llm route available");
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Run the auto-tag pass over retained memories.
|
|
68
|
+
* @param {object} opts
|
|
69
|
+
* @param {object} opts.ctx — { llm, logger }
|
|
70
|
+
* @param {object} opts.service — service handle (all/transaction/applyMemoryTags)
|
|
71
|
+
* @param {object} opts.config — plugin config (autoTagMaxPerRun, dreamProvider…)
|
|
72
|
+
* @param {object} [opts.route] — already-resolved consolidation route
|
|
73
|
+
* @returns {Promise<{ok: boolean, tagged: number, skipped: number, failed: number, skippedBy: boolean}>}
|
|
74
|
+
*/
|
|
75
|
+
export async function runAutoTag({ ctx, service, config, route }) {
|
|
76
|
+
const logger = ctx?.logger;
|
|
77
|
+
const maxPerRun = Number.isInteger(config?.autoTagMaxPerRun) && config.autoTagMaxPerRun > 0
|
|
78
|
+
? config.autoTagMaxPerRun
|
|
79
|
+
: 10;
|
|
80
|
+
// Retained = post-consolidation active memories, newest first, capped.
|
|
81
|
+
// Skip memories that already carry live tags — otherwise every run re-picks
|
|
82
|
+
// the same "newest" batch (tagging doesn't bump updated_at) and older
|
|
83
|
+
// memories starve; manual tags are also left untouched this way.
|
|
84
|
+
const memories = service.all()
|
|
85
|
+
.filter((m) => !m.forgotten && !m.archived && !m.session_disposed_at && m.type !== "summary")
|
|
86
|
+
.filter((m) => (service.getMemoryTags?.(m.id) ?? []).length === 0)
|
|
87
|
+
.sort((a, b) => {
|
|
88
|
+
const ta = String(a.updated_at ?? "");
|
|
89
|
+
const tb = String(b.updated_at ?? "");
|
|
90
|
+
if (ta < tb) return 1;
|
|
91
|
+
if (ta > tb) return -1;
|
|
92
|
+
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
|
|
93
|
+
})
|
|
94
|
+
.slice(0, Math.max(1, maxPerRun));
|
|
95
|
+
if (!memories.length) return { ok: true, tagged: 0, skipped: 0, failed: 0, skippedBy: "empty" };
|
|
96
|
+
const tagRoute = resolveTagRoute(route, config, logger);
|
|
97
|
+
if (!tagRoute) return { ok: false, tagged: 0, skipped: memories.length, failed: 0, skippedBy: "no-route" };
|
|
98
|
+
|
|
99
|
+
const listText = memories
|
|
100
|
+
.map((m) => `id=${m.id} | title=${m.title} | content=${m.content}`)
|
|
101
|
+
.join("\n");
|
|
102
|
+
const text = await streamText(ctx, {
|
|
103
|
+
provider: tagRoute.provider,
|
|
104
|
+
model: tagRoute.model,
|
|
105
|
+
purpose: "compaction",
|
|
106
|
+
maxTokens: Math.min(2048, config?.dreamMaxTokens ?? 2048),
|
|
107
|
+
messages: [
|
|
108
|
+
{ role: "system", content: [{ type: "text", text: TAG_PROMPT }] },
|
|
109
|
+
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
110
|
+
]
|
|
111
|
+
});
|
|
112
|
+
if (text === undefined) return { ok: false, tagged: 0, skipped: memories.length, failed: 0, skippedBy: "llm-failed" };
|
|
113
|
+
|
|
114
|
+
const entries = parseJsonArray(text);
|
|
115
|
+
if (!entries) {
|
|
116
|
+
logger?.warn?.("dsh-mneme auto-tag: no json array in llm output");
|
|
117
|
+
return { ok: false, tagged: 0, skipped: memories.length, failed: 0, skippedBy: "bad-json" };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Validate ids against the candidate set (unknown ids are ignored, never
|
|
121
|
+
// written — a stray id could otherwise tag an unrelated memory).
|
|
122
|
+
const candidateIds = new Set(memories.map((m) => m.id));
|
|
123
|
+
const toWrite = [];
|
|
124
|
+
let skipped = 0;
|
|
125
|
+
let failed = 0;
|
|
126
|
+
const seenIds = new Set();
|
|
127
|
+
for (const entry of entries) {
|
|
128
|
+
if (!entry || typeof entry !== "object") { failed++; continue; }
|
|
129
|
+
const id = typeof entry.id === "string" ? entry.id : undefined;
|
|
130
|
+
if (!id || !candidateIds.has(id)) { failed++; continue; }
|
|
131
|
+
if (seenIds.has(id)) continue; // first entry per id wins
|
|
132
|
+
seenIds.add(id);
|
|
133
|
+
// Reuse the shared sanitizer: strip, ≤20 chars, drop illegal, dedupe.
|
|
134
|
+
const tags = sanitizeTags(entry.tags);
|
|
135
|
+
if (!tags.length) { skipped++; continue; }
|
|
136
|
+
toWrite.push({ id, tags });
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Write the whole batch under one transaction: mirror re-renders once.
|
|
140
|
+
let tagged = 0;
|
|
141
|
+
try {
|
|
142
|
+
service.transaction(() => {
|
|
143
|
+
for (const { id, tags } of toWrite) {
|
|
144
|
+
// Merge with existing live tags (never clobber manual tags that landed
|
|
145
|
+
// between selection and write).
|
|
146
|
+
const existing = service.getMemoryTags?.(id) ?? [];
|
|
147
|
+
service.applyMemoryTags(id, [...new Set([...existing, ...tags])]);
|
|
148
|
+
tagged++;
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
} catch (error) {
|
|
152
|
+
logger?.warn?.(`dsh-mneme auto-tag: write failed: ${String(error)}`);
|
|
153
|
+
return { ok: false, tagged, skipped, failed, skippedBy: "write-failed" };
|
|
154
|
+
}
|
|
155
|
+
return { ok: true, tagged, skipped, failed, skippedBy: false };
|
|
156
|
+
}
|
package/src/dream.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { validateDecisions, applyDecisions } from "./dream/decisions.js";
|
|
2
2
|
import { clusterMemories, findPotentialConflicts } from "./dream/clustering.js";
|
|
3
|
+
import { runAutoTag } from "./dream/tag-extractor.js";
|
|
3
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
4
5
|
export { validateDecisions, applyDecisions, normalizeDecisions };
|
|
5
6
|
|
|
@@ -815,6 +816,25 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
815
816
|
// that only froze conflicts is not a noop.
|
|
816
817
|
const noChange = frozenCount === 0 && applied === 0 && committed.every((c) => c.action === "keep");
|
|
817
818
|
|
|
819
|
+
// v0.6.2 auto-tag: a light LLM pass over the retained (post-consolidation)
|
|
820
|
+
// memories. Opt-in via config.autoTagEnabled, bounded by autoTagMaxPerRun,
|
|
821
|
+
// and always fail-safe — a tag failure must never change the consolidation
|
|
822
|
+
// outcome reported below (it is logged and counted, nothing more).
|
|
823
|
+
let autoTagged = 0;
|
|
824
|
+
if (config.autoTagEnabled === true) {
|
|
825
|
+
try {
|
|
826
|
+
const tagResult = await runAutoTag({ ctx, service, config, route });
|
|
827
|
+
autoTagged = tagResult?.tagged ?? 0;
|
|
828
|
+
if (tagResult?.ok === false && tagResult?.skippedBy && tagResult.skippedBy !== "empty") {
|
|
829
|
+
logger?.warn?.(`dsh-mneme dream: auto-tag skipped (${tagResult.skippedBy})`);
|
|
830
|
+
} else if (autoTagged > 0) {
|
|
831
|
+
logger?.info?.(`[dsh-mneme] auto-tag: ${autoTagged} memory(ies) tagged`);
|
|
832
|
+
}
|
|
833
|
+
} catch (error) {
|
|
834
|
+
logger?.warn?.(`dsh-mneme dream: auto-tag failed: ${String(error)}`);
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
818
838
|
// Keep the vector index consistent with the post-dream store state.
|
|
819
839
|
if (semantic?.embedder && semantic?.vectorIndex) {
|
|
820
840
|
try {
|
package/src/mirror.js
CHANGED
|
@@ -31,6 +31,15 @@ function renderMemory(m) {
|
|
|
31
31
|
.digest("hex");
|
|
32
32
|
const lines = [];
|
|
33
33
|
lines.push(`## ${esc(m.title)}`);
|
|
34
|
+
// v0.6.2 tag line: entity_attrs-backed tags (attached by the service as
|
|
35
|
+
// `entityTags`) rendered as `#tag` space-separated under the title. No tags
|
|
36
|
+
// → no line. Legacy `- **标签**:` metadata below keeps the memories.tags
|
|
37
|
+
// column (still written by save/update) readable.
|
|
38
|
+
const entityTags = Array.isArray(m.entityTags) ? m.entityTags : [];
|
|
39
|
+
if (entityTags.length) {
|
|
40
|
+
lines.push("");
|
|
41
|
+
lines.push(entityTags.map((t) => `#${esc(t)}`).join(" "));
|
|
42
|
+
}
|
|
34
43
|
lines.push("");
|
|
35
44
|
lines.push(`- **ID**: \`${m.id}\``);
|
|
36
45
|
lines.push(`- **类型**: ${m.type}`);
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tag parser (v0.6.2). Given a memory's content, extract explicit hashtags of
|
|
3
|
+
* the form `#标签`. The allowed character set is `[a-zA-Z0-9_一-龥-]+` (ASCII
|
|
4
|
+
* letters/digits/underscore, CJK characters, and hyphen), so `#linux`,
|
|
5
|
+
* `#deepseek` and `#考研` are all valid tags.
|
|
6
|
+
*
|
|
7
|
+
* Rules:
|
|
8
|
+
* - multiple tags on one line are all extracted, in source order;
|
|
9
|
+
* - duplicates collapse to the first occurrence;
|
|
10
|
+
* - tags longer than MAX_TAG_LENGTH (20) characters are dropped as illegal;
|
|
11
|
+
* - edge hyphens (leading/trailing, e.g. a sentence-terminating `#tag-`) are
|
|
12
|
+
* stripped; a tag that collapses to empty is dropped;
|
|
13
|
+
* - markdown `## heading` never matches (the second `#`/space is not in the
|
|
14
|
+
* allowed set).
|
|
15
|
+
*
|
|
16
|
+
* Pure module: no store, no side effects — persistence (store.setMemoryTags)
|
|
17
|
+
* is a separate step. sanitizeTags shares the same validation so LLM-extracted
|
|
18
|
+
* tag arrays (autoDream tag-extractor) are filtered identically.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
export const MAX_TAG_LENGTH = 20;
|
|
22
|
+
|
|
23
|
+
// `一` (U+4E00) .. `龥` (U+9FA5) is the CJK Unified Ideographs block (the
|
|
24
|
+
// rule's `一-龥`). Matching requires a tag character right after `#` so a bare
|
|
25
|
+
// `#` and markdown `## heading` are naturally excluded (#/space not in set).
|
|
26
|
+
const TAG_RE = /#[a-zA-Z0-9_一-龥-]+/g;
|
|
27
|
+
const EDGE_HYPHEN = /^-+|-+$/g;
|
|
28
|
+
const VALID_TAG = /^[a-zA-Z0-9_一-龥-]+$/;
|
|
29
|
+
|
|
30
|
+
/** Validate + dedupe an array of tag strings. Non-strings, blanks, over-long
|
|
31
|
+
* and illegal-character tags are dropped (fail-safe). A leading `#` is
|
|
32
|
+
* tolerated so raw LLM output can be passed through directly. */
|
|
33
|
+
export function sanitizeTags(tags) {
|
|
34
|
+
if (!Array.isArray(tags)) return [];
|
|
35
|
+
const seen = new Set();
|
|
36
|
+
const out = [];
|
|
37
|
+
for (const t of tags) {
|
|
38
|
+
if (typeof t !== "string") continue;
|
|
39
|
+
let tag = t.trim();
|
|
40
|
+
if (tag.startsWith("#")) tag = tag.slice(1);
|
|
41
|
+
tag = tag.replace(EDGE_HYPHEN, "");
|
|
42
|
+
if (!tag || tag.length > MAX_TAG_LENGTH) continue;
|
|
43
|
+
if (!VALID_TAG.test(tag)) continue;
|
|
44
|
+
if (seen.has(tag)) continue; // 去重(保留首次出现顺序)
|
|
45
|
+
seen.add(tag);
|
|
46
|
+
out.push(tag);
|
|
47
|
+
}
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** @returns {string[]} validated tags in source order, deduplicated. */
|
|
52
|
+
export function parseTags(content) {
|
|
53
|
+
if (typeof content !== "string" || content.length === 0) return [];
|
|
54
|
+
const raw = [];
|
|
55
|
+
TAG_RE.lastIndex = 0;
|
|
56
|
+
let m;
|
|
57
|
+
while ((m = TAG_RE.exec(content)) !== null) raw.push(m[0]);
|
|
58
|
+
return sanitizeTags(raw);
|
|
59
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// v0.6.4 Tag-weighted re-rank helpers.
|
|
2
|
+
// Query tags come from `#hashtags` in the query plus any known tags the query
|
|
3
|
+
// mentions; candidates whose tags overlap the query tags or the session's
|
|
4
|
+
// hot-memory tags get their score boosted before the final top-K cut.
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Extract tags from a query: explicit `#hashtags` (CJK allowed) plus known
|
|
8
|
+
* tags whose lowercase form appears in the query. Deduped, order-preserving.
|
|
9
|
+
*/
|
|
10
|
+
export function extractQueryTags(query, knownTags = []) {
|
|
11
|
+
if (typeof query !== "string" || !query) return [];
|
|
12
|
+
const set = new Set();
|
|
13
|
+
const re = /#([a-zA-Z0-9_一-龥-]+)/g;
|
|
14
|
+
let m;
|
|
15
|
+
while ((m = re.exec(query)) !== null) set.add(m[1]);
|
|
16
|
+
const q = query.toLowerCase();
|
|
17
|
+
for (const t of knownTags) {
|
|
18
|
+
if (typeof t === "string" && q.includes(t.toLowerCase())) set.add(t);
|
|
19
|
+
}
|
|
20
|
+
return Array.from(set);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function hasOverlap(a, b) {
|
|
24
|
+
if (!Array.isArray(a) || !Array.isArray(b)) return false;
|
|
25
|
+
const setB = new Set(b);
|
|
26
|
+
return a.some((x) => setB.has(x));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Boost candidate scores by tag overlap with the query and/or session tags.
|
|
31
|
+
* Each candidate is expected to carry a `tags` array (enriched by the caller).
|
|
32
|
+
* Returns a new array sorted by boosted score descending; candidates that were
|
|
33
|
+
* boosted are marked `tagBoost: true`. Unchanged candidates keep their
|
|
34
|
+
* relative order (stable).
|
|
35
|
+
*/
|
|
36
|
+
export function applyTagBoost(
|
|
37
|
+
candidates,
|
|
38
|
+
{ queryTags = [], sessionTags = [], factor = 1.15, sessionFactor = 1.08 }
|
|
39
|
+
) {
|
|
40
|
+
const boosted = candidates.map((c) => {
|
|
41
|
+
const tags = c.tags ?? [];
|
|
42
|
+
let score = typeof c.score === "number" ? c.score : 0;
|
|
43
|
+
let didBoost = false;
|
|
44
|
+
|
|
45
|
+
if (queryTags.length && hasOverlap(tags, queryTags)) {
|
|
46
|
+
score = Math.min(1, score * factor);
|
|
47
|
+
didBoost = true;
|
|
48
|
+
}
|
|
49
|
+
if (sessionTags.length && hasOverlap(tags, sessionTags)) {
|
|
50
|
+
score = Math.min(1, score * sessionFactor);
|
|
51
|
+
didBoost = true;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const out = { ...c, score };
|
|
55
|
+
if (didBoost) out.tagBoost = true;
|
|
56
|
+
return out;
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
boosted.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
|
60
|
+
return boosted;
|
|
61
|
+
}
|
package/src/service.js
CHANGED
|
@@ -4,6 +4,7 @@ import { evaluateMemoryQuality } from "./quality-filter.js";
|
|
|
4
4
|
import { createBM25Index } from "./search/bm25.js";
|
|
5
5
|
import { adaptiveThreshold } from "./search/adaptive.js";
|
|
6
6
|
import { parseWikiLinks } from "./parser/wiki-link.js";
|
|
7
|
+
import { extractQueryTags, applyTagBoost } from "./search/tag-boost.js";
|
|
7
8
|
|
|
8
9
|
const INJECT_TYPES = new Set(["preference", "project", "decision", "summary"]);
|
|
9
10
|
|
|
@@ -327,6 +328,57 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
327
328
|
return hits;
|
|
328
329
|
}
|
|
329
330
|
|
|
331
|
+
/**
|
|
332
|
+
* Search for memories carrying a given tag set (v0.6.2). Multiple tag:
|
|
333
|
+
* tokens in one query use AND semantics — a memory must carry every tag.
|
|
334
|
+
* Combinable with the leftover query: an entity:/attr: prefix is intersected
|
|
335
|
+
* with the tag-matched set, and plain keyword text ranks it (only rows whose
|
|
336
|
+
* keyword score > 0 survive). Tags are first-class recall, not entity-gated:
|
|
337
|
+
* the tag match itself does not depend on config.entitySearchEnabled.
|
|
338
|
+
* @param {string[]} tagTokens
|
|
339
|
+
* @param {string} q — leftover query after tag: tokens were stripped
|
|
340
|
+
* @param {object} [options]
|
|
341
|
+
* @param {number} [options.topK=20]
|
|
342
|
+
* @returns {any[]}
|
|
343
|
+
*/
|
|
344
|
+
function searchByTags(tagTokens, q, options) {
|
|
345
|
+
const { topK = 20 } = options;
|
|
346
|
+
const tags = (Array.isArray(tagTokens) ? tagTokens : [])
|
|
347
|
+
.map((t) => String(t).trim()).filter(Boolean);
|
|
348
|
+
if (!tags.length) return [];
|
|
349
|
+
let hits = store.findMemoriesByTags(tags);
|
|
350
|
+
if (!hits.length) return [];
|
|
351
|
+
// Intersect with entity:/attr: prefixes present in the leftover query.
|
|
352
|
+
if (config?.entitySearchEnabled) {
|
|
353
|
+
if (q.startsWith("entity:")) {
|
|
354
|
+
const ids = new Set(searchByEntity(q.slice(7).trim(), { topK: 10000 }).map((m) => m.id));
|
|
355
|
+
hits = hits.filter((m) => ids.has(m.id));
|
|
356
|
+
}
|
|
357
|
+
if (q.startsWith("attr:")) {
|
|
358
|
+
const [key, value] = q.slice(5).split("=");
|
|
359
|
+
const ids = new Set(searchByAttr(key, value, { topK: 10000 }).map((m) => m.id));
|
|
360
|
+
hits = hits.filter((m) => ids.has(m.id));
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
// Leftover plain text ranks the tag-matched set; only rows whose title or
|
|
364
|
+
// content actually mentions the keyword survive (scoreKeyword's 0.3 base is
|
|
365
|
+
// a non-match, so the containment check is the real gate); otherwise
|
|
366
|
+
// preserve insertion order.
|
|
367
|
+
const keywordOnly = q && !q.startsWith("entity:") && !q.startsWith("attr:");
|
|
368
|
+
hits = keywordOnly
|
|
369
|
+
? hits
|
|
370
|
+
.map((m) => ({ ...m, score: scoreKeyword(m, q), source: "tag" }))
|
|
371
|
+
.filter((m) => (m.title ?? "").toLowerCase().includes(q.toLowerCase())
|
|
372
|
+
|| (m.content ?? "").toLowerCase().includes(q.toLowerCase()))
|
|
373
|
+
.sort((a, b) => b.score - a.score)
|
|
374
|
+
: hits.map((m) => ({ ...m, source: "tag" }));
|
|
375
|
+
const result = hits.slice(0, topK);
|
|
376
|
+
// Only count toward recall stats/forgetting when the caller asked for
|
|
377
|
+
// recording — the panel's `tag:` search must not pollute the curve.
|
|
378
|
+
if (options.recordRecall) touchRecalled(result);
|
|
379
|
+
return result;
|
|
380
|
+
}
|
|
381
|
+
|
|
330
382
|
/**
|
|
331
383
|
* Semantic-aware memory search: keyword recall (store.search) plus optional
|
|
332
384
|
* vector recall + rerank. mode:
|
|
@@ -433,8 +485,18 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
433
485
|
|
|
434
486
|
async function searchMemories(query, options = {}) {
|
|
435
487
|
const { mode = "auto", topK = 20, threshold, useRerank = true, recordRecall = false } = options;
|
|
436
|
-
const
|
|
437
|
-
if (!
|
|
488
|
+
const raw = String(query ?? "").trim();
|
|
489
|
+
if (!raw) return [];
|
|
490
|
+
|
|
491
|
+
// tag: 前缀(v0.6.2)。先把所有 tag: 令牌从 query 里剥出来,剩余的 q 仍可带
|
|
492
|
+
// entity:/attr: 前缀或普通关键词 —— searchByTags 负责交集/排序。没有 tag:
|
|
493
|
+
// 令牌则走下面的 entity:/attr:/文本原逻辑(完全向后兼容)。
|
|
494
|
+
const tagTokens = [];
|
|
495
|
+
const q = raw.replace(/\btag:([^\s]+)/g, (_, tok) => {
|
|
496
|
+
if (tok) tagTokens.push(tok);
|
|
497
|
+
return "";
|
|
498
|
+
}).trim();
|
|
499
|
+
if (tagTokens.length) return searchByTags(tagTokens, q, options);
|
|
438
500
|
|
|
439
501
|
// entity:/attr: 前缀路由(v0.3.0 Phase 3)。entitySearchEnabled 关闭时走原逻辑。
|
|
440
502
|
if (config?.entitySearchEnabled) {
|
|
@@ -539,7 +601,10 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
539
601
|
const ranked = [...byId.values()]
|
|
540
602
|
.sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
|
|
541
603
|
.map((r) => ({ ...r, score: Math.max(0, Math.min(1, r.score ?? 0)) }));
|
|
542
|
-
|
|
604
|
+
// Tag boost (v0.6.4) needs the full ranked pool, not just the top-lim
|
|
605
|
+
// slice, so a tagged candidate just below the line can be re-admitted
|
|
606
|
+
// after the boost. Without boost this is the legacy lim truncation.
|
|
607
|
+
merged = config.tagBoostEnabled === true ? ranked : ranked.slice(0, lim);
|
|
543
608
|
if (merged.length < lim && !merged.length) {
|
|
544
609
|
// Vector unavailable entirely: fall back to plain keyword.
|
|
545
610
|
merged = keyword.slice(0, lim);
|
|
@@ -565,7 +630,31 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
565
630
|
// it is the documented text-only path and must not be altered by
|
|
566
631
|
// embedding state.
|
|
567
632
|
merged = mode === "keyword" ? merged : semanticDeduplicate(merged);
|
|
633
|
+
|
|
634
|
+
// Tag-weighted re-rank (v0.6.4): boost candidates whose tags overlap the
|
|
635
|
+
// query tags or the current session's hot-memory tags — applied BEFORE the
|
|
636
|
+
// final top-K cut so tagged candidates just below the line can be
|
|
637
|
+
// re-admitted. Opt-in, and skipped entirely on the keyword-only path. When
|
|
638
|
+
// a reranker is configured (opt-in), it remains the final authority on
|
|
639
|
+
// order; the boost still shapes which candidates reach it.
|
|
640
|
+
if (config.tagBoostEnabled === true && mode !== "keyword" && merged.length) {
|
|
641
|
+
const ids = merged.map((m) => m.id);
|
|
642
|
+
const tagsMap = store.getMemoryTagsMap(ids);
|
|
643
|
+
const enriched = merged.map((m) => ({ ...m, tags: tagsMap.get(m.id) ?? [] }));
|
|
644
|
+
const knownTags = [...tagsMap.values()].flat();
|
|
645
|
+
const queryTags = extractQueryTags(q, knownTags);
|
|
646
|
+
const sessionTags = Array.isArray(options?.sessionTags) ? options.sessionTags : [];
|
|
647
|
+
if (queryTags.length || sessionTags.length) {
|
|
648
|
+
merged = applyTagBoost(enriched, {
|
|
649
|
+
queryTags,
|
|
650
|
+
sessionTags,
|
|
651
|
+
factor: config.tagBoostFactor,
|
|
652
|
+
sessionFactor: config.sessionTagBoostFactor,
|
|
653
|
+
}).map(({ tags, ...rest }) => rest);
|
|
654
|
+
}
|
|
655
|
+
}
|
|
568
656
|
merged = merged.slice(0, lim);
|
|
657
|
+
|
|
569
658
|
let result = useRerank && reranker && merged.length
|
|
570
659
|
? await rerankCandidates(q, merged, lim)
|
|
571
660
|
: merged;
|
|
@@ -1043,6 +1132,21 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1043
1132
|
}));
|
|
1044
1133
|
}
|
|
1045
1134
|
|
|
1135
|
+
/**
|
|
1136
|
+
* Directory view (v0.6.3): group live memories by tag. Delegates to
|
|
1137
|
+
* store.getDirectory (tag-sorted groups, importance/updated DESC members,
|
|
1138
|
+
* live-only filtering) and maps every memory to the wire DTO so the result
|
|
1139
|
+
* is JSON-safe for the API endpoint.
|
|
1140
|
+
* @returns {{groups: {tag: string, memories: object[]}[], untagged: object[]}}
|
|
1141
|
+
*/
|
|
1142
|
+
function getDirectory() {
|
|
1143
|
+
const { groups, untagged } = store.getDirectory();
|
|
1144
|
+
return {
|
|
1145
|
+
groups: groups.map((g) => ({ tag: g.tag, memories: toApiList(g.memories) })),
|
|
1146
|
+
untagged: toApiList(untagged)
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1046
1150
|
/**
|
|
1047
1151
|
* Three-way merge of in-flight human mirror edits before a re-render.
|
|
1048
1152
|
* Runs on every syncMirror, so a human edit made between two store writes is
|
|
@@ -1150,8 +1254,14 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1150
1254
|
// leaving committed files mislabeled as failed and masking partial state.
|
|
1151
1255
|
// Absent entries (a type with no memories) count as success: sync prunes
|
|
1152
1256
|
// the stale file, which is itself a completed physical state.
|
|
1257
|
+
// v0.6.2: attach entity_attrs-backed tags (entityTags) after the human-edit
|
|
1258
|
+
// merge so renderMemory can draw the `#tag` line under each title. The
|
|
1259
|
+
// bulk map is a single query, and a missing row simply renders no line.
|
|
1260
|
+
const reconciled = reconcileHumanEdits(list);
|
|
1261
|
+
const tagsMap = store.getMemoryTagsMap(reconciled.map((m) => m.id));
|
|
1262
|
+
const tagged = reconciled.map((m) => ({ ...m, entityTags: tagsMap.get(m.id) ?? [] }));
|
|
1153
1263
|
let allOk = true;
|
|
1154
|
-
const results = mirror.sync(
|
|
1264
|
+
const results = mirror.sync(tagged) ?? {};
|
|
1155
1265
|
for (const type of Object.keys(TYPE_FILE)) {
|
|
1156
1266
|
const r = results[type];
|
|
1157
1267
|
const ok = !r || r.ok === true;
|
|
@@ -1371,6 +1481,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1371
1481
|
injectCandidates,
|
|
1372
1482
|
mergeHumanEdits,
|
|
1373
1483
|
toApiList,
|
|
1484
|
+
getDirectory,
|
|
1374
1485
|
transaction,
|
|
1375
1486
|
enqueue,
|
|
1376
1487
|
setDreamHook(fn) { dreamHook = fn; },
|
|
@@ -1585,6 +1696,25 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1585
1696
|
saveWikiLinks: (r) => store.saveWikiLinks(r),
|
|
1586
1697
|
listEntities: (o) => store.listEntities(o),
|
|
1587
1698
|
getRelations: (id) => store.getRelations(id),
|
|
1699
|
+
// Tag system (v0.6.2). setMemoryTags is the manual/user path: gated by
|
|
1700
|
+
// config.manualTagEnabled (default true), re-renders the mirror and fires
|
|
1701
|
+
// the write hook. applyMemoryTags is the raw write used by the autoDream
|
|
1702
|
+
// tag pass (gated by autoTagEnabled, wrapped in a transaction by the
|
|
1703
|
+
// extractor so the mirror re-renders exactly once).
|
|
1704
|
+
setMemoryTags: (memoryId, tags) => {
|
|
1705
|
+
if (config.manualTagEnabled === false) {
|
|
1706
|
+
return { ok: false, error: "manualTagEnabled is off" };
|
|
1707
|
+
}
|
|
1708
|
+
const stored = store.setMemoryTags(memoryId, tags);
|
|
1709
|
+
afterSync("write");
|
|
1710
|
+
notifyWrite();
|
|
1711
|
+
return { ok: true, tags: stored };
|
|
1712
|
+
},
|
|
1713
|
+
getMemoryTags: (memoryId) => store.getMemoryTags(memoryId),
|
|
1714
|
+
// Read-only gate flag so the Web panel can hide tag editing when the
|
|
1715
|
+
// manual path is disabled (default: manual tagging is on).
|
|
1716
|
+
manualTagEnabled: () => config.manualTagEnabled !== false,
|
|
1717
|
+
applyMemoryTags: (memoryId, tags) => store.setMemoryTags(memoryId, tags),
|
|
1588
1718
|
saveAttr: (r) => store.saveAttr(r),
|
|
1589
1719
|
createEntity: (r) => store.createEntity(r),
|
|
1590
1720
|
findEntityByName: (n) => store.findEntityByName(n),
|