@unblocklabs/unblock-memory 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -11
- package/dist/src/analysis.d.ts +12 -2
- package/dist/src/analysis.js +106 -12
- package/dist/src/config.d.ts +12 -1
- package/dist/src/config.js +69 -15
- package/dist/src/curation.d.ts +70 -0
- package/dist/src/curation.js +191 -0
- package/dist/src/manager.d.ts +23 -0
- package/dist/src/manager.js +257 -29
- package/dist/src/plugin.js +80 -1
- package/dist/src/runtime.d.ts +8 -0
- package/dist/src/runtime.js +14 -2
- package/dist/src/skill-whisperer.d.ts +16 -0
- package/dist/src/skill-whisperer.js +113 -0
- package/dist/src/sources.d.ts +4 -3
- package/dist/src/sources.js +34 -1
- package/openclaw.plugin.json +40 -4
- package/package.json +2 -2
- package/skills/memory-curator/SKILL.md +12 -1
package/README.md
CHANGED
|
@@ -38,6 +38,10 @@ directories, or globs into named corpora:
|
|
|
38
38
|
slots: { memory: "unblock-memory" },
|
|
39
39
|
entries: {
|
|
40
40
|
"unblock-memory": {
|
|
41
|
+
hooks: {
|
|
42
|
+
// Required only when skillWhisperer.enabled is true.
|
|
43
|
+
allowConversationAccess: true,
|
|
44
|
+
},
|
|
41
45
|
config: {
|
|
42
46
|
// Default: avoid repeated model cold starts after idle periods.
|
|
43
47
|
keepEmbeddingModelWarm: true,
|
|
@@ -57,7 +61,24 @@ directories, or globs into named corpora:
|
|
|
57
61
|
kind: "files",
|
|
58
62
|
paths: ["knowledge/**/*.md"],
|
|
59
63
|
},
|
|
64
|
+
{
|
|
65
|
+
name: "skills",
|
|
66
|
+
kind: "skills",
|
|
67
|
+
paths: [
|
|
68
|
+
"skills/**/SKILL.md",
|
|
69
|
+
".agents/skills/**/SKILL.md",
|
|
70
|
+
"~/.agents/skills/**/SKILL.md",
|
|
71
|
+
"~/.openclaw/skills/**/SKILL.md",
|
|
72
|
+
"~/.openclaw/plugin-skills/**/SKILL.md",
|
|
73
|
+
],
|
|
74
|
+
},
|
|
60
75
|
],
|
|
76
|
+
skillWhisperer: {
|
|
77
|
+
enabled: false,
|
|
78
|
+
historyMessages: 5,
|
|
79
|
+
minScore: 0.4,
|
|
80
|
+
cooldownTurns: 10,
|
|
81
|
+
},
|
|
61
82
|
// Optional: omit unless the local analysis worker is installed.
|
|
62
83
|
analysis: {
|
|
63
84
|
executable: "/absolute/path/to/unblock-cluster/bin/unblock-memory-analysis",
|
|
@@ -79,11 +100,34 @@ corpus; other unique names may be added for custom material.
|
|
|
79
100
|
context resident after first use. Set it to `false` to restore QMD's five-minute
|
|
80
101
|
idle unload behavior.
|
|
81
102
|
|
|
82
|
-
`memory_search` searches every configured corpus by default. Pass
|
|
103
|
+
`memory_search` searches every configured non-skill corpus by default. Pass
|
|
83
104
|
`corpora: ["knowledge"]` to search selected corpora or `corpora: ["all"]` to
|
|
84
105
|
request all of them explicitly. Search results include their corpus name and
|
|
85
106
|
remain readable by passing the returned `qmd://` path to `memory_get`.
|
|
86
107
|
|
|
108
|
+
### Skill Whisperer
|
|
109
|
+
|
|
110
|
+
Skill Whisperer is an optional semantic reminder for user turns. Configure one
|
|
111
|
+
isolated `skills` corpus, set `skillWhisperer.enabled` to `true`, and authorize
|
|
112
|
+
`plugins.entries.unblock-memory.hooks.allowConversationAccess`. The feature
|
|
113
|
+
embeds the current prompt plus the configured number of prior user/assistant
|
|
114
|
+
messages, searches only skill files, and prepends at most one name/path hint
|
|
115
|
+
when the best eligible match reaches `minScore`. It never opens or invokes a
|
|
116
|
+
skill automatically.
|
|
117
|
+
|
|
118
|
+
The defaults use five prior messages, a calibrated score threshold of `0.4`,
|
|
119
|
+
and a ten-turn cooldown. A skill is cooling down after either a suggestion or a
|
|
120
|
+
successful direct `read` of its indexed `SKILL.md`; the next result is eligible
|
|
121
|
+
only when it independently meets the same score threshold. Cooldown state is
|
|
122
|
+
per session and intentionally resets with the Gateway. Shell-command reads are
|
|
123
|
+
not tracked.
|
|
124
|
+
|
|
125
|
+
The `skills` corpus shares the existing QMD store and warm embedding model but
|
|
126
|
+
is private to Skill Whisperer: it is excluded from ordinary `memory_search`
|
|
127
|
+
(including `corpora: ["all"]`), `memory_get`, clustering, and memory-maintenance
|
|
128
|
+
tasks. Paths are explicit by design; the plugin does not reconstruct
|
|
129
|
+
OpenClaw's effective skill inventory from `openclaw.json`.
|
|
130
|
+
|
|
87
131
|
Use `sessionFilter` to restrict session results by metadata while leaving file
|
|
88
132
|
corpora searchable. Supported fields are `startedFrom` and `startedTo`
|
|
89
133
|
(inclusive ISO 8601 timestamps), `provider`, `chatType`, `accountId`, and
|
|
@@ -122,9 +166,10 @@ does not sync sessions at startup or on a schedule; refreshes are manual through
|
|
|
122
166
|
`memory_sync_sessions`.
|
|
123
167
|
|
|
124
168
|
Indexes live at `~/.openclaw/agents/<agentId>/unblock-memory/index.sqlite` (or the
|
|
125
|
-
equivalent configured OpenClaw state directory).
|
|
126
|
-
|
|
127
|
-
|
|
169
|
+
equivalent configured OpenClaw state directory). Durable agent-supplied event
|
|
170
|
+
dates and maintenance proposals live separately in `curation.sqlite`, so a QMD
|
|
171
|
+
index rebuild does not discard them. The first lookup builds the index;
|
|
172
|
+
Markdown filesystem changes queue a debounced, serialized background refresh.
|
|
128
173
|
|
|
129
174
|
## Memory analysis
|
|
130
175
|
|
|
@@ -144,9 +189,10 @@ python3 -m venv .venv
|
|
|
144
189
|
Set `analysis.executable` to the absolute path of
|
|
145
190
|
`bin/unblock-memory-analysis` in that checkout. One worker installation can
|
|
146
191
|
serve every agent on the host. The plugin invokes it directly with
|
|
147
|
-
`--db <the agent's known index path
|
|
148
|
-
`--config-json <clustering options>` payload.
|
|
149
|
-
executable, shell command, or
|
|
192
|
+
`--db <the agent's known index path>`, the plugin's non-skill collection IDs,
|
|
193
|
+
and, when requested, a validated `--config-json <clustering options>` payload.
|
|
194
|
+
Agents cannot choose a database, executable, collection, shell command, or
|
|
195
|
+
arbitrary arguments.
|
|
150
196
|
|
|
151
197
|
Without the worker, `memory_list_clusters` reports that memory has not been
|
|
152
198
|
analyzed and `memory_recluster` reports that analysis is unavailable. Ordinary
|
|
@@ -175,10 +221,24 @@ and a deterministic seed. Omitting them uses the worker's defaults.
|
|
|
175
221
|
`memory_fetch_cluster` accepts `topK` (1–50), a zero-based `offset`, and
|
|
176
222
|
`sort`: `representative` (the default), `score_desc`, `score_asc`, `date_desc`,
|
|
177
223
|
or `date_asc`. Score is cluster membership probability for normal clusters and
|
|
178
|
-
outlier score for noise. Each member
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
224
|
+
outlier score for noise. Each member reports raw `sourceModifiedAt` separately
|
|
225
|
+
from `eventTime` and `eventTimeBasis`. Session start times and dated memory paths
|
|
226
|
+
resolve programmatically; reviewed annotations resolve otherwise ambiguous
|
|
227
|
+
chunks or whole documents. Date sorting uses resolved event time when available
|
|
228
|
+
and the clearly labeled source modification time only as a fallback. Responses
|
|
229
|
+
include page totals and the next offset when more members remain.
|
|
230
|
+
|
|
231
|
+
A chronological cluster read creates a coalesced maintenance proposal only for
|
|
232
|
+
returned documents whose event time remains ambiguous; it does not scan the
|
|
233
|
+
whole corpus for chores. Persisted exact-duplicate analysis can likewise create
|
|
234
|
+
review proposals for non-session Markdown. `memory_list_maintenance_tasks`
|
|
235
|
+
returns at most ten tasks, while `memory_update_maintenance_task` can resolve,
|
|
236
|
+
defer, or mark one irrelevant and optionally attach a supported event date.
|
|
237
|
+
For duplicate proposals, defer confirmed cleanup until the source change is
|
|
238
|
+
complete, mark intentional repetition irrelevant, and resolve only completed
|
|
239
|
+
work. These tools never edit or delete source Markdown. Duplicate cleanup
|
|
240
|
+
remains a reviewed source change outside the maintenance tool, and generated
|
|
241
|
+
session projections must never be edited directly.
|
|
182
242
|
|
|
183
243
|
Member excerpts are capped at 2 KB each and 12 KB across a response; source
|
|
184
244
|
aliases are capped at five per member and 50 across a response. These budgets
|
package/dist/src/analysis.d.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { QMDStore } from "@unblocklabs/qmd";
|
|
2
2
|
type AnalysisDatabase = QMDStore["internal"]["db"];
|
|
3
|
+
export type TemporalReadOptions = {
|
|
4
|
+
sessionCollection?: string;
|
|
5
|
+
};
|
|
3
6
|
export type MemoryReclusterOptions = {
|
|
4
7
|
space?: {
|
|
5
8
|
method?: "umap" | "none";
|
|
@@ -19,6 +22,7 @@ export type MemoryReclusterOptions = {
|
|
|
19
22
|
export type AnalysisRunner = (params: {
|
|
20
23
|
executable: string;
|
|
21
24
|
dbPath: string;
|
|
25
|
+
collections: readonly string[];
|
|
22
26
|
options?: MemoryReclusterOptions;
|
|
23
27
|
signal?: AbortSignal;
|
|
24
28
|
}) => Promise<void>;
|
|
@@ -30,7 +34,11 @@ type MemoryAnalysisMember = {
|
|
|
30
34
|
x: number;
|
|
31
35
|
y: number;
|
|
32
36
|
representativeRank: number | null;
|
|
33
|
-
|
|
37
|
+
sourceModifiedAt: string;
|
|
38
|
+
eventTime: string | null;
|
|
39
|
+
eventTimeBasis: "path" | "frontmatter" | "session" | "agent_verified" | null;
|
|
40
|
+
eventTimeSource: string;
|
|
41
|
+
contentFingerprint: string;
|
|
34
42
|
text: string;
|
|
35
43
|
sourcePaths: string[];
|
|
36
44
|
};
|
|
@@ -88,11 +96,13 @@ export declare function clusterReference(runId: string, clusterId: number): stri
|
|
|
88
96
|
export declare function runAnalysisWorker(params: {
|
|
89
97
|
executable: string;
|
|
90
98
|
dbPath: string;
|
|
99
|
+
collections: readonly string[];
|
|
91
100
|
options?: MemoryReclusterOptions;
|
|
92
101
|
signal?: AbortSignal;
|
|
93
102
|
}): Promise<void>;
|
|
94
103
|
export declare function latestAnalysisRunId(db: AnalysisDatabase): string | undefined;
|
|
104
|
+
export declare function latestAnalysisCollections(db: AnalysisDatabase): readonly string[] | undefined;
|
|
95
105
|
export declare function readAnalysisSummary(db: AnalysisDatabase): MemoryAnalysisSummary | undefined;
|
|
96
106
|
export declare function readClusters(db: AnalysisDatabase, requestedLimit?: number): MemoryClusterList;
|
|
97
|
-
export declare function readCluster(db: AnalysisDatabase, clusterReferenceId: string, requestedLimit?: number, requestedOffset?: number, sort?: MemoryClusterSort): MemoryClusterDetail;
|
|
107
|
+
export declare function readCluster(db: AnalysisDatabase, clusterReferenceId: string, requestedLimit?: number, requestedOffset?: number, sort?: MemoryClusterSort, temporal?: TemporalReadOptions): MemoryClusterDetail;
|
|
98
108
|
export {};
|
package/dist/src/analysis.js
CHANGED
|
@@ -48,6 +48,17 @@ export function ensureMemoryAnalysisSchema(db) {
|
|
|
48
48
|
CREATE INDEX IF NOT EXISTS idx_memory_analysis_memberships_cluster
|
|
49
49
|
ON memory_analysis_memberships(run_id, cluster_id, representative_rank);
|
|
50
50
|
|
|
51
|
+
CREATE TABLE IF NOT EXISTS memory_analysis_duplicate_occurrences (
|
|
52
|
+
run_id TEXT NOT NULL,
|
|
53
|
+
content_fingerprint TEXT NOT NULL,
|
|
54
|
+
canonical_hash TEXT NOT NULL,
|
|
55
|
+
canonical_seq INTEGER NOT NULL,
|
|
56
|
+
duplicate_hash TEXT NOT NULL,
|
|
57
|
+
duplicate_seq INTEGER NOT NULL,
|
|
58
|
+
PRIMARY KEY (run_id, duplicate_hash, duplicate_seq),
|
|
59
|
+
FOREIGN KEY (run_id) REFERENCES memory_analysis_runs(id) ON DELETE CASCADE
|
|
60
|
+
);
|
|
61
|
+
|
|
51
62
|
CREATE VIEW IF NOT EXISTS memory_analysis_available_memberships AS
|
|
52
63
|
SELECT
|
|
53
64
|
m.run_id, m.hash, m.seq, m.cluster_id, m.probability, m.outlier_score,
|
|
@@ -61,6 +72,18 @@ export function ensureMemoryAnalysisSchema(db) {
|
|
|
61
72
|
WHERE d.hash = m.hash AND d.active = 1
|
|
62
73
|
);
|
|
63
74
|
`);
|
|
75
|
+
db.exec(`
|
|
76
|
+
CREATE TEMP TABLE IF NOT EXISTS memory_temporal_annotations (
|
|
77
|
+
collection TEXT NOT NULL,
|
|
78
|
+
path TEXT NOT NULL,
|
|
79
|
+
qmd_hash TEXT,
|
|
80
|
+
qmd_seq INTEGER,
|
|
81
|
+
event_time TEXT NOT NULL,
|
|
82
|
+
basis TEXT NOT NULL,
|
|
83
|
+
document_wide INTEGER NOT NULL,
|
|
84
|
+
PRIMARY KEY (collection, path, qmd_hash, qmd_seq, document_wide)
|
|
85
|
+
);
|
|
86
|
+
`);
|
|
64
87
|
}
|
|
65
88
|
export function markMemoryAnalysisStale(db) {
|
|
66
89
|
db.prepare(`
|
|
@@ -81,7 +104,7 @@ export function clusterReference(runId, clusterId) {
|
|
|
81
104
|
export function runAnalysisWorker(params) {
|
|
82
105
|
return new Promise((resolve, reject) => {
|
|
83
106
|
params.signal?.throwIfAborted();
|
|
84
|
-
const args = ["--db", params.dbPath];
|
|
107
|
+
const args = ["--db", params.dbPath, "--collections-json", JSON.stringify(params.collections)];
|
|
85
108
|
if (params.options && Object.keys(params.options).length > 0) {
|
|
86
109
|
args.push("--config-json", JSON.stringify(params.options));
|
|
87
110
|
}
|
|
@@ -147,6 +170,26 @@ function latestRun(db) {
|
|
|
147
170
|
export function latestAnalysisRunId(db) {
|
|
148
171
|
return latestRun(db)?.id;
|
|
149
172
|
}
|
|
173
|
+
export function latestAnalysisCollections(db) {
|
|
174
|
+
const row = db.prepare(`
|
|
175
|
+
SELECT params_json
|
|
176
|
+
FROM memory_analysis_runs
|
|
177
|
+
WHERE completed_at IS NOT NULL
|
|
178
|
+
ORDER BY completed_at DESC, created_at DESC, id DESC
|
|
179
|
+
LIMIT 1
|
|
180
|
+
`).get();
|
|
181
|
+
if (!row)
|
|
182
|
+
return undefined;
|
|
183
|
+
try {
|
|
184
|
+
const collections = JSON.parse(row.params_json).collections;
|
|
185
|
+
return Array.isArray(collections) && collections.every((value) => typeof value === "string")
|
|
186
|
+
? collections
|
|
187
|
+
: undefined;
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
return undefined;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
150
193
|
function count(db, sql, runId) {
|
|
151
194
|
return db.prepare(sql).get(runId)?.count ?? 0;
|
|
152
195
|
}
|
|
@@ -208,7 +251,7 @@ function byteSlice(text, maxBytes) {
|
|
|
208
251
|
return "";
|
|
209
252
|
return bytes.subarray(0, maxBytes - 3).toString("utf8").replace(/\uFFFD$/u, "") + "…";
|
|
210
253
|
}
|
|
211
|
-
function members(db, runId, clusterId, limit, offset = 0, sort = "representative", maxExcerptBytes = MAX_EXCERPT_BYTES, maxTotalBytes = MAX_TOTAL_EXCERPT_BYTES, maxTotalAliases = MAX_TOTAL_ALIASES) {
|
|
254
|
+
function members(db, runId, clusterId, limit, offset = 0, sort = "representative", maxExcerptBytes = MAX_EXCERPT_BYTES, maxTotalBytes = MAX_TOTAL_EXCERPT_BYTES, maxTotalAliases = MAX_TOTAL_ALIASES, temporal = {}) {
|
|
212
255
|
const representativeOrder = clusterId === -1
|
|
213
256
|
? "m.outlier_score DESC, m.hash, m.seq"
|
|
214
257
|
: `CASE WHEN m.representative_rank IS NULL THEN 1 ELSE 0 END,
|
|
@@ -222,32 +265,79 @@ function members(db, runId, clusterId, limit, offset = 0, sort = "representative
|
|
|
222
265
|
representative: representativeOrder,
|
|
223
266
|
score_desc: `${score} DESC, m.hash, m.seq`,
|
|
224
267
|
score_asc: `${score} ASC, m.hash, m.seq`,
|
|
225
|
-
date_desc: "m.
|
|
226
|
-
date_asc: "m.
|
|
268
|
+
date_desc: "julianday(COALESCE(m.event_time, m.source_modified_at)) DESC, m.hash, m.seq",
|
|
269
|
+
date_asc: "julianday(COALESCE(m.event_time, m.source_modified_at)) ASC, m.hash, m.seq",
|
|
227
270
|
}[sort];
|
|
228
271
|
const rows = db.prepare(`
|
|
229
|
-
WITH
|
|
272
|
+
WITH candidate_times AS (
|
|
273
|
+
SELECT
|
|
274
|
+
m.hash,
|
|
275
|
+
m.seq,
|
|
276
|
+
d.collection,
|
|
277
|
+
d.path,
|
|
278
|
+
d.modified_at AS source_modified_at,
|
|
279
|
+
CASE
|
|
280
|
+
WHEN d.collection = ? THEN d.modified_at
|
|
281
|
+
WHEN d.path GLOB '*[12][0-9][0-9][0-9]-[01][0-9]-[0-3][0-9].md'
|
|
282
|
+
THEN substr(d.path, length(d.path) - 12, 10) || 'T00:00:00.000Z'
|
|
283
|
+
ELSE annotation.event_time
|
|
284
|
+
END AS event_time,
|
|
285
|
+
CASE
|
|
286
|
+
WHEN d.collection = ? THEN 'session'
|
|
287
|
+
WHEN d.path GLOB '*[12][0-9][0-9][0-9]-[01][0-9]-[0-3][0-9].md' THEN 'path'
|
|
288
|
+
ELSE annotation.basis
|
|
289
|
+
END AS event_time_basis,
|
|
290
|
+
CASE
|
|
291
|
+
WHEN d.collection = ? THEN 1
|
|
292
|
+
WHEN d.path GLOB '*[12][0-9][0-9][0-9]-[01][0-9]-[0-3][0-9].md' THEN 2
|
|
293
|
+
WHEN annotation.event_time IS NOT NULL THEN 3
|
|
294
|
+
ELSE 4
|
|
295
|
+
END AS priority
|
|
296
|
+
FROM memory_analysis_available_memberships m
|
|
297
|
+
JOIN documents d ON d.hash = m.hash AND d.active = 1
|
|
298
|
+
LEFT JOIN memory_temporal_annotations annotation
|
|
299
|
+
ON annotation.collection = d.collection
|
|
300
|
+
AND annotation.path = d.path
|
|
301
|
+
AND (annotation.document_wide = 1 OR
|
|
302
|
+
(annotation.qmd_hash = m.hash AND annotation.qmd_seq = m.seq))
|
|
303
|
+
WHERE m.run_id = ? AND m.cluster_id = ?
|
|
304
|
+
), ranked_times AS (
|
|
305
|
+
SELECT *, ROW_NUMBER() OVER (
|
|
306
|
+
PARTITION BY hash, seq
|
|
307
|
+
ORDER BY priority, julianday(COALESCE(event_time, source_modified_at)) DESC, collection, path
|
|
308
|
+
) AS rank
|
|
309
|
+
FROM candidate_times
|
|
310
|
+
), member_rows AS (
|
|
230
311
|
SELECT
|
|
231
312
|
m.hash, m.seq, m.probability, m.outlier_score, m.x, m.y,
|
|
232
313
|
m.representative_rank, m.pos, m.chunk_len, m.doc,
|
|
233
314
|
(
|
|
234
|
-
SELECT
|
|
315
|
+
SELECT d.modified_at
|
|
235
316
|
FROM documents d
|
|
236
317
|
WHERE d.hash = m.hash AND d.active = 1
|
|
237
|
-
|
|
318
|
+
ORDER BY julianday(d.modified_at) DESC, d.collection, d.path
|
|
319
|
+
LIMIT 1
|
|
320
|
+
) AS source_modified_at,
|
|
321
|
+
temporal.event_time,
|
|
322
|
+
temporal.event_time_basis,
|
|
323
|
+
temporal.collection AS event_collection,
|
|
324
|
+
temporal.path AS event_path
|
|
238
325
|
FROM memory_analysis_available_memberships m
|
|
326
|
+
JOIN ranked_times temporal
|
|
327
|
+
ON temporal.hash = m.hash AND temporal.seq = m.seq AND temporal.rank = 1
|
|
239
328
|
WHERE m.run_id = ? AND m.cluster_id = ?
|
|
240
329
|
)
|
|
241
330
|
SELECT * FROM member_rows m
|
|
242
331
|
ORDER BY ${order}
|
|
243
332
|
LIMIT ? OFFSET ?
|
|
244
|
-
`).all(runId, clusterId, limit, offset);
|
|
333
|
+
`).all(temporal.sessionCollection ?? "", temporal.sessionCollection ?? "", temporal.sessionCollection ?? "", runId, clusterId, runId, clusterId, limit, offset);
|
|
245
334
|
let remaining = maxTotalBytes;
|
|
246
335
|
let remainingAliases = maxTotalAliases;
|
|
247
336
|
return rows.map((row, index) => {
|
|
248
337
|
const remainingRows = rows.length - index;
|
|
249
338
|
const excerptBudget = Math.min(maxExcerptBytes, Math.floor(remaining / remainingRows));
|
|
250
|
-
const
|
|
339
|
+
const fullText = row.doc.slice(row.pos, row.pos + row.chunk_len);
|
|
340
|
+
const text = byteSlice(fullText, excerptBudget);
|
|
251
341
|
remaining -= Buffer.byteLength(text);
|
|
252
342
|
const aliasBudget = Math.min(MAX_ALIASES_PER_MEMBER, Math.floor(remainingAliases / remainingRows));
|
|
253
343
|
const aliases = sourcePaths(db, row.hash, aliasBudget);
|
|
@@ -260,7 +350,11 @@ function members(db, runId, clusterId, limit, offset = 0, sort = "representative
|
|
|
260
350
|
x: row.x,
|
|
261
351
|
y: row.y,
|
|
262
352
|
representativeRank: row.representative_rank,
|
|
263
|
-
|
|
353
|
+
sourceModifiedAt: row.source_modified_at,
|
|
354
|
+
eventTime: row.event_time,
|
|
355
|
+
eventTimeBasis: row.event_time_basis,
|
|
356
|
+
eventTimeSource: `qmd://${row.event_collection}/${row.event_path}`,
|
|
357
|
+
contentFingerprint: createHash("sha256").update(fullText).digest("hex"),
|
|
264
358
|
text,
|
|
265
359
|
sourcePaths: aliases,
|
|
266
360
|
};
|
|
@@ -380,7 +474,7 @@ function resolveClusterId(db, runId, reference) {
|
|
|
380
474
|
`).all(runId, runId);
|
|
381
475
|
return clusterIds.find((row) => clusterReference(runId, row.cluster_id) === reference)?.cluster_id;
|
|
382
476
|
}
|
|
383
|
-
export function readCluster(db, clusterReferenceId, requestedLimit = DEFAULT_MEMBER_LIMIT, requestedOffset = 0, sort = "representative") {
|
|
477
|
+
export function readCluster(db, clusterReferenceId, requestedLimit = DEFAULT_MEMBER_LIMIT, requestedOffset = 0, sort = "representative", temporal = {}) {
|
|
384
478
|
const run = latestValidRun(db);
|
|
385
479
|
if (!run) {
|
|
386
480
|
return { status: "not_analyzed", ...readMetadata() };
|
|
@@ -408,7 +502,7 @@ export function readCluster(db, clusterReferenceId, requestedLimit = DEFAULT_MEM
|
|
|
408
502
|
const limit = Math.max(1, Math.min(MAX_MEMBER_LIMIT, Math.floor(requestedLimit)));
|
|
409
503
|
const offset = Math.max(0, Math.floor(requestedOffset));
|
|
410
504
|
const total = availableSize(db, run.id, clusterId);
|
|
411
|
-
const pageMembers = members(db, run.id, clusterId, limit, offset, sort);
|
|
505
|
+
const pageMembers = members(db, run.id, clusterId, limit, offset, sort, MAX_EXCERPT_BYTES, MAX_TOTAL_EXCERPT_BYTES, MAX_TOTAL_ALIASES, temporal);
|
|
412
506
|
const nextOffset = offset + pageMembers.length;
|
|
413
507
|
const hasMore = nextOffset < total;
|
|
414
508
|
return {
|
package/dist/src/config.d.ts
CHANGED
|
@@ -3,6 +3,11 @@ export type FileCorpusConfig = {
|
|
|
3
3
|
kind: "files";
|
|
4
4
|
paths: readonly string[];
|
|
5
5
|
};
|
|
6
|
+
export type SkillCorpusConfig = {
|
|
7
|
+
name: "skills";
|
|
8
|
+
kind: "skills";
|
|
9
|
+
paths: readonly string[];
|
|
10
|
+
};
|
|
6
11
|
declare const CHAT_TYPES: readonly ["channel", "group", "direct"];
|
|
7
12
|
export type ChatType = typeof CHAT_TYPES[number];
|
|
8
13
|
type SessionCorpusConfig = {
|
|
@@ -10,7 +15,7 @@ type SessionCorpusConfig = {
|
|
|
10
15
|
kind: "sessions";
|
|
11
16
|
chatTypes: readonly ChatType[];
|
|
12
17
|
};
|
|
13
|
-
export type CorpusConfig = FileCorpusConfig | SessionCorpusConfig;
|
|
18
|
+
export type CorpusConfig = FileCorpusConfig | SkillCorpusConfig | SessionCorpusConfig;
|
|
14
19
|
export declare const DEFAULT_CORPORA: readonly FileCorpusConfig[];
|
|
15
20
|
export type UnblockMemoryConfig = {
|
|
16
21
|
corpora: readonly CorpusConfig[];
|
|
@@ -18,6 +23,12 @@ export type UnblockMemoryConfig = {
|
|
|
18
23
|
analysis: {
|
|
19
24
|
executable?: string;
|
|
20
25
|
};
|
|
26
|
+
skillWhisperer: {
|
|
27
|
+
enabled: boolean;
|
|
28
|
+
historyMessages: number;
|
|
29
|
+
minScore: number;
|
|
30
|
+
cooldownTurns: number;
|
|
31
|
+
};
|
|
21
32
|
};
|
|
22
33
|
export declare function resolveConfig(value: unknown): UnblockMemoryConfig;
|
|
23
34
|
export {};
|
package/dist/src/config.js
CHANGED
|
@@ -6,6 +6,12 @@ export const DEFAULT_CORPORA = [{
|
|
|
6
6
|
kind: "files",
|
|
7
7
|
paths: DEFAULT_PATHS,
|
|
8
8
|
}];
|
|
9
|
+
const DEFAULT_SKILL_WHISPERER = {
|
|
10
|
+
enabled: false,
|
|
11
|
+
historyMessages: 5,
|
|
12
|
+
minScore: 0.4,
|
|
13
|
+
cooldownTurns: 10,
|
|
14
|
+
};
|
|
9
15
|
function assertOnlyKeys(value, allowed, label) {
|
|
10
16
|
const unknown = Object.keys(value).find((key) => !allowed.includes(key));
|
|
11
17
|
if (unknown)
|
|
@@ -33,6 +39,17 @@ function resolveCorpora(value) {
|
|
|
33
39
|
if (names.has(name))
|
|
34
40
|
throw new Error(`unblock-memory corpus names must be unique: ${name}`);
|
|
35
41
|
names.add(name);
|
|
42
|
+
if (corpus.kind === "skills") {
|
|
43
|
+
assertOnlyKeys(corpus, ["name", "kind", "paths"], `corpora[${index}]`);
|
|
44
|
+
if (name !== "skills") {
|
|
45
|
+
throw new Error('unblock-memory skills corpus must be named "skills"');
|
|
46
|
+
}
|
|
47
|
+
if (!Array.isArray(corpus.paths) || corpus.paths.length === 0 ||
|
|
48
|
+
!corpus.paths.every((path) => typeof path === "string" && path.trim())) {
|
|
49
|
+
throw new Error("unblock-memory corpus skills paths must be a non-empty array of non-empty strings");
|
|
50
|
+
}
|
|
51
|
+
return { name: "skills", kind: "skills", paths: corpus.paths.map((path) => path.trim()) };
|
|
52
|
+
}
|
|
36
53
|
if (corpus.kind === "sessions") {
|
|
37
54
|
assertOnlyKeys(corpus, ["name", "kind", "chatTypes"], `corpora[${index}]`);
|
|
38
55
|
if (name !== "sessions") {
|
|
@@ -49,8 +66,11 @@ function resolveCorpora(value) {
|
|
|
49
66
|
if (name === "sessions") {
|
|
50
67
|
throw new Error('unblock-memory corpus named "sessions" must have kind "sessions"');
|
|
51
68
|
}
|
|
69
|
+
if (name === "skills") {
|
|
70
|
+
throw new Error('unblock-memory corpus named "skills" must have kind "skills"');
|
|
71
|
+
}
|
|
52
72
|
if (corpus.kind !== "files") {
|
|
53
|
-
throw new Error(`unblock-memory corpus ${name} must have kind "files" or "sessions"`);
|
|
73
|
+
throw new Error(`unblock-memory corpus ${name} must have kind "files", "skills", or "sessions"`);
|
|
54
74
|
}
|
|
55
75
|
if (!Array.isArray(corpus.paths) || corpus.paths.length === 0 ||
|
|
56
76
|
!corpus.paths.every((path) => typeof path === "string" && path.trim())) {
|
|
@@ -65,30 +85,64 @@ function resolveCorpora(value) {
|
|
|
65
85
|
}
|
|
66
86
|
export function resolveConfig(value) {
|
|
67
87
|
if (value === undefined || value === null) {
|
|
68
|
-
return {
|
|
88
|
+
return {
|
|
89
|
+
corpora: DEFAULT_CORPORA,
|
|
90
|
+
keepEmbeddingModelWarm: true,
|
|
91
|
+
analysis: {},
|
|
92
|
+
skillWhisperer: DEFAULT_SKILL_WHISPERER,
|
|
93
|
+
};
|
|
69
94
|
}
|
|
70
95
|
if (typeof value !== "object" || Array.isArray(value)) {
|
|
71
96
|
throw new Error("unblock-memory config must be an object");
|
|
72
97
|
}
|
|
73
98
|
const config = value;
|
|
74
|
-
assertOnlyKeys(config, ["corpora", "keepEmbeddingModelWarm", "analysis"], "config");
|
|
99
|
+
assertOnlyKeys(config, ["corpora", "keepEmbeddingModelWarm", "analysis", "skillWhisperer"], "config");
|
|
75
100
|
const corpora = resolveCorpora(config.corpora);
|
|
76
101
|
if (config.keepEmbeddingModelWarm !== undefined && typeof config.keepEmbeddingModelWarm !== "boolean") {
|
|
77
102
|
throw new Error("unblock-memory keepEmbeddingModelWarm must be a boolean");
|
|
78
103
|
}
|
|
79
104
|
const keepEmbeddingModelWarm = config.keepEmbeddingModelWarm ?? true;
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
105
|
+
let analysisConfig = {};
|
|
106
|
+
if (config.analysis !== undefined) {
|
|
107
|
+
if (!config.analysis || typeof config.analysis !== "object" || Array.isArray(config.analysis)) {
|
|
108
|
+
throw new Error("unblock-memory analysis must be an object");
|
|
109
|
+
}
|
|
110
|
+
const analysis = config.analysis;
|
|
111
|
+
assertOnlyKeys(analysis, ["executable"], "analysis");
|
|
112
|
+
const configured = analysis.executable;
|
|
113
|
+
if (configured !== undefined) {
|
|
114
|
+
if (typeof configured !== "string" || !configured.trim() || !isAbsolute(configured.trim())) {
|
|
115
|
+
throw new Error("unblock-memory analysis.executable must be an absolute non-empty path");
|
|
116
|
+
}
|
|
117
|
+
analysisConfig = { executable: configured.trim() };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
let skillWhisperer = DEFAULT_SKILL_WHISPERER;
|
|
121
|
+
if (config.skillWhisperer !== undefined) {
|
|
122
|
+
if (!config.skillWhisperer || typeof config.skillWhisperer !== "object" || Array.isArray(config.skillWhisperer)) {
|
|
123
|
+
throw new Error("unblock-memory skillWhisperer must be an object");
|
|
124
|
+
}
|
|
125
|
+
const value = config.skillWhisperer;
|
|
126
|
+
assertOnlyKeys(value, ["enabled", "historyMessages", "minScore", "cooldownTurns"], "skillWhisperer");
|
|
127
|
+
const enabled = value.enabled ?? false;
|
|
128
|
+
const historyMessages = value.historyMessages ?? 5;
|
|
129
|
+
const minScore = value.minScore ?? 0.4;
|
|
130
|
+
const cooldownTurns = value.cooldownTurns ?? 10;
|
|
131
|
+
if (typeof enabled !== "boolean")
|
|
132
|
+
throw new Error("unblock-memory skillWhisperer.enabled must be a boolean");
|
|
133
|
+
if (typeof historyMessages !== "number" || !Number.isInteger(historyMessages) || historyMessages < 0) {
|
|
134
|
+
throw new Error("unblock-memory skillWhisperer.historyMessages must be a non-negative integer");
|
|
135
|
+
}
|
|
136
|
+
if (typeof minScore !== "number" || !Number.isFinite(minScore) || minScore < 0 || minScore > 1) {
|
|
137
|
+
throw new Error("unblock-memory skillWhisperer.minScore must be between 0 and 1");
|
|
138
|
+
}
|
|
139
|
+
if (typeof cooldownTurns !== "number" || !Number.isInteger(cooldownTurns) || cooldownTurns < 0) {
|
|
140
|
+
throw new Error("unblock-memory skillWhisperer.cooldownTurns must be a non-negative integer");
|
|
141
|
+
}
|
|
142
|
+
skillWhisperer = { enabled, historyMessages, minScore, cooldownTurns };
|
|
84
143
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
const configured = analysis.executable;
|
|
88
|
-
if (configured === undefined)
|
|
89
|
-
return { corpora, keepEmbeddingModelWarm, analysis: {} };
|
|
90
|
-
if (typeof configured !== "string" || !configured.trim() || !isAbsolute(configured.trim())) {
|
|
91
|
-
throw new Error("unblock-memory analysis.executable must be an absolute non-empty path");
|
|
144
|
+
if (skillWhisperer.enabled && !corpora.some((corpus) => corpus.kind === "skills")) {
|
|
145
|
+
throw new Error('unblock-memory enabled skillWhisperer requires a corpus named "skills" with kind "skills"');
|
|
92
146
|
}
|
|
93
|
-
return { corpora, keepEmbeddingModelWarm, analysis:
|
|
147
|
+
return { corpora, keepEmbeddingModelWarm, analysis: analysisConfig, skillWhisperer };
|
|
94
148
|
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
declare const TEMPORAL_BASES: readonly ["path", "frontmatter", "session", "agent_verified"];
|
|
2
|
+
export type TemporalBasis = typeof TEMPORAL_BASES[number];
|
|
3
|
+
declare const MAINTENANCE_TASK_TYPES: readonly ["ambiguous_event_time", "exact_duplicate"];
|
|
4
|
+
export type MaintenanceTaskType = typeof MAINTENANCE_TASK_TYPES[number];
|
|
5
|
+
declare const MAINTENANCE_STATUSES: readonly ["pending", "resolved", "deferred", "irrelevant"];
|
|
6
|
+
export type MaintenanceStatus = typeof MAINTENANCE_STATUSES[number];
|
|
7
|
+
export type TemporalAnnotation = {
|
|
8
|
+
corpus: string;
|
|
9
|
+
collection: string;
|
|
10
|
+
path: string;
|
|
11
|
+
contentFingerprint: string;
|
|
12
|
+
eventTime: string;
|
|
13
|
+
basis: TemporalBasis;
|
|
14
|
+
evidence: string;
|
|
15
|
+
qmdHash: string | null;
|
|
16
|
+
qmdSeq: number | null;
|
|
17
|
+
createdAt: string;
|
|
18
|
+
updatedAt: string;
|
|
19
|
+
};
|
|
20
|
+
export type MaintenanceTask = {
|
|
21
|
+
id: string;
|
|
22
|
+
type: MaintenanceTaskType;
|
|
23
|
+
corpus: string;
|
|
24
|
+
collection: string;
|
|
25
|
+
path: string;
|
|
26
|
+
reason: string;
|
|
27
|
+
contentFingerprint: string;
|
|
28
|
+
detail: string | null;
|
|
29
|
+
resolutionNote: string | null;
|
|
30
|
+
status: MaintenanceStatus;
|
|
31
|
+
createdAt: string;
|
|
32
|
+
updatedAt: string;
|
|
33
|
+
};
|
|
34
|
+
export declare function chunkFingerprint(text: string): string;
|
|
35
|
+
export declare class CurationStore {
|
|
36
|
+
#private;
|
|
37
|
+
constructor(path: string);
|
|
38
|
+
close(): void;
|
|
39
|
+
annotations(): TemporalAnnotation[];
|
|
40
|
+
addTask(candidate: {
|
|
41
|
+
type: MaintenanceTaskType;
|
|
42
|
+
corpus: string;
|
|
43
|
+
collection: string;
|
|
44
|
+
path: string;
|
|
45
|
+
reason: string;
|
|
46
|
+
contentFingerprint?: string;
|
|
47
|
+
detail?: string;
|
|
48
|
+
}): void;
|
|
49
|
+
listTasks(params?: {
|
|
50
|
+
status?: MaintenanceStatus;
|
|
51
|
+
limit?: number;
|
|
52
|
+
}): MaintenanceTask[];
|
|
53
|
+
updateTask(params: {
|
|
54
|
+
id: string;
|
|
55
|
+
status: Exclude<MaintenanceStatus, "pending">;
|
|
56
|
+
note?: string;
|
|
57
|
+
annotation?: {
|
|
58
|
+
scope: "chunk" | "document";
|
|
59
|
+
eventTime: string;
|
|
60
|
+
basis: TemporalBasis;
|
|
61
|
+
evidence: string;
|
|
62
|
+
};
|
|
63
|
+
}): MaintenanceTask | undefined;
|
|
64
|
+
updateAnnotationLocation(params: {
|
|
65
|
+
annotation: TemporalAnnotation;
|
|
66
|
+
qmdHash: string | null;
|
|
67
|
+
qmdSeq: number | null;
|
|
68
|
+
}): void;
|
|
69
|
+
}
|
|
70
|
+
export {};
|