@unblocklabs/unblock-memory 0.3.27 → 0.3.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/src/agent-transcript.d.ts +8 -0
  2. package/dist/src/agent-transcript.js +29 -0
  3. package/dist/src/cluster-review.js +1 -1
  4. package/dist/src/config.d.ts +4 -0
  5. package/dist/src/config.js +14 -1
  6. package/dist/src/date-option.d.ts +1 -0
  7. package/dist/src/date-option.js +8 -0
  8. package/dist/src/diagnostics.d.ts +13 -4
  9. package/dist/src/diagnostics.js +5 -0
  10. package/dist/src/manager.d.ts +3 -1
  11. package/dist/src/manager.js +61 -57
  12. package/dist/src/memory-whisperer.d.ts +2 -1
  13. package/dist/src/memory-whisperer.js +241 -114
  14. package/dist/src/mlx-query.d.ts +22 -0
  15. package/dist/src/mlx-query.js +155 -0
  16. package/dist/src/people-dossier-review.js +1 -1
  17. package/dist/src/people-hooks.d.ts +1 -1
  18. package/dist/src/people-hooks.js +13 -7
  19. package/dist/src/people-primer-tool.js +1 -1
  20. package/dist/src/people-primer.js +3 -2
  21. package/dist/src/plugin.js +10 -5
  22. package/dist/src/quality-audit.js +86 -81
  23. package/dist/src/response-audit.js +1 -1
  24. package/dist/src/response-episodes.js +4 -11
  25. package/dist/src/response-judge.d.ts +1 -1
  26. package/dist/src/response-judge.js +18 -15
  27. package/dist/src/response-runtime.js +1 -8
  28. package/dist/src/response-stages.d.ts +8 -8
  29. package/dist/src/response-text.d.ts +5 -0
  30. package/dist/src/response-text.js +23 -1
  31. package/dist/src/retrieval-telemetry.d.ts +2 -2
  32. package/dist/src/retrieval-telemetry.js +1 -1
  33. package/dist/src/review-tools.js +1 -1
  34. package/dist/src/session-sync.js +8 -22
  35. package/dist/src/skill-whisperer.d.ts +1 -2
  36. package/dist/src/skill-whisperer.js +77 -72
  37. package/dist/src/training-candidates.d.ts +9 -1
  38. package/dist/src/training-candidates.js +12 -7
  39. package/dist/src/training-gate.js +3 -3
  40. package/dist/src/training-input.js +6 -34
  41. package/dist/src/training-judge.js +3 -3
  42. package/dist/src/training-queries.js +5 -4
  43. package/dist/src/training-runtime.js +1 -8
  44. package/dist/src/training.js +5 -4
  45. package/dist/src/typesafe-client.d.ts +15 -0
  46. package/dist/src/typesafe-client.js +75 -0
  47. package/dist/src/typesafe-review.d.ts +2 -6
  48. package/dist/src/typesafe-review.js +49 -55
  49. package/dist/src/typesafe.d.ts +3 -7
  50. package/dist/src/typesafe.js +108 -163
  51. package/dist/src/whisperer-context.d.ts +1 -0
  52. package/dist/src/whisperer-context.js +9 -0
  53. package/dist/src/whisperer-prompt.d.ts +3 -0
  54. package/dist/src/whisperer-prompt.js +21 -0
  55. package/docs/configuration.md +12 -2
  56. package/docs/peoplesql.md +3 -1
  57. package/docs/response-audit.md +4 -1
  58. package/docs/retrieval.md +128 -20
  59. package/openclaw.plugin.json +13 -4
  60. package/package.json +2 -1
  61. package/src/mlx-query-worker.py +104 -0
  62. package/dist/src/typesafe-transport.d.ts +0 -10
  63. package/dist/src/typesafe-transport.js +0 -26
@@ -3,10 +3,10 @@ import { existsSync, lstatSync, readFileSync, statSync } from "node:fs";
3
3
  import { chmod, mkdir, readFile, rename, unlink, utimes, writeFile } from "node:fs/promises";
4
4
  import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
5
5
  import { DatabaseSync } from "node:sqlite";
6
+ import { ACTIVE_EVENTS_FROM, agentTranscriptSchemaVersion, assertAgentTranscriptIdentity } from "./agent-transcript.js";
6
7
  import { projectSessionDocument, sessionDocumentPath, } from "./session-projector.js";
7
8
  const MANIFEST_VERSION = 1;
8
9
  export const PROJECTOR_VERSION = 7;
9
- const SUPPORTED_SCHEMA_VERSIONS = new Set([17, 18, 19]);
10
10
  // Source lives in src/, published code in dist/src/. Read our own pinned dependency
11
11
  // metadata, not QMD internals (which may also be substituted by runtime inspectors).
12
12
  const sourcePackage = new URL("../package.json", import.meta.url);
@@ -76,12 +76,7 @@ function projectionPath(outputDir, documentPath) {
76
76
  return target;
77
77
  }
78
78
  function assertSchema(db, expectedAgentId) {
79
- const pragma = db.prepare("PRAGMA user_version").get();
80
- const schemaVersion = pragma?.user_version;
81
- if (typeof schemaVersion !== "number" || !SUPPORTED_SCHEMA_VERSIONS.has(schemaVersion)) {
82
- throw new Error("unsupported OpenClaw agent database schema: expected one of 17, 18, 19, " +
83
- `found ${String(schemaVersion ?? "unknown")}`);
84
- }
79
+ const version = agentTranscriptSchemaVersion(db);
85
80
  for (const [table, required] of Object.entries(REQUIRED_COLUMNS)) {
86
81
  const columns = new Set(db.prepare(`PRAGMA table_info(${table})`).all()
87
82
  .map((column) => column.name));
@@ -89,14 +84,7 @@ function assertSchema(db, expectedAgentId) {
89
84
  if (missing)
90
85
  throw new Error(`unsupported OpenClaw agent database: missing ${table}.${missing}`);
91
86
  }
92
- const meta = db.prepare("SELECT role, schema_version AS schemaVersion, agent_id AS agentId " +
93
- "FROM schema_meta WHERE meta_key = 'primary' LIMIT 1").get();
94
- if (meta?.role !== "agent" || meta.schemaVersion !== schemaVersion) {
95
- throw new Error("unsupported OpenClaw agent database primary schema metadata");
96
- }
97
- if (meta.agentId !== expectedAgentId) {
98
- throw new Error(`OpenClaw agent database belongs to ${String(meta.agentId)}, not ${expectedAgentId}`);
99
- }
87
+ assertAgentTranscriptIdentity(db, expectedAgentId, version);
100
88
  }
101
89
  function readSnapshot(params) {
102
90
  const db = new DatabaseSync(params.databasePath, { readOnly: true });
@@ -129,13 +117,11 @@ function readSnapshot(params) {
129
117
  ORDER BY window.created_at, window.session_id
130
118
  `).all(...params.chatTypes);
131
119
  const readEvents = db.prepare(`
132
- SELECT active.session_id AS sessionId, event.event_json AS eventJson,
133
- event.created_at AS createdAt
134
- FROM session_transcript_active_events AS active
135
- JOIN transcript_events AS event
136
- ON event.session_id = active.session_id AND event.seq = active.event_seq
137
- WHERE active.session_id = ?
138
- ORDER BY active.active_position
120
+ SELECT a.session_id AS sessionId, e.event_json AS eventJson,
121
+ e.created_at AS createdAt
122
+ ${ACTIVE_EVENTS_FROM}
123
+ WHERE a.session_id = ?
124
+ ORDER BY a.active_position
139
125
  `);
140
126
  const events = new Map();
141
127
  const changed = new Set();
@@ -12,6 +12,5 @@ type SkillWhispererRuntime = {
12
12
  agentId: string;
13
13
  }, path: string): string | undefined;
14
14
  };
15
- export declare function buildSkillWhispererQuery(prompt: string, messages: readonly unknown[], historyMessages: number): string;
16
- export declare function registerSkillWhisperer(api: OpenClawPluginApi, runtime: SkillWhispererRuntime, config: UnblockMemoryConfig["skillWhisperer"], typesafe: UnblockMemoryConfig["typesafe"], diagnostics?: WhispererDiagnostics): void;
15
+ export declare function registerSkillWhisperer(api: OpenClawPluginApi, runtime: SkillWhispererRuntime, config: UnblockMemoryConfig["skillWhisperer"], typesafe: UnblockMemoryConfig["typesafe"], diagnostics?: WhispererDiagnostics): Parameters<typeof api.on<"before_prompt_build">>[1] | undefined;
17
16
  export {};
@@ -1,20 +1,13 @@
1
1
  import { basename } from "node:path";
2
- import { resolveTypeSafeApiKey, selectTypeSafeSkill } from "./typesafe.js";
3
- import { messageText } from "./whisperer-context.js";
2
+ import { selectTypeSafeSkill } from "./typesafe.js";
3
+ import { resolveTypeSafeApiKey, TypeSafeRequestError } from "./typesafe-client.js";
4
+ import { buildSkillWhispererQuery, messageText } from "./whisperer-context.js";
4
5
  const CANDIDATE_LIMIT = 10;
5
6
  const MAX_QUERY_CHARS = 12_000;
6
7
  const TYPESAFE_CANDIDATE_LIMIT = 3;
7
8
  function isRecord(value) {
8
9
  return value !== null && typeof value === "object" && !Array.isArray(value);
9
10
  }
10
- export function buildSkillWhispererQuery(prompt, messages, historyMessages) {
11
- const availableHistory = messages.flatMap((message) => {
12
- const parsed = messageText(message);
13
- return parsed ? [`${parsed.role}: ${parsed.text}`] : [];
14
- });
15
- const history = historyMessages === 0 ? [] : availableHistory.slice(-historyMessages);
16
- return [...history, `user: ${prompt.trim()}`].join("\n\n").slice(-MAX_QUERY_CHARS);
17
- }
18
11
  function typeSafeConversation(prompt, messages, historyMessages) {
19
12
  const currentRequest = prompt.trim().slice(-MAX_QUERY_CHARS);
20
13
  let remaining = MAX_QUERY_CHARS - currentRequest.length;
@@ -55,78 +48,89 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
55
48
  return state;
56
49
  };
57
50
  const active = (agentId) => ({ cfg: api.config, agentId });
58
- api.on("before_prompt_build", async (event, context) => {
51
+ const beforePrompt = async (event, context) => {
52
+ const { agentId, runId } = context;
59
53
  const scope = sessionScope(context);
60
- if (context.trigger !== "user" || !scope || !context.runId || !context.agentId)
54
+ if (context.trigger !== "user" || !scope || !runId || !agentId)
61
55
  return;
62
56
  const state = stateFor(scope);
63
- if (state.lastRunId === context.runId)
64
- return;
65
- state.lastRunId = context.runId;
57
+ if (state.lastRunId === runId)
58
+ return state.result;
59
+ state.lastRunId = runId;
66
60
  state.turn += 1;
67
- try {
68
- const runtimeParams = active(context.agentId);
69
- const apiKey = await resolveTypeSafeApiKey(typesafe);
70
- if (!apiKey)
71
- diagnostics?.record(context.agentId, "skill", typesafe.enabled ? "missing_key" : "typesafe_disabled");
72
- const candidates = await runtime.searchSkills(runtimeParams, buildSkillWhispererQuery(event.prompt, event.messages, config.historyMessages), apiKey ? -1 : config.minScore, CANDIDATE_LIMIT);
73
- const resolvedCandidates = candidates.flatMap((candidate) => {
74
- const canonicalPath = runtime.resolveSkillPath(runtimeParams, candidate.path);
75
- return canonicalPath ? [{ candidate, canonicalPath }] : [];
76
- });
77
- let resolved = resolvedCandidates[0];
78
- if (!resolved) {
79
- diagnostics?.record(context.agentId, "skill", "no_candidates");
80
- return;
81
- }
82
- if (apiKey) {
83
- const shortlist = resolvedCandidates.slice(0, TYPESAFE_CANDIDATE_LIMIT);
84
- const selectedIndex = await selectTypeSafeSkill({
85
- apiKey, timeoutMs: typesafe.timeoutMs,
86
- ...typeSafeConversation(event.prompt, event.messages, config.historyMessages),
87
- candidates: shortlist.map(({ candidate }) => candidate),
61
+ // Reuse even an in-flight result when the host rebuilds this run's prompt.
62
+ const run = async () => {
63
+ try {
64
+ const runtimeParams = active(agentId);
65
+ const apiKey = await resolveTypeSafeApiKey(typesafe);
66
+ if (!apiKey)
67
+ diagnostics?.record(agentId, "skill", typesafe.enabled ? "missing_key" : "typesafe_disabled");
68
+ const candidates = await runtime.searchSkills(runtimeParams, buildSkillWhispererQuery(event.prompt, event.messages, config.historyMessages), apiKey ? -1 : config.minScore, CANDIDATE_LIMIT);
69
+ const resolvedCandidates = candidates.flatMap((candidate) => {
70
+ const canonicalPath = runtime.resolveSkillPath(runtimeParams, candidate.path);
71
+ return canonicalPath ? [{ candidate, canonicalPath }] : [];
88
72
  });
89
- if (selectedIndex === undefined) {
90
- diagnostics?.record(context.agentId, "skill", "rejected");
73
+ let resolved = resolvedCandidates[0];
74
+ if (!resolved) {
75
+ diagnostics?.record(agentId, "skill", "no_candidates");
91
76
  return;
92
77
  }
93
- resolved = shortlist[selectedIndex];
94
- }
95
- else if (resolved && resolved.candidate.score < config.minScore) {
96
- diagnostics?.record(context.agentId, "skill", "rejected");
97
- return;
98
- }
99
- if (!resolved)
100
- return;
101
- // A selection completing after session teardown must not resurrect its hint.
102
- if (sessions.get(scope) !== state || state.lastRunId !== context.runId) {
103
- diagnostics?.record(context.agentId, "skill", "cancelled");
104
- return;
78
+ if (apiKey) {
79
+ const shortlist = resolvedCandidates.slice(0, TYPESAFE_CANDIDATE_LIMIT);
80
+ const selectedIndex = await selectTypeSafeSkill({
81
+ apiKey, timeoutMs: typesafe.timeoutMs,
82
+ ...typeSafeConversation(event.prompt, event.messages, config.historyMessages),
83
+ candidates: shortlist.map(({ candidate }) => candidate),
84
+ onCandidateFailure: (candidateIndex, error) => {
85
+ diagnostics?.record(agentId, "skill", "judge_candidate_failed");
86
+ api.logger.warn("unblock-memory skill_whisperer " + JSON.stringify({ event: "candidate_failed",
87
+ agentId, runId, candidateIndex, errorCode: error instanceof TypeSafeRequestError ? error.code : "unexpected",
88
+ httpStatus: error instanceof TypeSafeRequestError ? error.status : undefined }));
89
+ },
90
+ });
91
+ if (selectedIndex === undefined) {
92
+ diagnostics?.record(agentId, "skill", "rejected");
93
+ return;
94
+ }
95
+ resolved = shortlist[selectedIndex];
96
+ }
97
+ else if (resolved && resolved.candidate.score < config.minScore) {
98
+ diagnostics?.record(agentId, "skill", "rejected");
99
+ return;
100
+ }
101
+ if (!resolved)
102
+ return;
103
+ // A selection completing after session teardown must not resurrect its hint.
104
+ if (sessions.get(scope) !== state || state.lastRunId !== runId) {
105
+ diagnostics?.record(agentId, "skill", "cancelled");
106
+ return;
107
+ }
108
+ const { candidate: selected, canonicalPath } = resolved;
109
+ if (apiKey && runtime.resolveSkillPath(runtimeParams, selected.path) !== canonicalPath)
110
+ return;
111
+ const previous = state.skills.get(canonicalPath);
112
+ const lastSeen = Math.max(previous?.suggested ?? -Infinity, previous?.opened ?? -Infinity);
113
+ if (state.turn - lastSeen <= config.cooldownTurns) {
114
+ diagnostics?.record(agentId, "skill", "cooldown");
115
+ return;
116
+ }
117
+ const history = state.skills.get(canonicalPath) ?? {};
118
+ history.suggested = state.turn;
119
+ state.skills.set(canonicalPath, history);
120
+ diagnostics?.record(agentId, "skill", "emitted");
121
+ return {
122
+ appendContext: `<skill>This skill may be relevant: ${JSON.stringify(selected.path).replace(/</gu, "\\u003c").replace(/>/gu, "\\u003e")}</skill>`,
123
+ };
105
124
  }
106
- const { candidate: selected, canonicalPath } = resolved;
107
- if (apiKey && runtime.resolveSkillPath(runtimeParams, selected.path) !== canonicalPath)
108
- return;
109
- const previous = state.skills.get(canonicalPath);
110
- const lastSeen = Math.max(previous?.suggested ?? -Infinity, previous?.opened ?? -Infinity);
111
- if (state.turn - lastSeen <= config.cooldownTurns) {
112
- diagnostics?.record(context.agentId, "skill", "cooldown");
125
+ catch (error) {
126
+ diagnostics?.record(agentId, "skill", error instanceof TypeSafeRequestError && error.code === "timeout" ? "timed_out" : "failed");
127
+ api.logger.warn("unblock-memory skill whisperer failed; no hint emitted");
113
128
  return;
114
129
  }
115
- const history = state.skills.get(canonicalPath) ?? {};
116
- history.suggested = state.turn;
117
- state.skills.set(canonicalPath, history);
118
- diagnostics?.record(context.agentId, "skill", "emitted");
119
- return {
120
- prependContext: `A potentially relevant skill is available: ${JSON.stringify(selected.name)} ` +
121
- `at ${JSON.stringify(selected.path)}. Check it before proceeding if applicable.`,
122
- };
123
- }
124
- catch (error) {
125
- diagnostics?.record(context.agentId, "skill", error instanceof Error && error.message === "TypeSafe selection timed out" ? "timed_out" : "failed");
126
- api.logger.warn("unblock-memory skill whisperer failed; no hint emitted");
127
- return;
128
- }
129
- });
130
+ };
131
+ state.result = run();
132
+ return state.result;
133
+ };
130
134
  api.on("after_tool_call", (event, context) => {
131
135
  if (event.toolName !== "read" || event.error ||
132
136
  (isRecord(event.result) && event.result.isError === true) || !context.agentId)
@@ -155,4 +159,5 @@ export function registerSkillWhisperer(api, runtime, config, typesafe, diagnosti
155
159
  if (context.sessionKey)
156
160
  sessions.delete(context.sessionKey);
157
161
  });
162
+ return beforePrompt;
158
163
  }
@@ -8,6 +8,14 @@ type Candidate = {
8
8
  explain: {
9
9
  methods: string[];
10
10
  };
11
+ vector?: {
12
+ score: number;
13
+ rank: number;
14
+ };
15
+ bm25?: {
16
+ score: number;
17
+ rank: number;
18
+ };
11
19
  };
12
- export declare function trainingCandidates(qmd: QMDStore, query: string, collection: string, intent: string): Promise<Candidate[]>;
20
+ export declare function trainingCandidates(qmd: QMDStore, query: string, collection: string | string[], intent: string, signal?: AbortSignal): Promise<Candidate[]>;
13
21
  export {};
@@ -26,14 +26,15 @@ function lexicalChunk(chunks, body, highlighted, marker, intent) {
26
26
  return { chunk, matches, intentMatches: intentTerms.filter(term => lower.includes(term)).length };
27
27
  }).sort((a, b) => b.matches - a.matches || b.intentMatches - a.intentMatches || a.chunk.pos - b.chunk.pos)[0]?.chunk;
28
28
  }
29
- export async function trainingCandidates(qmd, query, collection, intent) {
29
+ export async function trainingCandidates(qmd, query, collection, intent, signal) {
30
+ signal?.throwIfAborted();
30
31
  if (!query.trim() || query.length > 12_000)
31
32
  throw new Error("Invalid training query");
32
33
  // QMD exposes its store but not these chunk helpers at the package root.
33
34
  // Resolve relative to its installed SDK, never a global QMD or modified copy.
34
35
  const chunksApi = await import(new URL("./store.js", import.meta.resolve("@unblocklabs/qmd")).href);
35
36
  const candidates = new Map();
36
- const add = (hit, method, rank) => {
37
+ const add = (hit, method, rank, rawScore) => {
37
38
  if (!hit.bestChunk.trim() || hit.bestChunk.length > 12_000)
38
39
  return;
39
40
  const key = JSON.stringify([hit.file, hit.bestChunk.trim()]), existing = candidates.get(key);
@@ -41,16 +42,19 @@ export async function trainingCandidates(qmd, query, collection, intent) {
41
42
  if (!existing.explain.methods.includes(method))
42
43
  existing.explain.methods.push(method);
43
44
  existing.score = Math.max(existing.score, 1 / (rank + 1));
45
+ existing[method] ??= { score: rawScore, rank: rank + 1 };
44
46
  }
45
47
  else
46
- candidates.set(key, { ...hit, score: 1 / (rank + 1), explain: { methods: [method] } });
48
+ candidates.set(key, { ...hit, score: 1 / (rank + 1), explain: { methods: [method] },
49
+ [method]: { score: rawScore, rank: rank + 1 } });
47
50
  };
48
51
  const vectors = await qmd.searchVector(query, { limit: 10, collection });
52
+ signal?.throwIfAborted();
49
53
  for (const [rank, hit] of vectors.entries()) {
50
54
  const pos = hit.chunkPos, len = hit.chunkLen, body = hit.body ?? "";
51
55
  if (pos === undefined || len === undefined || pos < 0 || len <= 0 || pos + len > body.length)
52
56
  continue;
53
- add({ file: hit.filepath, body, bestChunk: body.slice(pos, pos + len), bestChunkPos: pos }, "vector", rank);
57
+ add({ file: hit.filepath, body, bestChunk: body.slice(pos, pos + len), bestChunkPos: pos }, "vector", rank, hit.score);
54
58
  }
55
59
  const expression = queryTerms(query).map(term => `"${chunksApi.normalizeCjkForFTS(term).trim()}"`).join(" OR ");
56
60
  if (expression) {
@@ -58,9 +62,10 @@ export async function trainingCandidates(qmd, query, collection, intent) {
58
62
  const rows = qmd.internal.db.prepare(`SELECT d.collection,d.path,d.hash,c.doc,
59
63
  bm25(documents_fts,1.5,4.0,1.0) AS rank, highlight(documents_fts,2,?,?) AS highlighted
60
64
  FROM documents_fts JOIN documents d ON d.id=documents_fts.rowid JOIN content c ON c.hash=d.hash
61
- WHERE documents_fts MATCH ? AND d.active=1 AND d.collection=?
62
- ORDER BY rank,d.collection,d.path LIMIT 10`).all(marker, marker, expression, collection);
65
+ WHERE documents_fts MATCH ? AND d.active=1 AND d.collection IN (SELECT value FROM json_each(?))
66
+ ORDER BY rank,d.collection,d.path LIMIT 10`).all(marker, marker, expression, JSON.stringify(typeof collection === "string" ? [collection] : collection));
63
67
  for (const [rank, row] of rows.entries()) {
68
+ signal?.throwIfAborted();
64
69
  const file = `qmd://${row.collection}/${row.path}`;
65
70
  const stored = chunksApi.getStoredChunkSpans(qmd.internal.db, row.hash)
66
71
  .filter(span => span.pos >= 0 && span.chunk_len > 0 && span.pos + span.chunk_len <= row.doc.length)
@@ -68,7 +73,7 @@ export async function trainingCandidates(qmd, query, collection, intent) {
68
73
  const chunks = stored.length ? stored : await chunksApi.chunkDocumentAsync(row.doc, undefined, undefined, undefined, file);
69
74
  const selected = lexicalChunk(chunks, row.doc, row.highlighted, marker, intent);
70
75
  if (selected)
71
- add({ file, body: row.doc, bestChunk: selected.text, bestChunkPos: selected.pos }, "bm25", rank);
76
+ add({ file, body: row.doc, bestChunk: selected.text, bestChunkPos: selected.pos }, "bm25", rank, row.rank);
72
77
  }
73
78
  }
74
79
  return [...candidates.values()].sort((a, b) => b.score - a.score);
@@ -1,6 +1,6 @@
1
1
  import { Type } from "typebox";
2
2
  import { Value } from "typebox/value";
3
- import { postTypeSafe, TYPESAFE_MODEL } from "./typesafe-transport.js";
3
+ import { requestTypeSafe, TYPESAFE_MODEL, TypeSafeRequestError } from "./typesafe-client.js";
4
4
  export const TRAINING_GATE_VERSION = "historical-recall-v1";
5
5
  export const TRAINING_GATE_MODEL = TYPESAFE_MODEL;
6
6
  export const TRAINING_GATE_THRESHOLD = 0.7;
@@ -25,9 +25,9 @@ const resultSchema = Type.Object({
25
25
  usage: Type.Object({ input_tokens: Type.Integer({ minimum: 0 }), output_tokens: Type.Integer({ minimum: 0 }) }),
26
26
  });
27
27
  export async function judgeTrainingInput(input, apiKey, signal) {
28
- const result = await postTypeSafe({ apiKey, signal }, input, TRAINING_GATE_QUESTIONS);
28
+ const result = await requestTypeSafe({ apiKey, signal }, input, TRAINING_GATE_QUESTIONS);
29
29
  if (!Value.Check(resultSchema, result) || !Number.isFinite(result.answers.recall_needed.noul)) {
30
- throw new Error("Invalid training gate response");
30
+ throw new TypeSafeRequestError("Invalid training gate response", "invalid_response");
31
31
  }
32
32
  return { probability: result.answers.recall_needed.noul, model: result.model, usage: result.usage };
33
33
  }
@@ -1,7 +1,8 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { DatabaseSync } from "node:sqlite";
3
+ import { ACTIVE_EVENT_COUNT_SQL, ACTIVE_EVENT_ROWS_SQL, assertAgentTranscriptSchema } from "./agent-transcript.js";
3
4
  import { messageText } from "./whisperer-context.js";
4
- import { responseUserText } from "./response-text.js";
5
+ import { conversationUserText } from "./response-text.js";
5
6
  // Identical serialized inputs keep their checkpoints when eligibility broadens.
6
7
  export const TRAINING_PREPARATION = "visible-history-v1";
7
8
  // A deliberately conservative byte budget, NOT a tokenizer or a 32k-token target.
@@ -10,27 +11,6 @@ export const trainingHash = (value) => createHash("sha256").update(JSON.stringif
10
11
  function record(value) {
11
12
  return value !== null && typeof value === "object" && !Array.isArray(value) ? value : undefined;
12
13
  }
13
- /** Legacy sender IDs are useful for envelope cleanup, not an admission requirement. */
14
- function userText(raw, sender) {
15
- let senderId = typeof sender === "string" ? sender : undefined;
16
- if (senderId === undefined) {
17
- const header = /^Conversation info: ⟦openclaw:ctx⟧\r?\n```json\r?\n([^]*?)\r?\n```\r?\n/.exec(raw.trim());
18
- if (header) {
19
- let metadata;
20
- try {
21
- metadata = JSON.parse(header[1]);
22
- }
23
- catch {
24
- return;
25
- }
26
- const id = record(record(metadata)?.sender)?.id;
27
- if (typeof id === "string")
28
- senderId = id;
29
- }
30
- senderId ??= /^From: [^\r\n]+ \(([^()\r\n]+)\)\r?\n/.exec(raw.trim())?.[1];
31
- }
32
- return responseUserText(raw, senderId ?? "");
33
- }
34
14
  /** The following answer establishes eligibility, but is never part of that example's input. */
35
15
  export function trainingExamples(rows) {
36
16
  const examples = [];
@@ -88,7 +68,7 @@ export function trainingExamples(rows) {
88
68
  continue;
89
69
  }
90
70
  const raw = typeof meta?.upstreamUserText === "string" ? meta.upstreamUserText : messageText(message)?.text;
91
- const visible = raw ? userText(raw, meta?.senderId ?? message.senderId) : undefined;
71
+ const visible = raw ? conversationUserText(raw, meta?.senderId ?? message.senderId) : undefined;
92
72
  if (!visible || /^(?:\[OpenClaw heartbeat poll\]|\[Queued messages while agent was busy\]|\[Subagent Context\]|<relevant-memories>)/.test(visible.text)) {
93
73
  coverage.filtered++;
94
74
  boundary();
@@ -154,11 +134,7 @@ export class TrainingTranscriptReader {
154
134
  this.#db = new DatabaseSync(path, { readOnly: true });
155
135
  try {
156
136
  this.#db.exec("PRAGMA query_only=ON; PRAGMA busy_timeout=1000");
157
- const version = this.#db.prepare("PRAGMA user_version").get()?.user_version;
158
- const meta = this.#db.prepare("SELECT role,agent_id,schema_version FROM schema_meta WHERE meta_key='primary'").get();
159
- if (![17, 18, 19].includes(Number(version)) || meta?.role !== "agent" || meta.agent_id !== agentId || meta.schema_version !== version) {
160
- throw new Error("Unsupported training transcript schema or agent");
161
- }
137
+ assertAgentTranscriptSchema(this.#db, agentId, "Unsupported training transcript schema or agent");
162
138
  const columns = this.#db.prepare("PRAGMA table_info(session_windows)").all().map(c => c.name);
163
139
  this.#lineage = ["parent_session_key", "spawned_by", "plugin_owner_id", "hook_external_content_source"].every(c => columns.includes(c));
164
140
  }
@@ -181,14 +157,10 @@ export class TrainingTranscriptReader {
181
157
  /:(?:cron|subagent|heartbeat|hook)(?::|$)/i.test(String(session.session_key)) ||
182
158
  session.parent_session_key || session.spawned_by || session.plugin_owner_id || session.hook_external_content_source)
183
159
  return null;
184
- const size = this.#db.prepare(`SELECT COUNT(*) n,COALESCE(SUM(length(e.event_json)),0) bytes
185
- FROM session_transcript_active_events a JOIN transcript_events e ON e.session_id=a.session_id AND e.seq=a.event_seq
186
- WHERE a.session_id=?`).get(sessionId);
160
+ const size = this.#db.prepare(ACTIVE_EVENT_COUNT_SQL).get(sessionId);
187
161
  if (Number(size.n) > 50_000 || Number(size.bytes) > 32_000_000)
188
162
  return { oversized: true };
189
- const rows = this.#db.prepare(`SELECT e.seq,e.event_json eventJson,e.created_at createdAt
190
- FROM session_transcript_active_events a JOIN transcript_events e ON e.session_id=a.session_id AND e.seq=a.event_seq
191
- WHERE a.session_id=? ORDER BY a.active_position`).iterate(sessionId);
163
+ const rows = this.#db.prepare(ACTIVE_EVENT_ROWS_SQL).iterate(sessionId);
192
164
  return trainingExamples(rows);
193
165
  }
194
166
  finally {
@@ -1,8 +1,8 @@
1
1
  import { Type } from 'typebox';
2
2
  import { Value } from 'typebox/value';
3
- import { postTypeSafe } from './typesafe-transport.js';
3
+ import { requestTypeSafe, TYPESAFE_MODEL } from './typesafe-client.js';
4
4
  export const CONTEXT_JUDGE_VERSION = 'conversation-context-usefulness-v1';
5
- const CONTEXT_JUDGE_MODEL = 'jev-1.13.0';
5
+ const CONTEXT_JUDGE_MODEL = TYPESAFE_MODEL;
6
6
  const CONTEXT_QUESTIONS = {
7
7
  usefulness: {
8
8
  type: 'score',
@@ -53,5 +53,5 @@ export function parseContextJudgment(payload) {
53
53
  return { score: answer.score / 3, answer, model: payload.model, usage: payload.usage ?? null };
54
54
  }
55
55
  export async function judgeTrainingPassage(request, apiKey) {
56
- return parseContextJudgment(await postTypeSafe({ apiKey, signal: AbortSignal.timeout(30_000) }, request.state, request.questions));
56
+ return parseContextJudgment(await requestTypeSafe({ apiKey, timeoutMs: 30_000 }, request.state, request.questions));
57
57
  }
@@ -1,11 +1,10 @@
1
1
  import { collectTraining } from "./training.js";
2
2
  import { trainingHash } from "./training-input.js";
3
3
  import { TRAINING_GATE_THRESHOLD } from "./training-gate.js";
4
- import { resolveTypeSafeApiKey } from "./typesafe.js";
4
+ import { resolveTypeSafeApiKey, TypeSafeRequestError } from "./typesafe-client.js";
5
5
  import { trainingTeacher, trainingTeacherMessage, TRAINING_TEACHER_MODEL, TRAINING_TEACHER_PROMPT, TRAINING_TEACHER_VERSION } from "./training-models.js";
6
6
  import { historicalTrainingSearch, TRAINING_RETRIEVAL_VERSION, TRAINING_SEARCH_OPTIONS } from "./training-retrieval.js";
7
7
  import { contextJudgeRequest, judgeTrainingPassage, CONTEXT_JUDGE_VERSION } from "./training-judge.js";
8
- import { TypeSafeHttpError } from "./typesafe-transport.js";
9
8
  const SELECTION_VERSION = "conversation-utility-top5-sum-v2";
10
9
  export const TRAINING_EVALUATION_CONCURRENCY = 4;
11
10
  function bounds(options) {
@@ -41,9 +40,11 @@ async function checkpoint(store, step, result, operation) {
41
40
  const hostCode = error && typeof error === "object" && "code" in error && typeof error.code === "string" &&
42
41
  /^LLM_[A-Z_]+$/u.test(error.code) ? error.code : undefined;
43
42
  const status = hostCode === "LLM_COMPLETION_NOT_AUTHORIZED" ||
44
- (error instanceof TypeSafeHttpError && error.status >= 400 && error.status < 500) ? "failed" : "ambiguous";
43
+ (error instanceof TypeSafeRequestError && error.code === "http_error" &&
44
+ error.status !== undefined && error.status >= 400 && error.status < 500) ? "failed" : "ambiguous";
45
45
  store.finishStep(step.stage, step.id, attempt, { status,
46
- error: error instanceof TypeSafeHttpError ? `http_${error.status}` : hostCode ?? "request_or_response_uncertain" });
46
+ error: error instanceof TypeSafeRequestError && error.code === "http_error" ?
47
+ `http_${error.status}` : hostCode ?? "request_or_response_uncertain" });
47
48
  result[status]++;
48
49
  return;
49
50
  }
@@ -7,14 +7,7 @@ import { TrainingStore } from "./training-store.js";
7
7
  import { TRAINING_GATE_THRESHOLD } from "./training-gate.js";
8
8
  import { collectTraining, runTraining } from "./training.js";
9
9
  import { generateTrainingQueries, evaluateTrainingQueries, exportQueryTraining, TRAINING_EVALUATION_CONCURRENCY } from "./training-queries.js";
10
- function dateOption(value, fallback) {
11
- if (value === undefined)
12
- return fallback;
13
- if (!/^\d{4}-\d{2}-\d{2}$/.test(value) || !Number.isFinite(Date.parse(value)) || new Date(value).toISOString().slice(0, 10) !== value) {
14
- throw new Error("Dates must be valid YYYY-MM-DD UTC dates");
15
- }
16
- return Date.parse(value);
17
- }
10
+ import { dateOption } from "./date-option.js";
18
11
  function thresholdOption(value) {
19
12
  const result = Number(value);
20
13
  if (!value.trim() || !Number.isFinite(result) || result < 0 || result > 1)
@@ -1,5 +1,4 @@
1
- import { TypeSafeHttpError } from "./typesafe-transport.js";
2
- import { resolveTypeSafeApiKey } from "./typesafe.js";
1
+ import { TypeSafeRequestError, resolveTypeSafeApiKey } from "./typesafe-client.js";
3
2
  import { TrainingTranscriptReader } from "./training-input.js";
4
3
  import { judgeTrainingInput, TRAINING_GATE_QUESTIONS } from "./training-gate.js";
5
4
  /** A full active-branch rescan is cheap/local; only changed exact inputs need inference. */
@@ -78,9 +77,11 @@ export async function runTraining(source, store, config, options) {
78
77
  judgment = await judgeTrainingInput(input, key, AbortSignal.timeout(30_000));
79
78
  }
80
79
  catch (error) {
81
- const definite = error instanceof TypeSafeHttpError && error.status >= 400 && error.status < 500;
80
+ const definite = error instanceof TypeSafeRequestError && error.code === "http_error" &&
81
+ error.status !== undefined && error.status >= 400 && error.status < 500;
82
82
  const status = definite ? "failed" : "ambiguous";
83
- store.finish(job.id, attempt, { status, error: error instanceof TypeSafeHttpError ? `http_${error.status}` : "request_or_response_uncertain" });
83
+ store.finish(job.id, attempt, { status, error: error instanceof TypeSafeRequestError && error.code === "http_error" ?
84
+ `http_${error.status}` : "request_or_response_uncertain" });
84
85
  result[status]++;
85
86
  // Stop on the first failure rather than spending the rest of the budget during an outage.
86
87
  return;
@@ -0,0 +1,15 @@
1
+ import type { UnblockMemoryConfig } from "./config.js";
2
+ export declare const TYPESAFE_MODEL = "jev-1.13.0";
3
+ export declare class TypeSafeRequestError extends Error {
4
+ readonly code: "timeout" | "cancelled" | "http_error" | "invalid_response" | "network_error";
5
+ readonly status?: number | undefined;
6
+ constructor(message: string, code: "timeout" | "cancelled" | "http_error" | "invalid_response" | "network_error", status?: number | undefined);
7
+ }
8
+ /** Explicit credentials take precedence; a missing explicit file never selects another key. */
9
+ export declare function resolveTypeSafeApiKey(config: UnblockMemoryConfig["typesafe"]): Promise<string | undefined>;
10
+ /** One TypeSafe request. The caller owns the response schema and failure policy. */
11
+ export declare function requestTypeSafe(params: {
12
+ apiKey: string;
13
+ timeoutMs?: number;
14
+ signal?: AbortSignal;
15
+ }, state: unknown, questions: unknown): Promise<unknown>;
@@ -0,0 +1,75 @@
1
+ import { readFile } from "node:fs/promises";
2
+ import { parseEnv } from "node:util";
3
+ export const TYPESAFE_MODEL = "jev-1.13.0";
4
+ export class TypeSafeRequestError extends Error {
5
+ code;
6
+ status;
7
+ constructor(message, code, status) {
8
+ super(message);
9
+ this.code = code;
10
+ this.status = status;
11
+ }
12
+ }
13
+ /** Explicit credentials take precedence; a missing explicit file never selects another key. */
14
+ export async function resolveTypeSafeApiKey(config) {
15
+ if (!config.enabled)
16
+ return undefined;
17
+ if (config.apiKey)
18
+ return config.apiKey.trim() || undefined;
19
+ if (!config.apiKeyFile)
20
+ return process.env.TYPESAFE_API_KEY?.trim() || undefined;
21
+ let contents;
22
+ try {
23
+ contents = (await readFile(config.apiKeyFile, "utf8")).trim();
24
+ }
25
+ catch (error) {
26
+ if (error && typeof error === "object" && "code" in error && error.code === "ENOENT")
27
+ return undefined;
28
+ throw new Error("TypeSafe credential file could not be read");
29
+ }
30
+ if (!contents)
31
+ return undefined;
32
+ if (/^(?:export\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=/m.test(contents) || contents.startsWith("#")) {
33
+ return parseEnv(contents).TYPESAFE_API_KEY?.trim() || undefined;
34
+ }
35
+ if (/\s/.test(contents))
36
+ throw new Error("TypeSafe credential file must contain a key or dotenv entries");
37
+ return contents;
38
+ }
39
+ /** One TypeSafe request. The caller owns the response schema and failure policy. */
40
+ export async function requestTypeSafe(params, state, questions) {
41
+ const timeout = params.timeoutMs === undefined ? undefined : AbortSignal.timeout(params.timeoutMs);
42
+ const signal = params.signal && timeout ? AbortSignal.any([params.signal, timeout]) : params.signal ?? timeout;
43
+ try {
44
+ signal?.throwIfAborted();
45
+ const response = await fetch("https://api.typesafe.ai/v1/systemone", {
46
+ method: "POST", redirect: "error", signal,
47
+ headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json" },
48
+ body: JSON.stringify({ model: TYPESAFE_MODEL, state, questions }),
49
+ });
50
+ if (!response.ok) {
51
+ try {
52
+ await response.body?.cancel();
53
+ }
54
+ catch { /* Status is authoritative; discard unsafe cancellation detail. */ }
55
+ throw new TypeSafeRequestError(`TypeSafe HTTP ${response.status}`, "http_error", response.status);
56
+ }
57
+ try {
58
+ return await response.json();
59
+ }
60
+ catch (error) {
61
+ if (error instanceof SyntaxError)
62
+ throw new TypeSafeRequestError("TypeSafe returned invalid JSON", "invalid_response");
63
+ throw error;
64
+ }
65
+ }
66
+ catch (error) {
67
+ if (error instanceof TypeSafeRequestError)
68
+ throw error;
69
+ if (signal?.aborted) {
70
+ const timedOut = signal.reason instanceof Error && signal.reason.name === "TimeoutError";
71
+ throw new TypeSafeRequestError(timedOut ? "TypeSafe request timed out" : "TypeSafe request cancelled", timedOut ? "timeout" : "cancelled");
72
+ }
73
+ throw new TypeSafeRequestError("TypeSafe request failed", "network_error");
74
+ }
75
+ }
@@ -3,11 +3,7 @@ type RequestOptions = {
3
3
  timeoutMs: number;
4
4
  signal: AbortSignal;
5
5
  };
6
- type Json = string | number | boolean | null | Json[] | {
7
- [key: string]: Json;
8
- };
9
- export { TYPESAFE_MODEL as TYPESAFE_REVIEW_MODEL } from "./typesafe-transport.js";
10
- export declare function askTypeSafeReview(params: RequestOptions, state: Json, questions: Json): Promise<unknown>;
6
+ export { TYPESAFE_MODEL as TYPESAFE_REVIEW_MODEL } from "./typesafe-client.js";
11
7
  /** The source is an indexed snapshot, not proof of current truth or permission to write. */
12
8
  export declare function reviewTypeSafeClaim(params: RequestOptions & {
13
9
  claim: string;
@@ -34,9 +30,9 @@ export declare function reviewTypeSafeClaim(params: RequestOptions & {
34
30
  export declare function reviewMemoryRedundancy(params: RequestOptions & {
35
31
  excerpts: readonly string[];
36
32
  }): Promise<{
37
- redundant: number;
38
33
  earlier: number;
39
34
  later: number;
35
+ redundant: number;
40
36
  }[]>;
41
37
  export declare function complementaryIndices(count: number, pairs: readonly {
42
38
  earlier: number;