@try-works/dsh-memory 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +585 -585
- package/STATUS.md +294 -287
- package/cordis.patch.yml +69 -69
- package/package.json +1 -1
- package/scripts/check-package.mjs +76 -76
- package/scripts/inspect-log.ts +40 -0
- package/scripts/repair-ignorable.ts +130 -0
- package/scripts/scan-sessions.ts +43 -0
- package/scripts/verify-log.ts +26 -0
- package/src/SYSTEM.template.md +311 -311
- package/src/capture.js +242 -242
- package/src/client/index.js +886 -886
- package/src/config.js +57 -57
- package/src/consolidate-run.js +132 -132
- package/src/consolidate.js +330 -330
- package/src/context.js +88 -88
- package/src/decisions.js +106 -106
- package/src/defaults.js +93 -93
- package/src/digest.js +255 -255
- package/src/entry.js +179 -179
- package/src/eval.js +72 -72
- package/src/files.js +60 -60
- package/src/fixture.js +56 -56
- package/src/forget.js +71 -71
- package/src/freshness.js +65 -65
- package/src/fts.js +166 -166
- package/src/git.js +53 -53
- package/src/history.js +226 -226
- package/src/host.js +884 -866
- package/src/index.js +115 -94
- package/src/inject.js +108 -108
- package/src/layers.js +178 -178
- package/src/ledger.js +19 -19
- package/src/map.js +184 -184
- package/src/models.js +136 -136
- package/src/queue.js +602 -602
- package/src/rank.js +31 -31
- package/src/recover.js +91 -91
- package/src/refresh.js +80 -80
- package/src/remember.js +58 -58
- package/src/rerank.js +90 -90
- package/src/search.js +75 -75
- package/src/select.js +57 -57
- package/src/session.js +60 -60
- package/src/state.js +240 -240
- package/src/store.js +128 -128
- package/src/summarize.js +195 -195
- package/src/system-doc.js +171 -171
- package/src/tasks.js +617 -617
- package/src/telemetry.js +122 -122
- package/src/tools.js +828 -818
- package/src/working.js +288 -288
- package/src/workspace-key.js +68 -68
- package/src/workspace.js +87 -87
package/src/rerank.js
CHANGED
|
@@ -1,90 +1,90 @@
|
|
|
1
|
-
// Optional LLM rerank over search candidates.
|
|
2
|
-
//
|
|
3
|
-
// The lexical score cannot tell that a snippet mentioning a word in passing is less relevant
|
|
4
|
-
// than one about it. A model can. This is OFF by default because it costs a call per search,
|
|
5
|
-
// and it is strictly advisory: any failure - no model, unparseable reply, nonsense ordering -
|
|
6
|
-
// leaves the deterministic ranking untouched. Memory search must never depend on a model.
|
|
7
|
-
//
|
|
8
|
-
// The reply protocol is deliberately trivial: a list of candidate NUMBERS in the wanted order.
|
|
9
|
-
// An earlier nested-JSON contract failed repeatedly in practice, and an ordering is exactly the
|
|
10
|
-
// kind of answer a short line of digits expresses perfectly.
|
|
11
|
-
|
|
12
|
-
/** Only rerank when there is a real choice to make. */
|
|
13
|
-
export const RERANK_MIN_CANDIDATES = 4
|
|
14
|
-
/** Candidates sent to the model; more is slower with little gain. */
|
|
15
|
-
export const RERANK_MAX_CANDIDATES = 20
|
|
16
|
-
/** Characters of each snippet shown to the model. */
|
|
17
|
-
export const RERANK_SNIPPET_CHARS = 220
|
|
18
|
-
|
|
19
|
-
/** Instructions for the rerank model. */
|
|
20
|
-
export const RERANK_INSTRUCTIONS = [
|
|
21
|
-
'You are reranking memory search results for relevance to a query.',
|
|
22
|
-
'You receive the query and a numbered list of candidate snippets from a durable memory store.',
|
|
23
|
-
'',
|
|
24
|
-
'Reply with the candidate NUMBERS in order of usefulness to the query, most useful first.',
|
|
25
|
-
'- One line, numbers separated by spaces or commas. Example: 3 1 4 2',
|
|
26
|
-
'- Include every number exactly once. Do not explain, do not add prose, do not omit any.',
|
|
27
|
-
'- Judge by what would actually help someone act on the query, not by keyword overlap.',
|
|
28
|
-
'- Prefer concrete facts, decisions and procedures over passing mentions.',
|
|
29
|
-
].join('\n')
|
|
30
|
-
|
|
31
|
-
/** Build the rerank prompt. Snippets are clipped so the prompt stays bounded. */
|
|
32
|
-
export function buildRerankPrompt(query, candidates, options = {}) {
|
|
33
|
-
const limit = options.maxCandidates ?? RERANK_MAX_CANDIDATES
|
|
34
|
-
const chars = options.snippetChars ?? RERANK_SNIPPET_CHARS
|
|
35
|
-
const lines = [RERANK_INSTRUCTIONS, '', '--- QUERY ---', String(query ?? '').trim(), '', '--- CANDIDATES ---']
|
|
36
|
-
candidates.slice(0, limit).forEach((candidate, index) => {
|
|
37
|
-
const where = candidate.file ? candidate.file + ':' + candidate.line : ''
|
|
38
|
-
const text = String(candidate.snippet ?? '').replace(/\s+/g, ' ').trim().slice(0, chars)
|
|
39
|
-
lines.push((index + 1) + '. [' + where + '] ' + text)
|
|
40
|
-
})
|
|
41
|
-
lines.push('', 'Reply with the numbers in order. Nothing else.')
|
|
42
|
-
return lines.join('\n')
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Parse an ordering out of a model reply.
|
|
47
|
-
* @param text - the reply.
|
|
48
|
-
* @param count - how many candidates exist.
|
|
49
|
-
* @returns a complete, duplicate-free 0-based ordering, or undefined when the reply is unusable.
|
|
50
|
-
*/
|
|
51
|
-
export function parseRerankOrder(text, count) {
|
|
52
|
-
if (!Number.isFinite(count) || count < 2) return undefined
|
|
53
|
-
const numbers = (String(text ?? '').match(/\d+/g) ?? []).map(Number)
|
|
54
|
-
const order = []
|
|
55
|
-
const seen = new Set()
|
|
56
|
-
for (const n of numbers) {
|
|
57
|
-
if (n < 1 || n > count || seen.has(n)) continue
|
|
58
|
-
seen.add(n)
|
|
59
|
-
order.push(n - 1)
|
|
60
|
-
}
|
|
61
|
-
// Require a genuinely useful signal, not a token gesture. "1 2" out of four is agreement with
|
|
62
|
-
// the existing order, not a reranking, and accepting it would silently no-op every call.
|
|
63
|
-
const minimum = Math.max(2, Math.ceil(count * 0.75))
|
|
64
|
-
if (order.length < minimum) return undefined
|
|
65
|
-
// Anything the model left out keeps its original relative position, at the end.
|
|
66
|
-
for (let i = 0; i < count; i += 1) if (!seen.has(i + 1)) order.push(i)
|
|
67
|
-
return order
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* Reorder hits by a model's judgement.
|
|
72
|
-
* @param options.llm - (prompt) => Promise<string>; absent or throwing leaves the order alone.
|
|
73
|
-
* @param options.enabled - master switch (default false).
|
|
74
|
-
* @returns {Promise<{hits: object[], reranked: boolean, reason?: string}>}
|
|
75
|
-
*/
|
|
76
|
-
export async function rerankHits(query, hits, options = {}) {
|
|
77
|
-
const list = Array.isArray(hits) ? hits : []
|
|
78
|
-
if (!options.enabled) return { hits: list, reranked: false, reason: 'disabled' }
|
|
79
|
-
if (list.length < (options.minCandidates ?? RERANK_MIN_CANDIDATES)) return { hits: list, reranked: false, reason: 'too-few' }
|
|
80
|
-
if (typeof options.llm !== 'function') return { hits: list, reranked: false, reason: 'no-model' }
|
|
81
|
-
let reply
|
|
82
|
-
try {
|
|
83
|
-
reply = await options.llm(buildRerankPrompt(query, list, options))
|
|
84
|
-
} catch (error) {
|
|
85
|
-
return { hits: list, reranked: false, reason: 'error: ' + String((error && error.message) || error).slice(0, 120) }
|
|
86
|
-
}
|
|
87
|
-
const order = parseRerankOrder(reply, list.length)
|
|
88
|
-
if (!order) return { hits: list, reranked: false, reason: 'unparseable' }
|
|
89
|
-
return { hits: order.map((i) => list[i]), reranked: true }
|
|
90
|
-
}
|
|
1
|
+
// Optional LLM rerank over search candidates.
|
|
2
|
+
//
|
|
3
|
+
// The lexical score cannot tell that a snippet mentioning a word in passing is less relevant
|
|
4
|
+
// than one about it. A model can. This is OFF by default because it costs a call per search,
|
|
5
|
+
// and it is strictly advisory: any failure - no model, unparseable reply, nonsense ordering -
|
|
6
|
+
// leaves the deterministic ranking untouched. Memory search must never depend on a model.
|
|
7
|
+
//
|
|
8
|
+
// The reply protocol is deliberately trivial: a list of candidate NUMBERS in the wanted order.
|
|
9
|
+
// An earlier nested-JSON contract failed repeatedly in practice, and an ordering is exactly the
|
|
10
|
+
// kind of answer a short line of digits expresses perfectly.
|
|
11
|
+
|
|
12
|
+
/** Only rerank when there is a real choice to make. */
|
|
13
|
+
export const RERANK_MIN_CANDIDATES = 4
|
|
14
|
+
/** Candidates sent to the model; more is slower with little gain. */
|
|
15
|
+
export const RERANK_MAX_CANDIDATES = 20
|
|
16
|
+
/** Characters of each snippet shown to the model. */
|
|
17
|
+
export const RERANK_SNIPPET_CHARS = 220
|
|
18
|
+
|
|
19
|
+
/** Instructions for the rerank model. */
|
|
20
|
+
export const RERANK_INSTRUCTIONS = [
|
|
21
|
+
'You are reranking memory search results for relevance to a query.',
|
|
22
|
+
'You receive the query and a numbered list of candidate snippets from a durable memory store.',
|
|
23
|
+
'',
|
|
24
|
+
'Reply with the candidate NUMBERS in order of usefulness to the query, most useful first.',
|
|
25
|
+
'- One line, numbers separated by spaces or commas. Example: 3 1 4 2',
|
|
26
|
+
'- Include every number exactly once. Do not explain, do not add prose, do not omit any.',
|
|
27
|
+
'- Judge by what would actually help someone act on the query, not by keyword overlap.',
|
|
28
|
+
'- Prefer concrete facts, decisions and procedures over passing mentions.',
|
|
29
|
+
].join('\n')
|
|
30
|
+
|
|
31
|
+
/** Build the rerank prompt. Snippets are clipped so the prompt stays bounded. */
|
|
32
|
+
export function buildRerankPrompt(query, candidates, options = {}) {
|
|
33
|
+
const limit = options.maxCandidates ?? RERANK_MAX_CANDIDATES
|
|
34
|
+
const chars = options.snippetChars ?? RERANK_SNIPPET_CHARS
|
|
35
|
+
const lines = [RERANK_INSTRUCTIONS, '', '--- QUERY ---', String(query ?? '').trim(), '', '--- CANDIDATES ---']
|
|
36
|
+
candidates.slice(0, limit).forEach((candidate, index) => {
|
|
37
|
+
const where = candidate.file ? candidate.file + ':' + candidate.line : ''
|
|
38
|
+
const text = String(candidate.snippet ?? '').replace(/\s+/g, ' ').trim().slice(0, chars)
|
|
39
|
+
lines.push((index + 1) + '. [' + where + '] ' + text)
|
|
40
|
+
})
|
|
41
|
+
lines.push('', 'Reply with the numbers in order. Nothing else.')
|
|
42
|
+
return lines.join('\n')
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Parse an ordering out of a model reply.
|
|
47
|
+
* @param text - the reply.
|
|
48
|
+
* @param count - how many candidates exist.
|
|
49
|
+
* @returns a complete, duplicate-free 0-based ordering, or undefined when the reply is unusable.
|
|
50
|
+
*/
|
|
51
|
+
export function parseRerankOrder(text, count) {
|
|
52
|
+
if (!Number.isFinite(count) || count < 2) return undefined
|
|
53
|
+
const numbers = (String(text ?? '').match(/\d+/g) ?? []).map(Number)
|
|
54
|
+
const order = []
|
|
55
|
+
const seen = new Set()
|
|
56
|
+
for (const n of numbers) {
|
|
57
|
+
if (n < 1 || n > count || seen.has(n)) continue
|
|
58
|
+
seen.add(n)
|
|
59
|
+
order.push(n - 1)
|
|
60
|
+
}
|
|
61
|
+
// Require a genuinely useful signal, not a token gesture. "1 2" out of four is agreement with
|
|
62
|
+
// the existing order, not a reranking, and accepting it would silently no-op every call.
|
|
63
|
+
const minimum = Math.max(2, Math.ceil(count * 0.75))
|
|
64
|
+
if (order.length < minimum) return undefined
|
|
65
|
+
// Anything the model left out keeps its original relative position, at the end.
|
|
66
|
+
for (let i = 0; i < count; i += 1) if (!seen.has(i + 1)) order.push(i)
|
|
67
|
+
return order
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Reorder hits by a model's judgement.
|
|
72
|
+
* @param options.llm - (prompt) => Promise<string>; absent or throwing leaves the order alone.
|
|
73
|
+
* @param options.enabled - master switch (default false).
|
|
74
|
+
* @returns {Promise<{hits: object[], reranked: boolean, reason?: string}>}
|
|
75
|
+
*/
|
|
76
|
+
export async function rerankHits(query, hits, options = {}) {
|
|
77
|
+
const list = Array.isArray(hits) ? hits : []
|
|
78
|
+
if (!options.enabled) return { hits: list, reranked: false, reason: 'disabled' }
|
|
79
|
+
if (list.length < (options.minCandidates ?? RERANK_MIN_CANDIDATES)) return { hits: list, reranked: false, reason: 'too-few' }
|
|
80
|
+
if (typeof options.llm !== 'function') return { hits: list, reranked: false, reason: 'no-model' }
|
|
81
|
+
let reply
|
|
82
|
+
try {
|
|
83
|
+
reply = await options.llm(buildRerankPrompt(query, list, options))
|
|
84
|
+
} catch (error) {
|
|
85
|
+
return { hits: list, reranked: false, reason: 'error: ' + String((error && error.message) || error).slice(0, 120) }
|
|
86
|
+
}
|
|
87
|
+
const order = parseRerankOrder(reply, list.length)
|
|
88
|
+
if (!order) return { hits: list, reranked: false, reason: 'unparseable' }
|
|
89
|
+
return { hits: order.map((i) => list[i]), reranked: true }
|
|
90
|
+
}
|
package/src/search.js
CHANGED
|
@@ -1,75 +1,75 @@
|
|
|
1
|
-
// Search: FTS match -> scope filter -> rankBlend -> citations. In-process, read-only.
|
|
2
|
-
//
|
|
3
|
-
// Two indexes back it. The word index (unicode61) serves normal queries and ranks by bm25. The
|
|
4
|
-
// trigram index serves SUBSTRING queries - scripts written without spaces, which the word
|
|
5
|
-
// tokenizer cannot express at all. A word query that finds nothing also retries as a substring,
|
|
6
|
-
// so a partial word still matches without paying for the trigram scan on every search.
|
|
7
|
-
import { existsSync } from 'node:fs'
|
|
8
|
-
import { DatabaseSync } from 'node:sqlite'
|
|
9
|
-
import { buildFtsQuery, buildIndex, buildLikeQuery, buildTrigramQuery, indexPath, needsSubstring, SUBSTRING_TABLE } from './fts.js'
|
|
10
|
-
import { buildLedger } from './ledger.js'
|
|
11
|
-
import { rankBlend } from './rank.js'
|
|
12
|
-
|
|
13
|
-
/** Rows pulled from the word index before ranking. */
|
|
14
|
-
const CANDIDATES = 200
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* A LIKE scan over the indexed lines. Used only when a MATCH cannot express the query: a
|
|
18
|
-
* no-space script shorter than the trigram minimum. Bounded by the same candidate cap.
|
|
19
|
-
*/
|
|
20
|
-
function likeRows(db, pattern, table) {
|
|
21
|
-
try {
|
|
22
|
-
return db.prepare(
|
|
23
|
-
'SELECT file, line, section, content, 0 AS rank FROM ' + table + ' WHERE content LIKE ? ESCAPE \'\\\' LIMIT ?'
|
|
24
|
-
).all(pattern, CANDIDATES)
|
|
25
|
-
} catch { return [] }
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
/** One MATCH against one table; returns [] when the table or expression is unusable. */
|
|
29
|
-
function matchRows(db, expression, table) {
|
|
30
|
-
try {
|
|
31
|
-
return db.prepare(
|
|
32
|
-
'SELECT file, line, section, content, bm25(' + table + ') AS rank FROM ' + table + ' WHERE ' + table + ' MATCH ? ORDER BY rank LIMIT ?'
|
|
33
|
-
).all(expression, CANDIDATES)
|
|
34
|
-
} catch { return [] }
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
export function search(store, query, { limit = 10, scope, key } = {}) {
|
|
38
|
-
const dbPath = indexPath(store.root)
|
|
39
|
-
if (!existsSync(dbPath)) buildIndex(store.root)
|
|
40
|
-
const db = new DatabaseSync(dbPath, { readOnly: true })
|
|
41
|
-
try {
|
|
42
|
-
let rows = []
|
|
43
|
-
if (needsSubstring(query)) {
|
|
44
|
-
// No-space script: only the substring index can match it.
|
|
45
|
-
const expression = buildTrigramQuery(String(query).trim())
|
|
46
|
-
if (expression) rows = matchRows(db, expression, SUBSTRING_TABLE)
|
|
47
|
-
// Too short for trigram (common for two-character CJK words): scan instead.
|
|
48
|
-
if (rows.length === 0) {
|
|
49
|
-
const pattern = buildLikeQuery(query)
|
|
50
|
-
if (pattern) rows = likeRows(db, pattern, SUBSTRING_TABLE)
|
|
51
|
-
}
|
|
52
|
-
} else {
|
|
53
|
-
const match = buildFtsQuery(query)
|
|
54
|
-
if (match) rows = matchRows(db, match, 'mem_fts')
|
|
55
|
-
if (rows.length === 0) {
|
|
56
|
-
// A word query that matched nothing may still be a partial word: retry as substring.
|
|
57
|
-
const expression = buildTrigramQuery(String(query).trim())
|
|
58
|
-
if (expression) rows = matchRows(db, expression, SUBSTRING_TABLE)
|
|
59
|
-
if (rows.length === 0) {
|
|
60
|
-
const pattern = buildLikeQuery(query)
|
|
61
|
-
if (pattern) rows = likeRows(db, pattern, SUBSTRING_TABLE)
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
if (rows.length === 0) return []
|
|
66
|
-
let filtered = rows
|
|
67
|
-
if (scope === 'global') filtered = rows.filter((r) => r.file.startsWith('global/'))
|
|
68
|
-
else if (scope === 'workspace') filtered = rows.filter((r) => r.file.startsWith('workspaces/' + (key ?? '') + '/'))
|
|
69
|
-
const hits = filtered.map((r) => ({ file: r.file, line: r.line, section: r.section, snippet: String(r.content).trim() }))
|
|
70
|
-
return rankBlend(hits, buildLedger(store.root)).slice(0, limit)
|
|
71
|
-
.map((h) => ({ file: h.file, line: h.line, section: h.section ?? '', snippet: h.snippet, id: h.id ?? '', score: Math.round(h.score * 1000) / 1000 }))
|
|
72
|
-
} finally {
|
|
73
|
-
db.close()
|
|
74
|
-
}
|
|
75
|
-
}
|
|
1
|
+
// Search: FTS match -> scope filter -> rankBlend -> citations. In-process, read-only.
|
|
2
|
+
//
|
|
3
|
+
// Two indexes back it. The word index (unicode61) serves normal queries and ranks by bm25. The
|
|
4
|
+
// trigram index serves SUBSTRING queries - scripts written without spaces, which the word
|
|
5
|
+
// tokenizer cannot express at all. A word query that finds nothing also retries as a substring,
|
|
6
|
+
// so a partial word still matches without paying for the trigram scan on every search.
|
|
7
|
+
import { existsSync } from 'node:fs'
|
|
8
|
+
import { DatabaseSync } from 'node:sqlite'
|
|
9
|
+
import { buildFtsQuery, buildIndex, buildLikeQuery, buildTrigramQuery, indexPath, needsSubstring, SUBSTRING_TABLE } from './fts.js'
|
|
10
|
+
import { buildLedger } from './ledger.js'
|
|
11
|
+
import { rankBlend } from './rank.js'
|
|
12
|
+
|
|
13
|
+
/** Rows pulled from the word index before ranking. */
|
|
14
|
+
const CANDIDATES = 200
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* A LIKE scan over the indexed lines. Used only when a MATCH cannot express the query: a
|
|
18
|
+
* no-space script shorter than the trigram minimum. Bounded by the same candidate cap.
|
|
19
|
+
*/
|
|
20
|
+
function likeRows(db, pattern, table) {
|
|
21
|
+
try {
|
|
22
|
+
return db.prepare(
|
|
23
|
+
'SELECT file, line, section, content, 0 AS rank FROM ' + table + ' WHERE content LIKE ? ESCAPE \'\\\' LIMIT ?'
|
|
24
|
+
).all(pattern, CANDIDATES)
|
|
25
|
+
} catch { return [] }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** One MATCH against one table; returns [] when the table or expression is unusable. */
|
|
29
|
+
function matchRows(db, expression, table) {
|
|
30
|
+
try {
|
|
31
|
+
return db.prepare(
|
|
32
|
+
'SELECT file, line, section, content, bm25(' + table + ') AS rank FROM ' + table + ' WHERE ' + table + ' MATCH ? ORDER BY rank LIMIT ?'
|
|
33
|
+
).all(expression, CANDIDATES)
|
|
34
|
+
} catch { return [] }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function search(store, query, { limit = 10, scope, key } = {}) {
|
|
38
|
+
const dbPath = indexPath(store.root)
|
|
39
|
+
if (!existsSync(dbPath)) buildIndex(store.root)
|
|
40
|
+
const db = new DatabaseSync(dbPath, { readOnly: true })
|
|
41
|
+
try {
|
|
42
|
+
let rows = []
|
|
43
|
+
if (needsSubstring(query)) {
|
|
44
|
+
// No-space script: only the substring index can match it.
|
|
45
|
+
const expression = buildTrigramQuery(String(query).trim())
|
|
46
|
+
if (expression) rows = matchRows(db, expression, SUBSTRING_TABLE)
|
|
47
|
+
// Too short for trigram (common for two-character CJK words): scan instead.
|
|
48
|
+
if (rows.length === 0) {
|
|
49
|
+
const pattern = buildLikeQuery(query)
|
|
50
|
+
if (pattern) rows = likeRows(db, pattern, SUBSTRING_TABLE)
|
|
51
|
+
}
|
|
52
|
+
} else {
|
|
53
|
+
const match = buildFtsQuery(query)
|
|
54
|
+
if (match) rows = matchRows(db, match, 'mem_fts')
|
|
55
|
+
if (rows.length === 0) {
|
|
56
|
+
// A word query that matched nothing may still be a partial word: retry as substring.
|
|
57
|
+
const expression = buildTrigramQuery(String(query).trim())
|
|
58
|
+
if (expression) rows = matchRows(db, expression, SUBSTRING_TABLE)
|
|
59
|
+
if (rows.length === 0) {
|
|
60
|
+
const pattern = buildLikeQuery(query)
|
|
61
|
+
if (pattern) rows = likeRows(db, pattern, SUBSTRING_TABLE)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (rows.length === 0) return []
|
|
66
|
+
let filtered = rows
|
|
67
|
+
if (scope === 'global') filtered = rows.filter((r) => r.file.startsWith('global/'))
|
|
68
|
+
else if (scope === 'workspace') filtered = rows.filter((r) => r.file.startsWith('workspaces/' + (key ?? '') + '/'))
|
|
69
|
+
const hits = filtered.map((r) => ({ file: r.file, line: r.line, section: r.section, snippet: String(r.content).trim() }))
|
|
70
|
+
return rankBlend(hits, buildLedger(store.root)).slice(0, limit)
|
|
71
|
+
.map((h) => ({ file: h.file, line: h.line, section: h.section ?? '', snippet: h.snippet, id: h.id ?? '', score: Math.round(h.score * 1000) / 1000 }))
|
|
72
|
+
} finally {
|
|
73
|
+
db.close()
|
|
74
|
+
}
|
|
75
|
+
}
|
package/src/select.js
CHANGED
|
@@ -1,57 +1,57 @@
|
|
|
1
|
-
// Which memory goes in front of the agent.
|
|
2
|
-
//
|
|
3
|
-
// This used to be `.slice(-5)`: the last five entries in file order, whatever the turn was
|
|
4
|
-
// about. Measured, that meant the SAME five learnings arrived every turn through an entire
|
|
5
|
-
// queue-rewrite session, while the entries that actually described the work - the orphaned-as-
|
|
6
|
-
// retries bug, the state list, the queue design decisions - were never shown at all. A fixed
|
|
7
|
-
// dump is worse than no memory, because it looks like recall.
|
|
8
|
-
//
|
|
9
|
-
// The index that answers this already existed. Selection now asks it a question.
|
|
10
|
-
import { search } from './search.js'
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* The question to ask the index.
|
|
14
|
-
*
|
|
15
|
-
* The live TASK outranks the session GOAL, because a goal is set once and then ages: a goal
|
|
16
|
-
* recorded at the start of a long session describes what the session began as, not what the
|
|
17
|
-
* agent is doing now. The goal is kept as a fallback for a turn with no task recorded.
|
|
18
|
-
*/
|
|
19
|
-
export function relevanceQuery(workingMemory, goal) {
|
|
20
|
-
const task = (workingMemory?.sections ?? []).find((s) => /^task$/i.test(s.name))?.content ?? ''
|
|
21
|
-
const next = (workingMemory?.sections ?? []).find((s) => /^next steps$/i.test(s.name))?.content ?? ''
|
|
22
|
-
const parts = [task, next, goal].map((s) => String(s ?? '').trim()).filter(Boolean)
|
|
23
|
-
return parts.join(' ').slice(0, 400)
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Choose the entries to show, preferring RELEVANCE and falling back to recency.
|
|
28
|
-
*
|
|
29
|
-
* The fallback is not decoration. With no query, an unbuilt index, or nothing matching, the
|
|
30
|
-
* previous behaviour is exactly what should happen - a turn should never get LESS context
|
|
31
|
-
* because a ranking attempt found nothing.
|
|
32
|
-
* @returns { entries, relevant } - relevant says which rule actually ran, so a caller can say so.
|
|
33
|
-
*/
|
|
34
|
-
export function selectEntries(store, pool, query, limit, options = {}) {
|
|
35
|
-
const fallback = { entries: pool.slice(-limit), relevant: false }
|
|
36
|
-
if (!query || pool.length <= limit) return fallback;
|
|
37
|
-
try {
|
|
38
|
-
// A hit is LINE-level; the pool is entry-level, so hits are mapped onto files and each file
|
|
39
|
-
// keeps its BEST rank. One strongly matching line should lift its entry, not be diluted by
|
|
40
|
-
// the other lines of the same file.
|
|
41
|
-
const best = new Map()
|
|
42
|
-
for (const hit of search(store, query, { limit: limit * 8, scope: options.scope, key: options.key })) {
|
|
43
|
-
const file = String(hit.file ?? '').split('\\').join('/')
|
|
44
|
-
const rank = Number(hit.rank ?? 0)
|
|
45
|
-
if (!best.has(file) || rank > best.get(file)) best.set(file, rank)
|
|
46
|
-
}
|
|
47
|
-
if (best.size === 0) return fallback
|
|
48
|
-
const ranked = pool
|
|
49
|
-
.filter((entry) => best.has(String(entry.file ?? '').split('\\').join('/')))
|
|
50
|
-
.sort((a, b) => (best.get(b.file) ?? 0) - (best.get(a.file) ?? 0))
|
|
51
|
-
if (ranked.length === 0) return fallback
|
|
52
|
-
return { entries: ranked.slice(0, limit), relevant: true }
|
|
53
|
-
} catch {
|
|
54
|
-
// An index that is missing or mid-rebuild must not cost the agent its context.
|
|
55
|
-
return fallback
|
|
56
|
-
}
|
|
57
|
-
}
|
|
1
|
+
// Which memory goes in front of the agent.
|
|
2
|
+
//
|
|
3
|
+
// This used to be `.slice(-5)`: the last five entries in file order, whatever the turn was
|
|
4
|
+
// about. Measured, that meant the SAME five learnings arrived every turn through an entire
|
|
5
|
+
// queue-rewrite session, while the entries that actually described the work - the orphaned-as-
|
|
6
|
+
// retries bug, the state list, the queue design decisions - were never shown at all. A fixed
|
|
7
|
+
// dump is worse than no memory, because it looks like recall.
|
|
8
|
+
//
|
|
9
|
+
// The index that answers this already existed. Selection now asks it a question.
|
|
10
|
+
import { search } from './search.js'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The question to ask the index.
|
|
14
|
+
*
|
|
15
|
+
* The live TASK outranks the session GOAL, because a goal is set once and then ages: a goal
|
|
16
|
+
* recorded at the start of a long session describes what the session began as, not what the
|
|
17
|
+
* agent is doing now. The goal is kept as a fallback for a turn with no task recorded.
|
|
18
|
+
*/
|
|
19
|
+
export function relevanceQuery(workingMemory, goal) {
|
|
20
|
+
const task = (workingMemory?.sections ?? []).find((s) => /^task$/i.test(s.name))?.content ?? ''
|
|
21
|
+
const next = (workingMemory?.sections ?? []).find((s) => /^next steps$/i.test(s.name))?.content ?? ''
|
|
22
|
+
const parts = [task, next, goal].map((s) => String(s ?? '').trim()).filter(Boolean)
|
|
23
|
+
return parts.join(' ').slice(0, 400)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Choose the entries to show, preferring RELEVANCE and falling back to recency.
|
|
28
|
+
*
|
|
29
|
+
* The fallback is not decoration. With no query, an unbuilt index, or nothing matching, the
|
|
30
|
+
* previous behaviour is exactly what should happen - a turn should never get LESS context
|
|
31
|
+
* because a ranking attempt found nothing.
|
|
32
|
+
* @returns { entries, relevant } - relevant says which rule actually ran, so a caller can say so.
|
|
33
|
+
*/
|
|
34
|
+
export function selectEntries(store, pool, query, limit, options = {}) {
|
|
35
|
+
const fallback = { entries: pool.slice(-limit), relevant: false }
|
|
36
|
+
if (!query || pool.length <= limit) return fallback;
|
|
37
|
+
try {
|
|
38
|
+
// A hit is LINE-level; the pool is entry-level, so hits are mapped onto files and each file
|
|
39
|
+
// keeps its BEST rank. One strongly matching line should lift its entry, not be diluted by
|
|
40
|
+
// the other lines of the same file.
|
|
41
|
+
const best = new Map()
|
|
42
|
+
for (const hit of search(store, query, { limit: limit * 8, scope: options.scope, key: options.key })) {
|
|
43
|
+
const file = String(hit.file ?? '').split('\\').join('/')
|
|
44
|
+
const rank = Number(hit.rank ?? 0)
|
|
45
|
+
if (!best.has(file) || rank > best.get(file)) best.set(file, rank)
|
|
46
|
+
}
|
|
47
|
+
if (best.size === 0) return fallback
|
|
48
|
+
const ranked = pool
|
|
49
|
+
.filter((entry) => best.has(String(entry.file ?? '').split('\\').join('/')))
|
|
50
|
+
.sort((a, b) => (best.get(b.file) ?? 0) - (best.get(a.file) ?? 0))
|
|
51
|
+
if (ranked.length === 0) return fallback
|
|
52
|
+
return { entries: ranked.slice(0, limit), relevant: true }
|
|
53
|
+
} catch {
|
|
54
|
+
// An index that is missing or mid-rebuild must not cost the agent its context.
|
|
55
|
+
return fallback
|
|
56
|
+
}
|
|
57
|
+
}
|
package/src/session.js
CHANGED
|
@@ -1,60 +1,60 @@
|
|
|
1
|
-
// Resolve a session's working directory to a memory workspace key.
|
|
2
|
-
// A session belongs to a workspace only when DSH's own workspaceRegistry resolves its
|
|
3
|
-
// cwd to a registered workspace; otherwise it falls back to the global layer.
|
|
4
|
-
import { basename } from 'node:path'
|
|
5
|
-
import { associate, listWorkspaces, resolveKeyFor } from './workspace.js'
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* True when a session was created as a subagent child.
|
|
9
|
-
*
|
|
10
|
-
* Memory belongs to the MAIN agent. A subagent is a short-lived delegate working inside a task
|
|
11
|
-
* the parent already owns, and the parent's own transcript carries the delegation and its
|
|
12
|
-
* result, so recording the child as well duplicates the same work and buries the parent's
|
|
13
|
-
* account of it. Measured here: of 306 sessions with no memory, every one sampled from the
|
|
14
|
-
* post-install group was a subagent.
|
|
15
|
-
*
|
|
16
|
-
* Both fields are checked because they are set independently: `delegationDepth` is the recursion
|
|
17
|
-
* budget (absent on a resumed top-level session, parent+1 on a child), and `origin` is the
|
|
18
|
-
* product classification. Either one being present means the session is not the main agent.
|
|
19
|
-
*/
|
|
20
|
-
export function isSubagentSession(session) {
|
|
21
|
-
const header = session?.header
|
|
22
|
-
if (!header) return false
|
|
23
|
-
if (header.origin === 'subagent') return true
|
|
24
|
-
return typeof header.delegationDepth === 'number' && header.delegationDepth > 0
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* The main-agent predicate, phrased for the call sites that guard a WRITE.
|
|
29
|
-
* Reads are harmless from anywhere; only recording is restricted.
|
|
30
|
-
*/
|
|
31
|
-
export function isMainAgentSession(session) { return !isSubagentSession(session) }
|
|
32
|
-
|
|
33
|
-
/** Sync fallback (no workspaceRegistry): basename-derived key, or 'global'. */
|
|
34
|
-
export function workspaceFromSession(store, session) {
|
|
35
|
-
const cwd = session?.header?.cwd
|
|
36
|
-
if (!cwd) return 'global'
|
|
37
|
-
return resolveKeyFor(store.root, { cwd, basename: basename(cwd) }) ?? 'global'
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Resolve + lazily associate a session's workspace. Only a DSH-registered workspace
|
|
42
|
-
* (resolveByPath hit) triggers association; everything else stays global.
|
|
43
|
-
*/
|
|
44
|
-
export async function autoAssociateSession(store, ctx, session) {
|
|
45
|
-
const cwd = session?.header?.cwd
|
|
46
|
-
if (!cwd) return 'global'
|
|
47
|
-
const registry = ctx?.get('workspaceRegistry')
|
|
48
|
-
if (!registry || typeof registry.resolveByPath !== 'function') return 'global'
|
|
49
|
-
let ws
|
|
50
|
-
try { ws = await registry.resolveByPath(cwd) } catch { ws = undefined }
|
|
51
|
-
if (!ws) return 'global'
|
|
52
|
-
const key = resolveKeyFor(store.root, { cwd: ws.path, basename: basename(ws.path) })
|
|
53
|
-
if (!key) return 'global'
|
|
54
|
-
const known = listWorkspaces(store).some((w) => w.key === key)
|
|
55
|
-
if (!known) {
|
|
56
|
-
try { associate(store, { key, title: ws.title ?? key, cwd: ws.path, workspaceId: ws.id }) }
|
|
57
|
-
catch { /* keep going; the key is still usable */ }
|
|
58
|
-
}
|
|
59
|
-
return key
|
|
60
|
-
}
|
|
1
|
+
// Resolve a session's working directory to a memory workspace key.
|
|
2
|
+
// A session belongs to a workspace only when DSH's own workspaceRegistry resolves its
|
|
3
|
+
// cwd to a registered workspace; otherwise it falls back to the global layer.
|
|
4
|
+
import { basename } from 'node:path'
|
|
5
|
+
import { associate, listWorkspaces, resolveKeyFor } from './workspace.js'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* True when a session was created as a subagent child.
|
|
9
|
+
*
|
|
10
|
+
* Memory belongs to the MAIN agent. A subagent is a short-lived delegate working inside a task
|
|
11
|
+
* the parent already owns, and the parent's own transcript carries the delegation and its
|
|
12
|
+
* result, so recording the child as well duplicates the same work and buries the parent's
|
|
13
|
+
* account of it. Measured here: of 306 sessions with no memory, every one sampled from the
|
|
14
|
+
* post-install group was a subagent.
|
|
15
|
+
*
|
|
16
|
+
* Both fields are checked because they are set independently: `delegationDepth` is the recursion
|
|
17
|
+
* budget (absent on a resumed top-level session, parent+1 on a child), and `origin` is the
|
|
18
|
+
* product classification. Either one being present means the session is not the main agent.
|
|
19
|
+
*/
|
|
20
|
+
export function isSubagentSession(session) {
|
|
21
|
+
const header = session?.header
|
|
22
|
+
if (!header) return false
|
|
23
|
+
if (header.origin === 'subagent') return true
|
|
24
|
+
return typeof header.delegationDepth === 'number' && header.delegationDepth > 0
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The main-agent predicate, phrased for the call sites that guard a WRITE.
|
|
29
|
+
* Reads are harmless from anywhere; only recording is restricted.
|
|
30
|
+
*/
|
|
31
|
+
export function isMainAgentSession(session) { return !isSubagentSession(session) }
|
|
32
|
+
|
|
33
|
+
/** Sync fallback (no workspaceRegistry): basename-derived key, or 'global'. */
|
|
34
|
+
export function workspaceFromSession(store, session) {
|
|
35
|
+
const cwd = session?.header?.cwd
|
|
36
|
+
if (!cwd) return 'global'
|
|
37
|
+
return resolveKeyFor(store.root, { cwd, basename: basename(cwd) }) ?? 'global'
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Resolve + lazily associate a session's workspace. Only a DSH-registered workspace
|
|
42
|
+
* (resolveByPath hit) triggers association; everything else stays global.
|
|
43
|
+
*/
|
|
44
|
+
export async function autoAssociateSession(store, ctx, session) {
|
|
45
|
+
const cwd = session?.header?.cwd
|
|
46
|
+
if (!cwd) return 'global'
|
|
47
|
+
const registry = ctx?.get('workspaceRegistry')
|
|
48
|
+
if (!registry || typeof registry.resolveByPath !== 'function') return 'global'
|
|
49
|
+
let ws
|
|
50
|
+
try { ws = await registry.resolveByPath(cwd) } catch { ws = undefined }
|
|
51
|
+
if (!ws) return 'global'
|
|
52
|
+
const key = resolveKeyFor(store.root, { cwd: ws.path, basename: basename(ws.path) })
|
|
53
|
+
if (!key) return 'global'
|
|
54
|
+
const known = listWorkspaces(store).some((w) => w.key === key)
|
|
55
|
+
if (!known) {
|
|
56
|
+
try { associate(store, { key, title: ws.title ?? key, cwd: ws.path, workspaceId: ws.id }) }
|
|
57
|
+
catch { /* keep going; the key is still usable */ }
|
|
58
|
+
}
|
|
59
|
+
return key
|
|
60
|
+
}
|