dsh-context-guardian 0.1.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +555 -0
- package/LICENSE +21 -0
- package/README.md +412 -0
- package/THIRD_PARTY_NOTICES.md +30 -0
- package/cg_anchors.js +203 -0
- package/cg_autorecall.js +215 -0
- package/cg_configure.py +205 -0
- package/cg_doctor.py +449 -0
- package/cg_handoff.js +253 -0
- package/cg_memory.js +509 -0
- package/cg_postcheck.js +173 -0
- package/cg_provenance.js +206 -0
- package/cg_recall.js +559 -0
- package/cg_recall_cli.py +124 -0
- package/configure.py +10 -0
- package/context_guardian.py +1991 -0
- package/cordis.patch.yml +42 -0
- package/docs/dsh-integration.md +229 -0
- package/docs/dsh-upstream-append-only.md +37 -0
- package/engine.js +1407 -0
- package/index.js +255 -0
- package/modules/cg_bridge.py +251 -0
- package/package.json +88 -0
- package/requirements.txt +4 -0
- package/setup.mjs +629 -0
- package/update_check.js +184 -0
- package/vendor/LICENSE.dsh-compaction-instant +28 -0
- package/vendor/compiler.js +851 -0
- package/vendor/region.js +455 -0
package/cg_anchors.js
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
// cg_anchors.js -- anchor recovery for the context-guardian.
|
|
2
|
+
// After a conversation region is compacted into a checkpoint, find the named
|
|
3
|
+
// things (files, identifiers, URLs, code spans) that the checkpoint dropped but
|
|
4
|
+
// that later turns still reference, plus the constraints/decisions the summary
|
|
5
|
+
// forgot. Pure library: no imports, no I/O, no clock, no mutated globals.
|
|
6
|
+
|
|
7
|
+
export const ANCHORS_REV = 'cg-anchors-1'
|
|
8
|
+
|
|
9
|
+
const UNIQUE_CAP = 2000
|
|
10
|
+
const RETURN_CAP = 200
|
|
11
|
+
const LOST_CAP = 40
|
|
12
|
+
const ITEM_CAP = 20
|
|
13
|
+
|
|
14
|
+
const CARRIED_HEADER = '[context-guardian: carried facts -- named before this checkpoint and still referenced after it, but missing from the summary above]'
|
|
15
|
+
|
|
16
|
+
const RE_URL = /https?:\/\/[^\s)\]>"'`]+/g
|
|
17
|
+
const RE_PATH = /(?:[A-Za-z]:[\\/])?(?:[\w.\-]+[\\/])+[\w.\-]+\.[A-Za-z0-9]{1,8}\b/g
|
|
18
|
+
const RE_FILE = /\b[\w\-]+\.(py|js|mjs|cjs|ts|tsx|jsx|json|md|yml|yaml|toml|txt|sh|bat|ps1|csv|html|css|sql|go|rs|java|c|cpp|h)\b/g
|
|
19
|
+
const RE_CALL = /\b([A-Za-z_][A-Za-z0-9_]{2,})\(/g
|
|
20
|
+
|
|
21
|
+
const RE_TRAILING = /[.,;:!?]+$/
|
|
22
|
+
const RE_WHITESPACE = /\s+/g
|
|
23
|
+
|
|
24
|
+
const K_CODE = 1
|
|
25
|
+
const K_URL = 2
|
|
26
|
+
const K_PATH = 3
|
|
27
|
+
const K_FILE = 4
|
|
28
|
+
const K_CALL = 5
|
|
29
|
+
|
|
30
|
+
// Defensive coercion: never throw, never leak a non-string into a regex.
|
|
31
|
+
function asText(value) {
|
|
32
|
+
if (typeof value === 'string') return value
|
|
33
|
+
if (value === null || value === undefined) return ''
|
|
34
|
+
try {
|
|
35
|
+
return String(value)
|
|
36
|
+
} catch {
|
|
37
|
+
return ''
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function asList(value) {
|
|
42
|
+
if (Array.isArray(value)) return value
|
|
43
|
+
if (value === null || value === undefined) return []
|
|
44
|
+
return [value]
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function asItems(value) {
|
|
48
|
+
return asList(value).map((entry) => {
|
|
49
|
+
if (entry !== null && typeof entry === 'object') return entry
|
|
50
|
+
return { cat: '', text: asText(entry) }
|
|
51
|
+
})
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function norm(s) {
|
|
55
|
+
return asText(s).toLowerCase().replace(RE_WHITESPACE, ' ').trim().replace(RE_TRAILING, '')
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function pushCandidate(bag, text, kind, start) {
|
|
59
|
+
if (text === '') return
|
|
60
|
+
bag.push({ text, lc: text.toLowerCase(), kind, start, seq: bag.length })
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Kind 1: code spans, one per odd split segment that has a closing backtick.
|
|
64
|
+
function collectCodeSpans(text, bag) {
|
|
65
|
+
const segments = text.split('`')
|
|
66
|
+
let offset = 0
|
|
67
|
+
for (let i = 0; i < segments.length; i += 1) {
|
|
68
|
+
const segment = segments[i]
|
|
69
|
+
const start = offset
|
|
70
|
+
offset += segment.length + 1
|
|
71
|
+
if (i % 2 !== 1 || i >= segments.length - 1) continue
|
|
72
|
+
if (segment.includes('\n')) continue
|
|
73
|
+
const trimmed = segment.trim()
|
|
74
|
+
if (trimmed.length < 3 || trimmed.length > 120) continue
|
|
75
|
+
pushCandidate(bag, trimmed, K_CODE, start)
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function collectMatches(text, re, kind, bag, group) {
|
|
80
|
+
for (const m of text.matchAll(re)) {
|
|
81
|
+
let value = group === 0 ? m[0] : m[group]
|
|
82
|
+
if (value === undefined) continue
|
|
83
|
+
if (kind === K_URL) value = value.replace(RE_TRAILING, '')
|
|
84
|
+
if (value === '') continue
|
|
85
|
+
pushCandidate(bag, value, kind, m.index)
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// An occurrence is redundant only when it is a strict sub-string of a longer
|
|
90
|
+
// anchor AND it is not a distinct mention in its own right: either it was found
|
|
91
|
+
// by a different extractor (a path containing a bare file name), or the shorter
|
|
92
|
+
// match physically lies inside the longer one in this text.
|
|
93
|
+
function insideShorter(cand, other) {
|
|
94
|
+
return cand.start >= other.start && cand.start + cand.text.length <= other.start + other.text.length
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function isRedundant(cand, unique) {
|
|
98
|
+
for (const other of unique) {
|
|
99
|
+
if (other === cand) continue
|
|
100
|
+
if (other.lc.length <= cand.lc.length) continue
|
|
101
|
+
if (!other.lc.includes(cand.lc)) continue
|
|
102
|
+
if (cand.kind !== other.kind || insideShorter(cand, other)) return true
|
|
103
|
+
}
|
|
104
|
+
return false
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function extractAnchors(text) {
|
|
108
|
+
if (typeof text !== 'string' || text === '') return []
|
|
109
|
+
|
|
110
|
+
const bag = []
|
|
111
|
+
collectCodeSpans(text, bag)
|
|
112
|
+
collectMatches(text, RE_URL, K_URL, bag, 0)
|
|
113
|
+
collectMatches(text, RE_PATH, K_PATH, bag, 0)
|
|
114
|
+
collectMatches(text, RE_FILE, K_FILE, bag, 0)
|
|
115
|
+
collectMatches(text, RE_CALL, K_CALL, bag, 1)
|
|
116
|
+
if (bag.length === 0) return []
|
|
117
|
+
|
|
118
|
+
bag.sort((a, b) => (a.start - b.start) || (a.seq - b.seq))
|
|
119
|
+
|
|
120
|
+
const seen = new Set()
|
|
121
|
+
const unique = []
|
|
122
|
+
for (const cand of bag) {
|
|
123
|
+
if (seen.has(cand.lc)) continue
|
|
124
|
+
seen.add(cand.lc)
|
|
125
|
+
unique.push(cand)
|
|
126
|
+
if (unique.length >= UNIQUE_CAP) break
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const out = []
|
|
130
|
+
for (const cand of unique) {
|
|
131
|
+
if (isRedundant(cand, unique)) continue
|
|
132
|
+
out.push(cand.text)
|
|
133
|
+
if (out.length >= RETURN_CAP) break
|
|
134
|
+
}
|
|
135
|
+
return out
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function findLostAnchors({ regionTexts = [], laterTexts = [], checkpointText = '', memoryTexts = [] } = {}) {
|
|
139
|
+
const region = asList(regionTexts).map(asText).join('\n')
|
|
140
|
+
const later = asList(laterTexts).map(asText).join('\n').toLowerCase()
|
|
141
|
+
const haystack = (asText(checkpointText) + '\n' + asList(memoryTexts).map(asText).join('\n')).toLowerCase()
|
|
142
|
+
|
|
143
|
+
const recurring = []
|
|
144
|
+
for (const anchor of extractAnchors(region)) {
|
|
145
|
+
if (later.includes(anchor.toLowerCase())) recurring.push(anchor)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const kept = []
|
|
149
|
+
const lost = []
|
|
150
|
+
for (const anchor of recurring) {
|
|
151
|
+
if (haystack.includes(anchor.toLowerCase())) {
|
|
152
|
+
kept.push(anchor)
|
|
153
|
+
} else if (lost.length < LOST_CAP) {
|
|
154
|
+
lost.push(anchor)
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return { recurring, kept, lost }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export function lostConstraints({ regionItems = [], checkpointText = '' } = {}) {
|
|
161
|
+
const target = norm(checkpointText)
|
|
162
|
+
const out = []
|
|
163
|
+
for (const item of asItems(regionItems)) {
|
|
164
|
+
const cat = asText(item.cat)
|
|
165
|
+
if (cat !== 'constraints' && cat !== 'decisions') continue
|
|
166
|
+
const text = norm(item.text)
|
|
167
|
+
if (text === '') continue
|
|
168
|
+
if (target.includes(text)) continue
|
|
169
|
+
out.push(item)
|
|
170
|
+
if (out.length >= ITEM_CAP) break
|
|
171
|
+
}
|
|
172
|
+
return out
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export function renderCarried(lost = [], items = [], maxChars = 1600) {
|
|
176
|
+
const anchors = asList(lost).map(asText).filter((s) => s !== '')
|
|
177
|
+
const carried = []
|
|
178
|
+
for (const item of asItems(items)) {
|
|
179
|
+
const cat = asText(item.cat)
|
|
180
|
+
if (cat !== 'constraints' && cat !== 'decisions') continue
|
|
181
|
+
const text = asText(item.text)
|
|
182
|
+
if (text === '') continue
|
|
183
|
+
carried.push(cat === 'constraints' ? '- constraint: ' + text : '- decision: ' + text)
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const candidates = []
|
|
187
|
+
for (const anchor of anchors) candidates.push('- `' + anchor + '`')
|
|
188
|
+
for (const line of carried) candidates.push(line)
|
|
189
|
+
if (candidates.length === 0) return ''
|
|
190
|
+
|
|
191
|
+
const limit = Number.isFinite(maxChars) ? maxChars : 0
|
|
192
|
+
const lines = [CARRIED_HEADER]
|
|
193
|
+
let width = CARRIED_HEADER.length
|
|
194
|
+
for (const line of candidates) {
|
|
195
|
+
// join('\n') costs one separator per extra line: width + line + count.
|
|
196
|
+
if (width + 1 + line.length > limit) break
|
|
197
|
+
lines.push(line)
|
|
198
|
+
width += 1 + line.length
|
|
199
|
+
}
|
|
200
|
+
const leftOut = candidates.length - (lines.length - 1)
|
|
201
|
+
if (leftOut > 0) lines.push('- (+' + leftOut + ' more; recall by seq)')
|
|
202
|
+
return lines.join('\n')
|
|
203
|
+
}
|
package/cg_autorecall.js
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { pathToFileURL } from 'node:url'
|
|
2
|
+
import { renderMessage } from './cg_recall.js'
|
|
3
|
+
|
|
4
|
+
export const RECALL_TAG = '[context-guardian recall]'
|
|
5
|
+
export const STOP = Object.freeze(['JSON', 'YAML', 'HTML', 'HTTP', 'HTTPS', 'URL', 'API', 'CSV', 'SQL', 'TODO', 'NOTE', 'README'])
|
|
6
|
+
|
|
7
|
+
const MAX_NAMES = 12
|
|
8
|
+
const HIT_CHARS = 200
|
|
9
|
+
const WINDOW = 80
|
|
10
|
+
|
|
11
|
+
export function extractNames(text) {
|
|
12
|
+
const s = String(text ?? '')
|
|
13
|
+
const regexes = [
|
|
14
|
+
/\bdoc\d+\b/gi,
|
|
15
|
+
/\bL\d{3}\b/g,
|
|
16
|
+
/\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b/g,
|
|
17
|
+
/\b[A-Z][A-Z0-9]{2,}(?:[-_][A-Z0-9]+)*\b/g,
|
|
18
|
+
/"([^"\n]{3,60})"|'([^'\n]{3,60})'|`([^`\n]{3,60})`/g
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
const matches = []
|
|
22
|
+
|
|
23
|
+
for (let i = 0; i < regexes.length; i++) {
|
|
24
|
+
const re = regexes[i]
|
|
25
|
+
let match
|
|
26
|
+
while ((match = re.exec(s)) !== null) {
|
|
27
|
+
// For the quoted alternation the interesting value is the captured inner
|
|
28
|
+
// text; the other four keep the whole match.
|
|
29
|
+
const value = i === 4 ? (match[1] !== undefined ? match[1] : match[2] !== undefined ? match[2] : match[3]) : match[0]
|
|
30
|
+
matches.push({ index: match.index, value })
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
matches.sort((a, b) => a.index - b.index)
|
|
35
|
+
|
|
36
|
+
const seen = new Set()
|
|
37
|
+
const result = []
|
|
38
|
+
|
|
39
|
+
for (const m of matches) {
|
|
40
|
+
if (result.length >= MAX_NAMES) break
|
|
41
|
+
if (m.value === undefined) continue
|
|
42
|
+
if (seen.has(m.value) || STOP.includes(m.value)) continue
|
|
43
|
+
seen.add(m.value)
|
|
44
|
+
result.push(m.value)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return result
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function linePairs(text) {
|
|
51
|
+
const s = String(text ?? '')
|
|
52
|
+
const re = /\b(doc\d+)\b[^\n]{0,40}?\b(L\d{3})\b/gi
|
|
53
|
+
const pairs = []
|
|
54
|
+
const seen = new Set()
|
|
55
|
+
let match
|
|
56
|
+
|
|
57
|
+
while ((match = re.exec(s)) !== null) {
|
|
58
|
+
const doc = match[1]
|
|
59
|
+
const line = match[2].toUpperCase()
|
|
60
|
+
const key = doc.toLowerCase() + ' ' + line.toLowerCase()
|
|
61
|
+
if (seen.has(key)) continue
|
|
62
|
+
seen.add(key)
|
|
63
|
+
pairs.push({ doc, line })
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return pairs
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function staleRecall(input = {}) {
|
|
70
|
+
const inp = input && typeof input === 'object' ? input : {}
|
|
71
|
+
const userText = String(inp.userText ?? '')
|
|
72
|
+
const live = String(inp.liveText ?? '').toLowerCase()
|
|
73
|
+
const maxHits = Number.isFinite(inp.maxHits) ? inp.maxHits : 3
|
|
74
|
+
const maxChars = Number.isFinite(inp.maxChars) ? inp.maxChars : 600
|
|
75
|
+
|
|
76
|
+
const nodes = (Array.isArray(inp.nodes) ? inp.nodes : [])
|
|
77
|
+
.filter((n) => n && n.message)
|
|
78
|
+
.sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
|
|
79
|
+
|
|
80
|
+
const rendered = nodes.map((n) => {
|
|
81
|
+
let text
|
|
82
|
+
try {
|
|
83
|
+
text = renderMessage(n.message)
|
|
84
|
+
} catch (err) {
|
|
85
|
+
text = ''
|
|
86
|
+
}
|
|
87
|
+
return { seq: n.seq, text: String(text ?? '') }
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
const hits = []
|
|
91
|
+
let used = 0
|
|
92
|
+
|
|
93
|
+
function add(label, seq, text) {
|
|
94
|
+
if (hits.length >= maxHits) return
|
|
95
|
+
const t = String(text).slice(0, HIT_CHARS)
|
|
96
|
+
if (used + t.length > maxChars) return
|
|
97
|
+
hits.push({ label, seq, text: t })
|
|
98
|
+
used += t.length
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Step 1: "docNN LNNN" pairs pull the exact archived line back in.
|
|
102
|
+
const pairs = linePairs(userText)
|
|
103
|
+
const pairNames = new Set()
|
|
104
|
+
|
|
105
|
+
for (const { doc, line } of pairs) {
|
|
106
|
+
pairNames.add(doc.toLowerCase())
|
|
107
|
+
pairNames.add(line.toLowerCase())
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
for (const { doc, line } of pairs) {
|
|
111
|
+
const lowerDoc = doc.toLowerCase()
|
|
112
|
+
// Built from the upper-cased line number: the regex is not case-insensitive
|
|
113
|
+
// and archived lines start with "L017", not "l017".
|
|
114
|
+
const re = new RegExp('^\\s*' + line + '\\b.*$', 'm')
|
|
115
|
+
|
|
116
|
+
let foundEntry = null
|
|
117
|
+
for (const entry of rendered) {
|
|
118
|
+
if (entry.text.toLowerCase().includes(lowerDoc) && re.test(entry.text)) {
|
|
119
|
+
foundEntry = entry
|
|
120
|
+
break
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
if (!foundEntry) continue
|
|
124
|
+
|
|
125
|
+
const found = foundEntry.text.match(re)
|
|
126
|
+
const lineText = (found && found[0] ? found[0] : '').trim()
|
|
127
|
+
if (!lineText) continue
|
|
128
|
+
if (live.includes(lineText.toLowerCase())) continue
|
|
129
|
+
add(doc + ' ' + line, foundEntry.seq, lineText)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Step 2: every other name that is neither a pair name nor still live.
|
|
133
|
+
for (const name of extractNames(userText)) {
|
|
134
|
+
const key = name.toLowerCase()
|
|
135
|
+
if (pairNames.has(key) || live.includes(key)) continue
|
|
136
|
+
|
|
137
|
+
let foundEntry = null
|
|
138
|
+
for (const entry of rendered) {
|
|
139
|
+
if (entry.text.toLowerCase().includes(key)) {
|
|
140
|
+
foundEntry = entry
|
|
141
|
+
break
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (!foundEntry) continue
|
|
145
|
+
|
|
146
|
+
const at = foundEntry.text.toLowerCase().indexOf(key)
|
|
147
|
+
if (at === -1) continue
|
|
148
|
+
|
|
149
|
+
const len = foundEntry.text.length
|
|
150
|
+
const start = Math.max(0, at - WINDOW)
|
|
151
|
+
const end = Math.min(len, at + key.length + WINDOW)
|
|
152
|
+
const body = foundEntry.text.slice(start, end).replace(/[\n\t]+/g, ' ')
|
|
153
|
+
|
|
154
|
+
add(name, foundEntry.seq, (start > 0 ? '…' : '') + body + (end < len ? '…' : ''))
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
if (hits.length === 0) return ''
|
|
158
|
+
|
|
159
|
+
return RECALL_TAG + ' from earlier in this session (compacted out of view):\n' +
|
|
160
|
+
hits.map((h) => '- ' + h.label + ' (seq ' + h.seq + '): ' + h.text).join('\n')
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function userNode(seq, text) {
|
|
164
|
+
return { seq, message: { role: 'user', content: [{ type: 'text', text }] } }
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export function selftest() {
|
|
168
|
+
const FENCE = '`'.repeat(3)
|
|
169
|
+
const doc = (k) => Array.from({ length: 100 }, (_, i) => 'L' + String(i + 1).padStart(3, '0') + ' Word' + k + ' line ' + (i + 1) + '; code C' + k + String(i + 1).padStart(3, '0') + '.').join('\n')
|
|
170
|
+
const nodes = [1, 2, 4].map((k, i) => userNode(10 * (i + 1), 'Reading task ' + k + '. Below is lines 1-100 of the read-only file doc0' + k + '.txt.\n' + FENCE + '\n' + doc(k) + '\n' + FENCE))
|
|
171
|
+
const ask = 'Final check, part 2. Reply with ONLY a JSON object: {"doc01 L017": ?, "doc02 L058": ?, "doc04 L081": ?}.'
|
|
172
|
+
const checks = [
|
|
173
|
+
['names_in_order_unique_stoplisted', () => JSON.stringify(extractNames('see HeronNest and MAPLE-123, then "the blue file" and doc03 L044 and HeronNest again in JSON')) === JSON.stringify(['HeronNest', 'MAPLE-123', 'the blue file', 'doc03', 'L044'])],
|
|
174
|
+
['line_pairs', () => JSON.stringify(linePairs(ask)) === JSON.stringify([{ doc: 'doc01', line: 'L017' }, { doc: 'doc02', line: 'L058' }, { doc: 'doc04', line: 'L081' }])],
|
|
175
|
+
['bench_three_early_lines_recalled', () => {
|
|
176
|
+
const r = staleRecall({ userText: ask, liveText: 'summary: three reading tasks were done', nodes })
|
|
177
|
+
return r.startsWith(RECALL_TAG) && r.includes('code C1017') && r.includes('code C2058') && r.includes('code C4081')
|
|
178
|
+
}],
|
|
179
|
+
['live_line_not_recalled', () => {
|
|
180
|
+
const live = 'L017 Word1 line 17; code C1017.'
|
|
181
|
+
const r = staleRecall({ userText: ask, liveText: live, nodes })
|
|
182
|
+
return !r.includes('C1017') && r.includes('C2058')
|
|
183
|
+
}],
|
|
184
|
+
['max_three_hits_and_600_chars', () => {
|
|
185
|
+
const r = staleRecall({ userText: ask + ' also MAPLE-123', liveText: '', nodes: nodes.concat([userNode(50, 'the codename is MAPLE-123')]) })
|
|
186
|
+
return r.split('\n').length === 4 && r.length <= 600 + 200
|
|
187
|
+
}],
|
|
188
|
+
['name_absent_from_live_recalled', () => {
|
|
189
|
+
const r = staleRecall({ userText: 'what was MAPLE-123 again?', liveText: 'nothing here', nodes: [userNode(5, 'For reference: the release codename for project Heron is MAPLE-123.')] })
|
|
190
|
+
return r.includes('MAPLE-123 (seq 5)')
|
|
191
|
+
}],
|
|
192
|
+
['name_still_live_skipped', () => staleRecall({ userText: 'what was MAPLE-123 again?', liveText: 'Heron is MAPLE-123', nodes: [userNode(5, 'Heron is MAPLE-123')] }) === ''],
|
|
193
|
+
['nothing_found_empty_string', () => staleRecall({ userText: 'hello there', liveText: '', nodes }) === '' && staleRecall() === ''],
|
|
194
|
+
]
|
|
195
|
+
let passed = 0
|
|
196
|
+
for (const [name, fn] of checks) {
|
|
197
|
+
let ok = false
|
|
198
|
+
let label = name
|
|
199
|
+
try {
|
|
200
|
+
ok = Boolean(fn())
|
|
201
|
+
} catch (err) {
|
|
202
|
+
label = name + ' (raised ' + (err && err.name) + ': ' + (err && err.message) + ')'
|
|
203
|
+
}
|
|
204
|
+
if (ok) passed += 1
|
|
205
|
+
console.log((ok ? 'ok ' : 'FAIL ') + label)
|
|
206
|
+
}
|
|
207
|
+
const failed = checks.length - passed
|
|
208
|
+
console.log('cg_autorecall selftest: ' + checks.length + ' checks, ' + passed + ' passed, ' + failed + ' failed')
|
|
209
|
+
return { checks: checks.length, passed, failed }
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href && process.argv.includes('--selftest')) {
|
|
213
|
+
const r = selftest()
|
|
214
|
+
process.exit(r.failed === 0 && r.checks > 0 ? 0 : 1)
|
|
215
|
+
}
|
package/cg_configure.py
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
cg_configure.py (shipped in the wheel as `context-guardian-configure`; `python configure.py` in a clone still works)
|
|
4
|
+
============
|
|
5
|
+
Interactive first-run setup for Context Guardian. Asks a few questions about
|
|
6
|
+
YOUR hardware/backend and writes them to .env, so you never hand-edit a config
|
|
7
|
+
file and guess.
|
|
8
|
+
|
|
9
|
+
python configure.py
|
|
10
|
+
|
|
11
|
+
Every question has a sensible default (press Enter to accept). Numeric answers
|
|
12
|
+
are validated, so a typo tells you rather than crashing Guardian at startup.
|
|
13
|
+
Re-run any time to change your answers, or edit .env directly.
|
|
14
|
+
"""
|
|
15
|
+
import sys
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
REPO_ROOT = Path(__file__).resolve().parent
|
|
19
|
+
ENV_PATH = REPO_ROOT / ".env"
|
|
20
|
+
|
|
21
|
+
# Context-window presets, smallest to largest, with rough guidance.
|
|
22
|
+
#
|
|
23
|
+
# The VRAM notes are DELIBERATELY approximate and framed for the audience this
|
|
24
|
+
# tool serves: someone running a quantized ~7-14B model on a consumer GPU. The
|
|
25
|
+
# context window drives the KV-cache size, which is only PART of your VRAM use
|
|
26
|
+
# (the model weights are the other, larger part), and it scales with the
|
|
27
|
+
# model's size too -- a 30B model at 32K needs far more than an 8B at 32K. So
|
|
28
|
+
# treat these as "you'll want at least this much headroom on a typical mid-size
|
|
29
|
+
# model", not a guarantee. The real number for YOUR setup comes from
|
|
30
|
+
# `ollama ps` while your model is loaded. Bigger model => bump a tier.
|
|
31
|
+
CONTEXT_OPTIONS = [
|
|
32
|
+
(4096, "Light use / short chats. Roughly 8 GB+ VRAM on a small model."),
|
|
33
|
+
(8192, "Everyday agent use. Roughly 10 GB+ VRAM."),
|
|
34
|
+
(16384, "Comfortable coding/agent work. Roughly 12 GB+ VRAM."),
|
|
35
|
+
(32768, "Full agent sessions (default). Roughly 12-16 GB VRAM on a 7-14B model."),
|
|
36
|
+
(65536, "Large context. Roughly 20-24 GB VRAM, or heavy quant."),
|
|
37
|
+
(131072, "Very large context. 24 GB+ VRAM / aggressive quantization."),
|
|
38
|
+
]
|
|
39
|
+
DEFAULT_CTX = 32768
|
|
40
|
+
STANDARD_CTX = {tok for tok, _ in CONTEXT_OPTIONS}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def ask_str(prompt: str, default: str) -> str:
|
|
44
|
+
print("\n" + prompt)
|
|
45
|
+
return input(f" [{default}]: ").strip() or default
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def ask_int(prompt: str, default: int, lo: int = 1, hi: int = 10_000_000) -> int:
|
|
49
|
+
"""A validated positive integer. Re-asks on garbage instead of writing a
|
|
50
|
+
value that makes Guardian throw ValueError at import."""
|
|
51
|
+
while True:
|
|
52
|
+
raw = input(f"\n{prompt}\n [{default}]: ").strip()
|
|
53
|
+
if not raw:
|
|
54
|
+
return default
|
|
55
|
+
try:
|
|
56
|
+
val = int(raw.replace(",", "").replace("_", ""))
|
|
57
|
+
except ValueError:
|
|
58
|
+
print(f" '{raw}' is not a whole number. Digits only "
|
|
59
|
+
f"(e.g. 8192), no units or letters.")
|
|
60
|
+
continue
|
|
61
|
+
if not (lo <= val <= hi):
|
|
62
|
+
print(f" {val} is out of range. Expected {lo}..{hi}.")
|
|
63
|
+
continue
|
|
64
|
+
return val
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def ask_float(prompt: str, default: float, lo: float, hi: float) -> float:
|
|
68
|
+
while True:
|
|
69
|
+
raw = input(f"\n{prompt}\n [{default}]: ").strip()
|
|
70
|
+
if not raw:
|
|
71
|
+
return default
|
|
72
|
+
try:
|
|
73
|
+
val = float(raw)
|
|
74
|
+
except ValueError:
|
|
75
|
+
print(f" '{raw}' is not a number.")
|
|
76
|
+
continue
|
|
77
|
+
if not (lo <= val <= hi):
|
|
78
|
+
print(f" {val} is out of range. Expected {lo}..{hi}.")
|
|
79
|
+
continue
|
|
80
|
+
return val
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def ask_keep_recent():
|
|
84
|
+
"""None = the token-budget default (the key is NOT written, so the proxy
|
|
85
|
+
keeps the newest messages that fit in ~20% of the usable window, at least
|
|
86
|
+
4). A number = keep exactly that many messages (the old behaviour)."""
|
|
87
|
+
while True:
|
|
88
|
+
raw = input("\nWhat stays verbatim (never summarized)?\n"
|
|
89
|
+
" Enter = auto (newest ~20% of the window, at least 4 messages),\n"
|
|
90
|
+
" or a fixed number of messages\n [auto]: ").strip().lower()
|
|
91
|
+
if raw in ("", "auto"):
|
|
92
|
+
return None
|
|
93
|
+
if raw.isdigit() and 1 <= int(raw) <= 1000:
|
|
94
|
+
return int(raw)
|
|
95
|
+
print(f" '{raw}' is not a number from 1 to 1000 or 'auto'.")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def ask_context() -> int:
|
|
99
|
+
"""A menu of context-window presets with VRAM guidance, plus a custom
|
|
100
|
+
option that CONFIRMS a non-standard value back to the user."""
|
|
101
|
+
print("\nContext window size (in TOKENS) of the model you're actually running.")
|
|
102
|
+
print("This is the single most important setting -- it MUST match how your")
|
|
103
|
+
print("backend/model is really configured (check `ollama ps` while loaded),")
|
|
104
|
+
print("not the model's theoretical max.\n")
|
|
105
|
+
for i, (tok, blurb) in enumerate(CONTEXT_OPTIONS, 1):
|
|
106
|
+
star = " <- default" if tok == DEFAULT_CTX else ""
|
|
107
|
+
print(f" {i}) {tok:>7,} tokens {blurb}{star}")
|
|
108
|
+
print(f" {len(CONTEXT_OPTIONS) + 1}) other Enter a custom number")
|
|
109
|
+
|
|
110
|
+
default_idx = [i for i, (t, _) in enumerate(CONTEXT_OPTIONS, 1)
|
|
111
|
+
if t == DEFAULT_CTX][0]
|
|
112
|
+
while True:
|
|
113
|
+
raw = input(f"\n Choose 1-{len(CONTEXT_OPTIONS) + 1} "
|
|
114
|
+
f"[{default_idx}]: ").strip()
|
|
115
|
+
if not raw:
|
|
116
|
+
return DEFAULT_CTX
|
|
117
|
+
if not raw.isdigit():
|
|
118
|
+
print(" Enter the NUMBER of an option above.")
|
|
119
|
+
continue
|
|
120
|
+
choice = int(raw)
|
|
121
|
+
if 1 <= choice <= len(CONTEXT_OPTIONS):
|
|
122
|
+
return CONTEXT_OPTIONS[choice - 1][0]
|
|
123
|
+
if choice == len(CONTEXT_OPTIONS) + 1:
|
|
124
|
+
# Ceiling deliberately far above any real context window: an
|
|
125
|
+
# unusually large number should reach the CONFIRMATION below
|
|
126
|
+
# (showing the user their actual figure), not be silently rejected
|
|
127
|
+
# as out of range. Letters/negatives are still refused by ask_int.
|
|
128
|
+
custom = ask_int(" Custom context window, in tokens", DEFAULT_CTX,
|
|
129
|
+
lo=256, hi=100_000_000)
|
|
130
|
+
if custom in STANDARD_CTX:
|
|
131
|
+
return custom
|
|
132
|
+
# Non-standard value: confirm it back with the ACTUAL number, so a
|
|
133
|
+
# fat-fingered 132768 or 13212348 is caught before it is written.
|
|
134
|
+
print(f"\n {custom:,} is not one of the standard sizes.")
|
|
135
|
+
ok = input(f" Are you sure you want a context window of "
|
|
136
|
+
f"{custom:,} tokens? [y/N]: ").strip().lower()
|
|
137
|
+
if ok == "y":
|
|
138
|
+
return custom
|
|
139
|
+
print(" Okay, let's pick again.")
|
|
140
|
+
continue
|
|
141
|
+
print(f" {choice} is not one of the options.")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def main(argv=None) -> int:
|
|
145
|
+
argv = sys.argv[1:] if argv is None else list(argv)
|
|
146
|
+
if "-h" in argv or "--help" in argv:
|
|
147
|
+
print((__doc__ or "").strip())
|
|
148
|
+
print(f"\n.env is written to: {ENV_PATH}")
|
|
149
|
+
return 0
|
|
150
|
+
print("Context Guardian -- first-run setup")
|
|
151
|
+
print("=" * 40)
|
|
152
|
+
print("Press Enter on any question to accept the default in [brackets].")
|
|
153
|
+
|
|
154
|
+
if ENV_PATH.exists():
|
|
155
|
+
if input(f"\n.env already exists at {ENV_PATH}. Overwrite? [y/N]: "
|
|
156
|
+
).strip().lower() != "y":
|
|
157
|
+
print("Left the existing .env untouched. Edit it by hand to tweak "
|
|
158
|
+
"one value.")
|
|
159
|
+
return 0
|
|
160
|
+
|
|
161
|
+
answers = {}
|
|
162
|
+
answers["GUARDIAN_UPSTREAM_URL"] = ask_str(
|
|
163
|
+
"OpenAI-compatible backend Guardian should forward to\n"
|
|
164
|
+
" (your Ollama /v1 endpoint, or another proxy in front of it)",
|
|
165
|
+
"http://localhost:11434/v1")
|
|
166
|
+
answers["GUARDIAN_NUM_CTX"] = str(ask_context())
|
|
167
|
+
answers["GUARDIAN_PORT"] = str(ask_int(
|
|
168
|
+
"Port Context Guardian itself should listen on", 8786,
|
|
169
|
+
lo=1, hi=65535))
|
|
170
|
+
answers["GUARDIAN_COMPACT_THRESHOLD"] = str(ask_float(
|
|
171
|
+
"Fraction of the context window at which Guardian compacts (0.1-1.0)",
|
|
172
|
+
0.85, lo=0.1, hi=1.0))
|
|
173
|
+
keep = ask_keep_recent()
|
|
174
|
+
if keep is not None:
|
|
175
|
+
answers["GUARDIAN_KEEP_RECENT_MESSAGES"] = str(keep)
|
|
176
|
+
answers["GUARDIAN_UPSTREAM_TIMEOUT"] = str(ask_int(
|
|
177
|
+
"Seconds to wait for the upstream backend to respond\n"
|
|
178
|
+
" (raise this if a slow local \"thinking\" model gives 500s only on "
|
|
179
|
+
"real requests)", 600, lo=1, hi=86400))
|
|
180
|
+
answers["GUARDIAN_LOG_PATH"] = ask_str(
|
|
181
|
+
"Where to log compaction events (JSON lines)",
|
|
182
|
+
"logs/context_guardian_log.json")
|
|
183
|
+
|
|
184
|
+
lines = ["# Generated by configure.py -- edit freely, or re-run configure.py.\n"]
|
|
185
|
+
for k, v in answers.items():
|
|
186
|
+
lines.append(f"{k}={v}\n")
|
|
187
|
+
ENV_PATH.write_text("".join(lines), encoding="utf-8")
|
|
188
|
+
|
|
189
|
+
ctx = int(answers["GUARDIAN_NUM_CTX"])
|
|
190
|
+
print(f"\nWrote {ENV_PATH}")
|
|
191
|
+
if ctx not in STANDARD_CTX:
|
|
192
|
+
print(f" (custom context window: {ctx:,} tokens)")
|
|
193
|
+
print("\nNext steps:")
|
|
194
|
+
print(" 1. pip install -r requirements.txt")
|
|
195
|
+
print(" 2. Make sure your real backend is already running.")
|
|
196
|
+
print(" 3. python context_guardian.py")
|
|
197
|
+
print(f" 4. Point your CLI's OPENAI_BASE_URL at "
|
|
198
|
+
f"http://localhost:{answers['GUARDIAN_PORT']}/v1")
|
|
199
|
+
print("\nSee README.md's \"Testing before you trust it\" section before "
|
|
200
|
+
"pointing a real session at it.")
|
|
201
|
+
return 0
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
if __name__ == "__main__":
|
|
205
|
+
sys.exit(main())
|