@gotcos/glasses-server 6.21.35 → 6.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +107 -0
- package/README.md +52 -2
- package/package.json +1 -1
- package/server/lib/context-files.ts +436 -0
- package/server/lib/cos-context-browser.ts +120 -9
- package/server/lib/python-bridge.ts +124 -15
- package/server/routes/memory.ts +17 -5
- package/server/routes/threads.ts +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,110 @@
|
|
|
1
|
+
## 6.22.1
|
|
2
|
+
|
|
3
|
+
Notes attached from somewhere else by a symlink are now read properly. Found by
|
|
4
|
+
Queen within hours of 6.22.0, on the very first real setup.
|
|
5
|
+
|
|
6
|
+
- **A symlinked subfolder or file was silently skipped.** `readdirSync` reports a
|
|
7
|
+
symlink as `isSymbolicLink()`, never as `isDirectory()` or `isFile()`, so the walk
|
|
8
|
+
ignored every link it met. A top-level `memory -> /elsewhere` link worked only by
|
|
9
|
+
accident, because the folder LOOKUP uses `statSync` and follows links while the
|
|
10
|
+
walk did not. Anything linked one level deeper vanished with no error at all.
|
|
11
|
+
- Attaching an existing store is a primary way to adopt this feature, not an edge
|
|
12
|
+
case, so a link now behaves like whatever it points at: a linked folder is walked,
|
|
13
|
+
a linked note is read, and a broken link is skipped without failing the read.
|
|
14
|
+
- **Cycles terminate.** Following links makes `memory/loop -> memory` fatal, so every
|
|
15
|
+
directory is now visited once by resolved real path. The same identity check stops
|
|
16
|
+
a store reached through two different links being counted twice.
|
|
17
|
+
- **No depth limit.** An earlier draft of this fix capped nesting at 16 levels; that
|
|
18
|
+
guarded nothing real (cycles terminate on identity, and the file cap already bounds
|
|
19
|
+
the work) while silently hiding notes nested deeper. Removed.
|
|
20
|
+
- The status counter and the reader walk the same way, so the header count and the
|
|
21
|
+
list can no longer disagree.
|
|
22
|
+
|
|
23
|
+
## 6.22.0
|
|
24
|
+
|
|
25
|
+
Memory and Threads now work without a Python bridge, a venv, or a vector
|
|
26
|
+
database. Point COS Data at a folder holding `memory/` or `threads/` markdown and
|
|
27
|
+
they are browsable immediately.
|
|
28
|
+
|
|
29
|
+
This is the move that made Meetings adoptable, applied to context: the
|
|
30
|
+
requirement collapses to markdown files in folders. Any nesting, any filename,
|
|
31
|
+
front matter optional. `type` comes from front matter, else the containing folder
|
|
32
|
+
name, else `note`; ordering from a front-matter date, else a date in the filename,
|
|
33
|
+
else mtime.
|
|
34
|
+
|
|
35
|
+
- **Backwards compatible by construction, not by promise.** The file tier is
|
|
36
|
+
reachable only from the branch `callPython` takes when the bridge is ABSENT. An
|
|
37
|
+
install with a working venv and `cos_api_bridge.py` never executes a line of it,
|
|
38
|
+
so its behaviour cannot change. There is no merged resolver to get wrong, no
|
|
39
|
+
migration, no reindex, and no profile edit. A test reads `python-bridge.ts` and
|
|
40
|
+
fails if the bridge path ever references the file tier.
|
|
41
|
+
- **The routes were the real gate.** `/api/memory`, `/api/memory/:id`,
|
|
42
|
+
`/api/memory/overview`, `/api/threads` and `/api/threads/:id` returned 503
|
|
43
|
+
before `callPython` was ever called, so a fallback inside the bridge would have
|
|
44
|
+
changed nothing observable. They now gate on whether ANY source can answer.
|
|
45
|
+
- **`/api/context/status` reports a file store as available**, with
|
|
46
|
+
`source: "bridge" | "files"` so a client can say which tier it is showing
|
|
47
|
+
instead of implying a vector store that is not there. Absent on older servers.
|
|
48
|
+
`stale` is 0 for file threads because nothing computed staleness — that is the
|
|
49
|
+
truth, not a default.
|
|
50
|
+
- **File ids are namespaced `file_`,** disjoint from `mem_`, so a reference is
|
|
51
|
+
never ambiguous about which store it addresses. `MEMORY_ID_PATTERN` accepts
|
|
52
|
+
both; it previously required `mem_`, which silently dropped every file-backed
|
|
53
|
+
row from the list and returned a 200 containing nothing.
|
|
54
|
+
- **`COS_CONTEXT_DIR`** points the tier anywhere and is exclusive when set.
|
|
55
|
+
Otherwise `COS_OPERATIONS_DIR`, `COS_MEETINGS_ROOT` and its parent, the parent
|
|
56
|
+
of `COS_SCRIPTS_DIR`, then `~/.cos-glasses` — so `mkdir ~/.cos-glasses/memory`
|
|
57
|
+
is a complete setup.
|
|
58
|
+
- Amendments are NOT in this release. Reads only.
|
|
59
|
+
|
|
60
|
+
## 6.21.36
|
|
61
|
+
|
|
62
|
+
Security and accuracy fixes for the 6.21.35 context browser. Two claims in that
|
|
63
|
+
release's notes were false; both are corrected here and the claims restated
|
|
64
|
+
honestly.
|
|
65
|
+
|
|
66
|
+
- **Local filesystem paths were reaching the lens.** 6.21.35 said "no filesystem
|
|
67
|
+
paths are exposed." Three shapes leaked, each reproduced against the live store:
|
|
68
|
+
a tilde path lost only its `~/` and shipped the rest
|
|
69
|
+
(`~/.cos-glasses/data/voice-profiles.json`); a path with no whitespace before it
|
|
70
|
+
never matched at all (`KEY=/Users/...`, `>/Users/...`, `,/Users/...`); and a path
|
|
71
|
+
containing spaces stopped at the first space. The pattern is now anchored to real
|
|
72
|
+
filesystem roots, which also stops it corrupting API routes — `/api/health` and
|
|
73
|
+
`/v1/chat/completions` were being replaced with `[local path hidden]`, and that
|
|
74
|
+
same redacted string is sent to the model as evidence, so a follow-up about a
|
|
75
|
+
route lost its subject. UNC paths are covered too.
|
|
76
|
+
|
|
77
|
+
- **Eleven credential families were passing through**, each demonstrated with a
|
|
78
|
+
correctly-shaped value: HubSpot `pat-`, GitHub `github_pat_`, Slack `xapp-` and
|
|
79
|
+
`xoxd-`, Google `GOCSPX-`, `npm_`, Stripe `(sk|rk|pk)_live_`, AWS `ASIA` key ids
|
|
80
|
+
and unprefixed 40-char secret keys, GitLab `glpat-`, SSH2/PuTTY key headers, and
|
|
81
|
+
`redis://:password@host` where the username is empty. `PWD=` and `PASS=` are now
|
|
82
|
+
treated as credential names.
|
|
83
|
+
|
|
84
|
+
Conversely `MAX_THINKING_TOKENS=31999` was being redacted as a secret — a bare
|
|
85
|
+
integer is not a credential, and redacting a real setting corrupts evidence
|
|
86
|
+
without protecting anything.
|
|
87
|
+
|
|
88
|
+
- **Malformed bridge protocols were relabeled as protocol 1** — precisely what
|
|
89
|
+
6.21.35's notes said it prevented. `finiteInteger` ran `Math.trunc(Number(v))`
|
|
90
|
+
before the comparison, so `'1'`, `1.5`, `1.9`, `true` and `[1]` all became `1`
|
|
91
|
+
and were served as compatible, with the reported field rewritten to `1` so
|
|
92
|
+
Control could not see what it had been handed. The check is now strict and
|
|
93
|
+
pre-coercion, and an incompatible protocol is reported as its raw integer or 0,
|
|
94
|
+
never a truncated 1.
|
|
95
|
+
|
|
96
|
+
- **Three browse routes could CREATE the Qdrant collection.** `/api/memory`,
|
|
97
|
+
`/api/memory/:id` and `/api/memory/overview` called `ensure_collection()`, so a
|
|
98
|
+
read-only G2 browse against a machine with no collection wrote a new empty one
|
|
99
|
+
and made a broken setup look healthy. They now pass `ensure=False`.
|
|
100
|
+
`get_summary_stats` already documented this exact reasoning and deliberately
|
|
101
|
+
omitted the call; the browse paths did not follow it.
|
|
102
|
+
|
|
103
|
+
Tests were written to fail first: 21 new assertions reproduced real leaks against
|
|
104
|
+
6.21.35 before any pattern changed. One pre-existing assertion changed by a single
|
|
105
|
+
character — the old pattern's segment class included a comma and reported
|
|
106
|
+
`file.txt,` as the filename.
|
|
107
|
+
|
|
1
108
|
## 6.21.35
|
|
2
109
|
|
|
3
110
|
- Adds authenticated `/api/context/status` proof so Control and the companion
|
package/README.md
CHANGED
|
@@ -307,8 +307,58 @@ complete Bot Memory count/type split, bounded recent summaries, exact logical
|
|
|
307
307
|
memory IDs, and existing tracked/manual threads. Exact detail requests are
|
|
308
308
|
resolved by stable ID so a spoken follow-up can carry the selected snapshot as
|
|
309
309
|
context. Embeddings, vector-store point IDs, cache files, secrets, and local
|
|
310
|
-
paths never cross the API boundary.
|
|
311
|
-
|
|
310
|
+
paths never cross the API boundary.
|
|
311
|
+
|
|
312
|
+
### Memory and Threads from plain markdown (6.22.0)
|
|
313
|
+
|
|
314
|
+
You do not need a Python bridge, a virtual environment, or a vector database.
|
|
315
|
+
Make a folder with a `memory/` or `threads/` subfolder, put markdown files in it,
|
|
316
|
+
and point COS Data at it (or set `COS_CONTEXT_DIR`):
|
|
317
|
+
|
|
318
|
+
```
|
|
319
|
+
notes/
|
|
320
|
+
memory/ any nesting, any filenames
|
|
321
|
+
2026-08-09-hiring-call.md
|
|
322
|
+
decisions/pricing.md folder name becomes the type
|
|
323
|
+
threads/
|
|
324
|
+
website-rebuild.md
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
Front matter is optional and every field degrades rather than rejecting:
|
|
328
|
+
|
|
329
|
+
```markdown
|
|
330
|
+
---
|
|
331
|
+
type: decision # else the containing folder name, else "note"
|
|
332
|
+
date: 2026-08-09 # else a YYYY-MM-DD in the filename, else file mtime
|
|
333
|
+
status: resolved # threads only
|
|
334
|
+
---
|
|
335
|
+
# Held Rain POS for v25
|
|
336
|
+
|
|
337
|
+
Body text. The first heading becomes the summary.
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
Two tiers, and the bridge always wins when it is present:
|
|
341
|
+
|
|
342
|
+
| Tier | Requires | Provides |
|
|
343
|
+
| --- | --- | --- |
|
|
344
|
+
| Files | a folder of markdown | Browse, read, and reference memories and threads |
|
|
345
|
+
| Bridge | + venv, `cos_api_bridge.py`, vector store | Adds semantic recall, dedup, type statistics, retention |
|
|
346
|
+
|
|
347
|
+
`/api/context/status` reports `source: "bridge"` or `source: "files"` so a client
|
|
348
|
+
can say which tier it is showing. Fields a file-backed record cannot have are
|
|
349
|
+
empty rather than invented: a file thread has `velocity: ""`, `meeting_count: 0`
|
|
350
|
+
and `stale: 0` because nothing computed them.
|
|
351
|
+
|
|
352
|
+
Root resolution, first match wins: `COS_CONTEXT_DIR` (exclusive when set), then
|
|
353
|
+
`COS_OPERATIONS_DIR`, `COS_MEETINGS_ROOT` and its parent, the parent of
|
|
354
|
+
`COS_SCRIPTS_DIR`, then `~/.cos-glasses` — so `mkdir ~/.cos-glasses/memory` is a
|
|
355
|
+
complete setup. The file tier is read from the code path taken only when no
|
|
356
|
+
bridge is configured, so adding it cannot change the behaviour of an install that
|
|
357
|
+
already has one.
|
|
358
|
+
|
|
359
|
+
The API is read-only in both tiers. Standalone installs with neither a bridge nor
|
|
360
|
+
a notes folder report the feature as unavailable without affecting messages,
|
|
361
|
+
meetings, transcription, or agents.
|
|
312
362
|
|
|
313
363
|
The first server start downloads the real-time turbo model. True HQ additionally
|
|
314
364
|
requires the full `ggml-large-v3.bin` model (about 3.1 GB):
|
package/package.json
CHANGED
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
// Memory and Threads from plain markdown, with no venv, bridge, or vector store.
|
|
2
|
+
//
|
|
3
|
+
// WHY. Meetings became adoptable when the requirement collapsed to "markdown files
|
|
4
|
+
// in folders" — any domain name, any capitalisation, spaces allowed, with the only
|
|
5
|
+
// structural demand being a YYYY-MM directory because that is the shape the lister
|
|
6
|
+
// reads. Memory and Threads never got that treatment: the COS Data picker requires
|
|
7
|
+
// `operations/scripts/cos_api_bridge.py` AND an executable `venv/bin/python3`, and
|
|
8
|
+
// below that sit Docker and OpenAI embeddings. So the honest answer to "how do I
|
|
9
|
+
// start using memory?" was "clone a workspace and run a vector database."
|
|
10
|
+
//
|
|
11
|
+
// BACKWARDS COMPATIBILITY IS STRUCTURAL, NOT PROMISED. This module is only ever
|
|
12
|
+
// reached from `standaloneNoop` — the branch `callPython` takes when the bridge is
|
|
13
|
+
// ABSENT. An install with a working bridge never enters this code at all, so its
|
|
14
|
+
// behaviour cannot change. That is deliberate: the alternative, a resolver that
|
|
15
|
+
// merges both sources, would put new code in the path of an existing setup.
|
|
16
|
+
//
|
|
17
|
+
// The files are also the IMPORT surface. Someone brings notes in their own shape,
|
|
18
|
+
// COS reads them immediately, and later indexes those same files into their own
|
|
19
|
+
// vector store. The files were always the substrate — which is exactly why an
|
|
20
|
+
// amendment must write only here and never to the derived store.
|
|
21
|
+
|
|
22
|
+
import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from 'node:fs'
|
|
23
|
+
import { homedir } from 'node:os'
|
|
24
|
+
import { basename, dirname, extname, join, resolve } from 'node:path'
|
|
25
|
+
|
|
26
|
+
/** Where a file-tier install keeps its notes, relative to the chosen COS Data dir. */
|
|
27
|
+
const MEMORY_DIRS = ['memory', 'memories'] as const
|
|
28
|
+
const THREAD_DIRS = ['threads', 'thread'] as const
|
|
29
|
+
|
|
30
|
+
/** Bounded so a large folder cannot make a G2 browse slow or a payload huge. */
|
|
31
|
+
const MAX_FILES_SCANNED = 2_000
|
|
32
|
+
const MAX_BODY_CHARS = 32_000
|
|
33
|
+
const SUMMARY_CHARS = 240
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Date embedded in a filename, e.g. `2026-08-09-decision.md` or
|
|
37
|
+
* `2026-08-09_thing.md`. Used for ordering before falling back to mtime, because a
|
|
38
|
+
* filename date survives a file copy and mtime does not.
|
|
39
|
+
*/
|
|
40
|
+
const FILENAME_DATE = /(\d{4})-(\d{2})-(\d{2})/
|
|
41
|
+
|
|
42
|
+
export interface FileMemory {
|
|
43
|
+
id: string
|
|
44
|
+
type: string
|
|
45
|
+
summary: string
|
|
46
|
+
content: string
|
|
47
|
+
created_at: string
|
|
48
|
+
domain: string
|
|
49
|
+
refs: Record<string, string[]>
|
|
50
|
+
reference_available: true
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface FileThread {
|
|
54
|
+
id: string
|
|
55
|
+
name: string
|
|
56
|
+
domain: string
|
|
57
|
+
is_manual: boolean
|
|
58
|
+
topics: string[]
|
|
59
|
+
meeting_count: number
|
|
60
|
+
first_seen: string
|
|
61
|
+
last_seen: string
|
|
62
|
+
velocity: string
|
|
63
|
+
age_days: number
|
|
64
|
+
is_stale: boolean
|
|
65
|
+
is_resolved: boolean
|
|
66
|
+
meetings: Array<{ name: string; date: string }>
|
|
67
|
+
manual_updates: Array<{ content: string; timestamp: string; source: string }>
|
|
68
|
+
stakeholders: string[]
|
|
69
|
+
milestones: string[]
|
|
70
|
+
sources: string[]
|
|
71
|
+
target_date: string
|
|
72
|
+
serves_goal: string
|
|
73
|
+
created_at: string
|
|
74
|
+
created_by: string
|
|
75
|
+
access_count: number
|
|
76
|
+
reference_available: true
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** First existing candidate directory, or null. */
|
|
80
|
+
function findDir(root: string, names: readonly string[]): string | null {
|
|
81
|
+
for (const name of names) {
|
|
82
|
+
const candidate = join(root, name)
|
|
83
|
+
try { if (statSync(candidate).isDirectory()) return candidate } catch { /* next */ }
|
|
84
|
+
}
|
|
85
|
+
return null
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Extensions the readers treat as notes. */
|
|
89
|
+
const NOTE_EXTENSIONS = ['.md', '.markdown', '.txt']
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Every markdown file under `dir`, recursively, bounded, FOLLOWING SYMLINKS.
|
|
93
|
+
*
|
|
94
|
+
* Any nesting is accepted on purpose — a user's notes may be organised by year, by
|
|
95
|
+
* project, or not at all, and rejecting a shape is what made COS Data unusable for
|
|
96
|
+
* anyone but its author.
|
|
97
|
+
*
|
|
98
|
+
* SYMLINKS ARE FIRST-CLASS, and that is a correction. A `Dirent` reports a symlink
|
|
99
|
+
* as `isSymbolicLink()`, NOT as `isDirectory()` or `isFile()`, so the first version
|
|
100
|
+
* of this walk silently skipped every linked file and linked subfolder. Queen hit
|
|
101
|
+
* exactly that on 2026-08-09: she wired `operations/memory` at her real note store
|
|
102
|
+
* with a symlink, which worked only because `findDir` uses `statSync` and follows
|
|
103
|
+
* links — anything linked one level deeper would have vanished with no error.
|
|
104
|
+
*
|
|
105
|
+
* Attaching notes that live somewhere else is a PRIMARY use case, not an edge case,
|
|
106
|
+
* so a link has to behave like the thing it points at.
|
|
107
|
+
*
|
|
108
|
+
* Cycles are the cost of following them: `a -> ../a` recurses forever. Every
|
|
109
|
+
* directory is recorded by its resolved real path and visited once, which also
|
|
110
|
+
* stops the same store being counted twice when two links reach it.
|
|
111
|
+
*
|
|
112
|
+
* There is deliberately NO depth limit. An earlier draft capped nesting at 16 as
|
|
113
|
+
* "bounding a pathological tree", but real-path identity already makes cycles
|
|
114
|
+
* terminate and MAX_FILES_SCANNED already bounds the work — so the cap's only real
|
|
115
|
+
* effect was to silently hide notes nested deeper than that. That is the same
|
|
116
|
+
* invisible-skip defect this function was just fixed for, moved further out, and a
|
|
117
|
+
* mutation removing the cap changed no test, which is how it got noticed.
|
|
118
|
+
*/
|
|
119
|
+
function walkMarkdown(
|
|
120
|
+
dir: string,
|
|
121
|
+
budget = { left: MAX_FILES_SCANNED },
|
|
122
|
+
seen: Set<string> = new Set(),
|
|
123
|
+
): string[] {
|
|
124
|
+
const out: string[] = []
|
|
125
|
+
// Identity by real path, so a link and its target are the same directory.
|
|
126
|
+
let real: string
|
|
127
|
+
try { real = realpathSync(dir) } catch { return out }
|
|
128
|
+
if (seen.has(real)) return out
|
|
129
|
+
seen.add(real)
|
|
130
|
+
|
|
131
|
+
let entries: import('node:fs').Dirent[]
|
|
132
|
+
try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return out }
|
|
133
|
+
for (const entry of entries) {
|
|
134
|
+
if (budget.left <= 0) break
|
|
135
|
+
if (entry.name.startsWith('.')) continue
|
|
136
|
+
const full = join(dir, entry.name)
|
|
137
|
+
const isNote = NOTE_EXTENSIONS.includes(extname(entry.name).toLowerCase())
|
|
138
|
+
if (entry.isDirectory()) {
|
|
139
|
+
out.push(...walkMarkdown(full, budget, seen))
|
|
140
|
+
} else if (entry.isFile()) {
|
|
141
|
+
if (!isNote) continue
|
|
142
|
+
budget.left -= 1
|
|
143
|
+
out.push(full)
|
|
144
|
+
} else if (entry.isSymbolicLink()) {
|
|
145
|
+
// Resolve to decide: a linked folder is walked, a linked note is read.
|
|
146
|
+
let target: import('node:fs').Stats
|
|
147
|
+
try { target = statSync(full) } catch { continue } // broken link, skip quietly
|
|
148
|
+
if (target.isDirectory()) {
|
|
149
|
+
out.push(...walkMarkdown(full, budget, seen))
|
|
150
|
+
} else if (target.isFile() && isNote) {
|
|
151
|
+
budget.left -= 1
|
|
152
|
+
out.push(full)
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return out
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Minimal YAML-ish front matter. Absent or malformed degrades to {}. */
|
|
160
|
+
function parseFrontMatter(text: string): { data: Record<string, string>; body: string } {
|
|
161
|
+
if (!text.startsWith('---')) return { data: {}, body: text }
|
|
162
|
+
const end = text.indexOf('\n---', 3)
|
|
163
|
+
if (end === -1) return { data: {}, body: text }
|
|
164
|
+
const raw = text.slice(3, end)
|
|
165
|
+
const data: Record<string, string> = {}
|
|
166
|
+
for (const line of raw.split('\n')) {
|
|
167
|
+
const i = line.indexOf(':')
|
|
168
|
+
if (i <= 0) continue
|
|
169
|
+
const key = line.slice(0, i).trim().toLowerCase()
|
|
170
|
+
const value = line.slice(i + 1).trim().replace(/^["']|["']$/g, '')
|
|
171
|
+
if (key) data[key] = value
|
|
172
|
+
}
|
|
173
|
+
return { data, body: text.slice(end + 4).replace(/^\n+/, '') }
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** ISO date for ordering: front matter, else filename, else mtime. */
|
|
177
|
+
function resolveDate(path: string, front: Record<string, string>): string {
|
|
178
|
+
const explicit = front.date || front.created_at || front.created
|
|
179
|
+
if (explicit && /^\d{4}-\d{2}-\d{2}/.test(explicit)) return explicit
|
|
180
|
+
const m = FILENAME_DATE.exec(basename(path))
|
|
181
|
+
if (m) return `${m[1]}-${m[2]}-${m[3]}`
|
|
182
|
+
try { return new Date(statSync(path).mtimeMs).toISOString() } catch { return '' }
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Stable id for a file-backed record.
|
|
187
|
+
*
|
|
188
|
+
* Derived from the path so it survives restarts and can be resolved back to a file
|
|
189
|
+
* without an index. Prefixed `file_` so it can never be mistaken for a `mem_` id
|
|
190
|
+
* from the vector store — two different stores must not share an id space.
|
|
191
|
+
*/
|
|
192
|
+
function fileId(root: string, path: string): string {
|
|
193
|
+
const rel = path.startsWith(root) ? path.slice(root.length).replace(/^\//, '') : basename(path)
|
|
194
|
+
const safe = rel.replace(/[^A-Za-z0-9._-]+/g, '_')
|
|
195
|
+
// MEMORY_ID_PATTERN caps the id at 120 characters after the prefix and
|
|
196
|
+
// `cleanContextText` truncates at 128, so a deep path must be shortened HERE.
|
|
197
|
+
// Truncating alone would collide across two long sibling paths and resolve the
|
|
198
|
+
// wrong note, so the discarded head is folded into a short hash.
|
|
199
|
+
if (safe.length <= 100) return `file_${safe}`
|
|
200
|
+
let hash = 5381
|
|
201
|
+
for (let i = 0; i < safe.length; i++) hash = ((hash * 33) ^ safe.charCodeAt(i)) >>> 0
|
|
202
|
+
return `file_${hash.toString(36)}_${safe.slice(-88)}`
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function firstHeading(body: string): string {
|
|
206
|
+
for (const line of body.split('\n')) {
|
|
207
|
+
const m = /^#{1,3}\s+(.+)$/.exec(line.trim())
|
|
208
|
+
if (m) return m[1].trim()
|
|
209
|
+
}
|
|
210
|
+
return ''
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function summarise(body: string): string {
|
|
214
|
+
const flat = body.replace(/^#{1,6}\s+/gm, '').replace(/\s+/g, ' ').trim()
|
|
215
|
+
return flat.length <= SUMMARY_CHARS ? flat : `${flat.slice(0, SUMMARY_CHARS)}…`
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Newest first, by resolved date then path for determinism. */
|
|
219
|
+
function byNewest<T extends { created_at: string; id: string }>(a: T, b: T): number {
|
|
220
|
+
return b.created_at.localeCompare(a.created_at) || a.id.localeCompare(b.id)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function envPath(name: string): string | null {
|
|
224
|
+
const raw = process.env[name]?.trim()
|
|
225
|
+
return raw ? resolve(raw) : null
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Where this install keeps plain-text notes, or null.
|
|
230
|
+
*
|
|
231
|
+
* Candidates in priority order, first one holding a `memory/` or `threads/`
|
|
232
|
+
* folder wins. Env is read on every call rather than captured at import, so a
|
|
233
|
+
* COS Control settings change followed by a restart-free re-read behaves, and so
|
|
234
|
+
* tests can point it somewhere without module-load ordering games.
|
|
235
|
+
*
|
|
236
|
+
* `COS_MEETINGS_ROOT` contributes its PARENT as well as itself, because a direct
|
|
237
|
+
* meetings library is `.../personal/meetings` while notes sit beside it at
|
|
238
|
+
* `.../personal/memory`.
|
|
239
|
+
*
|
|
240
|
+
* The data home is last and is the zero-configuration answer: someone with no
|
|
241
|
+
* COS repo at all can `mkdir ~/.cos-glasses/memory` and start browsing. It is
|
|
242
|
+
* also the only candidate guaranteed to survive a server update, which is where
|
|
243
|
+
* anything durable belongs.
|
|
244
|
+
*/
|
|
245
|
+
export function resolveContextFilesRoot(): string | null {
|
|
246
|
+
// An EXPLICIT setting is exclusive, including when it turns out to hold nothing.
|
|
247
|
+
// Falling through from an empty explicit root to the data home would mean a
|
|
248
|
+
// user who pointed us at one folder could be served notes from another, and it
|
|
249
|
+
// would make every test's answer depend on whether `~/.cos-glasses/memory`
|
|
250
|
+
// happens to exist on the machine running it.
|
|
251
|
+
const explicit = envPath('COS_CONTEXT_DIR')
|
|
252
|
+
if (explicit) {
|
|
253
|
+
return hasFileMemory(explicit) || hasFileThreads(explicit) ? explicit : null
|
|
254
|
+
}
|
|
255
|
+
const meetings = envPath('COS_MEETINGS_ROOT')
|
|
256
|
+
const scripts = envPath('COS_SCRIPTS_DIR')
|
|
257
|
+
const dataHome = envPath('COS_DATA_DIR') ?? join(homedir(), '.cos-glasses')
|
|
258
|
+
const candidates = [
|
|
259
|
+
envPath('COS_OPERATIONS_DIR'),
|
|
260
|
+
meetings,
|
|
261
|
+
meetings ? dirname(meetings) : null,
|
|
262
|
+
scripts ? dirname(scripts) : null,
|
|
263
|
+
dataHome,
|
|
264
|
+
]
|
|
265
|
+
for (const candidate of candidates) {
|
|
266
|
+
if (candidate && (hasFileMemory(candidate) || hasFileThreads(candidate))) return candidate
|
|
267
|
+
}
|
|
268
|
+
return null
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Markdown files under `dir` with content, counted without reading any of them. */
|
|
272
|
+
function countMarkdown(dir: string | null): number {
|
|
273
|
+
if (!dir) return 0
|
|
274
|
+
let total = 0
|
|
275
|
+
for (const path of walkMarkdown(dir)) {
|
|
276
|
+
// `size > 0` instead of a read: this runs on the status route, which the
|
|
277
|
+
// companion polls, and reading every note to produce one integer is how a
|
|
278
|
+
// browse surface becomes a performance problem. A whitespace-only file is
|
|
279
|
+
// counted here but skipped by the readers, so the total can exceed the
|
|
280
|
+
// browsable list by the number of blank files. That is the only known
|
|
281
|
+
// discrepancy and it is bounded by files the user created empty.
|
|
282
|
+
try { if (statSync(path).size > 0) total += 1 } catch { /* vanished */ }
|
|
283
|
+
}
|
|
284
|
+
return total
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Counts for the status surface.
|
|
289
|
+
*
|
|
290
|
+
* Memory is counted by path. Threads are READ, because the status header needs
|
|
291
|
+
* the active/resolved split and `status: resolved` lives in front matter — and a
|
|
292
|
+
* thread store is one file per project, so tens of files, not thousands. The
|
|
293
|
+
* MAX_FILES_SCANNED bound still applies.
|
|
294
|
+
*/
|
|
295
|
+
export function fileTierStatus(root: string | null): {
|
|
296
|
+
memory: { present: boolean; total: number }
|
|
297
|
+
threads: { present: boolean; total: number; active: number; resolved: number }
|
|
298
|
+
} {
|
|
299
|
+
const memoryDir = root ? findDir(root, MEMORY_DIRS) : null
|
|
300
|
+
const threads = root && hasFileThreads(root) ? readFileThreads(root, MAX_FILES_SCANNED) : []
|
|
301
|
+
return {
|
|
302
|
+
memory: { present: !!memoryDir, total: countMarkdown(memoryDir) },
|
|
303
|
+
threads: {
|
|
304
|
+
present: !!root && hasFileThreads(root),
|
|
305
|
+
total: threads.length,
|
|
306
|
+
active: threads.filter(t => !t.is_resolved).length,
|
|
307
|
+
resolved: threads.filter(t => t.is_resolved).length,
|
|
308
|
+
},
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** Is a file-backed memory store present under this root? */
|
|
313
|
+
export function hasFileMemory(root: string | null): boolean {
|
|
314
|
+
return !!root && findDir(root, MEMORY_DIRS) !== null
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** Is a file-backed thread store present under this root? */
|
|
318
|
+
export function hasFileThreads(root: string | null): boolean {
|
|
319
|
+
return !!root && findDir(root, THREAD_DIRS) !== null
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Memories from markdown, newest first.
|
|
324
|
+
*
|
|
325
|
+
* `type` comes from front matter when present, else the containing folder name,
|
|
326
|
+
* else 'note' — a folder called `decisions/` is a reasonable signal and asking the
|
|
327
|
+
* user to add front matter before anything works would defeat the point.
|
|
328
|
+
*/
|
|
329
|
+
export function readFileMemories(root: string, limit = 30): FileMemory[] {
|
|
330
|
+
const dir = findDir(root, MEMORY_DIRS)
|
|
331
|
+
if (!dir) return []
|
|
332
|
+
const out: FileMemory[] = []
|
|
333
|
+
for (const path of walkMarkdown(dir)) {
|
|
334
|
+
let text: string
|
|
335
|
+
try { text = readFileSync(path, 'utf-8') } catch { continue }
|
|
336
|
+
const { data, body } = parseFrontMatter(text)
|
|
337
|
+
const trimmed = body.trim()
|
|
338
|
+
if (!trimmed) continue
|
|
339
|
+
const parent = basename(join(path, '..'))
|
|
340
|
+
out.push({
|
|
341
|
+
id: data.id || fileId(root, path),
|
|
342
|
+
type: (data.type || (parent && parent !== basename(dir) ? parent : '') || 'note').toLowerCase(),
|
|
343
|
+
summary: data.summary || firstHeading(trimmed) || summarise(trimmed),
|
|
344
|
+
content: trimmed.slice(0, MAX_BODY_CHARS),
|
|
345
|
+
created_at: resolveDate(path, data),
|
|
346
|
+
domain: data.domain || '',
|
|
347
|
+
refs: {},
|
|
348
|
+
reference_available: true,
|
|
349
|
+
})
|
|
350
|
+
}
|
|
351
|
+
return out.sort(byNewest).slice(0, Math.max(1, limit))
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** One file-backed memory by its `file_` id, or null. */
|
|
355
|
+
export function readFileMemoryById(root: string, id: string): FileMemory | null {
|
|
356
|
+
// Scanning rather than path-reconstructing: the id is sanitised, so mapping it
|
|
357
|
+
// back to a path would mean trusting a lossy transform to address the filesystem.
|
|
358
|
+
return readFileMemories(root, MAX_FILES_SCANNED).find(m => m.id === id) ?? null
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/** Total count and per-type split, for the overview surface. */
|
|
362
|
+
export function fileMemoryOverview(root: string): { total: number; by_type: Record<string, number> } {
|
|
363
|
+
const all = readFileMemories(root, MAX_FILES_SCANNED)
|
|
364
|
+
const by_type: Record<string, number> = {}
|
|
365
|
+
for (const m of all) by_type[m.type] = (by_type[m.type] ?? 0) + 1
|
|
366
|
+
return { total: all.length, by_type }
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Threads from markdown, newest first.
|
|
371
|
+
*
|
|
372
|
+
* Every derived field a vector-tier thread carries is present and honestly empty —
|
|
373
|
+
* velocity '', meeting_count 0 — rather than absent, so the same display code
|
|
374
|
+
* renders both tiers without branching. A file thread has no computed velocity
|
|
375
|
+
* because nothing computed it; that is a real difference, not a missing value to
|
|
376
|
+
* be invented.
|
|
377
|
+
*/
|
|
378
|
+
export function readFileThreads(root: string, limit = 30): FileThread[] {
|
|
379
|
+
const dir = findDir(root, THREAD_DIRS)
|
|
380
|
+
if (!dir) return []
|
|
381
|
+
const out: FileThread[] = []
|
|
382
|
+
for (const path of walkMarkdown(dir)) {
|
|
383
|
+
let text: string
|
|
384
|
+
try { text = readFileSync(path, 'utf-8') } catch { continue }
|
|
385
|
+
const { data, body } = parseFrontMatter(text)
|
|
386
|
+
const trimmed = body.trim()
|
|
387
|
+
if (!trimmed) continue
|
|
388
|
+
const created = resolveDate(path, data)
|
|
389
|
+
const topics = (data.topics || data.tags || '')
|
|
390
|
+
.split(/[,;]/).map(t => t.trim()).filter(Boolean)
|
|
391
|
+
out.push({
|
|
392
|
+
id: data.id || fileId(root, path),
|
|
393
|
+
name: data.name || data.title || firstHeading(trimmed) || basename(path, extname(path)),
|
|
394
|
+
domain: data.domain || '',
|
|
395
|
+
is_manual: true,
|
|
396
|
+
topics,
|
|
397
|
+
meeting_count: 0,
|
|
398
|
+
first_seen: created,
|
|
399
|
+
last_seen: created,
|
|
400
|
+
velocity: '',
|
|
401
|
+
age_days: 0,
|
|
402
|
+
is_stale: false,
|
|
403
|
+
is_resolved: (data.status || '').toLowerCase() === 'resolved',
|
|
404
|
+
meetings: [],
|
|
405
|
+
manual_updates: [{ content: trimmed.slice(0, MAX_BODY_CHARS), timestamp: created, source: 'file' }],
|
|
406
|
+
stakeholders: (data.stakeholders || '').split(/[,;]/).map(s => s.trim()).filter(Boolean),
|
|
407
|
+
milestones: [],
|
|
408
|
+
sources: [],
|
|
409
|
+
target_date: data.target_date || '',
|
|
410
|
+
serves_goal: data.goal || data.serves_goal || '',
|
|
411
|
+
created_at: created,
|
|
412
|
+
created_by: data.created_by || 'file',
|
|
413
|
+
access_count: 0,
|
|
414
|
+
reference_available: true,
|
|
415
|
+
})
|
|
416
|
+
}
|
|
417
|
+
return out.sort(byNewest).slice(0, Math.max(1, limit))
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/** One file-backed thread by id, or null. */
|
|
421
|
+
export function readFileThreadById(root: string, id: string): FileThread | null {
|
|
422
|
+
return readFileThreads(root, MAX_FILES_SCANNED).find(t => t.id === id) ?? null
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** Does this root hold anything the file tier can serve? */
|
|
426
|
+
export function fileTierState(root: string | null): 'ready' | 'empty' | 'absent' {
|
|
427
|
+
if (!root || !existsSync(root)) return 'absent'
|
|
428
|
+
const dirs = [findDir(root, MEMORY_DIRS), findDir(root, THREAD_DIRS)].filter((d): d is string => !!d)
|
|
429
|
+
if (dirs.length === 0) return 'absent'
|
|
430
|
+
// Existence, not content: `walkMarkdown` stops at the first hit, so this stays
|
|
431
|
+
// a couple of readdir calls even on a large tree. 'empty' is a HEALTHY state —
|
|
432
|
+
// a configured store with nothing in it yet — and must be distinguishable from
|
|
433
|
+
// 'absent', which is what the picker and the wizard branch on.
|
|
434
|
+
const any = dirs.some(dir => walkMarkdown(dir, { left: 1 }).length > 0)
|
|
435
|
+
return any ? 'ready' : 'empty'
|
|
436
|
+
}
|
|
@@ -1,16 +1,98 @@
|
|
|
1
|
-
|
|
1
|
+
// `mem_` addresses the vector store, `file_` addresses a note on disk. Two stores,
|
|
2
|
+
// one id space, disjoint prefixes — so a reference is never ambiguous about which
|
|
3
|
+
// one it means. `.` is permitted because a file id carries its extension; it
|
|
4
|
+
// cannot traverse, since `/` is excluded and a bare `..` cannot match the prefix.
|
|
5
|
+
export const MEMORY_ID_PATTERN = /^(?:mem|file)_[A-Za-z0-9._:-]{1,120}$/
|
|
2
6
|
export const THREAD_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/
|
|
3
7
|
|
|
4
8
|
const CONTROL_CHARS = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g
|
|
5
|
-
|
|
9
|
+
/**
|
|
10
|
+
* Filesystem roots. Anchoring to real roots does two jobs at once: it catches the
|
|
11
|
+
* shapes the old pattern missed, and it stops redacting API routes.
|
|
12
|
+
*
|
|
13
|
+
* The old pattern was `(^|[\s("'`])(?:~\/|\/(?!\/)…)`. Three failures, all
|
|
14
|
+
* reproduced against 6.21.35 on the live store before this rewrite:
|
|
15
|
+
* 1. `~\/` had no segment requirement, so `~/.cos-glasses/data/x.json` lost only
|
|
16
|
+
* the two characters "~/" and shipped the rest to the lens.
|
|
17
|
+
* 2. The leading `(^|[\s("'`])` meant `KEY=/Users/...`, `>/Users/...` and
|
|
18
|
+
* `,/Users/...` never matched at all.
|
|
19
|
+
* 3. Being root-agnostic, it redacted `/api/context/status` and `/v1/chat/…`
|
|
20
|
+
* — and that same redacted string is sent to the model as evidence, so a
|
|
21
|
+
* follow-up about a route lost its subject.
|
|
22
|
+
*/
|
|
23
|
+
const FS_ROOTS = 'Users|home|root|opt|usr|var|private|tmp|Volumes|etc|Library|System|Applications'
|
|
24
|
+
/**
|
|
25
|
+
* Greedy to a hard delimiter, then trimmed back in the replacer. A path may
|
|
26
|
+
* legitimately contain spaces — this machine's own repo root does — and no regex
|
|
27
|
+
* can tell "space inside a path" from "space before the next word", so the
|
|
28
|
+
* boundary decision is made in code where it can be tested.
|
|
29
|
+
*/
|
|
30
|
+
const ABSOLUTE_PATH = new RegExp(`(?:~|/(?:${FS_ROOTS}))(?:/[^,;)"'\`\r\n]*)?`, 'g')
|
|
31
|
+
const UNC_PATH = /\\\\[A-Za-z0-9._-]+\\(?:[^\\\r\n]+\\?)*/g
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Trim a greedy path match back to its last plausible component.
|
|
35
|
+
*
|
|
36
|
+
* Keeps `…/Ukaoma Chief Of Staff/MU/ops/comp.md` whole (every space is followed by
|
|
37
|
+
* more path) while not swallowing the trailing prose in `/Users/me/x.md here`.
|
|
38
|
+
* A component qualifies if it is followed by a `/` or contains a dot.
|
|
39
|
+
*/
|
|
40
|
+
function pathMatchEnd(match: string): number {
|
|
41
|
+
const lastSlash = match.lastIndexOf('/')
|
|
42
|
+
if (lastSlash <= 0) return match.length
|
|
43
|
+
const tail = match.slice(lastSlash + 1)
|
|
44
|
+
const space = tail.indexOf(' ')
|
|
45
|
+
// No space in the final component: the whole match is path.
|
|
46
|
+
if (space === -1) return match.length
|
|
47
|
+
// A dotted filename before the space is the end of the path.
|
|
48
|
+
const beforeSpace = tail.slice(0, space)
|
|
49
|
+
return beforeSpace.length > 0 ? lastSlash + 1 + beforeSpace.length : lastSlash
|
|
50
|
+
}
|
|
6
51
|
const SECRET_TOKEN = /\b(?:sk-[A-Za-z0-9_-]{12,}|(?:bearer|token|api[_ -]?key)\s*[:=]\s*[A-Za-z0-9._-]{12,})\b/gi
|
|
7
52
|
const WINDOWS_PATH = /\b[A-Za-z]:\\(?:[^\\\r\n]+\\)*[^\\\r\n]*/g
|
|
8
|
-
|
|
53
|
+
// 4 OR 5 dashes: SSH2/PuTTY exports use four, and PuTTY uses a bare header line.
|
|
54
|
+
const PEM_BLOCK = /-{4,5} ?BEGIN [^-\r\n]*(?:PRIVATE KEY|KEY)[^-\r\n]*-{4,5}[\s\S]*?(?:-{4,5} ?END [^-\r\n]+-{4,5}|$)/g
|
|
55
|
+
const PUTTY_KEY = /\bPuTTY-User-Key-File-\d+:.*/g
|
|
9
56
|
const JWT_TOKEN = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g
|
|
10
|
-
|
|
11
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Credential prefixes. Every family added here was demonstrated LEAKING against
|
|
59
|
+
* 6.21.35 with a correctly-shaped fabricated value. `pat-` is the one that
|
|
60
|
+
* mattered most: HubSpot PAK rotation is a recurring topic in this store, and
|
|
61
|
+
* `/end-session` writes rotation notes into it.
|
|
62
|
+
*/
|
|
63
|
+
const PREFIXED_SECRET = new RegExp([
|
|
64
|
+
'gh[pousr]_[A-Za-z0-9]{20,}',
|
|
65
|
+
'github_pat_[A-Za-z0-9_]{30,}',
|
|
66
|
+
'xox[baprsde]-[A-Za-z0-9-]{15,}', // +d (cookie) +e (refresh)
|
|
67
|
+
'xapp-[A-Za-z0-9-]{15,}',
|
|
68
|
+
'AIza[A-Za-z0-9_-]{20,}',
|
|
69
|
+
'AKIA[A-Z0-9]{16}',
|
|
70
|
+
'ASIA[A-Z0-9]{16}',
|
|
71
|
+
'GOCSPX-[A-Za-z0-9_-]{16,}',
|
|
72
|
+
'npm_[A-Za-z0-9]{30,}',
|
|
73
|
+
'(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{16,}',
|
|
74
|
+
'pat-[a-z0-9]{2,4}-[0-9a-fA-F-]{20,}',
|
|
75
|
+
'glpat-[A-Za-z0-9_-]{16,}',
|
|
76
|
+
].join('|'), 'g')
|
|
77
|
+
/**
|
|
78
|
+
* AWS secret access keys have no prefix: exactly 40 chars of the base64 alphabet.
|
|
79
|
+
*
|
|
80
|
+
* A `/` or `+` is REQUIRED, which is a deliberate trade-off. Without it, any
|
|
81
|
+
* 40-char alphanumeric run matches — and a SHA-1 hex digest is exactly 40 chars,
|
|
82
|
+
* so every commit sha in this store would be redacted as a credential. The cost
|
|
83
|
+
* is that an all-alphanumeric AWS key is missed; the alternative corrupts far more
|
|
84
|
+
* real evidence than it protects.
|
|
85
|
+
*/
|
|
86
|
+
const AWS_SECRET_KEY = /(?<![A-Za-z0-9/+])(?=[A-Za-z0-9/+]{40}(?![A-Za-z0-9/+]))(?=[A-Za-z0-9]*[/+])[A-Za-z0-9/+]{40}/g
|
|
87
|
+
// Username may be EMPTY — `redis://:password@host` defeated the old `+`.
|
|
88
|
+
const URL_CREDENTIALS = /(\b[a-z][a-z0-9+.-]{0,20}:\/\/)[^\s/@:]*:[^\s/@]+@/gi
|
|
12
89
|
const QUERY_SECRET = /([?&](?:access_token|api_key|apikey|token|secret|password|key)=)[^&#\s]+/gi
|
|
13
|
-
|
|
90
|
+
/**
|
|
91
|
+
* Env-style assignments. Added PWD and PASS; requires a value that is not purely
|
|
92
|
+
* numeric, because `MAX_THINKING_TOKENS=31999` is a real setting in this store and
|
|
93
|
+
* redacting it corrupted evidence without protecting anything.
|
|
94
|
+
*/
|
|
95
|
+
const ENV_SECRET = /\b([A-Z0-9_]*(?:SECRET|TOKEN|PASSWORD|PASSWD|PWD|PASS|API_KEY|PRIVATE_KEY|DATABASE_URL)[A-Z0-9_]*\s*=)\s*(?!\d+\b)[^\s]+/gi
|
|
14
96
|
const BEARER_SECRET = /\b(Bearer\s+)[A-Za-z0-9._~+\/-]{12,}/gi
|
|
15
97
|
const FILE_URI = /\bfile:\/\/(?:localhost)?\/(?:[^\s)"'`]+\/?)+/gi
|
|
16
98
|
const LABELED_PATH = /\b(path|file|folder|directory)\s*[:=]\s*(?:~\/|\/(?!\/)[^\s,;)}\]"'`]+)/gi
|
|
@@ -64,6 +146,7 @@ export function cleanContextText(value: unknown, limit: number): string {
|
|
|
64
146
|
text = text.slice(0, Math.max(limit, Math.min(64_000, limit * 2)))
|
|
65
147
|
text = text.replace(CONTROL_CHARS, '')
|
|
66
148
|
text = text.replace(PEM_BLOCK, '[secret hidden]')
|
|
149
|
+
text = text.replace(PUTTY_KEY, '[secret hidden]')
|
|
67
150
|
text = text.replace(JWT_TOKEN, '[secret hidden]')
|
|
68
151
|
text = text.replace(PREFIXED_SECRET, '[secret hidden]')
|
|
69
152
|
text = text.replace(URL_CREDENTIALS, '$1[credentials hidden]@')
|
|
@@ -73,7 +156,15 @@ export function cleanContextText(value: unknown, limit: number): string {
|
|
|
73
156
|
text = text.replace(FILE_URI, '[local path hidden]')
|
|
74
157
|
text = text.replace(LABELED_PATH, '$1: [local path hidden]')
|
|
75
158
|
text = text.replace(WINDOWS_PATH, '[local path hidden]')
|
|
76
|
-
text = text.replace(
|
|
159
|
+
text = text.replace(UNC_PATH, '[local path hidden]')
|
|
160
|
+
// Trimmed in code, not by the regex: a path may contain spaces, so the greedy
|
|
161
|
+
// match is walked back to its last plausible component.
|
|
162
|
+
text = text.replace(ABSOLUTE_PATH, (match: string) => {
|
|
163
|
+
const end = pathMatchEnd(match)
|
|
164
|
+
return `[local path hidden]${match.slice(end)}`
|
|
165
|
+
})
|
|
166
|
+
// AFTER the path rules, so a 40-char path segment cannot be mistaken for a key.
|
|
167
|
+
text = text.replace(AWS_SECRET_KEY, '[secret hidden]')
|
|
77
168
|
text = text.replace(SECRET_TOKEN, '[secret hidden]')
|
|
78
169
|
return text.trim().slice(0, limit)
|
|
79
170
|
}
|
|
@@ -265,6 +356,13 @@ export interface ContextBrowserStatus {
|
|
|
265
356
|
available: boolean
|
|
266
357
|
protocol: number
|
|
267
358
|
state?: string
|
|
359
|
+
/**
|
|
360
|
+
* WHICH tier answered: 'bridge' is a Python/vector pipeline, 'files' is plain
|
|
361
|
+
* markdown on disk. Present so a client can say what it is showing instead of
|
|
362
|
+
* implying a vector store that may not exist. Absent means the field was not
|
|
363
|
+
* supplied, which is every response from a server older than 6.22.0.
|
|
364
|
+
*/
|
|
365
|
+
source?: 'bridge' | 'files'
|
|
268
366
|
memory: { available: boolean; total: number; state: string; reason?: string }
|
|
269
367
|
threads: { available: boolean; total: number; active: number; stale: number; resolved: number; state: string; reason?: string }
|
|
270
368
|
}
|
|
@@ -277,8 +375,16 @@ export function normalizeContextBrowserStatus(value: unknown): ContextBrowserSta
|
|
|
277
375
|
const threads = source.threads && typeof source.threads === 'object' && !Array.isArray(source.threads)
|
|
278
376
|
? source.threads as Record<string, unknown> : {}
|
|
279
377
|
const cleanState = (candidate: unknown, fallback: string) => cleanContextText(candidate, 64) || fallback
|
|
280
|
-
|
|
281
|
-
|
|
378
|
+
// STRICT and PRE-COERCION. finiteInteger() does Math.trunc(Number(v)), so
|
|
379
|
+
// 6.21.35 turned '1', 1.5, 1.9, true and [1] all into 1 and served them as
|
|
380
|
+
// compatible — and rewrote the reported field to 1, so Control could not even
|
|
381
|
+
// see what it had been handed. The changelog claimed the opposite.
|
|
382
|
+
const protocolCompatible = source.protocol === 1
|
|
383
|
+
// Report the RAW value when it is a clean integer, else 0. Truncating 1.5 to 1
|
|
384
|
+
// is what let an incompatible bridge look compatible in Control's own display.
|
|
385
|
+
const protocol = protocolCompatible
|
|
386
|
+
? 1
|
|
387
|
+
: (Number.isInteger(source.protocol) ? source.protocol as number : 0)
|
|
282
388
|
const memoryAvailable = protocolCompatible && memory.available === true
|
|
283
389
|
const threadsAvailable = protocolCompatible && threads.available === true
|
|
284
390
|
const incompatibleState = protocolCompatible ? '' : 'bridge_outdated'
|
|
@@ -290,6 +396,11 @@ export function normalizeContextBrowserStatus(value: unknown): ContextBrowserSta
|
|
|
290
396
|
...(!protocolCompatible
|
|
291
397
|
? { state: 'bridge_outdated' }
|
|
292
398
|
: source.state ? { state: cleanState(source.state, 'unavailable') } : {}),
|
|
399
|
+
// Allowlisted, not passed through: an unrecognised value would let a future
|
|
400
|
+
// or hostile payload put arbitrary text on a surface a client renders.
|
|
401
|
+
...(protocolCompatible && (source.source === 'bridge' || source.source === 'files')
|
|
402
|
+
? { source: source.source }
|
|
403
|
+
: {}),
|
|
293
404
|
memory: {
|
|
294
405
|
available: memoryAvailable,
|
|
295
406
|
total: memoryAvailable ? finiteInteger(memory.total) : 0,
|
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
import { execFile } from 'node:child_process'
|
|
2
2
|
import { resolve } from 'node:path'
|
|
3
3
|
import { existsSync } from 'node:fs'
|
|
4
|
+
import {
|
|
5
|
+
fileMemoryOverview,
|
|
6
|
+
fileTierState,
|
|
7
|
+
fileTierStatus,
|
|
8
|
+
hasFileMemory,
|
|
9
|
+
hasFileThreads,
|
|
10
|
+
readFileMemories,
|
|
11
|
+
readFileMemoryById,
|
|
12
|
+
readFileThreadById,
|
|
13
|
+
readFileThreads,
|
|
14
|
+
resolveContextFilesRoot,
|
|
15
|
+
} from './context-files.js'
|
|
4
16
|
|
|
5
17
|
// Optional COS pipeline bridge.
|
|
6
18
|
//
|
|
@@ -40,6 +52,21 @@ export function pythonBridgeState(): 'ready' | 'pipeline_missing' | 'bridge_miss
|
|
|
40
52
|
return COS_SCRIPTS_DIR ? 'bridge_missing' : 'pipeline_missing'
|
|
41
53
|
}
|
|
42
54
|
|
|
55
|
+
/**
|
|
56
|
+
* Which context source can answer a memory/threads request: the Python bridge,
|
|
57
|
+
* plain files, or nothing.
|
|
58
|
+
*
|
|
59
|
+
* The routes gate on THIS rather than on `pythonBridgeAvailable()`, which is what
|
|
60
|
+
* made the file tier unreachable in the first draft — every route returned 503
|
|
61
|
+
* before `callPython` was ever called, so wiring the fallback into the bridge
|
|
62
|
+
* alone changed nothing observable. Bridge first, always: an install with a venv
|
|
63
|
+
* behaves exactly as it did.
|
|
64
|
+
*/
|
|
65
|
+
export function contextSourceAvailable(): 'bridge' | 'files' | null {
|
|
66
|
+
if (pythonAvailable) return 'bridge'
|
|
67
|
+
return fileTierState(resolveContextFilesRoot()) === 'absent' ? null : 'files'
|
|
68
|
+
}
|
|
69
|
+
|
|
43
70
|
if (pythonAvailable) {
|
|
44
71
|
console.log('[python-bridge] COS pipeline detected — sourcing live context')
|
|
45
72
|
} else if (COS_SCRIPTS_DIR) {
|
|
@@ -58,37 +85,119 @@ export function callPython(args: string[], timeoutMs = 30_000): Promise<unknown>
|
|
|
58
85
|
return Promise.resolve(standaloneNoop(args))
|
|
59
86
|
}
|
|
60
87
|
|
|
61
|
-
/**
|
|
88
|
+
/**
|
|
89
|
+
* No bridge configured. Try the FILE tier first, then fall back to empty shapes.
|
|
90
|
+
*
|
|
91
|
+
* This function is the only door to the file tier, and it is only reached when
|
|
92
|
+
* `pythonAvailable` is false — so an install with a working venv and bridge never
|
|
93
|
+
* executes any of it. That is what makes the file tier backwards compatible by
|
|
94
|
+
* construction rather than by promise: there is no merged resolver to get wrong.
|
|
95
|
+
*
|
|
96
|
+
* A user with a `memory/` or `threads/` folder gets browsable content with no venv,
|
|
97
|
+
* no bridge, and no vector store. When they later build the pipeline, the SAME files
|
|
98
|
+
* become the thing it indexes — nothing to refactor, because the files were always
|
|
99
|
+
* the substrate.
|
|
100
|
+
*/
|
|
62
101
|
function standaloneNoop(args: string[]): unknown {
|
|
102
|
+
const root = resolveContextFilesRoot()
|
|
63
103
|
switch (args[0]) {
|
|
64
104
|
case 'calendar': return { events: [] }
|
|
65
105
|
case 'tasks': return {}
|
|
66
|
-
case 'threads':
|
|
67
|
-
|
|
106
|
+
case 'threads': {
|
|
107
|
+
if (root && hasFileThreads(root)) {
|
|
108
|
+
const threads = readFileThreads(root, argLimit(args, 30))
|
|
109
|
+
return {
|
|
110
|
+
threads,
|
|
111
|
+
active_count: threads.filter(t => !t.is_resolved).length,
|
|
112
|
+
stale_count: 0,
|
|
113
|
+
resolved_count: threads.filter(t => t.is_resolved).length,
|
|
114
|
+
source: 'files',
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return { threads: [], active_count: 0, stale_count: 0, resolved_count: 0 }
|
|
118
|
+
}
|
|
119
|
+
case 'thread-detail': {
|
|
120
|
+
if (root && hasFileThreads(root)) {
|
|
121
|
+
const hit = readFileThreadById(root, args[1] ?? '')
|
|
122
|
+
if (hit) return hit
|
|
123
|
+
}
|
|
124
|
+
return { error: 'cos_pipeline_not_configured' }
|
|
125
|
+
}
|
|
68
126
|
case 'context-status': {
|
|
69
|
-
const
|
|
127
|
+
const bridgeState = pythonBridgeState()
|
|
128
|
+
const tier = fileTierState(root)
|
|
129
|
+
// 'files' is a real, working configuration — not a degraded bridge. Reporting
|
|
130
|
+
// it as unavailable is what made the G2 render "Unavailable." for a user who
|
|
131
|
+
// had perfectly readable notes.
|
|
132
|
+
if (tier !== 'absent') {
|
|
133
|
+
const counts = fileTierStatus(root)
|
|
134
|
+
return {
|
|
135
|
+
available: true,
|
|
136
|
+
protocol: 1,
|
|
137
|
+
state: tier,
|
|
138
|
+
source: 'files',
|
|
139
|
+
memory: {
|
|
140
|
+
available: counts.memory.present,
|
|
141
|
+
total: counts.memory.total,
|
|
142
|
+
state: counts.memory.present ? tier : 'absent',
|
|
143
|
+
},
|
|
144
|
+
threads: {
|
|
145
|
+
available: counts.threads.present,
|
|
146
|
+
total: counts.threads.total,
|
|
147
|
+
active: counts.threads.active,
|
|
148
|
+
// Staleness is a computed property of meeting cadence. Nothing
|
|
149
|
+
// computed it for a file thread, so 0 is the truth, not a default.
|
|
150
|
+
stale: 0,
|
|
151
|
+
resolved: counts.threads.resolved,
|
|
152
|
+
state: counts.threads.present ? tier : 'absent',
|
|
153
|
+
},
|
|
154
|
+
}
|
|
155
|
+
}
|
|
70
156
|
return {
|
|
71
157
|
available: false,
|
|
72
158
|
protocol: 1,
|
|
73
|
-
state,
|
|
74
|
-
memory: { available: false, total: 0, state },
|
|
75
|
-
threads: { available: false, total: 0, active: 0, stale: 0, resolved: 0, state },
|
|
159
|
+
state: bridgeState,
|
|
160
|
+
memory: { available: false, total: 0, state: bridgeState },
|
|
161
|
+
threads: { available: false, total: 0, active: 0, stale: 0, resolved: 0, state: bridgeState },
|
|
76
162
|
}
|
|
77
163
|
}
|
|
78
|
-
case 'memory':
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
164
|
+
case 'memory': {
|
|
165
|
+
if (root && hasFileMemory(root)) return readFileMemories(root, argLimit(args, 30))
|
|
166
|
+
return []
|
|
167
|
+
}
|
|
168
|
+
case 'memory-overview': {
|
|
169
|
+
if (root && hasFileMemory(root)) {
|
|
170
|
+
const o = fileMemoryOverview(root)
|
|
171
|
+
return { available: true, collection: 'files', total: o.total, by_type: o.by_type, source: 'files' }
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
available: false,
|
|
175
|
+
collection: 'cos_memory',
|
|
176
|
+
total: 0,
|
|
177
|
+
by_type: {},
|
|
178
|
+
reason: 'cos_pipeline_not_configured',
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
case 'memory-detail': {
|
|
182
|
+
if (root && hasFileMemory(root)) {
|
|
183
|
+
const hit = readFileMemoryById(root, args[1] ?? '')
|
|
184
|
+
if (hit) return hit
|
|
185
|
+
}
|
|
186
|
+
return { error: 'cos_pipeline_not_configured' }
|
|
85
187
|
}
|
|
86
|
-
case 'memory-detail': return { error: 'cos_pipeline_not_configured' }
|
|
87
188
|
case 'badges': return {}
|
|
88
189
|
default: return {}
|
|
89
190
|
}
|
|
90
191
|
}
|
|
91
192
|
|
|
193
|
+
/** `--limit N` from a bridge-style argv, bounded. */
|
|
194
|
+
function argLimit(args: string[], fallback: number): number {
|
|
195
|
+
const i = args.indexOf('--limit')
|
|
196
|
+
if (i === -1) return fallback
|
|
197
|
+
const n = Number(args[i + 1])
|
|
198
|
+
return Number.isFinite(n) && n > 0 ? Math.min(Math.trunc(n), 50) : fallback
|
|
199
|
+
}
|
|
200
|
+
|
|
92
201
|
/** Full Python bridge — requires the user's venv + cos_api_bridge.py. */
|
|
93
202
|
function callPythonDirect(args: string[], timeoutMs: number): Promise<unknown> {
|
|
94
203
|
return new Promise((resolvePromise, reject) => {
|
package/server/routes/memory.ts
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
import { Router } from 'express'
|
|
2
|
-
import { callPython,
|
|
2
|
+
import { callPython, contextSourceAvailable, pythonBridgeState } from '../lib/python-bridge.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Is there anything to serve — a Python bridge OR plain files on disk?
|
|
6
|
+
*
|
|
7
|
+
* These routes used to gate on `pythonBridgeAvailable()`, which returned 503
|
|
8
|
+
* before `callPython` was ever reached. That is why "I selected COS Memory and
|
|
9
|
+
* the G2 says Unavailable" was the experience for anyone without a venv and a
|
|
10
|
+
* vector database: the answer was decided two layers above the data.
|
|
11
|
+
*/
|
|
12
|
+
function contextConfigured(): boolean {
|
|
13
|
+
return contextSourceAvailable() !== null
|
|
14
|
+
}
|
|
3
15
|
import {
|
|
4
16
|
MEMORY_ID_PATTERN,
|
|
5
17
|
normalizeMemoryDetail,
|
|
@@ -12,7 +24,7 @@ export const memoryRouter = Router()
|
|
|
12
24
|
let overviewCache: { expiresAt: number; value: ReturnType<typeof normalizeMemoryOverview> } | null = null
|
|
13
25
|
|
|
14
26
|
memoryRouter.get('/context/status', async (_req, res) => {
|
|
15
|
-
if (!
|
|
27
|
+
if (!contextConfigured()) {
|
|
16
28
|
const state = pythonBridgeState()
|
|
17
29
|
res.json(normalizeContextBrowserStatus({
|
|
18
30
|
available: false, protocol: 1, state,
|
|
@@ -40,7 +52,7 @@ function boundedInteger(value: unknown, fallback: number, min: number, max: numb
|
|
|
40
52
|
}
|
|
41
53
|
|
|
42
54
|
memoryRouter.get('/memory/overview', async (_req, res) => {
|
|
43
|
-
if (!
|
|
55
|
+
if (!contextConfigured()) {
|
|
44
56
|
res.status(503).json(normalizeMemoryOverview({
|
|
45
57
|
available: false, reason: pythonBridgeState(), total: 0, by_type: {},
|
|
46
58
|
}))
|
|
@@ -71,7 +83,7 @@ memoryRouter.get('/memory/:id', async (req, res) => {
|
|
|
71
83
|
res.status(400).json({ error: 'invalid_memory_id' })
|
|
72
84
|
return
|
|
73
85
|
}
|
|
74
|
-
if (!
|
|
86
|
+
if (!contextConfigured()) {
|
|
75
87
|
res.status(503).json({ error: pythonBridgeState() })
|
|
76
88
|
return
|
|
77
89
|
}
|
|
@@ -95,7 +107,7 @@ memoryRouter.get('/memory/:id', async (req, res) => {
|
|
|
95
107
|
memoryRouter.get('/memory', async (req, res) => {
|
|
96
108
|
const days = boundedInteger(req.query.days, 30, 1, 3650)
|
|
97
109
|
const limit = boundedInteger(req.query.limit, 20, 1, 50)
|
|
98
|
-
if (!
|
|
110
|
+
if (!contextConfigured()) {
|
|
99
111
|
res.status(503).json({ error: pythonBridgeState() })
|
|
100
112
|
return
|
|
101
113
|
}
|
package/server/routes/threads.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Router } from 'express'
|
|
2
|
-
import { callPython,
|
|
2
|
+
import { callPython, contextSourceAvailable, pythonBridgeState } from '../lib/python-bridge.js'
|
|
3
3
|
import { THREAD_ID_PATTERN, normalizeThreadDetail, normalizeThreads } from '../lib/cos-context-browser.js'
|
|
4
4
|
|
|
5
5
|
export const threadsRouter = Router()
|
|
@@ -9,7 +9,7 @@ threadsRouter.get('/threads/:id', async (req, res) => {
|
|
|
9
9
|
res.status(400).json({ error: 'invalid_thread_id' })
|
|
10
10
|
return
|
|
11
11
|
}
|
|
12
|
-
if (
|
|
12
|
+
if (contextSourceAvailable() === null) {
|
|
13
13
|
res.status(503).json({ error: pythonBridgeState() })
|
|
14
14
|
return
|
|
15
15
|
}
|
|
@@ -33,7 +33,7 @@ threadsRouter.get('/threads/:id', async (req, res) => {
|
|
|
33
33
|
threadsRouter.get('/threads', async (req, res) => {
|
|
34
34
|
const parsed = Number(req.query.limit)
|
|
35
35
|
const limit = Number.isFinite(parsed) ? Math.max(1, Math.min(50, Math.trunc(parsed))) : 30
|
|
36
|
-
if (
|
|
36
|
+
if (contextSourceAvailable() === null) {
|
|
37
37
|
res.status(503).json({
|
|
38
38
|
error: pythonBridgeState(), available: false,
|
|
39
39
|
generated_at: '', active_count: 0, stale_count: 0, resolved_count: 0, threads: [],
|