@isparling/engram-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +187 -0
- package/README.md +41 -0
- package/bin/engram +14 -0
- package/package.json +32 -0
- package/release/engram-release.ts +1493 -0
- package/src/atomicWrite.ts +80 -0
- package/src/candidate.ts +229 -0
- package/src/classify.ts +275 -0
- package/src/cli.ts +709 -0
- package/src/contentHash.ts +54 -0
- package/src/deepFreeze.ts +13 -0
- package/src/diff.ts +81 -0
- package/src/guardedRetrieval.ts +128 -0
- package/src/guardedRetrievalInternal.ts +321 -0
- package/src/knowledgeRecord.ts +256 -0
- package/src/knowledgeRetrieval.ts +564 -0
- package/src/knowledgeRollup.ts +479 -0
- package/src/knowledgeTransaction.ts +683 -0
- package/src/knowledgeTypes.ts +249 -0
- package/src/knowledgeValidation.ts +269 -0
- package/src/markdownRecord.ts +265 -0
- package/src/packLoader.ts +188 -0
- package/src/packTypes.ts +12 -0
- package/src/presentation.ts +673 -0
- package/src/qmdConfigGuard.ts +245 -0
- package/src/qmdRunner.ts +392 -0
- package/src/realPath.ts +47 -0
- package/src/spaceBinding.ts +37 -0
- package/src/spaceRegistry.ts +1139 -0
- package/src/submit.ts +228 -0
- package/src/symlinkGuard.ts +71 -0
- package/src/transactionLock.ts +188 -0
- package/src/types.ts +38 -0
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
// qmd configuration guard: a bound config directory or cache home that is the
|
|
2
|
+
// user's real default is refused, and a config declaring an `update:` field is
|
|
3
|
+
// refused, so every qmd invocation stays scoped to a space.
|
|
4
|
+
//
|
|
5
|
+
// The bound qmd config directory holds `index.yml`, and that file is
|
|
6
|
+
// executable content, not passive configuration: qmd's `updateCollections`
|
|
7
|
+
// runs any collection's `update:` field through
|
|
8
|
+
// `Bun.spawn(["/usr/bin/env", "bash", "-c", yamlCol.update], ...)` on every
|
|
9
|
+
// `qmd update`. If anything ever wrote (or a local binding pointed at a
|
|
10
|
+
// config with) an `update:` field, refresh would
|
|
11
|
+
// be arbitrary shell execution. This module is the gate: every qmd
|
|
12
|
+
// invocation that would read this file is checked against it first, and
|
|
13
|
+
// checked for the config directory not being the user's real default.
|
|
14
|
+
//
|
|
15
|
+
// The same class of danger applies to the bound qmd cache home: qmd writes its
|
|
16
|
+
// sqlite index under XDG_CACHE_HOME/qmd, so a binding pointed at the
|
|
17
|
+
// user's real cache home would make refresh operate on the personal
|
|
18
|
+
// index. `isDefaultQmdCacheHome` refuses that the same way.
|
|
19
|
+
//
|
|
20
|
+
// Both default-location checks compare by FILESYSTEM IDENTITY (device +
|
|
21
|
+
// inode via realPath.ts's isSameDirectory), not string equality. On macOS,
|
|
22
|
+
// APFS firmlinks mean `/Users/<user>/.config/qmd` and
|
|
23
|
+
// `/System/Volumes/Data/Users/<user>/.config/qmd` are the same directory
|
|
24
|
+
// but compare unequal as strings (and realpath() does not collapse the
|
|
25
|
+
// firmlink either — verified directly), so a string- or realpath-based
|
|
26
|
+
// check misses a real, reachable way to point at the same dangerous
|
|
27
|
+
// location.
|
|
28
|
+
//
|
|
29
|
+
// This is a deliberately strict, ALLOWLIST-only reader for the exact shape
|
|
30
|
+
// `qmd collection add` produces for a single collection — it is not a
|
|
31
|
+
// general YAML parser, and does not try to be one. Anything outside that
|
|
32
|
+
// shape is refused, not tolerated.
|
|
33
|
+
|
|
34
|
+
import { readFile } from "node:fs/promises";
|
|
35
|
+
import { homedir } from "node:os";
|
|
36
|
+
import { resolve } from "node:path";
|
|
37
|
+
import { isSameDirectory, realOrResolvedPath } from "./realPath.ts";
|
|
38
|
+
import { err, ok, type Result } from "./types.ts";
|
|
39
|
+
import type { SpaceBinding } from "./spaceBinding.ts";
|
|
40
|
+
|
|
41
|
+
export function defaultQmdConfigDir(): string {
|
|
42
|
+
return resolve(homedir(), ".config", "qmd");
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export async function isDefaultQmdConfigDir(qmdConfigDir: string): Promise<boolean> {
|
|
46
|
+
return isSameDirectory(qmdConfigDir, defaultQmdConfigDir());
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** qmd's own fallback when XDG_CACHE_HOME is unset (store.ts:
|
|
50
|
+
* `Bun.env.XDG_CACHE_HOME || resolve(homedir(), ".cache")`) — this is the
|
|
51
|
+
* root a binding's qmd cache home must not collide with. */
|
|
52
|
+
export function defaultQmdCacheHome(): string {
|
|
53
|
+
return resolve(homedir(), ".cache");
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export async function isDefaultQmdCacheHome(qmdCacheHome: string): Promise<boolean> {
|
|
57
|
+
return isSameDirectory(qmdCacheHome, defaultQmdCacheHome());
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// qmd passes a collection's `pattern` straight to `Bun.Glob.scan({ cwd })`,
|
|
61
|
+
// so the pattern decides what the space actually covers — and the symlink
|
|
62
|
+
// guard cannot see it, since the tree is clean and the pattern is what
|
|
63
|
+
// reaches out of it.
|
|
64
|
+
//
|
|
65
|
+
// Two shapes reach above the records root. An ABSOLUTE pattern matches
|
|
66
|
+
// wherever it points and survives qmd's own filtering; this is the live one.
|
|
67
|
+
// A parent-relative pattern is resolved out of cwd by Bun (verified: `..`
|
|
68
|
+
// forms return sibling files), but qmd then discards any result with a
|
|
69
|
+
// dot-prefixed path component, and `..` is dot-prefixed — so in this
|
|
70
|
+
// installed version those results are dropped before being read.
|
|
71
|
+
//
|
|
72
|
+
// Both are refused here regardless. The `..` rule does not depend on qmd's
|
|
73
|
+
// incidental dotfile filter continuing to exist, and relying on a downstream
|
|
74
|
+
// consumer's unrelated exclusion rule for containment would be fragile.
|
|
75
|
+
const ALLOWED_PATTERN_CHARS = /^[A-Za-z0-9_\-./*?{},[\]]+$/;
|
|
76
|
+
|
|
77
|
+
export function validateGlobPattern(pattern: string): Result<void> {
|
|
78
|
+
const errors: string[] = [];
|
|
79
|
+
|
|
80
|
+
if (pattern.trim() === "") {
|
|
81
|
+
errors.push("collection pattern is empty");
|
|
82
|
+
return err(errors);
|
|
83
|
+
}
|
|
84
|
+
if (!ALLOWED_PATTERN_CHARS.test(pattern)) {
|
|
85
|
+
errors.push(
|
|
86
|
+
`collection pattern contains disallowed characters: ${JSON.stringify(pattern)} (allowed: letters, digits, _ - . / * ? { } , [ ])`,
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
if (pattern.startsWith("/")) {
|
|
90
|
+
errors.push(`collection pattern must be relative to the records root, not absolute: ${JSON.stringify(pattern)}`);
|
|
91
|
+
}
|
|
92
|
+
if (pattern.split("/").some((segment) => segment === "..")) {
|
|
93
|
+
errors.push(
|
|
94
|
+
`collection pattern escapes the records root via a ".." segment: ${JSON.stringify(pattern)}`,
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (errors.length > 0) return err(errors);
|
|
99
|
+
return ok(undefined);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
type ParsedLine = { indent: number; content: string };
|
|
103
|
+
|
|
104
|
+
function toIndentedLines(raw: string): ParsedLine[] {
|
|
105
|
+
const result: ParsedLine[] = [];
|
|
106
|
+
for (const line of raw.split("\n")) {
|
|
107
|
+
if (line.trim() === "") continue;
|
|
108
|
+
const indent = line.length - line.trimStart().length;
|
|
109
|
+
result.push({ indent, content: line.trim() });
|
|
110
|
+
}
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function stripQuotes(value: string): string {
|
|
115
|
+
if (value.length >= 2) {
|
|
116
|
+
const first = value[0];
|
|
117
|
+
const last = value[value.length - 1];
|
|
118
|
+
if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
|
|
119
|
+
return value.slice(1, -1);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return value;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Validates that the qmd config at `configPath` declares exactly one
|
|
127
|
+
* collection, named and pathed to match `binding`, with no `update:`
|
|
128
|
+
* field (or any field beyond `path`/`pattern`) anywhere. Call only after
|
|
129
|
+
* confirming the file exists — a missing config is a bootstrap case
|
|
130
|
+
* handled by the caller (ensureBoundCollection), not a validation failure
|
|
131
|
+
* here.
|
|
132
|
+
*/
|
|
133
|
+
export async function validateBoundQmdConfig(configPath: string, binding: SpaceBinding): Promise<Result<void>> {
|
|
134
|
+
let raw: string;
|
|
135
|
+
try {
|
|
136
|
+
raw = await readFile(configPath, "utf8");
|
|
137
|
+
} catch (error) {
|
|
138
|
+
return err([
|
|
139
|
+
`failed to read qmd config at ${configPath}: ${error instanceof Error ? error.message : String(error)}`,
|
|
140
|
+
]);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const errors: string[] = [];
|
|
144
|
+
|
|
145
|
+
// Defense in depth ahead of the structural parse below: an `update:` key
|
|
146
|
+
// at any indentation is refused outright, parse-tree position aside.
|
|
147
|
+
if (/^[ \t]*update[ \t]*:/m.test(raw)) {
|
|
148
|
+
errors.push(
|
|
149
|
+
"qmd config declares an 'update:' field; qmd runs this via bash on every 'qmd update', so it is refused unconditionally",
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const lines = toIndentedLines(raw);
|
|
154
|
+
if (lines.length === 0) {
|
|
155
|
+
errors.push("qmd config is empty");
|
|
156
|
+
return err(errors);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const first = lines[0];
|
|
160
|
+
if (first === undefined || first.indent !== 0 || first.content !== "collections:") {
|
|
161
|
+
errors.push('qmd config must begin with a top-level "collections:" key and no other top-level key');
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const collectionNames: string[] = [];
|
|
165
|
+
const collectionProps = new Map<string, Map<string, string>>();
|
|
166
|
+
let currentCollection: string | null = null;
|
|
167
|
+
|
|
168
|
+
for (let i = 1; i < lines.length; i++) {
|
|
169
|
+
const line = lines[i];
|
|
170
|
+
if (line === undefined) continue;
|
|
171
|
+
const { indent, content } = line;
|
|
172
|
+
|
|
173
|
+
if (indent === 0) {
|
|
174
|
+
errors.push(`qmd config has an unexpected top-level key: ${JSON.stringify(content)}`);
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if (indent === 2) {
|
|
179
|
+
const match = /^([^:]+):\s*$/.exec(content);
|
|
180
|
+
if (!match) {
|
|
181
|
+
errors.push(`unexpected line under "collections:": ${JSON.stringify(content)}`);
|
|
182
|
+
currentCollection = null;
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
const name = match[1] === undefined ? "" : match[1].trim();
|
|
186
|
+
collectionNames.push(name);
|
|
187
|
+
collectionProps.set(name, new Map());
|
|
188
|
+
currentCollection = name;
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (indent === 4 && currentCollection !== null) {
|
|
193
|
+
const match = /^([a-zA-Z_][a-zA-Z0-9_]*):\s*(.*)$/.exec(content);
|
|
194
|
+
if (!match) {
|
|
195
|
+
errors.push(`unexpected line under collection ${JSON.stringify(currentCollection)}: ${JSON.stringify(content)}`);
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
const key = match[1] === undefined ? "" : match[1];
|
|
199
|
+
const value = match[2] === undefined ? "" : match[2].trim();
|
|
200
|
+
if (key !== "path" && key !== "pattern") {
|
|
201
|
+
errors.push(`collection ${JSON.stringify(currentCollection)} has a disallowed field: ${JSON.stringify(key)}`);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
const props = collectionProps.get(currentCollection);
|
|
205
|
+
if (props) props.set(key, stripQuotes(value));
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
errors.push(`unexpected indentation in qmd config: ${JSON.stringify(content)}`);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (collectionNames.length !== 1) {
|
|
213
|
+
errors.push(`qmd config must declare exactly one collection; found ${collectionNames.length}`);
|
|
214
|
+
} else {
|
|
215
|
+
const name = collectionNames[0];
|
|
216
|
+
const props = name === undefined ? undefined : collectionProps.get(name);
|
|
217
|
+
if (name !== binding.qmdCollectionName) {
|
|
218
|
+
errors.push(
|
|
219
|
+
`qmd config's collection name ${JSON.stringify(name)} does not match the bound collection ${JSON.stringify(binding.qmdCollectionName)}`,
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
// qmd stores the collection's REAL path (symlinks resolved) when it
|
|
223
|
+
// writes this file, so the comparison must resolve symlinks on both
|
|
224
|
+
// sides too, or a symlinked temp-directory component (e.g. macOS's
|
|
225
|
+
// /var -> /private/var) produces a false mismatch.
|
|
226
|
+
const expectedPath = await realOrResolvedPath(binding.recordsRoot);
|
|
227
|
+
const configuredPath = props?.get("path");
|
|
228
|
+
const resolvedConfiguredPath = configuredPath ? await realOrResolvedPath(configuredPath) : undefined;
|
|
229
|
+
if (!configuredPath || resolvedConfiguredPath !== expectedPath) {
|
|
230
|
+
errors.push(
|
|
231
|
+
`qmd config's collection path ${JSON.stringify(configuredPath ?? null)} does not match the bound records root ${JSON.stringify(expectedPath)}`,
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
const configuredPattern = props?.get("pattern");
|
|
235
|
+
if (configuredPattern === undefined) {
|
|
236
|
+
errors.push(`qmd config's collection is missing a "pattern" field`);
|
|
237
|
+
} else {
|
|
238
|
+
const patternResult = validateGlobPattern(configuredPattern);
|
|
239
|
+
if (!patternResult.ok) errors.push(...patternResult.errors);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
if (errors.length > 0) return err(errors);
|
|
244
|
+
return ok(undefined);
|
|
245
|
+
}
|
package/src/qmdRunner.ts
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
// Every qmd invocation made on behalf of a space runs with QMD_CONFIG_DIR
|
|
2
|
+
// and XDG_CACHE_HOME set to space-owned paths, and nothing else in this
|
|
3
|
+
// codebase is permitted to spawn qmd.
|
|
4
|
+
//
|
|
5
|
+
// SAFETY BOUNDARY — read before touching this file:
|
|
6
|
+
// `runQmd` is the ONLY function in this codebase allowed to spawn the
|
|
7
|
+
// qmd child process. Every other module that needs qmd (refresh, and
|
|
8
|
+
// test fixture setup) must call through `runQmd`, never `child_process`
|
|
9
|
+
// directly. This is what makes it structurally impossible — not merely
|
|
10
|
+
// policy — for a defect elsewhere to widen scope into the user's real
|
|
11
|
+
// personal qmd collections.
|
|
12
|
+
//
|
|
13
|
+
// Before it spawns anything, `runQmd` runs a preflight check
|
|
14
|
+
// (`preflightCheck`): the bound config directory must not be the user's
|
|
15
|
+
// real default `~/.config/qmd`, and — if a config file already exists at
|
|
16
|
+
// the bound location — its content must validate (exactly one collection,
|
|
17
|
+
// matching name/path, no `update:` field; see qmdConfigGuard.ts). For
|
|
18
|
+
// commands that make qmd scan the filesystem (`update`, `collection add`),
|
|
19
|
+
// it also refuses if any entry under the records root is a symlink
|
|
20
|
+
// resolving outside it (symlinkGuard.ts). A refusal never spawns qmd.
|
|
21
|
+
//
|
|
22
|
+
// `buildQmdInvocation` stays pure (no process spawned) so tests can assert
|
|
23
|
+
// on the constructed command/args/env/cwd directly. `runQmd` accepts an
|
|
24
|
+
// injectable `spawnFn` (defaulting to the real `child_process.spawn`) so
|
|
25
|
+
// tests can also assert on what the spawn boundary itself receives and
|
|
26
|
+
// simulate spawn success/failure deterministically, without mocking
|
|
27
|
+
// `child_process` globally.
|
|
28
|
+
|
|
29
|
+
import { spawn as nodeSpawn } from "node:child_process";
|
|
30
|
+
import { existsSync } from "node:fs";
|
|
31
|
+
import { join } from "node:path";
|
|
32
|
+
import { isDefaultQmdCacheHome, isDefaultQmdConfigDir, validateBoundQmdConfig } from "./qmdConfigGuard.ts";
|
|
33
|
+
import type { SpaceBinding } from "./spaceBinding.ts";
|
|
34
|
+
import { verifyNoSymlinkEscape } from "./symlinkGuard.ts";
|
|
35
|
+
import { err, ok, type EnvLike, type Result } from "./types.ts";
|
|
36
|
+
|
|
37
|
+
export type QmdInvocation = {
|
|
38
|
+
command: string;
|
|
39
|
+
args: string[];
|
|
40
|
+
cwd: string;
|
|
41
|
+
env: Record<string, string>;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Pure: constructs the exact command/args/cwd/env that would be spawned,
|
|
46
|
+
* without spawning anything. `baseEnv` defaults to the real process
|
|
47
|
+
* environment but is overridable so tests can inject a hostile-looking
|
|
48
|
+
* base environment (e.g. one that already has INDEX_PATH or
|
|
49
|
+
* QMD_CONFIG_DIR pointed at real personal collections) and assert those
|
|
50
|
+
* values never survive into the constructed invocation.
|
|
51
|
+
*/
|
|
52
|
+
export function buildQmdInvocation(args: string[], binding: SpaceBinding, baseEnv: EnvLike = process.env): QmdInvocation {
|
|
53
|
+
const env: Record<string, string> = {};
|
|
54
|
+
for (const [key, value] of Object.entries(baseEnv)) {
|
|
55
|
+
if (value !== undefined) env[key] = value;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// qmd allows INDEX_PATH to override the index location outright (see
|
|
59
|
+
// qmd's store.ts getDefaultDbPath). If that var leaked in from the
|
|
60
|
+
// invoking host's environment it would silently bypass XDG_CACHE_HOME
|
|
61
|
+
// scoping, so it is always cleared here regardless of baseEnv.
|
|
62
|
+
delete env.INDEX_PATH;
|
|
63
|
+
|
|
64
|
+
env.QMD_CONFIG_DIR = binding.qmdConfigDir;
|
|
65
|
+
env.XDG_CACHE_HOME = binding.qmdCacheHome;
|
|
66
|
+
|
|
67
|
+
// qmd reads process.env.PWD (falling back to the real cwd) in several
|
|
68
|
+
// places that resolve relative/"." paths. child_process.spawn sets the
|
|
69
|
+
// OS-level cwd correctly regardless, but a stale inherited PWD would
|
|
70
|
+
// still be visible to qmd's own process.env.PWD lookup, so it is set to
|
|
71
|
+
// match cwd explicitly rather than left as whatever the harness process
|
|
72
|
+
// happened to inherit.
|
|
73
|
+
env.PWD = binding.recordsRoot;
|
|
74
|
+
|
|
75
|
+
return { command: "qmd", args, cwd: binding.recordsRoot, env };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The minimal surface of a spawned child process this module needs. Node's
|
|
79
|
+
* real ChildProcess (returned by child_process.spawn) satisfies this
|
|
80
|
+
* structurally; tests can substitute a much smaller fake without needing to
|
|
81
|
+
* implement all of ChildProcess. */
|
|
82
|
+
export type QmdChildProcess = {
|
|
83
|
+
readonly stdout: NodeJS.ReadableStream | null;
|
|
84
|
+
readonly stderr: NodeJS.ReadableStream | null;
|
|
85
|
+
on(event: "spawn", listener: () => void): unknown;
|
|
86
|
+
on(event: "error", listener: (error: Error) => void): unknown;
|
|
87
|
+
on(event: "close", listener: (code: number | null) => void): unknown;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
export type SpawnFn = (
|
|
91
|
+
command: string,
|
|
92
|
+
args: string[],
|
|
93
|
+
options: { cwd: string; env: Record<string, string> },
|
|
94
|
+
) => QmdChildProcess;
|
|
95
|
+
|
|
96
|
+
const realSpawn: SpawnFn = (command, args, options) => nodeSpawn(command, args, options);
|
|
97
|
+
|
|
98
|
+
export type QmdExecution = {
|
|
99
|
+
/** True iff the OS actually started the process (Node's 'spawn' event
|
|
100
|
+
* fired). False means qmd never ran at all — either the preflight
|
|
101
|
+
* refused before spawning, or the spawn itself failed (e.g. ENOENT). */
|
|
102
|
+
ranProcess: boolean;
|
|
103
|
+
code: number | null;
|
|
104
|
+
stdout: string;
|
|
105
|
+
stderr: string;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
async function executeQmdInvocation(invocation: QmdInvocation, spawnFn: SpawnFn): Promise<QmdExecution> {
|
|
109
|
+
return new Promise((resolvePromise) => {
|
|
110
|
+
const child = spawnFn(invocation.command, invocation.args, { cwd: invocation.cwd, env: invocation.env });
|
|
111
|
+
|
|
112
|
+
let stdout = "";
|
|
113
|
+
let stderr = "";
|
|
114
|
+
let ranProcess = false;
|
|
115
|
+
let settled = false;
|
|
116
|
+
|
|
117
|
+
child.stdout?.on("data", (chunk: Buffer) => {
|
|
118
|
+
stdout += chunk.toString("utf8");
|
|
119
|
+
});
|
|
120
|
+
child.stderr?.on("data", (chunk: Buffer) => {
|
|
121
|
+
stderr += chunk.toString("utf8");
|
|
122
|
+
});
|
|
123
|
+
child.on("spawn", () => {
|
|
124
|
+
ranProcess = true;
|
|
125
|
+
});
|
|
126
|
+
child.on("error", (error: Error) => {
|
|
127
|
+
if (settled) return;
|
|
128
|
+
settled = true;
|
|
129
|
+
resolvePromise({ ranProcess, code: null, stdout, stderr: stderr || error.message });
|
|
130
|
+
});
|
|
131
|
+
child.on("close", (code) => {
|
|
132
|
+
if (settled) return;
|
|
133
|
+
settled = true;
|
|
134
|
+
resolvePromise({ ranProcess, code, stdout, stderr });
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function commandScansFilesystem(args: string[]): boolean {
|
|
140
|
+
if (args[0] === "update") return true;
|
|
141
|
+
if (args[0] === "collection" && args[1] === "add") return true;
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async function preflightCheck(args: string[], binding: SpaceBinding): Promise<Result<void>> {
|
|
146
|
+
if (await isDefaultQmdConfigDir(binding.qmdConfigDir)) {
|
|
147
|
+
return err([
|
|
148
|
+
"the bound qmd config directory resolves to the user's default (~/.config/qmd); refusing to touch it",
|
|
149
|
+
]);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (await isDefaultQmdCacheHome(binding.qmdCacheHome)) {
|
|
153
|
+
return err([
|
|
154
|
+
"the bound qmd cache home resolves to the user's default (~/.cache); refusing to touch it, since qmd would write to the personal index",
|
|
155
|
+
]);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const configPath = join(binding.qmdConfigDir, "index.yml");
|
|
159
|
+
if (existsSync(configPath)) {
|
|
160
|
+
const configResult = await validateBoundQmdConfig(configPath, binding);
|
|
161
|
+
if (!configResult.ok) return configResult;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (commandScansFilesystem(args)) {
|
|
165
|
+
const symlinkResult = await verifyNoSymlinkEscape(binding.recordsRoot);
|
|
166
|
+
if (!symlinkResult.ok) return symlinkResult;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
return ok(undefined);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* THE single function in this codebase permitted to spawn qmd. Refresh,
|
|
174
|
+
* collection bootstrap, and test-fixture setup all route through this, so
|
|
175
|
+
* the preflight check and the scoped environment built by
|
|
176
|
+
* buildQmdInvocation are never bypassed. A preflight refusal returns a
|
|
177
|
+
* QmdExecution with ranProcess: false and never calls spawnFn.
|
|
178
|
+
*/
|
|
179
|
+
export async function runQmd(args: string[], binding: SpaceBinding, spawnFn: SpawnFn = realSpawn): Promise<QmdExecution> {
|
|
180
|
+
const preflight = await preflightCheck(args, binding);
|
|
181
|
+
if (!preflight.ok) {
|
|
182
|
+
return { ranProcess: false, code: null, stdout: "", stderr: `refused before invocation: ${preflight.errors.join("; ")}` };
|
|
183
|
+
}
|
|
184
|
+
const invocation = buildQmdInvocation(args, binding);
|
|
185
|
+
return executeQmdInvocation(invocation, spawnFn);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Either the collection already existed (nothing was run), or it didn't
|
|
190
|
+
* and a `collection add` call was actually started — carrying that
|
|
191
|
+
* call's QmdExecution so the caller can assess success/freshness from it
|
|
192
|
+
* directly, instead of also running `update` right afterward.
|
|
193
|
+
*/
|
|
194
|
+
export type EnsureCollectionOutcome = { provisioned: false } | { provisioned: true; execution: QmdExecution };
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Creates the bound qmd collection if it doesn't exist yet. Binding
|
|
198
|
+
* registration deliberately performs no index scan, so the first committed
|
|
199
|
+
* operation bootstraps the collection as its one refresh.
|
|
200
|
+
*
|
|
201
|
+
* `qmd collection add` indexes the collection as part of creating it
|
|
202
|
+
* (verified against qmd's own collectionAdd, which calls indexFiles
|
|
203
|
+
* internally and emits the same "Indexed: ..." line `update` does). This
|
|
204
|
+
* only reports whether the process ran at all (ranProcess: false is a
|
|
205
|
+
* genuine "provisioning never happened" failure); it does NOT judge
|
|
206
|
+
* success/freshness from the exit code, because the caller
|
|
207
|
+
* (refreshQmdCollection) needs to apply that judgment uniformly to
|
|
208
|
+
* whichever call — this one or a plain `update` — ends up being this
|
|
209
|
+
* invocation's one refresh.
|
|
210
|
+
*/
|
|
211
|
+
export async function ensureBoundCollection(
|
|
212
|
+
binding: SpaceBinding,
|
|
213
|
+
spawnFn: SpawnFn = realSpawn,
|
|
214
|
+
): Promise<Result<EnsureCollectionOutcome>> {
|
|
215
|
+
const configPath = join(binding.qmdConfigDir, "index.yml");
|
|
216
|
+
if (existsSync(configPath)) {
|
|
217
|
+
return ok({ provisioned: false });
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const execution = await runQmd(
|
|
221
|
+
["collection", "add", binding.recordsRoot, "--name", binding.qmdCollectionName, "--mask", "*.md"],
|
|
222
|
+
binding,
|
|
223
|
+
spawnFn,
|
|
224
|
+
);
|
|
225
|
+
|
|
226
|
+
if (!execution.ranProcess) {
|
|
227
|
+
return err([`qmd collection setup never ran: ${execution.stderr.trim().slice(0, 300)}`]);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
return ok({ provisioned: true, execution });
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
type RefreshReportBase = {
|
|
234
|
+
/** Number of `qmd update` invocations that actually ran to completion
|
|
235
|
+
* for this refresh — computed from whether the process really started
|
|
236
|
+
* and exited, never a hard-coded literal. 0 covers both "refresh was
|
|
237
|
+
* never attempted" and "the update process never started"; the detail
|
|
238
|
+
* string distinguishes the two. */
|
|
239
|
+
detail: string;
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
export type AttemptedRefreshReport = RefreshReportBase & {
|
|
243
|
+
attempted: true;
|
|
244
|
+
count: 0 | 1;
|
|
245
|
+
state: "fresh" | "index-stale";
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
export type SkippedStaleRefreshReport = RefreshReportBase & {
|
|
249
|
+
attempted: false;
|
|
250
|
+
count: 0;
|
|
251
|
+
state: "index-stale";
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
export type NotAttemptedRefreshReport = RefreshReportBase & {
|
|
255
|
+
attempted: false;
|
|
256
|
+
count: 0;
|
|
257
|
+
state: "not-attempted";
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
export type FreshnessRefreshReport = AttemptedRefreshReport | SkippedStaleRefreshReport;
|
|
261
|
+
|
|
262
|
+
export type RefreshReport = FreshnessRefreshReport | NotAttemptedRefreshReport;
|
|
263
|
+
|
|
264
|
+
export const REFRESH_NOT_ATTEMPTED: NotAttemptedRefreshReport = {
|
|
265
|
+
attempted: false,
|
|
266
|
+
count: 0,
|
|
267
|
+
state: "not-attempted",
|
|
268
|
+
detail: "no write occurred, so refresh was not attempted",
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
function stripAnsi(text: string): string {
|
|
272
|
+
return text
|
|
273
|
+
// OSC sequences: ESC ] ... (BEL | ESC \)
|
|
274
|
+
.replace(/\x1b\][^\x07\x1b]*(\x07|\x1b\\)/g, "")
|
|
275
|
+
// CSI sequences: ESC [ ... letter
|
|
276
|
+
.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "")
|
|
277
|
+
.replace(/\r/g, "");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const INDEXED_LINE_PATTERN = /Indexed:\s*\d+ new,\s*\d+ updated,\s*\d+ unchanged,\s*\d+ removed/;
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Refreshes the space's bound qmd collection exactly once — meaning
|
|
284
|
+
* exactly one qmd indexing pass, not exactly one qmd subcommand. When the
|
|
285
|
+
* collection has never been provisioned, `collection add` itself scans
|
|
286
|
+
* and indexes the tree, so THAT call is this invocation's one refresh and
|
|
287
|
+
* `update` is not also run afterward (that would be a second, redundant
|
|
288
|
+
* scan of the same records root, reported as if it were still "once").
|
|
289
|
+
* When the collection already exists, the one call is a plain `update`.
|
|
290
|
+
*
|
|
291
|
+
* Fails closed either way: a nonzero exit code, a process that never
|
|
292
|
+
* started, a preflight refusal, or stdout that does not contain the
|
|
293
|
+
* expected "Indexed: ..." line all report `index-stale` rather than being
|
|
294
|
+
* assumed fresh. qmd's own output is human prose with terminal escape
|
|
295
|
+
* sequences and no machine-readable mode, so this is a
|
|
296
|
+
* best-effort parse, not a guarantee.
|
|
297
|
+
*/
|
|
298
|
+
export async function refreshQmdCollection(binding: SpaceBinding, spawnFn: SpawnFn = realSpawn): Promise<AttemptedRefreshReport> {
|
|
299
|
+
const ensureResult = await ensureBoundCollection(binding, spawnFn);
|
|
300
|
+
if (!ensureResult.ok) {
|
|
301
|
+
return {
|
|
302
|
+
attempted: true,
|
|
303
|
+
count: 0,
|
|
304
|
+
state: "index-stale",
|
|
305
|
+
detail: `qmd collection provisioning never ran, so refresh never ran: ${ensureResult.errors.join("; ")}`,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const outcome = ensureResult.value;
|
|
310
|
+
const execution = outcome.provisioned ? outcome.execution : await runQmd(["update"], binding, spawnFn);
|
|
311
|
+
const ranCommand = outcome.provisioned ? "collection add" : "update";
|
|
312
|
+
|
|
313
|
+
if (!execution.ranProcess) {
|
|
314
|
+
return {
|
|
315
|
+
attempted: true,
|
|
316
|
+
count: 0,
|
|
317
|
+
state: "index-stale",
|
|
318
|
+
detail: `qmd ${ranCommand} never ran: ${execution.stderr.trim().slice(0, 500)}`,
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
const cleanStdout = stripAnsi(execution.stdout);
|
|
323
|
+
const sawIndexedLine = INDEXED_LINE_PATTERN.test(cleanStdout);
|
|
324
|
+
|
|
325
|
+
if (execution.code === 0 && sawIndexedLine) {
|
|
326
|
+
return { attempted: true, count: 1, state: "fresh", detail: cleanStdout.trim() };
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
return {
|
|
330
|
+
attempted: true,
|
|
331
|
+
count: 1,
|
|
332
|
+
state: "index-stale",
|
|
333
|
+
detail:
|
|
334
|
+
`qmd ${ranCommand} ran but did not report success: exit code ${execution.code}; ` +
|
|
335
|
+
(sawIndexedLine ? "" : "no recognizable 'Indexed: ...' line in stdout; ") +
|
|
336
|
+
`stderr: ${execution.stderr.trim().slice(0, 500)}`,
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/** Outcome of the vector-embedding pass. Reported separately from a refresh
|
|
341
|
+
* because embedding is a different operation on a different derived artifact:
|
|
342
|
+
* `update` maintains the full-text index, `embed` maintains the vectors that
|
|
343
|
+
* back `vsearch`. A caller that refreshes without embedding leaves semantic
|
|
344
|
+
* search stale for the records it just wrote, so the two states must not be
|
|
345
|
+
* collapsed into one "qmd is fine" flag. */
|
|
346
|
+
export type EmbedReport = {
|
|
347
|
+
attempted: boolean;
|
|
348
|
+
state: "embedded" | "embeddings-stale";
|
|
349
|
+
detail: string;
|
|
350
|
+
};
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Rebuilds vector embeddings for the bound space.
|
|
354
|
+
*
|
|
355
|
+
* Deliberately NOT called by `refreshQmdCollection`, `submitKnowledgeCandidate`,
|
|
356
|
+
* or the knowledge transaction. Two reasons, both load-bearing:
|
|
357
|
+
*
|
|
358
|
+
* 1. `refreshQmdCollection`'s pinned property is exactly one indexing pass per
|
|
359
|
+
* committed change. Embedding is not an indexing pass — it reads the index
|
|
360
|
+
* that pass just wrote — but folding it in would still make every caller
|
|
361
|
+
* spawn a second qmd process, which is not what "refresh" means.
|
|
362
|
+
* 2. Embedding runs an embedding model over every changed chunk. Doing it once
|
|
363
|
+
* per committed record would run it N times for a batch of N writes. It
|
|
364
|
+
* belongs at the boundary of a batch, which is where a caller invokes it.
|
|
365
|
+
*
|
|
366
|
+
* `qmd embed` takes no collection argument: it embeds whatever the resolved
|
|
367
|
+
* configuration contains. That is correct because a space's
|
|
368
|
+
* configuration contains only that space's collections — and it is exactly why
|
|
369
|
+
* this must go through `runQmd`, which scopes `QMD_CONFIG_DIR` and
|
|
370
|
+
* `XDG_CACHE_HOME`. Spawned directly it would reach the ambient personal index.
|
|
371
|
+
*/
|
|
372
|
+
export async function embedBoundCollection(binding: SpaceBinding, spawnFn: SpawnFn = realSpawn): Promise<EmbedReport> {
|
|
373
|
+
const execution = await runQmd(["embed"], binding, spawnFn);
|
|
374
|
+
|
|
375
|
+
if (!execution.ranProcess) {
|
|
376
|
+
return {
|
|
377
|
+
attempted: true,
|
|
378
|
+
state: "embeddings-stale",
|
|
379
|
+
detail: `qmd embed never started: ${execution.stderr.trim().slice(0, 500)}`,
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
if (execution.code === 0) {
|
|
384
|
+
return { attempted: true, state: "embedded", detail: stripAnsi(execution.stdout).trim() };
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
return {
|
|
388
|
+
attempted: true,
|
|
389
|
+
state: "embeddings-stale",
|
|
390
|
+
detail: `qmd embed ran but exit code ${execution.code}; stderr: ${execution.stderr.trim().slice(0, 500)}`,
|
|
391
|
+
};
|
|
392
|
+
}
|
package/src/realPath.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Real-path helpers that resolve symlinks before containment checks, so an
|
|
2
|
+
// escaping path is caught no matter how it is spelled.
|
|
3
|
+
//
|
|
4
|
+
// qmd resolves symlinks before storing or comparing collection paths (see
|
|
5
|
+
// qmd.ts's getRealPath: realpath, falling back to a plain resolve() if
|
|
6
|
+
// the path doesn't exist yet) — e.g. it stores the collection root's
|
|
7
|
+
// REAL path in index.yml, not whatever path string it was given. Any
|
|
8
|
+
// comparison against something qmd wrote must apply the same resolution,
|
|
9
|
+
// or a symlinked path component (macOS puts the whole OS temp directory
|
|
10
|
+
// under /var, itself a symlink to /private/var) produces spurious
|
|
11
|
+
// mismatches: two paths that are the same real location compare unequal
|
|
12
|
+
// because only one of them had its symlinks followed.
|
|
13
|
+
|
|
14
|
+
import { stat, realpath } from "node:fs/promises";
|
|
15
|
+
import { resolve } from "node:path";
|
|
16
|
+
|
|
17
|
+
export async function realOrResolvedPath(path: string): Promise<string> {
|
|
18
|
+
try {
|
|
19
|
+
return await realpath(path);
|
|
20
|
+
} catch {
|
|
21
|
+
return resolve(path);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* True iff two paths name the same directory on disk, compared by
|
|
27
|
+
* filesystem identity (device + inode) rather than string equality.
|
|
28
|
+
* String/realpath comparison alone is insufficient on macOS: APFS
|
|
29
|
+
* firmlinks make `/Users/<user>/.config/qmd` and
|
|
30
|
+
* `/System/Volumes/Data/Users/<user>/.config/qmd` two different path
|
|
31
|
+
* strings (and different realpath() results — realpath does not resolve
|
|
32
|
+
* firmlinks) that are nonetheless the same directory.
|
|
33
|
+
*
|
|
34
|
+
* If either path can't be stat'd (doesn't exist, no permission), identity
|
|
35
|
+
* can't be proven either way, so this falls back to a resolved-path
|
|
36
|
+
* string comparison rather than silently treating "unprovable" as either
|
|
37
|
+
* "same" or "different".
|
|
38
|
+
*/
|
|
39
|
+
export async function isSameDirectory(pathA: string, pathB: string): Promise<boolean> {
|
|
40
|
+
try {
|
|
41
|
+
const [statA, statB] = await Promise.all([stat(pathA), stat(pathB)]);
|
|
42
|
+
return statA.dev === statB.dev && statA.ino === statB.ino;
|
|
43
|
+
} catch {
|
|
44
|
+
const [resolvedA, resolvedB] = await Promise.all([realOrResolvedPath(pathA), realOrResolvedPath(pathB)]);
|
|
45
|
+
return resolvedA === resolvedB;
|
|
46
|
+
}
|
|
47
|
+
}
|