memtrails 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +39 -0
  2. package/README.md +20 -13
  3. package/dist/core/consolidate.d.ts +23 -8
  4. package/dist/core/consolidate.js +52 -25
  5. package/dist/core/episodes.d.ts +1 -1
  6. package/dist/core/episodes.js +2 -8
  7. package/dist/core/extract.js +5 -5
  8. package/dist/core/gather.d.ts +7 -0
  9. package/dist/core/gather.js +38 -14
  10. package/dist/core/model.d.ts +8 -4
  11. package/dist/core/model.js +29 -40
  12. package/dist/core/profiles.d.ts +1 -1
  13. package/dist/core/profiles.js +7 -6
  14. package/dist/core/qdrant-binary.d.ts +39 -5
  15. package/dist/core/qdrant-binary.js +185 -48
  16. package/dist/core/qdrant-client.d.ts +6 -18
  17. package/dist/core/qdrant-client.js +38 -90
  18. package/dist/core/qdrant-process.d.ts +22 -1
  19. package/dist/core/qdrant-process.js +190 -22
  20. package/dist/core/rebuild.d.ts +5 -1
  21. package/dist/core/rebuild.js +19 -4
  22. package/dist/core/recall-log.d.ts +3 -0
  23. package/dist/core/recall-log.js +6 -1
  24. package/dist/core/recall.js +14 -5
  25. package/dist/core/redact.d.ts +8 -0
  26. package/dist/core/redact.js +48 -9
  27. package/dist/core/sleep.d.ts +4 -3
  28. package/dist/core/sleep.js +19 -11
  29. package/dist/core/store.d.ts +7 -1
  30. package/dist/core/store.js +101 -33
  31. package/dist/core/text.d.ts +4 -2
  32. package/dist/core/text.js +17 -5
  33. package/dist/core/words.js +6 -2
  34. package/dist/core/writer.d.ts +2 -1
  35. package/dist/core/writer.js +9 -3
  36. package/dist/importers/files.js +53 -10
  37. package/dist/importers/git.js +3 -1
  38. package/dist/importers/mbox.js +22 -7
  39. package/dist/importers/slack.js +5 -2
  40. package/dist/importers/text.js +85 -14
  41. package/dist/llm.d.ts +1 -8
  42. package/dist/llm.js +11 -5
  43. package/dist/memory.d.ts +9 -3
  44. package/dist/memory.js +216 -48
  45. package/dist/records.js +6 -3
  46. package/dist/tools/agent-tools.d.ts +2 -5
  47. package/dist/tools/agent-tools.js +37 -21
  48. package/package.json +1 -1
@@ -4,12 +4,10 @@
4
4
  * pinned size and SHA-256. 218 MB. The model is Google's, under the Gemma Terms of Use (ai.google.dev/gemma/terms);
5
5
  * memtrails does not ship it.
6
6
  */
7
- import { createHash } from 'node:crypto';
8
- import { createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
7
+ import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
9
8
  import { dirname, join } from 'node:path';
10
- import { Readable } from 'node:stream';
11
- import { pipeline } from 'node:stream/promises';
12
9
  import { cacheDir } from './paths.js';
10
+ import { downloadVerified, withLock } from './qdrant-binary.js';
13
11
  export const MODEL_REPO = 'onnx-community/embeddinggemma-300m-ONNX';
14
12
  export const MODEL_REVISION = '5090578d9565bb06545b4552f76e6bc2c93e4a66';
15
13
  export const MODEL_FILES = [
@@ -23,52 +21,43 @@ export function defaultModelDir() {
23
21
  }
24
22
  const marker = (dir) => join(dir, '.verified');
25
23
  const stamp = MODEL_FILES.map((f) => `${f.rel}:${f.size}:${f.sha256}`).join('\n');
24
+ /** Every file in place at its pinned size, and the marker written after the last of them was verified. */
26
25
  export function modelReady(dir = defaultModelDir()) {
27
26
  try {
28
- return readFileSync(marker(dir), 'utf8') === stamp && MODEL_FILES.every((f) => existsSync(join(dir, f.rel)));
27
+ return readFileSync(marker(dir), 'utf8') === stamp &&
28
+ MODEL_FILES.every((f) => existsSync(join(dir, f.rel)) && statSync(join(dir, f.rel)).size === f.size);
29
29
  }
30
30
  catch {
31
31
  return false;
32
32
  }
33
33
  }
34
- /** The model in `dir` (fetched and checked unless it is already there); the directory, ready for `modelDir`. */
34
+ /**
35
+ * The model in `dir` (fetched and checked unless it is already there); the directory, ready for `modelDir`. Each file
36
+ * is fetched to a temp of its own, checked on disk and renamed into place; one run at a time per `dir` (withLock), and
37
+ * the marker is taken away first, so a set left half-replaced is never taken as ready.
38
+ */
35
39
  export async function ensureModel(opts = {}) {
36
40
  const dir = opts.dir ?? defaultModelDir();
37
41
  if (modelReady(dir))
38
42
  return dir;
39
- opts.log?.(`[memtrails] fetching ${MODEL_REPO} (218 MB, Gemma Terms of Use)`);
40
- for (const f of MODEL_FILES) {
41
- const path = join(dir, f.rel);
42
- mkdirSync(dirname(path), { recursive: true });
43
- const part = `${path}.part`;
44
- rmSync(part, { force: true });
45
- const res = await (opts.fetchImpl ?? fetch)(`https://huggingface.co/${MODEL_REPO}/resolve/${MODEL_REVISION}/${f.rel}`, { signal: opts.signal, redirect: 'follow' });
46
- if (!res.ok || !res.body)
47
- throw new Error(`HTTP ${res.status} for ${f.rel}`);
48
- const hash = createHash('sha256');
49
- let size = 0;
50
- try {
51
- await pipeline(Readable.fromWeb(res.body), async function* (src) {
52
- for await (const chunk of src) {
53
- const b = chunk;
54
- hash.update(b);
55
- size += b.length;
56
- yield b;
57
- }
58
- }, createWriteStream(part));
59
- }
60
- catch (err) {
61
- rmSync(part, { force: true });
62
- throw err;
43
+ mkdirSync(dir, { recursive: true });
44
+ return withLock(dir, async () => {
45
+ if (modelReady(dir))
46
+ return dir; // another run finished it while this one waited
47
+ opts.log?.(`[memtrails] fetching ${MODEL_REPO} (218 MB, Gemma Terms of Use)`);
48
+ rmSync(marker(dir), { force: true });
49
+ for (const f of MODEL_FILES) {
50
+ const path = join(dir, f.rel);
51
+ const { file, tmp } = await downloadVerified(`https://huggingface.co/${MODEL_REPO}/resolve/${MODEL_REVISION}/${f.rel}`, dirname(path), f, opts);
52
+ try {
53
+ renameSync(file, path);
54
+ }
55
+ finally {
56
+ rmSync(tmp, { recursive: true, force: true });
57
+ }
63
58
  }
64
- const digest = hash.digest('hex');
65
- if (size !== f.size || digest !== f.sha256) {
66
- rmSync(part, { force: true });
67
- throw new Error(`${f.rel}: got ${size} bytes sha256 ${digest}, expected ${f.size} ${f.sha256}`);
68
- }
69
- renameSync(part, path);
70
- }
71
- writeFileSync(marker(dir), stamp);
72
- opts.log?.('[memtrails] embedding model ready');
73
- return dir;
59
+ writeFileSync(marker(dir), stamp);
60
+ opts.log?.('[memtrails] embedding model ready');
61
+ return dir;
62
+ }, { signal: opts.signal });
74
63
  }
@@ -1,4 +1,4 @@
1
- import type { ConsolidateDeps } from './consolidate.js';
1
+ import { type ConsolidateDeps } from './consolidate.js';
2
2
  import type { MemoryItem, MemoryScope } from './store.js';
3
3
  export declare const PROFILE_MIN_ITEMS = 3;
4
4
  export declare const PROFILE_BROAD = 0.2;
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Profiles — what memory knows about one subject, gathered in one place (sleep's last step).
3
3
  *
4
- * A question about a subject whose facts were told over many conversations ("when did John join the brigade, and what
5
- * did he do there?") needs the dated facts of all of them, and a top 10 holds a few. For every concept enough long-term
4
+ * A question about a subject whose facts were told over many conversations ("when did Dana join the choir, and what
5
+ * did she sing there?") needs the dated facts of all of them, and a top 10 holds a few. For every concept enough long-term
6
6
  * items share — at least PROFILE_MIN_ITEMS of a project's items, and on at most PROFILE_BROAD of them (not the person
7
7
  * themselves, not the project's own name) — the fast model gathers what they say into one dated list of facts, word for
8
8
  * word: a `profile` item, keyed and searched like any other, whose provenance is every source it read.
@@ -18,6 +18,7 @@
18
18
  import { createHash } from 'node:crypto';
19
19
  import { redactString } from './redact.js';
20
20
  import { defaultAsk } from '../llm.js';
21
+ import { stamped } from './consolidate.js';
21
22
  import { conceptKey, cut } from './text.js';
22
23
  import { writeLtm } from './writer.js';
23
24
  export const PROFILE_MIN_ITEMS = 3;
@@ -86,15 +87,15 @@ export function profileSubjects(items, sizes) {
86
87
  return out;
87
88
  }
88
89
  const promptOf = (s) => `SUBJECT: ${s.concept}\n\nMEMORIES:\n${s.read.map((it) => `- (${when(it)}) ${it.title}: ${cut(it.summary.replace(/\s+/g, ' '), PROFILE_LINE)}`).join('\n')}`;
89
- function profileItem(s, title, facts) {
90
+ function profileItem(s, title, facts, now) {
90
91
  const newest = s.read[s.read.length - 1];
91
- return {
92
+ return stamped({
92
93
  uid: s.uid, kind: 'profile', scope: s.scope, project: s.project, engine: newest.engine, session: null, cwd: newest.cwd,
93
94
  where: s.concept, ts: Math.max(...s.read.map((it) => it.ts)), firstTs: Math.min(...s.read.map((it) => it.firstTs || it.ts)),
94
95
  title: cut(title.trim() || s.concept, 120), summary: facts.join('\n'), body: '', files: [], appliesWhen: '',
95
96
  confidence: 'curated', importance: 3, status: 'active', provenance: [...new Set(s.read.flatMap((it) => it.provenance))],
96
97
  seen: 1, fingerprint: s.fingerprint, concepts: [s.concept],
97
- };
98
+ }, now);
98
99
  }
99
100
  /** Profiles written for the subjects whose items changed (at most `max`, several asks at once), and archived for the
100
101
  * subjects that no longer qualify. An ask the model did not answer stops the night's profiles; the next night asks
@@ -137,7 +138,7 @@ export async function writeProfiles(d, max = PROFILES_PER_SLEEP) {
137
138
  const facts = (Array.isArray(answer.facts) ? answer.facts : []).filter((f) => typeof f === 'string' && !!f.trim()).map((f) => f.trim());
138
139
  if (!facts.length)
139
140
  continue;
140
- const it = profileItem(s, typeof answer.title === 'string' ? answer.title : '', facts);
141
+ const it = profileItem(s, typeof answer.title === 'string' ? answer.title : '', facts, (d.now ?? Date.now)());
141
142
  await writeLtm(d, [it]);
142
143
  d.ltm.put([it]);
143
144
  r.profiled++;
@@ -12,14 +12,48 @@ export declare function qdrantAsset(platform?: string, arch?: string): QdrantAss
12
12
  export declare function qdrantDir(): string;
13
13
  export declare function qdrantBinaryPath(a?: QdrantAsset | null): string | null;
14
14
  export declare const QDRANT_RELEASE_BASE = "https://github.com/qdrant/qdrant/releases/download/v1.20.0";
15
- /** The binary is in place and its archive was verified when it was unpacked. */
15
+ /** The binary is in place, its archive was verified when it was unpacked, and no one else could have changed it since. */
16
16
  export declare function qdrantReady(a?: QdrantAsset | null): boolean;
17
+ /** A download given up after this long without a byte (the answer's headers included). */
18
+ export declare const DOWNLOAD_STALL_MS = 60000;
19
+ export interface DownloadOptions {
20
+ signal?: AbortSignal;
21
+ fetchImpl?: typeof fetch;
22
+ stallMs?: number;
23
+ }
24
+ /**
25
+ * `url` fetched into a fresh temp folder inside `dir`, never to a fixed name: two first runs at once (two processes, or
26
+ * two memories opened together in one) wrote the same `.part` and could rename each other's half-written file. The
27
+ * file is then checked as it is on disk, not as it streamed past, against its pinned size and SHA-256. Returns the
28
+ * file and its folder; the caller renames the file into place and removes the folder. Throws (leaving nothing) on a
29
+ * failure, a mismatch, or `stallMs` without a byte.
30
+ */
31
+ export declare function downloadVerified(url: string, dir: string, pin: {
32
+ size: number;
33
+ sha256: string;
34
+ }, o?: DownloadOptions): Promise<{
35
+ file: string;
36
+ tmp: string;
37
+ }>;
38
+ /** A lock not touched for this long is a dead holder's (the holder touches it every quarter of this). */
39
+ export declare const LOCK_STALE_MS = 120000;
17
40
  /**
18
- * Fetch, verify and unpack the binary unless it is already there. Throws on a network failure, a size or hash
19
- * mismatch (the download is deleted), or a platform Qdrant does not ship for; the caller retries with backoff.
41
+ * `f` run holding `<path>.lock`, created with O_EXCL: a second first run, in another process or this one, waits and
42
+ * then finds the first one's result. A lock whose process is gone, or untouched for `staleMs`, is taken over. Nothing
43
+ * rests on the lock for correctness (every file is verified in a private temp and renamed into place); it keeps two
44
+ * runs from fetching the same hundreds of MB, and from swapping folders under each other.
20
45
  */
21
- export declare function ensureQdrant(opts?: {
46
+ export declare function withLock<T>(path: string, f: () => Promise<T>, o?: {
22
47
  signal?: AbortSignal;
48
+ staleMs?: number;
49
+ pollMs?: number;
50
+ }): Promise<T>;
51
+ /**
52
+ * Fetch, verify and unpack the binary unless it is already there. Throws on a network failure or stall, a size or
53
+ * hash mismatch (the download is deleted), a platform Qdrant does not ship for, or a folder others could change; the
54
+ * caller retries with backoff. `asset` is for tests: another archive with its own pins.
55
+ */
56
+ export declare function ensureQdrant(opts?: DownloadOptions & {
23
57
  log?: (line: string) => void;
24
- fetchImpl?: typeof fetch;
58
+ asset?: QdrantAsset;
25
59
  }): Promise<string>;
@@ -1,15 +1,16 @@
1
1
  /**
2
2
  * The Qdrant binary memory runs when you give it no Qdrant server of your own: one pinned release, fetched on first use
3
3
  * from Qdrant's GitHub release, verified against the size and SHA-256 pinned below, unpacked next to it (cache dir,
4
- * paths.ts), and trusted after that through a `.verified` marker.
4
+ * paths.ts), and trusted after that through a `.verified` marker while the folder is this user's alone.
5
5
  *
6
6
  * Qdrant publishes no checksums, so the hashes are pinned here from its release. Upgrading means a new version and new
7
7
  * hashes; Qdrant reads storage only from the minor version before it, so after a version change the index is written
8
8
  * again from the logs (rebuild.ts) rather than migrated: it is derived data.
9
9
  */
10
- import { createHash } from 'node:crypto';
11
- import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
12
- import { join } from 'node:path';
10
+ import { createHash, randomBytes } from 'node:crypto';
11
+ import { chmodSync, closeSync, createReadStream, createWriteStream, existsSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, renameSync, rmSync, statSync, utimesSync, writeFileSync, writeSync, } from 'node:fs';
12
+ import { hostname } from 'node:os';
13
+ import { dirname, join } from 'node:path';
13
14
  import { Readable } from 'node:stream';
14
15
  import { pipeline } from 'node:stream/promises';
15
16
  import { spawn } from 'node:child_process';
@@ -34,75 +35,211 @@ export function qdrantBinaryPath(a = qdrantAsset()) {
34
35
  }
35
36
  export const QDRANT_RELEASE_BASE = `https://github.com/qdrant/qdrant/releases/download/v${QDRANT_VERSION}`;
36
37
  const assetUrl = (a) => `${QDRANT_RELEASE_BASE}/${a.asset}`;
37
- /** The binary is in place and its archive was verified when it was unpacked. */
38
+ /**
39
+ * Run only what this user alone could have put there: the folder, the binary and its marker owned by this user, no
40
+ * symlink, writable by no one else. The archive's hash is pinned but not the unpacked binary's (that needs every
41
+ * platform's archive to compute), so on a shared machine this is what stops a binary swapped in after unpacking.
42
+ * Windows has no owner bits; there the user's profile ACLs keep others out.
43
+ */
44
+ function ownedSafely(path) {
45
+ if (typeof process.getuid !== 'function')
46
+ return true;
47
+ const st = lstatSync(path);
48
+ return !st.isSymbolicLink() && st.uid === process.getuid() && (st.mode & 0o022) === 0;
49
+ }
50
+ /** The binary is in place, its archive was verified when it was unpacked, and no one else could have changed it since. */
38
51
  export function qdrantReady(a = qdrantAsset()) {
39
52
  const bin = qdrantBinaryPath(a);
40
- if (!a || !bin || !existsSync(bin))
53
+ if (!a || !bin)
41
54
  return false;
55
+ const marker = join(qdrantDir(), '.verified');
42
56
  try {
43
- return readFileSync(join(qdrantDir(), '.verified'), 'utf8').trim() === `${a.size}:${a.sha256}`;
57
+ return readFileSync(marker, 'utf8').trim() === `${a.size}:${a.sha256}` && statSync(bin).size > 0 &&
58
+ [qdrantDir(), bin, marker].every(ownedSafely);
44
59
  }
45
60
  catch {
46
61
  return false;
47
62
  }
48
63
  }
64
+ /** The system's tar; on Windows by full path, so a `tar.exe` in the working folder or on a planted PATH is not run. */
65
+ function tarProgram() {
66
+ return process.platform === 'win32' ? join(process.env.SystemRoot || 'C:\\Windows', 'System32', 'tar.exe') : 'tar';
67
+ }
49
68
  /** Unpack one archive with the system's tar (bsdtar on macOS and Windows reads zip too). */
50
69
  function untar(archive, into) {
51
70
  return new Promise((resolve, reject) => {
52
- const child = spawn('tar', ['-xf', archive, '-C', into], { stdio: 'ignore' });
71
+ const child = spawn(tarProgram(), ['-xf', archive, '-C', into], { stdio: 'ignore' });
53
72
  child.once('error', reject);
54
73
  child.once('exit', (code) => (code === 0 ? resolve() : reject(new Error(`tar exited ${code}`))));
55
74
  });
56
75
  }
76
+ /** A download given up after this long without a byte (the answer's headers included). */
77
+ export const DOWNLOAD_STALL_MS = 60_000;
57
78
  /**
58
- * Fetch, verify and unpack the binary unless it is already there. Throws on a network failure, a size or hash
59
- * mismatch (the download is deleted), or a platform Qdrant does not ship for; the caller retries with backoff.
79
+ * `url` fetched into a fresh temp folder inside `dir`, never to a fixed name: two first runs at once (two processes, or
80
+ * two memories opened together in one) wrote the same `.part` and could rename each other's half-written file. The
81
+ * file is then checked as it is on disk, not as it streamed past, against its pinned size and SHA-256. Returns the
82
+ * file and its folder; the caller renames the file into place and removes the folder. Throws (leaving nothing) on a
83
+ * failure, a mismatch, or `stallMs` without a byte.
60
84
  */
61
- export async function ensureQdrant(opts = {}) {
62
- const a = qdrantAsset();
63
- if (!a)
64
- throw new Error(`no Qdrant build for ${process.platform}-${process.arch}`);
65
- const bin = qdrantBinaryPath(a);
66
- if (qdrantReady(a))
67
- return bin;
68
- const dir = qdrantDir();
85
+ export async function downloadVerified(url, dir, pin, o = {}) {
69
86
  mkdirSync(dir, { recursive: true });
70
- const archive = join(dir, a.asset);
71
- const part = `${archive}.part`;
72
- rmSync(part, { force: true });
73
- opts.log?.(`[memtrails] fetching Qdrant ${QDRANT_VERSION} (${Math.round(a.size / 1e6)} MB)`);
74
- const res = await (opts.fetchImpl ?? fetch)(assetUrl(a), { signal: opts.signal, redirect: 'follow' });
75
- if (!res.ok || !res.body)
76
- throw new Error(`HTTP ${res.status} for ${a.asset}`);
77
- const hash = createHash('sha256');
78
- let size = 0;
87
+ const tmp = mkdtempSync(join(dir, '.download-'));
88
+ const file = join(tmp, 'file');
89
+ const name = url.slice(url.lastIndexOf('/') + 1);
90
+ const stallMs = o.stallMs ?? DOWNLOAD_STALL_MS;
91
+ const stalled = new AbortController();
92
+ let timer;
93
+ const arm = () => {
94
+ clearTimeout(timer);
95
+ timer = setTimeout(() => stalled.abort(new Error(`${name}: no data for ${Math.round(stallMs / 1000)} s`)), stallMs);
96
+ };
97
+ const signal = o.signal ? AbortSignal.any([o.signal, stalled.signal]) : stalled.signal;
79
98
  try {
99
+ arm();
100
+ const res = await (o.fetchImpl ?? fetch)(url, { signal, redirect: 'follow' });
101
+ if (!res.ok || !res.body)
102
+ throw new Error(`HTTP ${res.status} for ${name}`);
103
+ let size = 0;
80
104
  await pipeline(Readable.fromWeb(res.body), async function* (src) {
81
105
  for await (const chunk of src) {
82
- const b = chunk;
83
- hash.update(b);
84
- size += b.length;
85
- yield b;
106
+ arm();
107
+ size += chunk.length;
108
+ if (size > pin.size)
109
+ throw new Error(`${name}: more than the expected ${pin.size} bytes`);
110
+ yield chunk;
86
111
  }
87
- }, createWriteStream(part));
112
+ }, createWriteStream(file, { flags: 'wx', mode: 0o600 }), { signal });
113
+ clearTimeout(timer);
114
+ const hash = createHash('sha256');
115
+ for await (const chunk of createReadStream(file))
116
+ hash.update(chunk);
117
+ const got = { size: statSync(file).size, sha256: hash.digest('hex') };
118
+ if (got.size !== pin.size || got.sha256 !== pin.sha256) {
119
+ throw new Error(`${name}: got ${got.size} bytes sha256 ${got.sha256}, expected ${pin.size} ${pin.sha256}`);
120
+ }
121
+ return { file, tmp };
88
122
  }
89
123
  catch (err) {
90
- rmSync(part, { force: true });
91
- throw err;
124
+ rmSync(tmp, { recursive: true, force: true });
125
+ throw stalled.signal.aborted ? stalled.signal.reason : err;
126
+ }
127
+ finally {
128
+ clearTimeout(timer);
129
+ }
130
+ }
131
+ /** A lock not touched for this long is a dead holder's (the holder touches it every quarter of this). */
132
+ export const LOCK_STALE_MS = 120_000;
133
+ function lockStale(lock, staleMs) {
134
+ try {
135
+ const [pid, host] = readFileSync(lock, 'utf8').split(' ');
136
+ if (host === hostname() && Number(pid) !== process.pid) {
137
+ try {
138
+ process.kill(Number(pid), 0);
139
+ }
140
+ catch (err) {
141
+ if (err.code === 'ESRCH')
142
+ return true; // its process is gone
143
+ }
144
+ }
145
+ return Date.now() - statSync(lock).mtimeMs > staleMs;
146
+ }
147
+ catch {
148
+ return false; // gone meanwhile: just try again
92
149
  }
93
- const digest = hash.digest('hex');
94
- if (size !== a.size || digest !== a.sha256) {
95
- rmSync(part, { force: true });
96
- throw new Error(`${a.asset}: got ${size} bytes sha256 ${digest}, expected ${a.size} ${a.sha256}`);
150
+ }
151
+ /**
152
+ * `f` run holding `<path>.lock`, created with O_EXCL: a second first run, in another process or this one, waits and
153
+ * then finds the first one's result. A lock whose process is gone, or untouched for `staleMs`, is taken over. Nothing
154
+ * rests on the lock for correctness (every file is verified in a private temp and renamed into place); it keeps two
155
+ * runs from fetching the same hundreds of MB, and from swapping folders under each other.
156
+ */
157
+ export async function withLock(path, f, o = {}) {
158
+ const lock = `${path}.lock`;
159
+ const staleMs = o.staleMs ?? LOCK_STALE_MS;
160
+ const mine = `${process.pid} ${hostname()} ${randomBytes(8).toString('hex')}`;
161
+ mkdirSync(dirname(lock), { recursive: true });
162
+ for (;;) {
163
+ try {
164
+ const fd = openSync(lock, 'wx', 0o600);
165
+ writeSync(fd, mine);
166
+ closeSync(fd);
167
+ break;
168
+ }
169
+ catch (err) {
170
+ if (err.code !== 'EEXIST')
171
+ throw err;
172
+ }
173
+ if (lockStale(lock, staleMs)) {
174
+ rmSync(lock, { force: true });
175
+ continue;
176
+ }
177
+ o.signal?.throwIfAborted();
178
+ await new Promise((ok) => setTimeout(ok, o.pollMs ?? 250));
97
179
  }
98
- renameSync(part, archive);
99
- await untar(archive, dir);
100
- rmSync(archive, { force: true });
101
- if (!existsSync(bin) || statSync(bin).size === 0)
102
- throw new Error(`${a.asset} did not contain ${a.exe}`);
103
- if (process.platform !== 'win32')
104
- chmodSync(bin, 0o755);
105
- writeFileSync(join(dir, '.verified'), `${a.size}:${a.sha256}\n`);
106
- opts.log?.(`[memtrails] Qdrant ${QDRANT_VERSION} ready`);
107
- return bin;
180
+ const beat = setInterval(() => {
181
+ try {
182
+ utimesSync(lock, new Date(), new Date());
183
+ }
184
+ catch { /* taken over: nothing to keep alive */ }
185
+ }, Math.max(10, staleMs / 4));
186
+ beat.unref();
187
+ try {
188
+ return await f();
189
+ }
190
+ finally {
191
+ clearInterval(beat);
192
+ try {
193
+ if (readFileSync(lock, 'utf8') === mine)
194
+ rmSync(lock, { force: true }); // not one taken over meanwhile
195
+ }
196
+ catch { /* already gone */ }
197
+ }
198
+ }
199
+ /**
200
+ * Fetch, verify and unpack the binary unless it is already there. Throws on a network failure or stall, a size or
201
+ * hash mismatch (the download is deleted), a platform Qdrant does not ship for, or a folder others could change; the
202
+ * caller retries with backoff. `asset` is for tests: another archive with its own pins.
203
+ */
204
+ export async function ensureQdrant(opts = {}) {
205
+ const a = opts.asset ?? qdrantAsset();
206
+ if (!a)
207
+ throw new Error(`no Qdrant build for ${process.platform}-${process.arch}`);
208
+ const bin = qdrantBinaryPath(a);
209
+ if (qdrantReady(a))
210
+ return bin;
211
+ const dir = qdrantDir();
212
+ const cache = dirname(dir);
213
+ mkdirSync(cache, { recursive: true, mode: 0o700 });
214
+ return withLock(dir, async () => {
215
+ if (qdrantReady(a))
216
+ return bin; // another run finished it while this one waited
217
+ opts.log?.(`[memtrails] fetching Qdrant ${QDRANT_VERSION} (${Math.round(a.size / 1e6)} MB)`);
218
+ const { file, tmp } = await downloadVerified(assetUrl(a), cache, a, opts);
219
+ try {
220
+ // Unpacked into a fresh folder and moved into place whole: never a half-unpacked binary under the real name.
221
+ const archive = join(tmp, a.asset);
222
+ renameSync(file, archive);
223
+ const unpacked = join(tmp, 'unpacked');
224
+ mkdirSync(unpacked);
225
+ await untar(archive, unpacked);
226
+ const exe = join(unpacked, a.exe);
227
+ if (!existsSync(exe) || !lstatSync(exe).isFile() || statSync(exe).size === 0)
228
+ throw new Error(`${a.asset} did not contain ${a.exe}`);
229
+ chmodSync(exe, 0o755);
230
+ writeFileSync(join(unpacked, '.verified'), `${a.size}:${a.sha256}\n`, { mode: 0o644 });
231
+ chmodSync(join(unpacked, '.verified'), 0o644);
232
+ chmodSync(unpacked, 0o755);
233
+ if (existsSync(dir))
234
+ renameSync(dir, join(tmp, 'old')); // unverified or not this user's alone; removed with tmp
235
+ renameSync(unpacked, dir);
236
+ }
237
+ finally {
238
+ rmSync(tmp, { recursive: true, force: true });
239
+ }
240
+ if (!qdrantReady(a))
241
+ throw new Error(`${dir} is not this user's alone (its owner or permissions): not running a binary others could change`);
242
+ opts.log?.(`[memtrails] Qdrant ${QDRANT_VERSION} ready`);
243
+ return bin;
244
+ }, { signal: opts.signal });
108
245
  }
@@ -1,15 +1,14 @@
1
1
  import type { QdrantEndpoint } from './qdrant-process.js';
2
2
  export declare const COLLECTION = "memory";
3
- /** The payload field a tenant client keys every point by, and the original point id it keeps beside the scoped one. */
4
- export declare const TENANT_FIELD = "user";
5
3
  export declare const DIMS: 768;
6
4
  /** What the index is made of; a change empties it and it is written in again from the logs (rebuild.ts).
7
5
  * 2: binary quantisation with rescoring (½ the RAM of int8, recall 0.98–1.00 measured at 1–3M points), keyword terms
8
6
  * folded and segmented for every language (text.ts); 3: on-disk binary copy, HNSW and payload indexes;
9
7
  * 4: remote EmbeddingGemma GPU vectors (768 dimensions, explicit runtime/pooling tag in embedder.ts); 5: nothing cut —
10
8
  * every chunk of a body (no 12-chunk limit), short-term entries chunked too, cues numbered below the key, full payloads;
11
- * 6: every captured request indexed for good — however old, consolidated or not (the raw record is the truth). */
12
- export declare const INDEX_LAYOUT = 6;
9
+ * 6: every captured request indexed for good — however old, consolidated or not (the raw record is the truth);
10
+ * 7: a record's kind (`engine`) on every point of it and indexed, so a kinds filter finds a record by its chunks. */
11
+ export declare const INDEX_LAYOUT = 7;
13
12
  export interface SparseVector {
14
13
  indices: number[];
15
14
  values: number[];
@@ -33,25 +32,14 @@ export declare class QdrantError extends Error {
33
32
  export interface QdrantClientOptions {
34
33
  /** The collection (COLLECTION when not given). */
35
34
  collection?: string;
36
- /** Many users in one collection: the user every call is for — read on each call from the caller's context; null or ''
37
- * refuses the call. */
38
- tenant?: () => string | null | undefined;
39
35
  }
40
36
  export declare class QdrantClient {
41
37
  private readonly endpoint;
42
38
  private readonly timeoutMs;
43
- private readonly opts;
44
- private readonly collection;
39
+ /** `/collections/<name>`, the name encoded: a name you give is a path segment, never more of the path. */
40
+ private readonly at;
45
41
  /** `endpoint` may start Qdrant (the service runs it only while memory has work): every call waits for it. */
46
42
  constructor(endpoint: () => QdrantEndpoint | null | Promise<QdrantEndpoint | null>, timeoutMs?: number, opts?: QdrantClientOptions);
47
- /** The user this call is for (a tenant client), or null (one person's own index). Throws when a tenant client has none. */
48
- private user;
49
- /** A filter with the user added — unchanged for one person's own index. */
50
- private scoped;
51
- /** A point's id in a shared collection: the user's and the item's (pointId of both), so users never share a point. */
52
- private scopedId;
53
- /** What a read hands back: the point's own id (as the writer named it) and only this user's points. */
54
- private own;
55
43
  private call;
56
44
  /** Drop the whole collection: the index is derived, written again from the logs (rebuild.ts). */
57
45
  dropCollection(): Promise<void>;
@@ -78,7 +66,7 @@ export declare class QdrantClient {
78
66
  * per-family similarities. An item with no such point is not in the map. */
79
67
  bestPerItem(vector: number[], items: string[], part: string, timeoutMs?: number): Promise<Map<string, number>>;
80
68
  /** `nearest` for several vectors in one call. */
81
- /** `fields`: only these payload fields (and the item and its user), for a caller that needs no more — a walk over
69
+ /** `fields`: only these payload fields (and the item), for a caller that needs no more — a walk over
82
70
  * many neighbours reads ids, not the bodies of long requests. */
83
71
  nearestMany(vectors: number[][], filter: Filter, limit?: number, timeoutMs?: number, fields?: string[]): Promise<Array<Array<{
84
72
  item: string;