memonaut 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-main.d.ts.map +1 -1
- package/dist/cli-main.js +133 -5
- package/dist/cli-main.js.map +1 -1
- package/dist/db.d.ts +9 -0
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +18 -1
- package/dist/db.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/indexer.d.ts +66 -1
- package/dist/indexer.d.ts.map +1 -1
- package/dist/indexer.js +138 -6
- package/dist/indexer.js.map +1 -1
- package/dist/resources.d.ts +2 -0
- package/dist/resources.d.ts.map +1 -0
- package/dist/resources.js +25 -0
- package/dist/resources.js.map +1 -0
- package/dist/skills.d.ts +34 -0
- package/dist/skills.d.ts.map +1 -0
- package/dist/skills.js +59 -0
- package/dist/skills.js.map +1 -0
- package/package.json +4 -3
- package/skills/memonaut/SKILL.md +122 -0
- package/src/cli-main.ts +165 -5
- package/src/db.ts +27 -1
- package/src/index.ts +1 -0
- package/src/indexer.ts +185 -7
- package/src/resources.ts +26 -0
- package/src/skills.ts +86 -0
- package/dist/silence-sqlite-warning.d.ts +0 -2
- package/dist/silence-sqlite-warning.d.ts.map +0 -1
- package/dist/silence-sqlite-warning.js +0 -9
- package/dist/silence-sqlite-warning.js.map +0 -1
package/src/cli-main.ts
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
tildify,
|
|
14
14
|
type Style,
|
|
15
15
|
} from './format.js';
|
|
16
|
-
import {index} from './indexer.js';
|
|
16
|
+
import {freshness, index} from './indexer.js';
|
|
17
17
|
import {TIERS, type ChunkKind, type SearchHit, type Tier} from './model.js';
|
|
18
18
|
import {readRawEntry} from './pi-source.js';
|
|
19
19
|
import {
|
|
@@ -22,6 +22,13 @@ import {
|
|
|
22
22
|
regexSearch,
|
|
23
23
|
} from './regex.js';
|
|
24
24
|
import {indexStats, readThread, resolveThread, search} from './search.js';
|
|
25
|
+
import {
|
|
26
|
+
availableSkills,
|
|
27
|
+
destinationFor,
|
|
28
|
+
installSkills,
|
|
29
|
+
isInstalled,
|
|
30
|
+
type Scope,
|
|
31
|
+
} from './skills.js';
|
|
25
32
|
|
|
26
33
|
const USAGE = `memonaut — search your agent conversation transcripts
|
|
27
34
|
|
|
@@ -33,9 +40,12 @@ COMMANDS
|
|
|
33
40
|
search <query...> Search transcripts, grouped by fork lineage
|
|
34
41
|
search --regex <pat> Exact regex search over the original transcripts
|
|
35
42
|
show <ref> Print a thread (file id, session uuid prefix, name or path)
|
|
43
|
+
(show and sql sync first too; --no-sync skips it)
|
|
36
44
|
sql <query> Run a read-only SQL query against the index
|
|
37
45
|
stats Summarise what is indexed
|
|
46
|
+
status How fresh the index is, and how much is pending
|
|
38
47
|
config Show config paths, or write a starter config
|
|
48
|
+
skills [install] List or install the agent skill (into ~/.agents/skills)
|
|
39
49
|
|
|
40
50
|
SEARCH OPTIONS
|
|
41
51
|
--cwd <glob> Filter by working directory (repeatable, globs, ~ ok)
|
|
@@ -80,6 +90,9 @@ QUERY SYNTAX
|
|
|
80
90
|
match in order, OR / NOT / NEAR(a b, 5) work, and trailing * is a prefix.
|
|
81
91
|
If that fails to parse, the query is retried as quoted literal tokens.
|
|
82
92
|
|
|
93
|
+
SKILLS OPTIONS
|
|
94
|
+
--project Install into ./.agents/skills instead of ~/.agents/skills
|
|
95
|
+
|
|
83
96
|
EXAMPLES
|
|
84
97
|
recall search steering queue --project wherever --since 30d
|
|
85
98
|
recall search '"fork point" OR parentSession' --threads all
|
|
@@ -149,15 +162,28 @@ function withTier(config: Config, tier: string | undefined): Config {
|
|
|
149
162
|
return {...config, tier: tier as Tier};
|
|
150
163
|
}
|
|
151
164
|
|
|
152
|
-
/**
|
|
165
|
+
/**
|
|
166
|
+
* Cheap incremental catch-up so a query is never answered from a stale index.
|
|
167
|
+
*
|
|
168
|
+
* A missing index builds itself here rather than failing. Telling somebody who
|
|
169
|
+
* just typed a search to go and type a different command first is a worse
|
|
170
|
+
* answer than taking ~35 s once and saying so.
|
|
171
|
+
*/
|
|
153
172
|
function sync(config: Config, quiet: boolean): void {
|
|
154
|
-
|
|
155
|
-
|
|
173
|
+
// "Cold" is about whether a build has ever FINISHED, not whether a file is
|
|
174
|
+
// there: an interrupted first build leaves a file behind, and judging by
|
|
175
|
+
// existence alone would swallow the banner and hand the user a silent
|
|
176
|
+
// 35-second hang the second time too.
|
|
177
|
+
const cold = lastIndexedAt(config) === null;
|
|
178
|
+
if (cold && !quiet) {
|
|
179
|
+
process.stderr.write(
|
|
180
|
+
`recall: no index at ${tildify(config.dbPath)} yet, building it once (well under a minute)…\n`,
|
|
181
|
+
);
|
|
156
182
|
}
|
|
157
183
|
const stats = index({config, onProgress: () => {}});
|
|
158
184
|
if (!quiet && stats.filesIndexed > 0) {
|
|
159
185
|
process.stderr.write(
|
|
160
|
-
`recall: synced ${stats.filesIndexed} transcript(s) in ${stats.durationMs}ms\n`,
|
|
186
|
+
`recall: ${cold ? 'indexed' : 'synced'} ${stats.filesIndexed} transcript(s) in ${stats.durationMs}ms\n`,
|
|
161
187
|
);
|
|
162
188
|
}
|
|
163
189
|
}
|
|
@@ -400,6 +426,7 @@ function cmdShow(argv: string[]): void {
|
|
|
400
426
|
limit: {type: 'string', default: '40'},
|
|
401
427
|
full: {type: 'boolean', default: false},
|
|
402
428
|
json: {type: 'boolean', default: false},
|
|
429
|
+
'no-sync': {type: 'boolean', default: false},
|
|
403
430
|
},
|
|
404
431
|
allowPositionals: true,
|
|
405
432
|
});
|
|
@@ -410,6 +437,10 @@ function cmdShow(argv: string[]): void {
|
|
|
410
437
|
);
|
|
411
438
|
|
|
412
439
|
const config = loadConfig();
|
|
440
|
+
// A thread can have grown since the search that found it, so this syncs too:
|
|
441
|
+
// "every query syncs before it answers" has to be true of all of them, or the
|
|
442
|
+
// sentence is not worth writing down.
|
|
443
|
+
if (!values['no-sync']) sync(config, Boolean(values.json));
|
|
413
444
|
const db = openDb(config.dbPath, {readOnly: true});
|
|
414
445
|
const thread = resolveThread(db, ref);
|
|
415
446
|
if (!thread) fail(`no thread matching "${ref}"`);
|
|
@@ -478,12 +509,14 @@ function cmdSql(argv: string[]): void {
|
|
|
478
509
|
options: {
|
|
479
510
|
json: {type: 'boolean', default: false},
|
|
480
511
|
limit: {type: 'string', default: '200'},
|
|
512
|
+
'no-sync': {type: 'boolean', default: false},
|
|
481
513
|
},
|
|
482
514
|
allowPositionals: true,
|
|
483
515
|
});
|
|
484
516
|
const sql = positionals.join(' ').trim();
|
|
485
517
|
if (!sql) fail('no SQL given');
|
|
486
518
|
const config = loadConfig();
|
|
519
|
+
if (!values['no-sync']) sync(config, Boolean(values.json));
|
|
487
520
|
const db = openDb(config.dbPath, {readOnly: true});
|
|
488
521
|
let rows: Array<Record<string, unknown>>;
|
|
489
522
|
try {
|
|
@@ -544,6 +577,85 @@ function cmdStats(argv: string[]): void {
|
|
|
544
577
|
db.close();
|
|
545
578
|
}
|
|
546
579
|
|
|
580
|
+
/** Epoch ms of the last COMPLETED catch-up, or null if one never finished. */
|
|
581
|
+
function lastIndexedAt(config: Config): number | null {
|
|
582
|
+
if (!fs.existsSync(config.dbPath)) return null;
|
|
583
|
+
try {
|
|
584
|
+
const db = openDb(config.dbPath, {readOnly: true});
|
|
585
|
+
try {
|
|
586
|
+
const value = Number(getMeta(db, 'indexed_at') ?? NaN);
|
|
587
|
+
return Number.isFinite(value) ? value : null;
|
|
588
|
+
} finally {
|
|
589
|
+
db.close();
|
|
590
|
+
}
|
|
591
|
+
} catch {
|
|
592
|
+
return null;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* `recall status`: how far behind the index is, without touching it.
|
|
598
|
+
*
|
|
599
|
+
* Every read path syncs before it answers, so this is not what keeps results
|
|
600
|
+
* correct. It exists so a human (or a scheduler asking whether it is earning
|
|
601
|
+
* its keep) can see the lag, and tell "nothing to do" apart from "the next
|
|
602
|
+
* query pays for a month of catch-up".
|
|
603
|
+
*/
|
|
604
|
+
function cmdStatus(argv: string[]): void {
|
|
605
|
+
const {values} = parseArgs({
|
|
606
|
+
args: argv,
|
|
607
|
+
options: {json: {type: 'boolean', default: false}},
|
|
608
|
+
allowPositionals: false,
|
|
609
|
+
});
|
|
610
|
+
const config = loadConfig();
|
|
611
|
+
const report = freshness(config);
|
|
612
|
+
const pending = report.changed + report.unseen + report.vanished;
|
|
613
|
+
if (values.json) {
|
|
614
|
+
process.stdout.write(
|
|
615
|
+
JSON.stringify(
|
|
616
|
+
{
|
|
617
|
+
...report,
|
|
618
|
+
pending,
|
|
619
|
+
indexedAt: report.indexedAt
|
|
620
|
+
? new Date(report.indexedAt).toISOString()
|
|
621
|
+
: null,
|
|
622
|
+
},
|
|
623
|
+
null,
|
|
624
|
+
2,
|
|
625
|
+
) + '\n',
|
|
626
|
+
);
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
const style = makeStyle(colorsEnabled());
|
|
630
|
+
if (!report.exists) {
|
|
631
|
+
process.stdout.write(
|
|
632
|
+
[
|
|
633
|
+
`${style.bold('no index yet')} at ${tildify(report.dbPath)}`,
|
|
634
|
+
`${report.transcriptsOnDisk} transcript(s) on disk waiting to be indexed`,
|
|
635
|
+
style.dim('Run `recall index`, or just search: any query builds it.'),
|
|
636
|
+
].join('\n') + '\n',
|
|
637
|
+
);
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
const verdict =
|
|
641
|
+
pending === 0
|
|
642
|
+
? style.bold('up to date')
|
|
643
|
+
: style.bold(`${pending} transcript(s) pending`);
|
|
644
|
+
process.stdout.write(
|
|
645
|
+
[
|
|
646
|
+
`${verdict} · last indexed ${relativeTime(report.indexedAt ? new Date(report.indexedAt).toISOString() : null)}`,
|
|
647
|
+
`${report.transcriptsIndexed} indexed · ${report.changed} changed · ${report.unseen} never seen · ${report.vanished} gone · ${report.transcriptsOnDisk} on disk`,
|
|
648
|
+
`tier ${report.tier ?? '?'} · db ${(report.dbBytes / 1e6).toFixed(1)} MB`,
|
|
649
|
+
style.dim(tildify(report.dbPath)),
|
|
650
|
+
style.dim(
|
|
651
|
+
pending === 0
|
|
652
|
+
? 'Every search syncs first, so this is informational.'
|
|
653
|
+
: `Every search syncs first, so those ${pending} will be folded in by the next query.`,
|
|
654
|
+
),
|
|
655
|
+
].join('\n') + '\n',
|
|
656
|
+
);
|
|
657
|
+
}
|
|
658
|
+
|
|
547
659
|
function cmdConfig(argv: string[]): void {
|
|
548
660
|
const {values} = parseArgs({
|
|
549
661
|
args: argv,
|
|
@@ -575,6 +687,50 @@ function cmdConfig(argv: string[]): void {
|
|
|
575
687
|
);
|
|
576
688
|
}
|
|
577
689
|
|
|
690
|
+
/**
|
|
691
|
+
* `recall skills [list|install]`.
|
|
692
|
+
*
|
|
693
|
+
* Kept flag-light on purpose: the only real decision is user versus project scope.
|
|
694
|
+
*/
|
|
695
|
+
function cmdSkills(argv: string[]): void {
|
|
696
|
+
const scope: Scope = argv.includes('--project') ? 'project' : 'user';
|
|
697
|
+
const verb = argv.find((argument) => !argument.startsWith('-')) ?? 'list';
|
|
698
|
+
if (verb !== 'list' && verb !== 'install')
|
|
699
|
+
fail(`unknown skills command "${verb}". Try \`list\` or \`install\``);
|
|
700
|
+
|
|
701
|
+
const available = availableSkills();
|
|
702
|
+
if (available.length === 0) {
|
|
703
|
+
fail(
|
|
704
|
+
'no skills found beside this package. A published install carries them; a checkout keeps them at the repo root',
|
|
705
|
+
);
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
const style = makeStyle(colorsEnabled());
|
|
709
|
+
if (verb === 'list') {
|
|
710
|
+
process.stdout.write(tildify(destinationFor(scope)) + '\n\n');
|
|
711
|
+
for (const skill of available) {
|
|
712
|
+
const mark = isInstalled(skill, scope) ? 'installed' : 'not installed';
|
|
713
|
+
process.stdout.write(
|
|
714
|
+
` ${style.bold(skill.name)} ${style.dim(`(${mark})`)}\n ${skill.description}\n\n`,
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
process.stdout.write('Run `recall skills install` to copy them in.\n');
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
const installed = installSkills(scope);
|
|
722
|
+
for (const entry of installed) {
|
|
723
|
+
process.stdout.write(
|
|
724
|
+
` ${entry.replaced ? 'replaced' : 'installed'} ${tildify(entry.to)}\n`,
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
process.stdout.write(
|
|
728
|
+
style.dim(
|
|
729
|
+
`\n${installed.length} skill${installed.length === 1 ? '' : 's'} copied. They are copies, so re-run this after upgrading memonaut.\n`,
|
|
730
|
+
),
|
|
731
|
+
);
|
|
732
|
+
}
|
|
733
|
+
|
|
578
734
|
async function main(): Promise<void> {
|
|
579
735
|
const argv = process.argv.slice(2);
|
|
580
736
|
const command = argv[0];
|
|
@@ -591,8 +747,12 @@ async function main(): Promise<void> {
|
|
|
591
747
|
return cmdSql(rest);
|
|
592
748
|
case 'stats':
|
|
593
749
|
return cmdStats(rest);
|
|
750
|
+
case 'status':
|
|
751
|
+
return cmdStatus(rest);
|
|
594
752
|
case 'config':
|
|
595
753
|
return cmdConfig(rest);
|
|
754
|
+
case 'skills':
|
|
755
|
+
return cmdSkills(rest);
|
|
596
756
|
case '--version':
|
|
597
757
|
case '-v':
|
|
598
758
|
process.stdout.write(version() + '\n');
|
package/src/db.ts
CHANGED
|
@@ -12,6 +12,19 @@ export const SCHEMA_VERSION = 1;
|
|
|
12
12
|
|
|
13
13
|
export type DB = DatabaseSync;
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* How long a connection waits for a writer to finish before giving up.
|
|
17
|
+
*
|
|
18
|
+
* Several processes share this index by design: the human's CLI, a pi
|
|
19
|
+
* extension flushing on shutdown, and any number of concurrent agents, each of
|
|
20
|
+
* which syncs before it queries. Without a busy timeout SQLite returns
|
|
21
|
+
* SQLITE_BUSY on the FIRST contended write, which surfaces to an agent as a
|
|
22
|
+
* failed recall. A warm catch-up takes ~120 ms and a cold build ~35 s, so 5 s
|
|
23
|
+
* covers every realistic overlap except a first-ever build, which is exactly
|
|
24
|
+
* the case where failing fast and reporting it is the honest answer.
|
|
25
|
+
*/
|
|
26
|
+
const BUSY_TIMEOUT_MS = 5_000;
|
|
27
|
+
|
|
15
28
|
const SCHEMA = `
|
|
16
29
|
CREATE TABLE IF NOT EXISTS meta(
|
|
17
30
|
key TEXT PRIMARY KEY,
|
|
@@ -107,6 +120,15 @@ export interface OpenOptions {
|
|
|
107
120
|
readOnly?: boolean;
|
|
108
121
|
/** Throw instead of rebuilding when the schema version differs. */
|
|
109
122
|
noRebuild?: boolean;
|
|
123
|
+
/**
|
|
124
|
+
* Leave `query_only` off on a read-only connection, so TEMP objects can be
|
|
125
|
+
* created. The index file itself is still unwritable, because that comes
|
|
126
|
+
* from opening it read-only, not from the pragma.
|
|
127
|
+
*
|
|
128
|
+
* This exists for one caller: the SQL tool shadows the real tables with temp
|
|
129
|
+
* views so arbitrary queries cannot reach private transcripts.
|
|
130
|
+
*/
|
|
131
|
+
writableTemp?: boolean;
|
|
110
132
|
}
|
|
111
133
|
|
|
112
134
|
export function getMeta(db: DB, key: string): string | undefined {
|
|
@@ -133,13 +155,17 @@ export function openDb(dbPath: string, opts: OpenOptions = {}): DB {
|
|
|
133
155
|
throw new Error(`no index at ${dbPath}. Run \`recall index\` first.`);
|
|
134
156
|
}
|
|
135
157
|
const db = new DatabaseSync(dbPath, {readOnly: true});
|
|
136
|
-
db.exec('PRAGMA query_only = 1');
|
|
158
|
+
if (!opts.writableTemp) db.exec('PRAGMA query_only = 1');
|
|
159
|
+
// Readers do not block on writers under WAL, but they do contend with
|
|
160
|
+
// checkpointing, so the timeout matters here too.
|
|
161
|
+
db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
137
162
|
return db;
|
|
138
163
|
}
|
|
139
164
|
|
|
140
165
|
fs.mkdirSync(path.dirname(dbPath), {recursive: true});
|
|
141
166
|
const fresh = !fs.existsSync(dbPath);
|
|
142
167
|
const db = new DatabaseSync(dbPath);
|
|
168
|
+
db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
143
169
|
db.exec('PRAGMA journal_mode = WAL');
|
|
144
170
|
db.exec('PRAGMA synchronous = NORMAL');
|
|
145
171
|
db.exec('PRAGMA foreign_keys = OFF');
|
package/src/index.ts
CHANGED
package/src/indexer.ts
CHANGED
|
@@ -20,6 +20,21 @@ export interface IndexOptions {
|
|
|
20
20
|
full?: boolean;
|
|
21
21
|
db?: DB;
|
|
22
22
|
onProgress?: (progress: IndexProgress) => void;
|
|
23
|
+
/**
|
|
24
|
+
* Stop ingesting once this many ms have passed, leaving the rest for the
|
|
25
|
+
* next run. Checked BETWEEN lineages, never inside one, so the per-lineage
|
|
26
|
+
* transaction and the watermark-written-last rule both still hold.
|
|
27
|
+
*
|
|
28
|
+
* For callers that must not hang: a session shutting down after a long gap
|
|
29
|
+
* could otherwise sit through a multi-second catch-up on the way out.
|
|
30
|
+
*
|
|
31
|
+
* It bounds the INGEST LOOP, not the whole run: the scan and the identity
|
|
32
|
+
* pass ahead of it are ~120 ms on a 4,831-file corpus and always run. A full
|
|
33
|
+
* rebuild is refused outright rather than truncated, see below.
|
|
34
|
+
*/
|
|
35
|
+
budgetMs?: number;
|
|
36
|
+
/** Clock seam, so the budget can be tested without racing a real one. */
|
|
37
|
+
now?: () => number;
|
|
23
38
|
}
|
|
24
39
|
|
|
25
40
|
export interface IndexProgress {
|
|
@@ -45,6 +60,8 @@ export interface IndexStats {
|
|
|
45
60
|
bytesRead: number;
|
|
46
61
|
durationMs: number;
|
|
47
62
|
fullRebuild: boolean;
|
|
63
|
+
/** True when `budgetMs` cut the run short and work is still outstanding. */
|
|
64
|
+
budgetExhausted: boolean;
|
|
48
65
|
}
|
|
49
66
|
|
|
50
67
|
interface FileRow {
|
|
@@ -78,7 +95,8 @@ function probe(file: string): {
|
|
|
78
95
|
}
|
|
79
96
|
|
|
80
97
|
export function index(opts: IndexOptions): IndexStats {
|
|
81
|
-
const
|
|
98
|
+
const clock = opts.now ?? Date.now;
|
|
99
|
+
const started = clock();
|
|
82
100
|
const {config} = opts;
|
|
83
101
|
const db = opts.db ?? openDb(config.dbPath);
|
|
84
102
|
const report = opts.onProgress ?? (() => {});
|
|
@@ -99,6 +117,7 @@ export function index(opts: IndexOptions): IndexStats {
|
|
|
99
117
|
bytesRead: 0,
|
|
100
118
|
durationMs: 0,
|
|
101
119
|
fullRebuild: false,
|
|
120
|
+
budgetExhausted: false,
|
|
102
121
|
};
|
|
103
122
|
|
|
104
123
|
// A tier change alters what text exists at all, so it forces a rebuild.
|
|
@@ -107,6 +126,18 @@ export function index(opts: IndexOptions): IndexStats {
|
|
|
107
126
|
if (storedTier !== undefined && storedTier !== config.tier) full = true;
|
|
108
127
|
stats.fullRebuild = full;
|
|
109
128
|
|
|
129
|
+
// A rebuild is all-or-nothing: it DELETEs every chunk, entry and membership
|
|
130
|
+
// before reading a byte, so truncating one would leave an index that later
|
|
131
|
+
// queries would answer from as though it were complete. A budget means the
|
|
132
|
+
// caller cannot afford to finish, so the honest move is not to start.
|
|
133
|
+
// Reachable without `opts.full`: a tier change forces one (line above).
|
|
134
|
+
if (full && opts.budgetMs !== undefined) {
|
|
135
|
+
stats.budgetExhausted = true;
|
|
136
|
+
stats.durationMs = clock() - started;
|
|
137
|
+
if (!opts.db) db.close();
|
|
138
|
+
return stats;
|
|
139
|
+
}
|
|
140
|
+
|
|
110
141
|
const isIgnored = matcher(config.ignore);
|
|
111
142
|
const isPrivate = matcher(config.private);
|
|
112
143
|
|
|
@@ -397,6 +428,12 @@ export function index(opts: IndexOptions): IndexStats {
|
|
|
397
428
|
|
|
398
429
|
let done = 0;
|
|
399
430
|
for (const unit of work) {
|
|
431
|
+
// `>=` so that a budget of 0 means "do nothing", even on a machine fast
|
|
432
|
+
// enough to reach this in the same millisecond it started.
|
|
433
|
+
if (opts.budgetMs !== undefined && clock() - started >= opts.budgetMs) {
|
|
434
|
+
stats.budgetExhausted = true;
|
|
435
|
+
break;
|
|
436
|
+
}
|
|
400
437
|
db.exec('BEGIN');
|
|
401
438
|
try {
|
|
402
439
|
const lineageId = idOf.get(unit.lineageRoot);
|
|
@@ -493,10 +530,13 @@ export function index(opts: IndexOptions): IndexStats {
|
|
|
493
530
|
}
|
|
494
531
|
|
|
495
532
|
setMeta(db, 'tier', config.tier);
|
|
496
|
-
|
|
533
|
+
// Only claim to be caught up when we actually are. `indexed_at` is what the
|
|
534
|
+
// sync TTL reads, so advancing it after a budget-truncated run would let the
|
|
535
|
+
// next query skip the catch-up and answer from an index we KNOW is behind.
|
|
536
|
+
if (!stats.budgetExhausted) setMeta(db, 'indexed_at', String(Date.now()));
|
|
497
537
|
setMeta(db, 'roots', JSON.stringify(config.sources.map((s) => s.root)));
|
|
498
538
|
|
|
499
|
-
stats.durationMs =
|
|
539
|
+
stats.durationMs = clock() - started;
|
|
500
540
|
if (!opts.db) db.close();
|
|
501
541
|
return stats;
|
|
502
542
|
}
|
|
@@ -512,6 +552,22 @@ export interface SyncResult {
|
|
|
512
552
|
stats?: IndexStats;
|
|
513
553
|
}
|
|
514
554
|
|
|
555
|
+
/**
|
|
556
|
+
* There is no index yet, as opposed to there being one that could not be read.
|
|
557
|
+
*
|
|
558
|
+
* The distinction is the whole point of the type. "No index" is fixed by
|
|
559
|
+
* building one; "index is locked" or "index is corrupt" is not, and a caller
|
|
560
|
+
* that collapses the two tells the user to run a command that will not help.
|
|
561
|
+
*/
|
|
562
|
+
export class MissingIndexError extends Error {
|
|
563
|
+
readonly dbPath: string;
|
|
564
|
+
constructor(dbPath: string) {
|
|
565
|
+
super(`no index at ${dbPath}`);
|
|
566
|
+
this.name = 'MissingIndexError';
|
|
567
|
+
this.dbPath = dbPath;
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
515
571
|
/**
|
|
516
572
|
* Catch the index up, unless it was already synced within `ttlMs`.
|
|
517
573
|
*
|
|
@@ -520,15 +576,137 @@ export interface SyncResult {
|
|
|
520
576
|
* turn. The TTL is what makes "always fresh" affordable for callers that query
|
|
521
577
|
* in a burst.
|
|
522
578
|
*/
|
|
523
|
-
export function syncIfStale(
|
|
524
|
-
|
|
525
|
-
|
|
579
|
+
export function syncIfStale(
|
|
580
|
+
config: Config,
|
|
581
|
+
ttlMs = 15_000,
|
|
582
|
+
budgetMs?: number,
|
|
583
|
+
): SyncResult {
|
|
584
|
+
if (!fs.existsSync(config.dbPath)) throw new MissingIndexError(config.dbPath);
|
|
526
585
|
const db = openDb(config.dbPath);
|
|
527
586
|
try {
|
|
528
587
|
const last = Number(getMeta(db, 'indexed_at') ?? 0);
|
|
529
588
|
if (Number.isFinite(last) && Date.now() - last < ttlMs) return {ran: false};
|
|
530
|
-
return {ran: true, stats: index({config, db})};
|
|
589
|
+
return {ran: true, stats: index({config, db, budgetMs})};
|
|
531
590
|
} finally {
|
|
532
591
|
db.close();
|
|
533
592
|
}
|
|
534
593
|
}
|
|
594
|
+
|
|
595
|
+
export interface Freshness {
|
|
596
|
+
dbPath: string;
|
|
597
|
+
exists: boolean;
|
|
598
|
+
dbBytes: number;
|
|
599
|
+
tier: string | null;
|
|
600
|
+
/** When the last catch-up finished, epoch ms. */
|
|
601
|
+
indexedAt: number | null;
|
|
602
|
+
ageMs: number | null;
|
|
603
|
+
/** Transcripts on disk, before ignore rules. */
|
|
604
|
+
transcriptsOnDisk: number;
|
|
605
|
+
/** Transcripts the index knows about. */
|
|
606
|
+
transcriptsIndexed: number;
|
|
607
|
+
/** Known transcripts that have grown or changed since their watermark. */
|
|
608
|
+
changed: number;
|
|
609
|
+
/**
|
|
610
|
+
* Transcripts whose CONTENT is not in the index: never discovered, or
|
|
611
|
+
* discovered but not yet ingested (a run cut short by its time budget
|
|
612
|
+
* registers the file before reading it). Ignore rules applied.
|
|
613
|
+
*/
|
|
614
|
+
unseen: number;
|
|
615
|
+
/** Indexed transcripts no longer on disk, which the next sync will delete. */
|
|
616
|
+
vanished: number;
|
|
617
|
+
sources: string[];
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
/**
|
|
621
|
+
* How far behind the index is, without changing it.
|
|
622
|
+
*
|
|
623
|
+
* This is a `stat` pass over the corpus plus a header read for the few files
|
|
624
|
+
* the index has never seen, because whether a new file is ignored can only be
|
|
625
|
+
* decided from its header `cwd`, never from its mangled directory name.
|
|
626
|
+
* Measured at ~70 ms over 4,831 transcripts.
|
|
627
|
+
*
|
|
628
|
+
* Every read path already syncs before it answers, so this is not how anything
|
|
629
|
+
* stays correct: it is how a human or an agent can SEE the lag, and how they
|
|
630
|
+
* can tell "nothing to do" apart from "about to pay for a month of catch-up".
|
|
631
|
+
*/
|
|
632
|
+
export function freshness(config: Config): Freshness {
|
|
633
|
+
const report: Freshness = {
|
|
634
|
+
dbPath: config.dbPath,
|
|
635
|
+
exists: fs.existsSync(config.dbPath),
|
|
636
|
+
dbBytes: 0,
|
|
637
|
+
tier: null,
|
|
638
|
+
indexedAt: null,
|
|
639
|
+
ageMs: null,
|
|
640
|
+
transcriptsOnDisk: 0,
|
|
641
|
+
transcriptsIndexed: 0,
|
|
642
|
+
changed: 0,
|
|
643
|
+
unseen: 0,
|
|
644
|
+
vanished: 0,
|
|
645
|
+
sources: config.sources.map((s) => s.root),
|
|
646
|
+
};
|
|
647
|
+
|
|
648
|
+
// Null size/mtime means "row exists, content never ingested", which is a
|
|
649
|
+
// different thing from "ingested and since changed", so the nulls are kept.
|
|
650
|
+
const known = new Map<string, {size: number | null; mtime: number | null}>();
|
|
651
|
+
if (report.exists) {
|
|
652
|
+
report.dbBytes = fs.statSync(config.dbPath).size;
|
|
653
|
+
const db = openDb(config.dbPath, {readOnly: true});
|
|
654
|
+
try {
|
|
655
|
+
report.tier = getMeta(db, 'tier') ?? null;
|
|
656
|
+
const at = Number(getMeta(db, 'indexed_at') ?? NaN);
|
|
657
|
+
if (Number.isFinite(at)) {
|
|
658
|
+
report.indexedAt = at;
|
|
659
|
+
report.ageMs = Date.now() - at;
|
|
660
|
+
}
|
|
661
|
+
for (const row of db
|
|
662
|
+
.prepare('SELECT path, size, mtime FROM file')
|
|
663
|
+
.all() as unknown as Array<{
|
|
664
|
+
path: string;
|
|
665
|
+
size: number | null;
|
|
666
|
+
mtime: number | null;
|
|
667
|
+
}>) {
|
|
668
|
+
known.set(row.path, {
|
|
669
|
+
size: row.size === null ? null : num(row.size),
|
|
670
|
+
mtime: row.mtime === null ? null : num(row.mtime),
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
} finally {
|
|
674
|
+
db.close();
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
report.transcriptsIndexed = known.size;
|
|
678
|
+
|
|
679
|
+
const isIgnored = matcher(config.ignore);
|
|
680
|
+
// Tracked so a deletion counts as pending work too: reporting "up to date"
|
|
681
|
+
// while the next sync still has N `deleteFile` calls to make would undercut
|
|
682
|
+
// the only thing this command is for.
|
|
683
|
+
const seen = new Set<string>();
|
|
684
|
+
for (const source of config.sources) {
|
|
685
|
+
if (!fs.existsSync(source.root)) continue;
|
|
686
|
+
for (const file of listTranscripts(source.root)) {
|
|
687
|
+
report.transcriptsOnDisk++;
|
|
688
|
+
let st: fs.Stats;
|
|
689
|
+
try {
|
|
690
|
+
st = fs.statSync(file);
|
|
691
|
+
} catch {
|
|
692
|
+
continue;
|
|
693
|
+
}
|
|
694
|
+
seen.add(file);
|
|
695
|
+
const row = known.get(file);
|
|
696
|
+
if (!row) {
|
|
697
|
+
// Only a file the index has never heard of has to be opened, and only
|
|
698
|
+
// because the ignore rules match the header cwd, never the mangled
|
|
699
|
+
// directory name.
|
|
700
|
+
const header = readHeader(file);
|
|
701
|
+
if (header && !isIgnored(header.cwd ?? file)) report.unseen++;
|
|
702
|
+
continue;
|
|
703
|
+
}
|
|
704
|
+
if (row.size === null || row.mtime === null) report.unseen++;
|
|
705
|
+
else if (row.size !== st.size || row.mtime !== Math.floor(st.mtimeMs))
|
|
706
|
+
report.changed++;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
for (const knownPath of known.keys())
|
|
710
|
+
if (!seen.has(knownPath)) report.vanished++;
|
|
711
|
+
return report;
|
|
712
|
+
}
|
package/src/resources.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import {fileURLToPath} from 'node:url';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Finding files that ship beside the package rather than inside `dist`.
|
|
6
|
+
*
|
|
7
|
+
* A published install carries `skills/` next to `dist/`, staged there at build time. A checkout
|
|
8
|
+
* keeps the same directory at the repo root, three levels above `packages/memonaut/dist`. Both are
|
|
9
|
+
* searched instead of one being made canonical, so `recall skills` behaves the same whether it is
|
|
10
|
+
* run from a global install, from the workspace, or through `tsx src/cli.ts`.
|
|
11
|
+
*/
|
|
12
|
+
const ROOTS = ['../', '../../../'];
|
|
13
|
+
|
|
14
|
+
export function resolvePackageResource(relative: string): string | undefined {
|
|
15
|
+
for (const root of ROOTS) {
|
|
16
|
+
try {
|
|
17
|
+
const candidate = fileURLToPath(
|
|
18
|
+
new URL(`${root}${relative}`, import.meta.url),
|
|
19
|
+
);
|
|
20
|
+
if (fs.existsSync(candidate)) return candidate;
|
|
21
|
+
} catch {
|
|
22
|
+
continue;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
package/src/skills.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import {resolvePackageResource} from './resources.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Installing the agent skill that ships with this package.
|
|
8
|
+
*
|
|
9
|
+
* pi loads the skill itself from `memonaut-pi` (its `pi.skills` entry), so this command exists for
|
|
10
|
+
* every other agent: the skill is the same file, and `~/.agents/skills/<name>` is where agents that
|
|
11
|
+
* are not pi look for it. One directory, no fan-out to twenty agent-specific locations, and the
|
|
12
|
+
* report names every path written.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export interface Skill {
|
|
16
|
+
name: string;
|
|
17
|
+
description: string;
|
|
18
|
+
from: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export type Scope = 'user' | 'project';
|
|
22
|
+
|
|
23
|
+
/** Where skills live, by convention. `project` keeps them beside the code that needs them. */
|
|
24
|
+
export function destinationFor(scope: Scope, cwd = process.cwd()): string {
|
|
25
|
+
return scope === 'user'
|
|
26
|
+
? path.join(os.homedir(), '.agents', 'skills')
|
|
27
|
+
: path.join(cwd, '.agents', 'skills');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function skillsSource(): string | undefined {
|
|
31
|
+
return resolvePackageResource('skills/');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Name and description come from the SKILL.md front matter, so there is one source of truth. */
|
|
35
|
+
export function availableSkills(): Skill[] {
|
|
36
|
+
const root = skillsSource();
|
|
37
|
+
if (!root) return [];
|
|
38
|
+
const skills: Skill[] = [];
|
|
39
|
+
for (const entry of fs.readdirSync(root, {withFileTypes: true})) {
|
|
40
|
+
if (!entry.isDirectory()) continue;
|
|
41
|
+
const manifest = path.join(root, entry.name, 'SKILL.md');
|
|
42
|
+
if (!fs.existsSync(manifest)) continue;
|
|
43
|
+
const text = fs.readFileSync(manifest, 'utf8');
|
|
44
|
+
skills.push({
|
|
45
|
+
name: /^name:\s*(.+)$/m.exec(text)?.[1]?.trim() ?? entry.name,
|
|
46
|
+
description: /^description:\s*(.+)$/m.exec(text)?.[1]?.trim() ?? '',
|
|
47
|
+
from: path.join(root, entry.name),
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
return skills.sort((a, b) => a.name.localeCompare(b.name));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface Installed {
|
|
54
|
+
name: string;
|
|
55
|
+
to: string;
|
|
56
|
+
replaced: boolean;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Copies rather than symlinks, so an installed skill survives `node_modules` being deleted.
|
|
61
|
+
*
|
|
62
|
+
* The cost is that upgrading the package does not upgrade the installed skill, which is why the
|
|
63
|
+
* report says `replaced` versus `installed`: a skill that is quietly stale is worse than one you
|
|
64
|
+
* know you have to refresh.
|
|
65
|
+
*/
|
|
66
|
+
export function installSkills(scope: Scope, cwd = process.cwd()): Installed[] {
|
|
67
|
+
const destination = destinationFor(scope, cwd);
|
|
68
|
+
const installed: Installed[] = [];
|
|
69
|
+
for (const skill of availableSkills()) {
|
|
70
|
+
const to = path.join(destination, skill.name);
|
|
71
|
+
const replaced = fs.existsSync(to);
|
|
72
|
+
if (replaced) fs.rmSync(to, {recursive: true, force: true});
|
|
73
|
+
fs.mkdirSync(destination, {recursive: true});
|
|
74
|
+
fs.cpSync(skill.from, to, {recursive: true});
|
|
75
|
+
installed.push({name: skill.name, to, replaced});
|
|
76
|
+
}
|
|
77
|
+
return installed;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function isInstalled(
|
|
81
|
+
skill: Skill,
|
|
82
|
+
scope: Scope,
|
|
83
|
+
cwd = process.cwd(),
|
|
84
|
+
): boolean {
|
|
85
|
+
return fs.existsSync(path.join(destinationFor(scope, cwd), skill.name));
|
|
86
|
+
}
|