@yurtsever/capsa 0.1.0-alpha.6 → 0.1.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +299 -97
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -10074,7 +10074,10 @@ function loadSqlite() {
|
|
|
10074
10074
|
}
|
|
10075
10075
|
var DB_DIR = ".capsa";
|
|
10076
10076
|
var DB_FILE = "index.db";
|
|
10077
|
-
var SCHEMA_VERSION =
|
|
10077
|
+
var SCHEMA_VERSION = 7;
|
|
10078
|
+
var META_INDEXED_AT = "indexed_at";
|
|
10079
|
+
var META_INDEX_FAILED_AT = "index_failed_at";
|
|
10080
|
+
var META_INDEX_ERROR = "index_error";
|
|
10078
10081
|
var CONTEXT_LOG_DDL = `
|
|
10079
10082
|
-- Every context delivery. This is the measurement for the MVP gate and
|
|
10080
10083
|
-- the seed of the organisation tier's audit log. Alone among these
|
|
@@ -10088,6 +10091,12 @@ var CONTEXT_LOG_DDL = `
|
|
|
10088
10091
|
query TEXT NOT NULL,
|
|
10089
10092
|
delivered TEXT NOT NULL,
|
|
10090
10093
|
tokens_est INTEGER NOT NULL,
|
|
10094
|
+
-- The ceiling this call ran under. Without it a row says what was
|
|
10095
|
+
-- spent but not what it was allowed to spend, and two deliveries are
|
|
10096
|
+
-- only comparable when that number matches. NULL on rows written
|
|
10097
|
+
-- before v6: they ran under a budget nobody recorded, and filling in
|
|
10098
|
+
-- today's default would be a guess wearing the costume of data.
|
|
10099
|
+
tokens_budget INTEGER,
|
|
10091
10100
|
duration_ms INTEGER NOT NULL
|
|
10092
10101
|
);
|
|
10093
10102
|
`;
|
|
@@ -10109,24 +10118,44 @@ var Store = class {
|
|
|
10109
10118
|
}
|
|
10110
10119
|
/** True when this index was written by an older schema and must be rebuilt. */
|
|
10111
10120
|
needsRebuild() {
|
|
10112
|
-
|
|
10113
|
-
|
|
10121
|
+
return this.getMeta("schema") !== String(SCHEMA_VERSION);
|
|
10122
|
+
}
|
|
10123
|
+
getMeta(key) {
|
|
10124
|
+
const row = this.db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
|
|
10125
|
+
return row?.value;
|
|
10126
|
+
}
|
|
10127
|
+
setMeta(key, value) {
|
|
10128
|
+
this.db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)").run(key, value);
|
|
10129
|
+
}
|
|
10130
|
+
deleteMeta(key) {
|
|
10131
|
+
this.db.prepare("DELETE FROM meta WHERE key = ?").run(key);
|
|
10114
10132
|
}
|
|
10115
10133
|
/**
|
|
10116
10134
|
* Drop all indexed content (not the log, not decisions) and mark the schema
|
|
10117
|
-
* current.
|
|
10118
|
-
* `CREATE
|
|
10119
|
-
*
|
|
10135
|
+
* current. Every one of these tables is dropped and recreated, not emptied,
|
|
10136
|
+
* because `CREATE TABLE IF NOT EXISTS` in migrate() is a no-op against a
|
|
10137
|
+
* table that already exists under its old shape. For the virtual tables that
|
|
10138
|
+
* once left FTS without stemming after v2; for `items` it meant a schema
|
|
10139
|
+
* bump could not add a column at all — v7 added `title` and the rebuild it
|
|
10140
|
+
* asked for failed with "table items has no column named title", because
|
|
10141
|
+
* DELETE had emptied the v6 table and left it standing.
|
|
10142
|
+
*
|
|
10143
|
+
* Nothing here is a source of truth: items and chunks are derived from files
|
|
10144
|
+
* on disk and from `decisions`, which is one of the two tables a reset
|
|
10145
|
+
* spares.
|
|
10120
10146
|
*/
|
|
10121
10147
|
reset() {
|
|
10122
10148
|
this.db.exec(`
|
|
10123
10149
|
DROP TABLE IF EXISTS chunks_fts;
|
|
10124
10150
|
DROP TABLE IF EXISTS chunks_vec;
|
|
10125
|
-
|
|
10126
|
-
|
|
10151
|
+
DROP TABLE IF EXISTS chunks;
|
|
10152
|
+
DROP TABLE IF EXISTS items;
|
|
10127
10153
|
`);
|
|
10128
10154
|
this.migrate();
|
|
10129
|
-
this.
|
|
10155
|
+
this.setMeta("schema", String(SCHEMA_VERSION));
|
|
10156
|
+
this.deleteMeta(META_INDEXED_AT);
|
|
10157
|
+
this.deleteMeta(META_INDEX_FAILED_AT);
|
|
10158
|
+
this.deleteMeta(META_INDEX_ERROR);
|
|
10130
10159
|
}
|
|
10131
10160
|
migrate() {
|
|
10132
10161
|
this.db.exec(`
|
|
@@ -10137,6 +10166,7 @@ var Store = class {
|
|
|
10137
10166
|
format TEXT NOT NULL,
|
|
10138
10167
|
kind TEXT NOT NULL,
|
|
10139
10168
|
name TEXT NOT NULL,
|
|
10169
|
+
title TEXT NOT NULL,
|
|
10140
10170
|
path TEXT NOT NULL UNIQUE,
|
|
10141
10171
|
rel_path TEXT NOT NULL,
|
|
10142
10172
|
content_hash TEXT NOT NULL,
|
|
@@ -10179,27 +10209,40 @@ var Store = class {
|
|
|
10179
10209
|
${CONTEXT_LOG_DDL}
|
|
10180
10210
|
`);
|
|
10181
10211
|
this.migrateContextLog();
|
|
10182
|
-
this.
|
|
10212
|
+
this.setMeta("dimensions", String(this.dimensions));
|
|
10183
10213
|
}
|
|
10184
10214
|
/**
|
|
10185
10215
|
* `context_log` outlives every rebuild, so it is the one table that has to
|
|
10186
|
-
* be migrated in place instead of recreated
|
|
10187
|
-
*
|
|
10188
|
-
*
|
|
10189
|
-
*
|
|
10190
|
-
*
|
|
10191
|
-
*
|
|
10216
|
+
* be migrated in place instead of recreated — twice now.
|
|
10217
|
+
*
|
|
10218
|
+
* v5 replaced `chunk_ids` with `delivered`: a chunk id is a rowid the next
|
|
10219
|
+
* `capsa index` reassigns, so an older row ended up naming whatever text
|
|
10220
|
+
* inherited its ids — the measurement quietly decayed. Rows written before
|
|
10221
|
+
* v5 keep everything that still means something (when, who asked, what for,
|
|
10222
|
+
* at what cost) and carry an empty delivery, because their ids can no longer
|
|
10223
|
+
* be resolved honestly.
|
|
10224
|
+
*
|
|
10225
|
+
* v6 added `tokens_budget`. A delivery of 1727 tokens against one of 758
|
|
10226
|
+
* reads like a saving until you learn the first ran on a larger ceiling;
|
|
10227
|
+
* that comparison was attempted on this repo's own log and could not be
|
|
10228
|
+
* settled, because the number was never written down. Old rows get NULL
|
|
10229
|
+
* rather than a backfilled default — an unknown budget is a fact about the
|
|
10230
|
+
* measurement, and inventing one would hide exactly the gap this closes.
|
|
10231
|
+
*
|
|
10232
|
+
* Keyed on the newest column, so a v4 database reaches v6 in one pass.
|
|
10192
10233
|
*/
|
|
10193
10234
|
migrateContextLog() {
|
|
10194
10235
|
const columns = this.db.prepare("PRAGMA table_info(context_log)").all();
|
|
10195
|
-
|
|
10236
|
+
const names = new Set(columns.map((c) => c.name));
|
|
10237
|
+
if (names.has("tokens_budget"))
|
|
10196
10238
|
return;
|
|
10239
|
+
const delivered = names.has("delivered") ? "delivered" : "'[]'";
|
|
10197
10240
|
this.db.exec(`
|
|
10198
|
-
ALTER TABLE context_log RENAME TO
|
|
10241
|
+
ALTER TABLE context_log RENAME TO context_log_old;
|
|
10199
10242
|
${CONTEXT_LOG_DDL}
|
|
10200
|
-
INSERT INTO context_log (id, created_at, source, query, delivered, tokens_est, duration_ms)
|
|
10201
|
-
SELECT id, created_at, source, query,
|
|
10202
|
-
DROP TABLE
|
|
10243
|
+
INSERT INTO context_log (id, created_at, source, query, delivered, tokens_est, tokens_budget, duration_ms)
|
|
10244
|
+
SELECT id, created_at, source, query, ${delivered}, tokens_est, NULL, duration_ms FROM context_log_old;
|
|
10245
|
+
DROP TABLE context_log_old;
|
|
10203
10246
|
`);
|
|
10204
10247
|
}
|
|
10205
10248
|
getItem(path) {
|
|
@@ -10217,13 +10260,14 @@ var Store = class {
|
|
|
10217
10260
|
this.db.prepare("DELETE FROM items WHERE id = ?").run(id);
|
|
10218
10261
|
}
|
|
10219
10262
|
upsertItem(row) {
|
|
10220
|
-
this.db.prepare(`INSERT INTO items (id, format, kind, name, path, rel_path, content_hash, status, updated_at, tags)
|
|
10221
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
10263
|
+
this.db.prepare(`INSERT INTO items (id, format, kind, name, title, path, rel_path, content_hash, status, updated_at, tags)
|
|
10264
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
10222
10265
|
ON CONFLICT(id) DO UPDATE SET
|
|
10223
10266
|
format = excluded.format, kind = excluded.kind, name = excluded.name,
|
|
10267
|
+
title = excluded.title,
|
|
10224
10268
|
rel_path = excluded.rel_path, content_hash = excluded.content_hash,
|
|
10225
10269
|
status = excluded.status, updated_at = excluded.updated_at, tags = excluded.tags,
|
|
10226
|
-
indexed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`).run(row.id, row.format, row.kind, row.name, row.path, row.rel_path, row.content_hash, row.status, row.updated_at, row.tags);
|
|
10270
|
+
indexed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')`).run(row.id, row.format, row.kind, row.name, row.title, row.path, row.rel_path, row.content_hash, row.status, row.updated_at, row.tags);
|
|
10227
10271
|
}
|
|
10228
10272
|
/**
|
|
10229
10273
|
* `text` is what the user sees; `searchText` (chunk + provenance header)
|
|
@@ -10254,7 +10298,7 @@ var Store = class {
|
|
|
10254
10298
|
return [];
|
|
10255
10299
|
const placeholders = ids.map(() => "?").join(",");
|
|
10256
10300
|
const rows = this.db.prepare(`SELECT c.id, c.item_id, c.idx, c.heading_path, c.text,
|
|
10257
|
-
i.id AS i_id, i.format, i.kind, i.name, i.path, i.rel_path, i.content_hash, i.status, i.updated_at, i.tags
|
|
10301
|
+
i.id AS i_id, i.format, i.kind, i.name, i.title, i.path, i.rel_path, i.content_hash, i.status, i.updated_at, i.tags
|
|
10258
10302
|
FROM chunks c JOIN items i ON i.id = c.item_id WHERE c.id IN (${placeholders})`).all(...ids);
|
|
10259
10303
|
return rows.map((r) => ({
|
|
10260
10304
|
id: r["id"],
|
|
@@ -10267,6 +10311,7 @@ var Store = class {
|
|
|
10267
10311
|
format: r["format"],
|
|
10268
10312
|
kind: r["kind"],
|
|
10269
10313
|
name: r["name"],
|
|
10314
|
+
title: r["title"],
|
|
10270
10315
|
path: r["path"],
|
|
10271
10316
|
rel_path: r["rel_path"],
|
|
10272
10317
|
content_hash: r["content_hash"],
|
|
@@ -10284,7 +10329,7 @@ var Store = class {
|
|
|
10284
10329
|
return this.db.prepare("SELECT * FROM items ORDER BY kind, rel_path").all();
|
|
10285
10330
|
}
|
|
10286
10331
|
logContext(entry) {
|
|
10287
|
-
this.db.prepare("INSERT INTO context_log (source, query, delivered, tokens_est, duration_ms) VALUES (?, ?, ?, ?, ?)").run(entry.source, entry.query, JSON.stringify(entry.delivered), entry.tokensEst, entry.durationMs);
|
|
10332
|
+
this.db.prepare("INSERT INTO context_log (source, query, delivered, tokens_est, tokens_budget, duration_ms) VALUES (?, ?, ?, ?, ?, ?)").run(entry.source, entry.query, JSON.stringify(entry.delivered), entry.tokensEst, entry.tokensBudget, entry.durationMs);
|
|
10288
10333
|
}
|
|
10289
10334
|
/**
|
|
10290
10335
|
* Returns the stored row, not just its id: `created_at` is SQL's to write,
|
|
@@ -10720,76 +10765,116 @@ function openStore(projectRoot, embedder) {
|
|
|
10720
10765
|
return new Store(dbPath(projectRoot), embedder.dimensions);
|
|
10721
10766
|
}
|
|
10722
10767
|
async function indexProject(projectRoot, embedder, onProgress) {
|
|
10723
|
-
const started = Date.now();
|
|
10724
10768
|
const store = openStore(projectRoot, embedder);
|
|
10725
10769
|
try {
|
|
10726
|
-
|
|
10727
|
-
onProgress?.({ phase: "rebuild" });
|
|
10728
|
-
store.reset();
|
|
10729
|
-
}
|
|
10730
|
-
onProgress?.({ phase: "scan" });
|
|
10731
|
-
const result = await scan(knowledgeAdapters(), { roots: [projectRoot] });
|
|
10732
|
-
const items = result.items.filter((i) => i.scope === "project" && isInside(projectRoot, i.path));
|
|
10733
|
-
let indexed = 0;
|
|
10734
|
-
let skipped = 0;
|
|
10735
|
-
const seen = /* @__PURE__ */ new Set();
|
|
10736
|
-
for (const item of items) {
|
|
10737
|
-
seen.add(item.path);
|
|
10738
|
-
const hash2 = sha1(item.content);
|
|
10739
|
-
const existing = store.getItem(item.path);
|
|
10740
|
-
if (existing && existing.content_hash === hash2) {
|
|
10741
|
-
skipped++;
|
|
10742
|
-
continue;
|
|
10743
|
-
}
|
|
10744
|
-
if (existing)
|
|
10745
|
-
store.deleteItem(existing.id);
|
|
10746
|
-
await indexItem(store, embedder, item, projectRoot, hash2);
|
|
10747
|
-
indexed++;
|
|
10748
|
-
onProgress?.({ phase: "embed", scanned: items.length, indexed, skipped });
|
|
10749
|
-
}
|
|
10750
|
-
const decisions = await indexDecisions(store, embedder, projectRoot);
|
|
10751
|
-
for (const path of decisions.paths)
|
|
10752
|
-
seen.add(path);
|
|
10753
|
-
indexed += decisions.indexed;
|
|
10754
|
-
skipped += decisions.skipped;
|
|
10755
|
-
let removed = 0;
|
|
10756
|
-
for (const row of store.listItems()) {
|
|
10757
|
-
if (!seen.has(row.path)) {
|
|
10758
|
-
store.deleteItem(row.id);
|
|
10759
|
-
removed++;
|
|
10760
|
-
}
|
|
10761
|
-
}
|
|
10762
|
-
const out = {
|
|
10763
|
-
scanned: items.length + decisions.paths.length,
|
|
10764
|
-
indexed,
|
|
10765
|
-
skipped,
|
|
10766
|
-
removed,
|
|
10767
|
-
issues: result.issues.map((i) => ({ path: i.path, message: i.message })),
|
|
10768
|
-
durationMs: Date.now() - started
|
|
10769
|
-
};
|
|
10770
|
-
onProgress?.({ phase: "done", ...out, issues: out.issues.length });
|
|
10771
|
-
return out;
|
|
10770
|
+
return await indexInto(store, projectRoot, embedder, { onProgress });
|
|
10772
10771
|
} finally {
|
|
10773
10772
|
store.close();
|
|
10774
10773
|
}
|
|
10775
10774
|
}
|
|
10776
|
-
|
|
10777
|
-
|
|
10778
|
-
|
|
10779
|
-
|
|
10780
|
-
|
|
10775
|
+
var TooManyChangesError = class extends Error {
|
|
10776
|
+
changed;
|
|
10777
|
+
limit;
|
|
10778
|
+
constructor(changed, limit) {
|
|
10779
|
+
super(`${changed} items changed since the last index \u2014 run \`capsa index\``);
|
|
10780
|
+
this.changed = changed;
|
|
10781
|
+
this.limit = limit;
|
|
10782
|
+
this.name = "TooManyChangesError";
|
|
10783
|
+
}
|
|
10784
|
+
};
|
|
10785
|
+
async function indexInto(store, projectRoot, embedder, options = {}) {
|
|
10786
|
+
const { onProgress, maxChanges } = options;
|
|
10787
|
+
const started = Date.now();
|
|
10788
|
+
if (store.needsRebuild()) {
|
|
10789
|
+
onProgress?.({ phase: "rebuild" });
|
|
10790
|
+
store.reset();
|
|
10791
|
+
}
|
|
10792
|
+
onProgress?.({ phase: "scan" });
|
|
10793
|
+
const result = await scan(knowledgeAdapters(), { roots: [projectRoot] });
|
|
10794
|
+
const items = result.items.filter((i) => i.scope === "project" && isInside(projectRoot, i.path));
|
|
10795
|
+
const all = [...items, ...decisionItems(store, projectRoot)];
|
|
10796
|
+
const { changed, skipped } = changedItems(store, all);
|
|
10797
|
+
if (maxChanges !== void 0 && changed.length > maxChanges) {
|
|
10798
|
+
throw new TooManyChangesError(changed.length, maxChanges);
|
|
10799
|
+
}
|
|
10800
|
+
let indexed = 0;
|
|
10801
|
+
for (const c of changed) {
|
|
10802
|
+
await indexItem(store, embedder, c.item, projectRoot, c.hash, c.previousId);
|
|
10803
|
+
indexed++;
|
|
10804
|
+
onProgress?.({ phase: "embed", scanned: all.length, indexed, skipped });
|
|
10805
|
+
}
|
|
10806
|
+
const seen = new Set(all.map((i) => i.path));
|
|
10807
|
+
let removed = 0;
|
|
10808
|
+
for (const row of store.listItems()) {
|
|
10809
|
+
if (!seen.has(row.path)) {
|
|
10810
|
+
store.deleteItem(row.id);
|
|
10811
|
+
removed++;
|
|
10812
|
+
}
|
|
10813
|
+
}
|
|
10814
|
+
const out = {
|
|
10815
|
+
scanned: all.length,
|
|
10816
|
+
indexed,
|
|
10817
|
+
skipped,
|
|
10818
|
+
removed,
|
|
10819
|
+
issues: result.issues.map((i) => ({ path: i.path, message: i.message })),
|
|
10820
|
+
durationMs: Date.now() - started
|
|
10821
|
+
};
|
|
10822
|
+
store.setMeta(META_INDEXED_AT, new Date(started).toISOString());
|
|
10823
|
+
store.deleteMeta(META_INDEX_FAILED_AT);
|
|
10824
|
+
store.deleteMeta(META_INDEX_ERROR);
|
|
10825
|
+
onProgress?.({ phase: "done", ...out, issues: out.issues.length });
|
|
10826
|
+
return out;
|
|
10827
|
+
}
|
|
10828
|
+
var DEFAULT_MAX_INDEX_AGE_MS = 6e4;
|
|
10829
|
+
var DEFAULT_MAX_REFRESH_CHANGES = 20;
|
|
10830
|
+
async function freshenIndex(store, projectRoot, embedder, options = {}) {
|
|
10831
|
+
const maxAge = options.maxAgeMs ?? DEFAULT_MAX_INDEX_AGE_MS;
|
|
10832
|
+
const maxChanges = options.maxChanges ?? DEFAULT_MAX_REFRESH_CHANGES;
|
|
10833
|
+
const now = Date.now();
|
|
10834
|
+
if (ageMs(store.getMeta(META_INDEXED_AT), now) <= maxAge)
|
|
10835
|
+
return { refreshed: false, stale: false };
|
|
10836
|
+
if (store.needsRebuild()) {
|
|
10837
|
+
return {
|
|
10838
|
+
refreshed: false,
|
|
10839
|
+
stale: true,
|
|
10840
|
+
reason: "this index was never built or its schema changed \u2014 run `capsa index`"
|
|
10841
|
+
};
|
|
10842
|
+
}
|
|
10843
|
+
if (ageMs(store.getMeta(META_INDEX_FAILED_AT), now) <= maxAge) {
|
|
10844
|
+
return { refreshed: false, stale: true, reason: store.getMeta(META_INDEX_ERROR) ?? "the last index run failed" };
|
|
10845
|
+
}
|
|
10846
|
+
try {
|
|
10847
|
+
const res = await indexInto(store, projectRoot, embedder, { maxChanges });
|
|
10848
|
+
return { refreshed: true, stale: false, indexed: res.indexed, removed: res.removed };
|
|
10849
|
+
} catch (err) {
|
|
10850
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
10851
|
+
store.setMeta(META_INDEX_FAILED_AT, (/* @__PURE__ */ new Date()).toISOString());
|
|
10852
|
+
store.setMeta(META_INDEX_ERROR, reason);
|
|
10853
|
+
return { refreshed: false, stale: true, reason };
|
|
10854
|
+
}
|
|
10855
|
+
}
|
|
10856
|
+
function ageMs(iso, now) {
|
|
10857
|
+
if (!iso)
|
|
10858
|
+
return Number.POSITIVE_INFINITY;
|
|
10859
|
+
const t = Date.parse(iso);
|
|
10860
|
+
return Number.isNaN(t) ? Number.POSITIVE_INFINITY : now - t;
|
|
10861
|
+
}
|
|
10862
|
+
function changedItems(store, items) {
|
|
10863
|
+
const changed = [];
|
|
10864
|
+
let skipped = 0;
|
|
10865
|
+
for (const item of items) {
|
|
10781
10866
|
const hash2 = sha1(item.content);
|
|
10782
10867
|
const existing = store.getItem(item.path);
|
|
10783
10868
|
if (existing && existing.content_hash === hash2) {
|
|
10784
|
-
|
|
10869
|
+
skipped++;
|
|
10785
10870
|
continue;
|
|
10786
10871
|
}
|
|
10787
|
-
|
|
10788
|
-
store.deleteItem(existing.id);
|
|
10789
|
-
await indexItem(store, embedder, item, projectRoot, hash2);
|
|
10790
|
-
out.indexed++;
|
|
10872
|
+
changed.push({ item, hash: hash2, previousId: existing?.id });
|
|
10791
10873
|
}
|
|
10792
|
-
return
|
|
10874
|
+
return { changed, skipped };
|
|
10875
|
+
}
|
|
10876
|
+
function decisionItems(store, projectRoot) {
|
|
10877
|
+
return store.listDecisions().map((row) => decisionItem(projectRoot, row));
|
|
10793
10878
|
}
|
|
10794
10879
|
async function recordDecision(store, embedder, projectRoot, input2) {
|
|
10795
10880
|
const row = store.recordDecision(input2.title, input2.body, input2.related);
|
|
@@ -10801,16 +10886,22 @@ async function recordDecision(store, embedder, projectRoot, input2) {
|
|
|
10801
10886
|
return { id: row.id, indexed: false, reason: err.message };
|
|
10802
10887
|
}
|
|
10803
10888
|
}
|
|
10804
|
-
async function indexItem(store, embedder, item, projectRoot, hash2) {
|
|
10889
|
+
async function indexItem(store, embedder, item, projectRoot, hash2, previousId) {
|
|
10805
10890
|
const chunks = chunkMarkdown(item.content);
|
|
10806
10891
|
const searchTexts = chunks.map((c) => withContext(item, c.headingPath, c.text));
|
|
10807
10892
|
const vectors = searchTexts.length ? await embedder.embed(searchTexts) : [];
|
|
10893
|
+
if (previousId)
|
|
10894
|
+
store.deleteItem(previousId);
|
|
10808
10895
|
const k = item.knowledge;
|
|
10809
10896
|
store.upsertItem({
|
|
10810
10897
|
id: item.id,
|
|
10811
10898
|
format: item.format,
|
|
10812
10899
|
kind: k?.kind ?? "instruction",
|
|
10813
10900
|
name: item.name,
|
|
10901
|
+
// The same expression the provenance header uses (see withContext): the
|
|
10902
|
+
// title an item claims for itself, falling back to its name when the
|
|
10903
|
+
// format has none to give.
|
|
10904
|
+
title: item.description ?? item.name,
|
|
10814
10905
|
path: item.path,
|
|
10815
10906
|
rel_path: relative5(projectRoot, item.path),
|
|
10816
10907
|
content_hash: hash2,
|
|
@@ -10968,9 +11059,17 @@ async function retrieve(store, embedder, query, options = {}) {
|
|
|
10968
11059
|
const pinnedIds = new Set(pinned.map((i) => i.id));
|
|
10969
11060
|
const neighbourCutoff = pinned.length ? Math.max(minRelative, 0.75) : minRelative;
|
|
10970
11061
|
const neighbourBudget = pinned.length ? Math.min(maxTokens, tokens + Math.floor(maxTokens / 3)) : maxTokens;
|
|
10971
|
-
const
|
|
10972
|
-
|
|
10973
|
-
|
|
11062
|
+
const searchText = neighbourQuery(query, pinned);
|
|
11063
|
+
let vec = [];
|
|
11064
|
+
let degraded;
|
|
11065
|
+
try {
|
|
11066
|
+
const [qv] = await embedder.embed([searchText]);
|
|
11067
|
+
if (qv)
|
|
11068
|
+
vec = store.vectorSearch(qv, candidates).map((r) => r.chunkId);
|
|
11069
|
+
} catch (err) {
|
|
11070
|
+
degraded = err instanceof Error ? err.message : String(err);
|
|
11071
|
+
}
|
|
11072
|
+
const fts = store.textSearch(searchText, candidates).map((r) => r.chunkId);
|
|
10974
11073
|
const fused = rrf([vec, fts], weights.rrfK);
|
|
10975
11074
|
const rows = store.chunksWithItems([...fused.keys()]).filter((r) => !pinnedIds.has(r.item_id));
|
|
10976
11075
|
const now = Date.now();
|
|
@@ -10979,10 +11078,33 @@ async function retrieve(store, embedder, query, options = {}) {
|
|
|
10979
11078
|
const factor = signalFactor({ kind: r.item.kind, status: r.item.status, updatedAt: r.item.updated_at }, weights, now);
|
|
10980
11079
|
return { row: r, fusedScore, factor, score: fusedScore * factor };
|
|
10981
11080
|
}).sort((a, b) => b.score - a.score);
|
|
10982
|
-
const
|
|
11081
|
+
const neighbourhood = [];
|
|
11082
|
+
for (const c of scored) {
|
|
11083
|
+
if (options.excludeRootInstructions && isRootInstruction(c.row.item.rel_path)) {
|
|
11084
|
+
if (options.debug) {
|
|
11085
|
+
debug.push({
|
|
11086
|
+
chunkId: c.row.id,
|
|
11087
|
+
relPath: c.row.item.rel_path,
|
|
11088
|
+
headingPath: JSON.parse(c.row.heading_path),
|
|
11089
|
+
kind: c.row.item.kind,
|
|
11090
|
+
status: c.row.item.status,
|
|
11091
|
+
vecRank: rankIn(vec, c.row.id),
|
|
11092
|
+
ftsRank: rankIn(fts, c.row.id),
|
|
11093
|
+
fused: c.fusedScore,
|
|
11094
|
+
factor: c.factor,
|
|
11095
|
+
score: c.score,
|
|
11096
|
+
kept: false,
|
|
11097
|
+
why: "in prompt"
|
|
11098
|
+
});
|
|
11099
|
+
}
|
|
11100
|
+
continue;
|
|
11101
|
+
}
|
|
11102
|
+
neighbourhood.push(c);
|
|
11103
|
+
}
|
|
11104
|
+
const best = neighbourhood[0]?.score ?? 0;
|
|
10983
11105
|
let cut = tokens >= maxTokens;
|
|
10984
11106
|
let kept = 0;
|
|
10985
|
-
for (const { row, fusedScore, factor, score } of
|
|
11107
|
+
for (const { row, fusedScore, factor, score } of neighbourhood) {
|
|
10986
11108
|
let why = "kept";
|
|
10987
11109
|
if (cut || kept > 0 && score < best * neighbourCutoff) {
|
|
10988
11110
|
cut = true;
|
|
@@ -11030,13 +11152,37 @@ async function retrieve(store, embedder, query, options = {}) {
|
|
|
11030
11152
|
}
|
|
11031
11153
|
const durationMs = Date.now() - started;
|
|
11032
11154
|
store.logContext({
|
|
11033
|
-
source
|
|
11155
|
+
// A keyword-only delivery is logged under its own source. The week-4
|
|
11156
|
+
// gate is read from this table, and a run with half the retriever
|
|
11157
|
+
// missing is not a sample of how the ranking performs — it should show
|
|
11158
|
+
// up as its own bucket in a `group by source`, not quietly average in.
|
|
11159
|
+
source: (options.source ?? "unknown") + (degraded ? ":degraded" : ""),
|
|
11034
11160
|
query,
|
|
11035
11161
|
delivered: chunks.map((c) => ({ rel_path: c.relPath, heading_path: c.headingPath, tokens: c.tokens })),
|
|
11036
11162
|
tokensEst: tokens,
|
|
11163
|
+
// The ceiling, not the spend: two rows are only comparable when this
|
|
11164
|
+
// matches, and retrieval stops at the relevance cliff well before it.
|
|
11165
|
+
tokensBudget: maxTokens,
|
|
11037
11166
|
durationMs
|
|
11038
11167
|
});
|
|
11039
|
-
return {
|
|
11168
|
+
return {
|
|
11169
|
+
query,
|
|
11170
|
+
searchText,
|
|
11171
|
+
chunks,
|
|
11172
|
+
tokensEst: tokens,
|
|
11173
|
+
durationMs,
|
|
11174
|
+
...degraded ? { degraded } : {},
|
|
11175
|
+
...options.debug ? { debug } : {}
|
|
11176
|
+
};
|
|
11177
|
+
}
|
|
11178
|
+
function neighbourQuery(query, pinned) {
|
|
11179
|
+
if (pinned.length === 0)
|
|
11180
|
+
return query;
|
|
11181
|
+
return pinned.map((i) => i.title).join("; ");
|
|
11182
|
+
}
|
|
11183
|
+
var ROOT_INSTRUCTION_FILES = /* @__PURE__ */ new Set(["CLAUDE.md", "CLAUDE.local.md", "AGENTS.md"]);
|
|
11184
|
+
function isRootInstruction(relPath) {
|
|
11185
|
+
return ROOT_INSTRUCTION_FILES.has(relPath);
|
|
11040
11186
|
}
|
|
11041
11187
|
function rankIn(list, id) {
|
|
11042
11188
|
const i = list.indexOf(id);
|
|
@@ -11045,7 +11191,11 @@ function rankIn(list, id) {
|
|
|
11045
11191
|
function formatDebug(result) {
|
|
11046
11192
|
if (!result.debug)
|
|
11047
11193
|
return "";
|
|
11048
|
-
const lines = [
|
|
11194
|
+
const lines = [
|
|
11195
|
+
`stage 2 searched: ${JSON.stringify(result.searchText)}${result.searchText === result.query ? "" : " (from the pinned titles)"}`,
|
|
11196
|
+
"",
|
|
11197
|
+
" vec fts factor score keep where"
|
|
11198
|
+
];
|
|
11049
11199
|
for (const d of result.debug) {
|
|
11050
11200
|
const where = [d.relPath, ...d.headingPath].join(" \u203A ").slice(0, 70);
|
|
11051
11201
|
lines.push([
|
|
@@ -40384,21 +40534,31 @@ var StdioServerTransport = class {
|
|
|
40384
40534
|
function createCapsaServer(options) {
|
|
40385
40535
|
const embedder = options.embedder ?? ollamaEmbedder();
|
|
40386
40536
|
const server = new McpServer({ name: "capsa", version: "0.0.1" });
|
|
40537
|
+
const ensureFresh = freshener(options, embedder);
|
|
40387
40538
|
server.registerTool("get_context", {
|
|
40388
40539
|
title: "Get project context",
|
|
40389
40540
|
description: "Return the smallest set of project knowledge (runbooks, tickets, plans, decisions, instruction files) relevant to a task. Call this before starting work on a task; pass the task as you understand it.",
|
|
40390
40541
|
inputSchema: {
|
|
40391
40542
|
task: external_exports.string().describe("The task or question, in one or two sentences"),
|
|
40392
|
-
|
|
40543
|
+
// 3000 is a ceiling on a ceiling: retrieval stops at the relevance
|
|
40544
|
+
// cliff long before it fills a budget, so a larger number only ever
|
|
40545
|
+
// buys a longer tail of near-misses.
|
|
40546
|
+
maxTokens: external_exports.number().int().min(200).max(3e3).optional().describe("Token budget, default 1500")
|
|
40393
40547
|
}
|
|
40394
40548
|
}, async ({ task, maxTokens }) => {
|
|
40549
|
+
const freshness = await ensureFresh();
|
|
40395
40550
|
const store = openStore(options.projectRoot, embedder);
|
|
40396
40551
|
try {
|
|
40397
|
-
const result = await retrieve(store, embedder, task, {
|
|
40552
|
+
const result = await retrieve(store, embedder, task, {
|
|
40553
|
+
maxTokens,
|
|
40554
|
+
source: "mcp:get_context",
|
|
40555
|
+
excludeRootInstructions: true
|
|
40556
|
+
});
|
|
40557
|
+
const caveat = caveats(freshness, result.degraded);
|
|
40398
40558
|
const footer = `
|
|
40399
40559
|
|
|
40400
40560
|
---
|
|
40401
|
-
_capsa: ${result.chunks.length} chunks, ~${result.tokensEst} tokens, ${result.durationMs}
|
|
40561
|
+
_capsa: ${result.chunks.length} chunks, ~${result.tokensEst} tokens, ${result.durationMs} ms${caveat ? ` \xB7 ${caveat}` : ""}_`;
|
|
40402
40562
|
return { content: [{ type: "text", text: formatContext(result) + footer }] };
|
|
40403
40563
|
} finally {
|
|
40404
40564
|
store.close();
|
|
@@ -40409,9 +40569,14 @@ _capsa: ${result.chunks.length} chunks, ~${result.tokensEst} tokens, ${result.du
|
|
|
40409
40569
|
description: "Where the project stands: open tickets and plans, recent decisions, recently changed knowledge.",
|
|
40410
40570
|
inputSchema: {}
|
|
40411
40571
|
}, async () => {
|
|
40572
|
+
const freshness = await ensureFresh();
|
|
40412
40573
|
const store = openStore(options.projectRoot, embedder);
|
|
40413
40574
|
try {
|
|
40414
|
-
|
|
40575
|
+
const caveat = caveats(freshness);
|
|
40576
|
+
const text = formatState(projectState(store)) + (caveat ? `
|
|
40577
|
+
|
|
40578
|
+
_${caveat}_` : "");
|
|
40579
|
+
return { content: [{ type: "text", text }] };
|
|
40415
40580
|
} finally {
|
|
40416
40581
|
store.close();
|
|
40417
40582
|
}
|
|
@@ -40440,6 +40605,41 @@ _capsa: ${result.chunks.length} chunks, ~${result.tokensEst} tokens, ${result.du
|
|
|
40440
40605
|
});
|
|
40441
40606
|
return server;
|
|
40442
40607
|
}
|
|
40608
|
+
function freshener(options, embedder) {
|
|
40609
|
+
const maxAgeMs = options.maxIndexAgeMs ?? DEFAULT_MAX_INDEX_AGE_MS;
|
|
40610
|
+
let inFlight = null;
|
|
40611
|
+
return () => {
|
|
40612
|
+
if (!inFlight) {
|
|
40613
|
+
const run = (async () => {
|
|
40614
|
+
let store;
|
|
40615
|
+
try {
|
|
40616
|
+
store = openStore(options.projectRoot, embedder);
|
|
40617
|
+
return await freshenIndex(store, options.projectRoot, embedder, { maxAgeMs });
|
|
40618
|
+
} catch (err) {
|
|
40619
|
+
return { refreshed: false, stale: true, reason: err instanceof Error ? err.message : String(err) };
|
|
40620
|
+
} finally {
|
|
40621
|
+
store?.close();
|
|
40622
|
+
}
|
|
40623
|
+
})();
|
|
40624
|
+
inFlight = run;
|
|
40625
|
+
void run.finally(() => {
|
|
40626
|
+
if (inFlight === run)
|
|
40627
|
+
inFlight = null;
|
|
40628
|
+
});
|
|
40629
|
+
}
|
|
40630
|
+
return inFlight;
|
|
40631
|
+
};
|
|
40632
|
+
}
|
|
40633
|
+
function caveats(freshness, degraded) {
|
|
40634
|
+
const parts = [];
|
|
40635
|
+
if (freshness.stale) {
|
|
40636
|
+
parts.push(`index may be out of date \u2014 ${freshness.reason ?? "the last index run did not finish"}`);
|
|
40637
|
+
}
|
|
40638
|
+
if (degraded) {
|
|
40639
|
+
parts.push(freshness.stale ? "keyword-only search (no embeddings)" : `keyword-only search \u2014 ${degraded}`);
|
|
40640
|
+
}
|
|
40641
|
+
return parts.join(" \xB7 ");
|
|
40642
|
+
}
|
|
40443
40643
|
async function serveStdio(options) {
|
|
40444
40644
|
const server = createCapsaServer(options);
|
|
40445
40645
|
await server.connect(new StdioServerTransport());
|
|
@@ -40516,6 +40716,8 @@ async function runSearchCommand(args) {
|
|
|
40516
40716
|
} else {
|
|
40517
40717
|
console.log(formatContext(res));
|
|
40518
40718
|
}
|
|
40719
|
+
if (res.degraded) console.error(`
|
|
40720
|
+
capsa: keyword-only search \u2014 ${res.degraded}`);
|
|
40519
40721
|
console.error(`
|
|
40520
40722
|
${res.chunks.length} chunks, ~${res.tokensEst} tokens, ${res.durationMs} ms`);
|
|
40521
40723
|
} finally {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yurtsever/capsa",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.7",
|
|
4
4
|
"description": "Your project's knowledge in one capsule. Local-first project memory and cockpit for AI coding agents: indexes runbooks, tickets, plans and instruction files, serves the smallest relevant context over MCP.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|