akm-cli 0.10.261124 → 0.10.261125
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/cli/usage.js +4 -0
- package/dist/cli.js +10 -0
- package/dist/commands/feedback-cli.js +12 -3
- package/dist/commands/health/semantic-search-check.js +12 -2
- package/dist/commands/health.js +8 -15
- package/dist/commands/lint/index.js +6 -6
- package/dist/commands/read/knowledge.js +8 -8
- package/dist/commands/read/related.js +3 -9
- package/dist/commands/read/show.js +17 -9
- package/dist/commands/sources/bundle-cli.js +1 -0
- package/dist/commands/sources/installed-stashes.js +36 -27
- package/dist/core/asset/resolve-ref.js +8 -0
- package/dist/core/common.js +21 -0
- package/dist/core/endpoint-url.js +4 -4
- package/dist/core/events.js +8 -2
- package/dist/core/file-lock.js +52 -1
- package/dist/core/state-db.js +47 -15
- package/dist/core/stdio-reader-gone.js +19 -0
- package/dist/core/write-source.js +2 -2
- package/dist/indexer/index-build.js +6 -3
- package/dist/indexer/index-readiness.js +30 -4
- package/dist/indexer/indexer.js +56 -8
- package/dist/indexer/search/db-search.js +7 -1
- package/dist/indexer/search/search-source.js +23 -25
- package/dist/indexer/usage/usage-events.js +13 -4
- package/dist/indexer/vector-file.js +3 -3
- package/dist/registry/origin-resolve.js +2 -3
- package/dist/scripts/akm-migrate-node.js +136 -88
- package/dist/scripts/akm-migrate.js +136 -88
- package/dist/setup/endpoint.js +1 -1
- package/dist/storage/database.js +29 -36
- package/dist/storage/repositories/index-connection.js +44 -41
- package/dist/storage/repositories/index-entries-repository.js +14 -7
- package/dist/storage/repositories/index-entry-schema.js +7 -5
- package/dist/storage/repositories/index-schema.js +23 -0
- package/dist/storage/sqlite-read-snapshot.js +2 -2
- package/dist/workspace/assets.js +24 -44
- package/dist/workspace/bundle-files.js +2 -2
- package/dist/workspace/documents.js +7 -12
- package/dist/workspace/graph-topics.js +249 -0
- package/dist/workspace/graph.js +68 -4
- package/docs/reference/workspace.md +21 -4
- package/package.json +1 -1
|
@@ -14,7 +14,7 @@ import fs from "node:fs";
|
|
|
14
14
|
import { ConfigError } from "../../core/errors.js";
|
|
15
15
|
import { classifyPathAccess, describeInaccessiblePath } from "../../core/path-access.js";
|
|
16
16
|
import { getDbPath } from "../../core/paths.js";
|
|
17
|
-
import {
|
|
17
|
+
import { warnOnce } from "../../core/warn.js";
|
|
18
18
|
import { openDatabase } from "../database.js";
|
|
19
19
|
import { openManagedDatabase } from "../managed-db.js";
|
|
20
20
|
import { SQLITE_BUSY_TIMEOUT_MS } from "../sqlite-pragmas.js";
|
|
@@ -22,28 +22,56 @@ import { openSqliteReadSnapshot, SqliteReadSnapshotUnavailableError } from "../s
|
|
|
22
22
|
import { CANONICAL_INDEX_DB_VERSION } from "./index-entry-schema.js";
|
|
23
23
|
import { ensureSchema, newerIndexLayoutError } from "./index-schema.js";
|
|
24
24
|
/**
|
|
25
|
-
* Whether `error` is SQLite reporting
|
|
26
|
-
* "database disk image is malformed"
|
|
27
|
-
*
|
|
28
|
-
*
|
|
25
|
+
* Whether `error` is SQLite reporting that the file is not a usable database
|
|
26
|
+
* (`SQLITE_CORRUPT*` "database disk image is malformed", `SQLITE_NOTADB` "file
|
|
27
|
+
* is not a database") rather than a permission, lock, or schema problem.
|
|
28
|
+
* Matched on both `code` (bun:sqlite, better-sqlite3) and message text, since
|
|
29
|
+
* driver error shapes are not perfectly uniform.
|
|
29
30
|
*/
|
|
30
|
-
function isCorruptionError(error) {
|
|
31
|
+
export function isCorruptionError(error) {
|
|
31
32
|
const code = error?.code;
|
|
32
|
-
if (code === "SQLITE_CORRUPT")
|
|
33
|
+
if (typeof code === "string" && (code.startsWith("SQLITE_CORRUPT") || code === "SQLITE_NOTADB"))
|
|
33
34
|
return true;
|
|
34
35
|
const message = error instanceof Error ? error.message : String(error);
|
|
35
|
-
return message.includes("database disk image is malformed") ||
|
|
36
|
+
return (message.includes("database disk image is malformed") ||
|
|
37
|
+
message.includes("file is not a database") ||
|
|
38
|
+
message.includes("SQLITE_CORRUPT") ||
|
|
39
|
+
message.includes("SQLITE_NOTADB"));
|
|
36
40
|
}
|
|
37
41
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
* corruption error instead of deleting a file a build or a reader may be using.
|
|
42
|
+
* Whether `index.db` itself is damaged: it cannot be opened, or `PRAGMA quick_check` reports a problem.
|
|
43
|
+
* A corruption error from a build or a read may come from another database (state.db), so callers ask
|
|
44
|
+
* this before deleting or blaming the index. An absent or unreadable-by-permission file is not damaged.
|
|
42
45
|
*/
|
|
46
|
+
export function indexDbIsDamaged(dbPath = getDbPath()) {
|
|
47
|
+
let db;
|
|
48
|
+
try {
|
|
49
|
+
db = openDatabase(dbPath, { readonly: true, create: false });
|
|
50
|
+
const row = db?.prepare("PRAGMA quick_check(1)").get();
|
|
51
|
+
return row !== undefined && row.quick_check !== "ok";
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
return isCorruptionError(error);
|
|
55
|
+
}
|
|
56
|
+
finally {
|
|
57
|
+
db?.close();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** Delete `index.db` and its sidecars; the build rebuilds it from the library. Best-effort per file. */
|
|
61
|
+
export function deleteIndexDatabase(dbPath) {
|
|
62
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
63
|
+
try {
|
|
64
|
+
fs.rmSync(`${dbPath}${suffix}`, { force: true });
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
// Best-effort cleanup; the next open still fails loudly if the file could not actually be removed.
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/** Open `index.db` for writing, creating and migrating it. Corruption raises; only the build, which holds the rebuild lock, deletes and rebuilds. */
|
|
43
72
|
export function openIndexDatabase(dbPath, options) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
path: resolvedPath,
|
|
73
|
+
return openManagedDatabase({
|
|
74
|
+
path: dbPath ?? getDbPath(),
|
|
47
75
|
init: (db) => {
|
|
48
76
|
// Source update uses this narrow lifecycle seam to ATTACH state.db and
|
|
49
77
|
// open its coordinator-owned outer transaction before ensureSchema or
|
|
@@ -51,32 +79,7 @@ export function openIndexDatabase(dbPath, options) {
|
|
|
51
79
|
options?.beforeSchema?.(db);
|
|
52
80
|
ensureSchema(db);
|
|
53
81
|
},
|
|
54
|
-
};
|
|
55
|
-
try {
|
|
56
|
-
return openManagedDatabase(spec);
|
|
57
|
-
}
|
|
58
|
-
catch (error) {
|
|
59
|
-
// index.db is a derived cache, fully regenerable from the stash on disk
|
|
60
|
-
// (see src/core/state-db.ts's "Why a separate database from index.db"
|
|
61
|
-
// note) — so real on-disk corruption is recovered by deleting the file
|
|
62
|
-
// and rebuilding, not by surfacing a raw SQLITE_CORRUPT to the caller or
|
|
63
|
-
// quietly falling through to an unreadable index (#865). This is the ONE
|
|
64
|
-
// from-scratch rebuild: an older layout is migrated in place by
|
|
65
|
-
// ensureSchema, never dropped.
|
|
66
|
-
if (!options?.rebuildCorrupt || !isCorruptionError(error))
|
|
67
|
-
throw error;
|
|
68
|
-
warn(`Index database is corrupt at ${resolvedPath} — rebuilding.`);
|
|
69
|
-
for (const suffix of ["", "-wal", "-shm"]) {
|
|
70
|
-
try {
|
|
71
|
-
fs.rmSync(`${resolvedPath}${suffix}`, { force: true });
|
|
72
|
-
}
|
|
73
|
-
catch {
|
|
74
|
-
// Best-effort cleanup; the retried open below still fails loudly if
|
|
75
|
-
// the file could not actually be removed.
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
return openManagedDatabase(spec);
|
|
79
|
-
}
|
|
82
|
+
});
|
|
80
83
|
}
|
|
81
84
|
export function openExistingDatabase(dbPath) {
|
|
82
85
|
// Existing-DB callers do not mutate schema or embedding metadata on open;
|
|
@@ -569,20 +569,27 @@ export function findEntryIdByRef(db, ref, bundleId) {
|
|
|
569
569
|
export function withMdVariants(name) {
|
|
570
570
|
return name.endsWith(".md") ? [name, name.slice(0, -3)] : [name, `${name}.md`];
|
|
571
571
|
}
|
|
572
|
+
/** True when `file` is a `.md` file named by the last segment of `conceptId`, up to its `.md`. */
|
|
573
|
+
export function namesMarkdownFile(conceptId, file) {
|
|
574
|
+
const last = (conceptId.split("/").pop() ?? "").replace(/\.md$/, "");
|
|
575
|
+
return path.extname(file) === ".md" && path.basename(file, ".md") === last;
|
|
576
|
+
}
|
|
572
577
|
/**
|
|
573
578
|
* Current (`[bundle//]conceptId`) id lookup: match `item_ref` exactly when
|
|
574
579
|
* bundle-qualified or by `//conceptId` suffix when short.
|
|
575
580
|
*/
|
|
576
581
|
function findEntryIdByBundleRef(db, ref, bundleId) {
|
|
577
582
|
const parsed = parseBundleRef(ref);
|
|
578
|
-
const conceptVariants = withMdVariants(parsed.conceptId);
|
|
579
583
|
// item_ref is the canonical stored spelling post-flip.
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
584
|
+
const exact = matchIdByItemRef(db, parsed.bundle, parsed.conceptId, bundleId);
|
|
585
|
+
if (exact !== undefined)
|
|
586
|
+
return exact;
|
|
587
|
+
// The `.md` toggle names a markdown file by its own name: `tasks/x.md` is not a spelling of `tasks/x.yml`,
|
|
588
|
+
// nor `skills/x.md` of `skills/x/SKILL.md`.
|
|
589
|
+
const [, toggled] = withMdVariants(parsed.conceptId);
|
|
590
|
+
const id = matchIdByItemRef(db, parsed.bundle, toggled, bundleId);
|
|
591
|
+
const file = id === undefined ? undefined : getEntryFilePathById(db, id);
|
|
592
|
+
return file !== undefined && namesMarkdownFile(parsed.conceptId, file) ? id : undefined;
|
|
586
593
|
}
|
|
587
594
|
/**
|
|
588
595
|
* Match a row `id` by `item_ref`: exact `bundle//conceptId` when `bundle` is
|
|
@@ -142,14 +142,16 @@ export function retiredEntryColumns(db) {
|
|
|
142
142
|
return RETIRED_ENTRY_COLUMNS.filter((column) => present.has(column));
|
|
143
143
|
}
|
|
144
144
|
/**
|
|
145
|
-
* Whether an index database has an `entries` table this release can read
|
|
146
|
-
* fresh or empty file has none; an index older
|
|
147
|
-
* by columns this release no longer reads (or
|
|
148
|
-
*
|
|
145
|
+
* Whether an index database has an `entries` table this release can read,
|
|
146
|
+
* with its full-text table. A fresh or empty file has none; an index older
|
|
147
|
+
* than layout 21 has one keyed by columns this release no longer reads (or
|
|
148
|
+
* lacks the ones it does); a file whose schema was cut short has entries
|
|
149
|
+
* without `entries_fts`. Each serves nothing until the next `akm index`.
|
|
149
150
|
*/
|
|
150
151
|
export function hasCurrentEntriesTable(db) {
|
|
151
152
|
try {
|
|
152
|
-
|
|
153
|
+
const fts = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'entries_fts'").get();
|
|
154
|
+
return Boolean(fts) && missingEntryColumns(db).length === 0 && retiredEntryColumns(db).length === 0;
|
|
153
155
|
}
|
|
154
156
|
catch {
|
|
155
157
|
return false;
|
|
@@ -205,7 +205,30 @@ function ensureColumn(db, table, column, type) {
|
|
|
205
205
|
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
|
|
206
206
|
return true;
|
|
207
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* Bring `index.db` to the current layout in one transaction, so no reader or
|
|
210
|
+
* crash can observe a half-migrated file (entries without their FTS table).
|
|
211
|
+
* One `BEGIN IMMEDIATE`, waiting out the connection's busy timeout once: a
|
|
212
|
+
* busy index surfaces to the caller (the build reports it as contention)
|
|
213
|
+
* rather than being retried here. Joins a transaction the caller already holds.
|
|
214
|
+
*/
|
|
208
215
|
export function ensureSchema(db) {
|
|
216
|
+
if (db.inTransaction) {
|
|
217
|
+
applySchema(db);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
db.exec("BEGIN IMMEDIATE");
|
|
221
|
+
try {
|
|
222
|
+
applySchema(db);
|
|
223
|
+
db.exec("COMMIT");
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
if (db.inTransaction)
|
|
227
|
+
db.exec("ROLLBACK");
|
|
228
|
+
throw error;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
function applySchema(db) {
|
|
209
232
|
db.exec(`
|
|
210
233
|
CREATE TABLE IF NOT EXISTS index_meta (
|
|
211
234
|
key TEXT PRIMARY KEY,
|
|
@@ -19,7 +19,7 @@ import fs from "node:fs";
|
|
|
19
19
|
import os from "node:os";
|
|
20
20
|
import path from "node:path";
|
|
21
21
|
import { sleepSync } from "../runtime.js";
|
|
22
|
-
import {
|
|
22
|
+
import { openDatabase } from "./database.js";
|
|
23
23
|
const SNAPSHOT_MAX_ATTEMPTS = 8;
|
|
24
24
|
const SNAPSHOT_BACKOFF_MS = 25;
|
|
25
25
|
export class SqliteReadSnapshotUnavailableError extends Error {
|
|
@@ -151,7 +151,7 @@ export function openSqliteReadSnapshot(dbPath) {
|
|
|
151
151
|
let db;
|
|
152
152
|
try {
|
|
153
153
|
copySqliteSnapshot(dbPath, snapshotPath);
|
|
154
|
-
db =
|
|
154
|
+
db = openDatabase(snapshotPath, { readonly: true, create: false });
|
|
155
155
|
const closeSnapshot = db.close.bind(db);
|
|
156
156
|
let closed = false;
|
|
157
157
|
const cleanup = (bestEffort) => {
|
package/dist/workspace/assets.js
CHANGED
|
@@ -13,12 +13,11 @@ import { loadConfig } from "../core/config/config.js";
|
|
|
13
13
|
import { resolveRefOwner } from "../core/env-secret-ref.js";
|
|
14
14
|
import { UsageError } from "../core/errors.js";
|
|
15
15
|
import { includedTypes, includesProposed } from "../core/include.js";
|
|
16
|
-
import { getDbPath } from "../core/paths.js";
|
|
17
16
|
import { projectView, resolveProject } from "../core/projects.js";
|
|
17
|
+
import { readUsableIndex } from "../indexer/index-readiness.js";
|
|
18
18
|
import { resolveReadSources } from "../indexer/read-preflight.js";
|
|
19
19
|
import { countBrowseEntries } from "../indexer/search/db-search.js";
|
|
20
20
|
import { findSourceForPath, resolveSourceEntries } from "../indexer/search/search-source.js";
|
|
21
|
-
import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
22
21
|
import { getEntryIdByFilePath, getItemRefById } from "../storage/repositories/index-entries-repository.js";
|
|
23
22
|
import { countFeedbackBySignal, listFeedbackEventRows } from "../storage/repositories/metrics-repository.js";
|
|
24
23
|
import { assetImprove } from "./asset-improve.js";
|
|
@@ -81,16 +80,9 @@ export function projectOverview(input) {
|
|
|
81
80
|
? local.map((source) => ({ name: source.registryId ?? source.path, source }))
|
|
82
81
|
: [project.from].flat().map((from) => ({ name: from, source: narrowSourcesToName(local, [from])[0] }));
|
|
83
82
|
const readable = selected.flatMap((bundle) => (bundle.source ? [bundle.source] : []));
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
try {
|
|
88
|
-
counts = countBrowseEntries(db, readable, project.filters, (config.search?.defaultExcludeTypes ?? ["session"]).filter((type) => !includedTypes(project.include).includes(type)), includesProposed(project.include));
|
|
89
|
-
}
|
|
90
|
-
finally {
|
|
91
|
-
closeDatabase(db);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
83
|
+
const counts = readable.length > 0
|
|
84
|
+
? readUsableIndex((db) => countBrowseEntries(db, readable, project.filters, (config.search?.defaultExcludeTypes ?? ["session"]).filter((type) => !includedTypes(project.include).includes(type)), includesProposed(project.include)), undefined)
|
|
85
|
+
: undefined;
|
|
94
86
|
const writeName = project.write.bundle ?? config.defaultWriteTarget ?? automaticWriteTarget(config)?.name ?? null;
|
|
95
87
|
const writeSource = writeName === null ? undefined : local.find((source) => source.registryId === writeName);
|
|
96
88
|
return {
|
|
@@ -121,16 +113,10 @@ export function projectOverview(input) {
|
|
|
121
113
|
}
|
|
122
114
|
/** The durable `bundle//conceptId` of the indexed asset at `filePath`, the key feedback rows use. */
|
|
123
115
|
function durableRefForPath(filePath) {
|
|
124
|
-
|
|
125
|
-
if (!db)
|
|
126
|
-
return null;
|
|
127
|
-
try {
|
|
116
|
+
return readUsableIndex((db) => {
|
|
128
117
|
const id = getEntryIdByFilePath(db, filePath);
|
|
129
118
|
return id === undefined ? null : getItemRefById(db, id);
|
|
130
|
-
}
|
|
131
|
-
finally {
|
|
132
|
-
closeDatabase(db);
|
|
133
|
-
}
|
|
119
|
+
}, null);
|
|
134
120
|
}
|
|
135
121
|
function feedbackFor(durableRef) {
|
|
136
122
|
const empty = { counts: {}, history: [] };
|
|
@@ -174,32 +160,26 @@ export async function show(input) {
|
|
|
174
160
|
const links = {};
|
|
175
161
|
if (shown.path && durableRef) {
|
|
176
162
|
const owner = parseBundleRef(durableRef).bundle;
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
try {
|
|
188
|
-
decoded = decodeURIComponent(file);
|
|
189
|
-
}
|
|
190
|
-
catch {
|
|
191
|
-
continue;
|
|
192
|
-
}
|
|
193
|
-
const id = getEntryIdByFilePath(db, path.resolve(path.dirname(shown.path), decoded));
|
|
194
|
-
const target = id === undefined ? undefined : getItemRefById(db, id);
|
|
195
|
-
if (target && parseBundleRef(target).bundle === owner)
|
|
196
|
-
links[raw] = target + (fragment ? `#${fragment}` : "");
|
|
163
|
+
readUsableIndex((db) => {
|
|
164
|
+
for (const raw of markdownLinkTargets(content, false)) {
|
|
165
|
+
if (/^(?:[a-z][a-z0-9+.-]*:|\/\/|#)/i.test(raw))
|
|
166
|
+
continue;
|
|
167
|
+
const [file, fragment] = raw.split("#", 2);
|
|
168
|
+
if (!file?.toLowerCase().endsWith(".md") || file.startsWith("/"))
|
|
169
|
+
continue;
|
|
170
|
+
let decoded;
|
|
171
|
+
try {
|
|
172
|
+
decoded = decodeURIComponent(file);
|
|
197
173
|
}
|
|
174
|
+
catch {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const id = getEntryIdByFilePath(db, path.resolve(path.dirname(shown.path), decoded));
|
|
178
|
+
const target = id === undefined ? undefined : getItemRefById(db, id);
|
|
179
|
+
if (target && parseBundleRef(target).bundle === owner)
|
|
180
|
+
links[raw] = target + (fragment ? `#${fragment}` : "");
|
|
198
181
|
}
|
|
199
|
-
|
|
200
|
-
closeDatabase(db);
|
|
201
|
-
}
|
|
202
|
-
}
|
|
182
|
+
}, undefined);
|
|
203
183
|
}
|
|
204
184
|
return {
|
|
205
185
|
redacted: false,
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import fs from "node:fs";
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
import { isWithin, safeRealpath } from "../core/common.js";
|
|
6
|
+
import { isWithin, relativeToRoot, safeRealpath } from "../core/common.js";
|
|
7
7
|
import { NotFoundError, UsageError } from "../core/errors.js";
|
|
8
8
|
import { findSourceForPath, resolveSourceEntries } from "../indexer/search/search-source.js";
|
|
9
9
|
import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
@@ -14,7 +14,7 @@ export function bundleFileLocation(filePath) {
|
|
|
14
14
|
const source = findSourceForPath(filePath, resolveSourceEntries());
|
|
15
15
|
if (!source?.registryId || source.unresolved || !isWithin(filePath, source.path))
|
|
16
16
|
return null;
|
|
17
|
-
const relative = path.relative(source.path, filePath);
|
|
17
|
+
const relative = relativeToRoot(filePath, source.path) ?? path.relative(source.path, filePath);
|
|
18
18
|
return {
|
|
19
19
|
bundle: source.registryId,
|
|
20
20
|
path: portable(relative),
|
|
@@ -24,7 +24,7 @@ import { appendEvent } from "../core/events.js";
|
|
|
24
24
|
import { resolveBundleWriteTarget } from "../core/mutation-target.js";
|
|
25
25
|
import { warn } from "../core/warn.js";
|
|
26
26
|
import { assertAkmAssetWrite } from "../core/write-source.js";
|
|
27
|
-
import {
|
|
27
|
+
import { NO_INDEX_MESSAGE, readUsableIndex } from "../indexer/index-readiness.js";
|
|
28
28
|
import { getEntryIdByFilePath, getItemRefById } from "../storage/repositories/index-entries-repository.js";
|
|
29
29
|
import { interactive } from "./lease.js";
|
|
30
30
|
import { redactUrlsInText } from "./redact.js";
|
|
@@ -37,17 +37,12 @@ async function resolveWritableIndexedAsset(ref) {
|
|
|
37
37
|
const shown = await akmShowUnified({ ref, readOnly: true });
|
|
38
38
|
if (!shown.path)
|
|
39
39
|
throw new UsageError("The document has no local source file.");
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
finally {
|
|
48
|
-
closeDatabase(db);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
40
|
+
const canonical = readUsableIndex((db) => {
|
|
41
|
+
const id = getEntryIdByFilePath(db, shown.path);
|
|
42
|
+
return id === undefined ? undefined : (getItemRefById(db, id) ?? undefined);
|
|
43
|
+
}, null);
|
|
44
|
+
if (canonical === null)
|
|
45
|
+
throw new UsageError(NO_INDEX_MESSAGE, "MISSING_REQUIRED_ARGUMENT", "akm index");
|
|
51
46
|
if (!canonical)
|
|
52
47
|
throw new UsageError("Only indexed documents can be edited.");
|
|
53
48
|
const target = resolveBundleWriteTarget(loadConfig(), parseBundleRef(canonical).bundle);
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* The library's topics and near-duplicates, from the stored embeddings: what `assets.graph` reports so a host can say
|
|
6
|
+
* what the library holds, where it overlaps, and which assets say the same thing. Nothing is embedded and nothing is
|
|
7
|
+
* written; the work yields to the event loop as it goes and is kept for one index build.
|
|
8
|
+
*/
|
|
9
|
+
/** Fewest embedded assets topics are found for. */
|
|
10
|
+
const MIN_ASSETS = 8;
|
|
11
|
+
/** Most topics, however large the library. */
|
|
12
|
+
const MAX_TOPICS = 36;
|
|
13
|
+
/** Most vectors the topic centres are found from; the rest are only assigned to them. */
|
|
14
|
+
const SAMPLE = 4000;
|
|
15
|
+
const ROUNDS = 16;
|
|
16
|
+
/** Cosine at or above which two assets of one topic are reported as near-duplicates. */
|
|
17
|
+
export const DUPLICATE_COSINE = 0.96;
|
|
18
|
+
/** Most near-duplicate pairs reported, the most alike first. */
|
|
19
|
+
const MAX_PAIRS = 2000;
|
|
20
|
+
/** Topics each topic names as nearest. */
|
|
21
|
+
const RELATED_TOPICS = 3;
|
|
22
|
+
const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
|
|
23
|
+
/** How many topics for `n` embedded assets: about the square root of n/2, from 2 to {@link MAX_TOPICS}. */
|
|
24
|
+
export function topicCount(n) {
|
|
25
|
+
return Math.max(2, Math.min(MAX_TOPICS, Math.round(Math.sqrt(n / 2))));
|
|
26
|
+
}
|
|
27
|
+
let cache;
|
|
28
|
+
/**
|
|
29
|
+
* Groups the assets by meaning (spherical k-means on the unit vectors: centres from an evenly spaced sample of at most
|
|
30
|
+
* 4,000, seeded deterministically, so the same vectors give the same topics), names each topic by the words of its
|
|
31
|
+
* assets' paths and titles that are most particular to it, and finds near-duplicate pairs inside each topic. `names[i]`
|
|
32
|
+
* is asset i's path and title. With a `key` (the index's build), the result is reused until the key changes.
|
|
33
|
+
*/
|
|
34
|
+
export async function findTopics(vectors, names, key) {
|
|
35
|
+
if (key !== undefined && cache?.key === key && cache.result.topic.length === vectors.length)
|
|
36
|
+
return cache.result;
|
|
37
|
+
const empty = {
|
|
38
|
+
topics: [],
|
|
39
|
+
topic: vectors.map(() => -1),
|
|
40
|
+
duplicates: { threshold: DUPLICATE_COSINE, a: [], b: [], cosine: [], truncated: false },
|
|
41
|
+
};
|
|
42
|
+
const dims = vectors.find((v) => v)?.length ?? 0;
|
|
43
|
+
const usable = [];
|
|
44
|
+
for (let i = 0; i < vectors.length; i++)
|
|
45
|
+
if (vectors[i]?.length === dims)
|
|
46
|
+
usable.push(i);
|
|
47
|
+
if (!dims || usable.length < MIN_ASSETS)
|
|
48
|
+
return empty;
|
|
49
|
+
// Unit vectors, packed: cosine is then a dot product.
|
|
50
|
+
const unit = new Float32Array(usable.length * dims);
|
|
51
|
+
usable.forEach((i, row) => {
|
|
52
|
+
const v = vectors[i];
|
|
53
|
+
let norm = 0;
|
|
54
|
+
for (let d = 0; d < dims; d++)
|
|
55
|
+
norm += v[d] * v[d];
|
|
56
|
+
norm = Math.sqrt(norm) || 1;
|
|
57
|
+
for (let d = 0; d < dims; d++)
|
|
58
|
+
unit[row * dims + d] = v[d] / norm;
|
|
59
|
+
});
|
|
60
|
+
const dot = (row, centre, c) => {
|
|
61
|
+
let sum = 0;
|
|
62
|
+
for (let d = 0; d < dims; d++)
|
|
63
|
+
sum += unit[row * dims + d] * centre[c * dims + d];
|
|
64
|
+
return sum;
|
|
65
|
+
};
|
|
66
|
+
const k = topicCount(usable.length);
|
|
67
|
+
const step = Math.max(1, usable.length / SAMPLE);
|
|
68
|
+
const sample = [];
|
|
69
|
+
for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
|
|
70
|
+
sample.push(Math.floor(at));
|
|
71
|
+
// k-means++ from a fixed start, with a deterministic generator, so a library always gets the same topics.
|
|
72
|
+
let seed = 7;
|
|
73
|
+
const random = () => {
|
|
74
|
+
seed = (seed * 1664525 + 1013904223) >>> 0;
|
|
75
|
+
return seed / 2 ** 32;
|
|
76
|
+
};
|
|
77
|
+
const centres = new Float32Array(k * dims);
|
|
78
|
+
const setCentre = (c, row) => centres.set(unit.subarray(row * dims, row * dims + dims), c * dims);
|
|
79
|
+
setCentre(0, sample[0]);
|
|
80
|
+
const far = new Float64Array(sample.length).fill(Number.POSITIVE_INFINITY);
|
|
81
|
+
for (let c = 1; c < k; c++) {
|
|
82
|
+
let total = 0;
|
|
83
|
+
for (let s = 0; s < sample.length; s++) {
|
|
84
|
+
far[s] = Math.min(far[s], Math.max(0, 1 - dot(sample[s], centres, c - 1)));
|
|
85
|
+
total += far[s] * far[s];
|
|
86
|
+
}
|
|
87
|
+
let r = random() * total;
|
|
88
|
+
let chosen = sample.length - 1;
|
|
89
|
+
for (let s = 0; s < sample.length; s++) {
|
|
90
|
+
r -= far[s] * far[s];
|
|
91
|
+
if (r <= 0) {
|
|
92
|
+
chosen = s;
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
setCentre(c, sample[chosen]);
|
|
97
|
+
}
|
|
98
|
+
const nearest = (row) => {
|
|
99
|
+
let best = 0;
|
|
100
|
+
let bestDot = Number.NEGATIVE_INFINITY;
|
|
101
|
+
for (let c = 0; c < k; c++) {
|
|
102
|
+
const value = dot(row, centres, c);
|
|
103
|
+
if (value > bestDot) {
|
|
104
|
+
bestDot = value;
|
|
105
|
+
best = c;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return best;
|
|
109
|
+
};
|
|
110
|
+
for (let round = 0; round < ROUNDS; round++) {
|
|
111
|
+
const sums = new Float64Array(k * dims);
|
|
112
|
+
for (const row of sample) {
|
|
113
|
+
const c = nearest(row);
|
|
114
|
+
for (let d = 0; d < dims; d++)
|
|
115
|
+
sums[c * dims + d] += unit[row * dims + d];
|
|
116
|
+
}
|
|
117
|
+
for (let c = 0; c < k; c++) {
|
|
118
|
+
let norm = 0;
|
|
119
|
+
for (let d = 0; d < dims; d++)
|
|
120
|
+
norm += sums[c * dims + d] ** 2;
|
|
121
|
+
norm = Math.sqrt(norm);
|
|
122
|
+
// A centre that lost every member keeps its place.
|
|
123
|
+
if (norm > 0)
|
|
124
|
+
for (let d = 0; d < dims; d++)
|
|
125
|
+
centres[c * dims + d] = sums[c * dims + d] / norm;
|
|
126
|
+
}
|
|
127
|
+
await yieldToLoop();
|
|
128
|
+
}
|
|
129
|
+
// Every asset to its nearest centre; topics are renumbered largest first, and empty ones dropped.
|
|
130
|
+
const raw = new Int32Array(usable.length);
|
|
131
|
+
for (let row = 0; row < usable.length; row++) {
|
|
132
|
+
raw[row] = nearest(row);
|
|
133
|
+
if (row % 2000 === 1999)
|
|
134
|
+
await yieldToLoop();
|
|
135
|
+
}
|
|
136
|
+
const sizes = new Array(k).fill(0);
|
|
137
|
+
for (const c of raw)
|
|
138
|
+
sizes[c]++;
|
|
139
|
+
const order = sizes
|
|
140
|
+
.map((size, c) => ({ size, c }))
|
|
141
|
+
.filter((t) => t.size > 0)
|
|
142
|
+
.sort((x, y) => y.size - x.size || x.c - y.c);
|
|
143
|
+
const renumber = new Map(order.map((t, n) => [t.c, n]));
|
|
144
|
+
const topic = vectors.map(() => -1);
|
|
145
|
+
usable.forEach((i, row) => {
|
|
146
|
+
topic[i] = renumber.get(raw[row]);
|
|
147
|
+
});
|
|
148
|
+
const members = order.map(() => []);
|
|
149
|
+
usable.forEach((i, row) => {
|
|
150
|
+
members[topic[i]].push(row);
|
|
151
|
+
});
|
|
152
|
+
const centreOf = (n) => order[n].c;
|
|
153
|
+
const topics = order.map((t, n) => ({
|
|
154
|
+
...nameTopic(members[n].map((row) => names[usable[row]] ?? ""), names),
|
|
155
|
+
size: t.size,
|
|
156
|
+
related: order
|
|
157
|
+
.map((_, m) => {
|
|
158
|
+
let cosine = 0;
|
|
159
|
+
for (let d = 0; d < dims; d++)
|
|
160
|
+
cosine += centres[centreOf(n) * dims + d] * centres[centreOf(m) * dims + d];
|
|
161
|
+
return { topic: m, cosine: Math.round(cosine * 1000) / 1000 };
|
|
162
|
+
})
|
|
163
|
+
.filter((r) => r.topic !== n)
|
|
164
|
+
.sort((x, y) => y.cosine - x.cosine)
|
|
165
|
+
.slice(0, RELATED_TOPICS),
|
|
166
|
+
}));
|
|
167
|
+
// Near-duplicates: every pair inside a topic. Pairs across topics are not compared; two assets that alike almost
|
|
168
|
+
// always share a topic.
|
|
169
|
+
// Only the best MAX_PAIRS are kept, so a library full of copies does not hold every pair in memory: the list is cut
|
|
170
|
+
// back whenever it grows to eight times that, to the same top pairs a full sort would keep.
|
|
171
|
+
let pairs = [];
|
|
172
|
+
let pairCount = 0;
|
|
173
|
+
const byAlike = (x, y) => y[2] - x[2] || x[0] - y[0] || x[1] - y[1];
|
|
174
|
+
for (const rows of members) {
|
|
175
|
+
for (let x = 0; x < rows.length; x++) {
|
|
176
|
+
const p = rows[x] * dims;
|
|
177
|
+
for (let y = x + 1; y < rows.length; y++) {
|
|
178
|
+
const q = rows[y] * dims;
|
|
179
|
+
let cosine = 0;
|
|
180
|
+
for (let d = 0; d < dims; d++)
|
|
181
|
+
cosine += unit[p + d] * unit[q + d];
|
|
182
|
+
if (cosine >= DUPLICATE_COSINE) {
|
|
183
|
+
pairs.push([usable[rows[x]], usable[rows[y]], cosine]);
|
|
184
|
+
pairCount++;
|
|
185
|
+
if (pairs.length >= MAX_PAIRS * 8)
|
|
186
|
+
pairs = pairs.sort(byAlike).slice(0, MAX_PAIRS);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (x % 200 === 199)
|
|
190
|
+
await yieldToLoop();
|
|
191
|
+
}
|
|
192
|
+
await yieldToLoop();
|
|
193
|
+
}
|
|
194
|
+
const kept = pairs.sort(byAlike).slice(0, MAX_PAIRS);
|
|
195
|
+
const result = {
|
|
196
|
+
topics,
|
|
197
|
+
topic,
|
|
198
|
+
duplicates: {
|
|
199
|
+
threshold: DUPLICATE_COSINE,
|
|
200
|
+
a: kept.map((p) => p[0]),
|
|
201
|
+
b: kept.map((p) => p[1]),
|
|
202
|
+
cosine: kept.map((p) => Math.round(p[2] * 10000) / 10000),
|
|
203
|
+
truncated: pairCount > MAX_PAIRS,
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
if (key !== undefined)
|
|
207
|
+
cache = { key, result };
|
|
208
|
+
return result;
|
|
209
|
+
}
|
|
210
|
+
const STOP = new Set("the and for with from into this that what when how why are was not use using about your you our its md readme index skill skills knowledge memory memories command commands agent agents script scripts workflow workflows fact facts lesson lessons references reference assets asset src lib test tests docs".split(" "));
|
|
211
|
+
/** The words of a path or title worth naming a topic by: lower case, three letters or more, no digits, no stop words. */
|
|
212
|
+
export function words(text) {
|
|
213
|
+
return text
|
|
214
|
+
.toLowerCase()
|
|
215
|
+
.split(/[^a-z0-9]+/)
|
|
216
|
+
.filter((w) => w.length >= 3 && !/\d/.test(w) && !STOP.has(w));
|
|
217
|
+
}
|
|
218
|
+
let documentFrequency;
|
|
219
|
+
/**
|
|
220
|
+
* A topic's distinctive words: those most frequent among its assets relative to how many assets of the whole library
|
|
221
|
+
* use them (each asset counts a word once). The label is the top two.
|
|
222
|
+
*/
|
|
223
|
+
export function nameTopic(memberNames, allNames) {
|
|
224
|
+
if (documentFrequency?.names !== allNames) {
|
|
225
|
+
const counts = new Map();
|
|
226
|
+
for (const name of allNames)
|
|
227
|
+
for (const w of new Set(words(name)))
|
|
228
|
+
counts.set(w, (counts.get(w) ?? 0) + 1);
|
|
229
|
+
documentFrequency = { names: allNames, counts };
|
|
230
|
+
}
|
|
231
|
+
const inTopic = new Map();
|
|
232
|
+
for (const name of memberNames)
|
|
233
|
+
for (const w of new Set(words(name)))
|
|
234
|
+
inTopic.set(w, (inTopic.get(w) ?? 0) + 1);
|
|
235
|
+
const total = allNames.length;
|
|
236
|
+
const terms = [...inTopic]
|
|
237
|
+
// A word one asset uses names nothing.
|
|
238
|
+
.filter(([, n]) => n >= Math.min(2, memberNames.length))
|
|
239
|
+
.map(([w, n]) => ({
|
|
240
|
+
w,
|
|
241
|
+
score: (n / memberNames.length) * Math.log(total / (documentFrequency.counts.get(w) ?? 1)),
|
|
242
|
+
}))
|
|
243
|
+
// A word every asset of the library has says nothing about this topic.
|
|
244
|
+
.filter((t) => t.score > 0)
|
|
245
|
+
.sort((x, y) => y.score - x.score || (x.w < y.w ? -1 : 1))
|
|
246
|
+
.slice(0, 5)
|
|
247
|
+
.map((t) => t.w);
|
|
248
|
+
return { label: terms.slice(0, 2).join(" · ") || "unnamed", terms };
|
|
249
|
+
}
|