akm-cli 0.10.261124 → 0.10.261125-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/cli.js +10 -0
- package/dist/commands/feedback-cli.js +12 -3
- package/dist/commands/health/semantic-search-check.js +12 -2
- package/dist/commands/health.js +8 -15
- package/dist/commands/lint/index.js +6 -6
- package/dist/commands/read/knowledge.js +8 -8
- package/dist/commands/read/related.js +3 -9
- package/dist/commands/read/show.js +17 -9
- package/dist/commands/sources/bundle-cli.js +1 -0
- package/dist/commands/sources/installed-stashes.js +36 -27
- package/dist/core/asset/resolve-ref.js +8 -0
- package/dist/core/common.js +21 -0
- package/dist/core/endpoint-url.js +4 -4
- package/dist/core/events.js +8 -2
- package/dist/core/file-lock.js +52 -1
- package/dist/core/state-db.js +47 -15
- package/dist/core/stdio-reader-gone.js +19 -0
- package/dist/core/write-source.js +2 -2
- package/dist/indexer/index-build.js +6 -3
- package/dist/indexer/index-readiness.js +30 -4
- package/dist/indexer/indexer.js +56 -8
- package/dist/indexer/search/db-search.js +7 -1
- package/dist/indexer/search/search-source.js +23 -25
- package/dist/indexer/usage/usage-events.js +13 -4
- package/dist/registry/origin-resolve.js +2 -3
- package/dist/scripts/akm-migrate-node.js +118 -72
- package/dist/scripts/akm-migrate.js +118 -72
- package/dist/setup/endpoint.js +1 -1
- package/dist/storage/repositories/index-connection.js +44 -41
- package/dist/storage/repositories/index-entries-repository.js +14 -7
- package/dist/storage/repositories/index-entry-schema.js +7 -5
- package/dist/storage/repositories/index-schema.js +23 -0
- package/dist/workspace/assets.js +24 -44
- package/dist/workspace/bundle-files.js +2 -2
- package/dist/workspace/documents.js +7 -12
- package/dist/workspace/graph-topics.js +249 -0
- package/dist/workspace/graph.js +68 -4
- package/docs/reference/workspace.md +21 -4
- package/package.json +1 -1
|
@@ -569,20 +569,27 @@ export function findEntryIdByRef(db, ref, bundleId) {
|
|
|
569
569
|
export function withMdVariants(name) {
|
|
570
570
|
return name.endsWith(".md") ? [name, name.slice(0, -3)] : [name, `${name}.md`];
|
|
571
571
|
}
|
|
572
|
+
/** True when `file` is a `.md` file named by the last segment of `conceptId`, up to its `.md`. */
|
|
573
|
+
export function namesMarkdownFile(conceptId, file) {
|
|
574
|
+
const last = (conceptId.split("/").pop() ?? "").replace(/\.md$/, "");
|
|
575
|
+
return path.extname(file) === ".md" && path.basename(file, ".md") === last;
|
|
576
|
+
}
|
|
572
577
|
/**
|
|
573
578
|
* Current (`[bundle//]conceptId`) id lookup: match `item_ref` exactly when
|
|
574
579
|
* bundle-qualified or by `//conceptId` suffix when short.
|
|
575
580
|
*/
|
|
576
581
|
function findEntryIdByBundleRef(db, ref, bundleId) {
|
|
577
582
|
const parsed = parseBundleRef(ref);
|
|
578
|
-
const conceptVariants = withMdVariants(parsed.conceptId);
|
|
579
583
|
// item_ref is the canonical stored spelling post-flip.
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
584
|
+
const exact = matchIdByItemRef(db, parsed.bundle, parsed.conceptId, bundleId);
|
|
585
|
+
if (exact !== undefined)
|
|
586
|
+
return exact;
|
|
587
|
+
// The `.md` toggle names a markdown file by its own name: `tasks/x.md` is not a spelling of `tasks/x.yml`,
|
|
588
|
+
// nor `skills/x.md` of `skills/x/SKILL.md`.
|
|
589
|
+
const [, toggled] = withMdVariants(parsed.conceptId);
|
|
590
|
+
const id = matchIdByItemRef(db, parsed.bundle, toggled, bundleId);
|
|
591
|
+
const file = id === undefined ? undefined : getEntryFilePathById(db, id);
|
|
592
|
+
return file !== undefined && namesMarkdownFile(parsed.conceptId, file) ? id : undefined;
|
|
586
593
|
}
|
|
587
594
|
/**
|
|
588
595
|
* Match a row `id` by `item_ref`: exact `bundle//conceptId` when `bundle` is
|
|
@@ -142,14 +142,16 @@ export function retiredEntryColumns(db) {
|
|
|
142
142
|
return RETIRED_ENTRY_COLUMNS.filter((column) => present.has(column));
|
|
143
143
|
}
|
|
144
144
|
/**
|
|
145
|
-
* Whether an index database has an `entries` table this release can read
|
|
146
|
-
* fresh or empty file has none; an index older
|
|
147
|
-
* by columns this release no longer reads (or
|
|
148
|
-
*
|
|
145
|
+
* Whether an index database has an `entries` table this release can read,
|
|
146
|
+
* with its full-text table. A fresh or empty file has none; an index older
|
|
147
|
+
* than layout 21 has one keyed by columns this release no longer reads (or
|
|
148
|
+
* lacks the ones it does); a file whose schema was cut short has entries
|
|
149
|
+
* without `entries_fts`. Each serves nothing until the next `akm index`.
|
|
149
150
|
*/
|
|
150
151
|
export function hasCurrentEntriesTable(db) {
|
|
151
152
|
try {
|
|
152
|
-
|
|
153
|
+
const fts = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'entries_fts'").get();
|
|
154
|
+
return Boolean(fts) && missingEntryColumns(db).length === 0 && retiredEntryColumns(db).length === 0;
|
|
153
155
|
}
|
|
154
156
|
catch {
|
|
155
157
|
return false;
|
|
@@ -205,7 +205,30 @@ function ensureColumn(db, table, column, type) {
|
|
|
205
205
|
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
|
|
206
206
|
return true;
|
|
207
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* Bring `index.db` to the current layout in one transaction, so no reader or
|
|
210
|
+
* crash can observe a half-migrated file (entries without their FTS table).
|
|
211
|
+
* One `BEGIN IMMEDIATE`, waiting out the connection's busy timeout once: a
|
|
212
|
+
* busy index surfaces to the caller (the build reports it as contention)
|
|
213
|
+
* rather than being retried here. Joins a transaction the caller already holds.
|
|
214
|
+
*/
|
|
208
215
|
export function ensureSchema(db) {
|
|
216
|
+
if (db.inTransaction) {
|
|
217
|
+
applySchema(db);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
db.exec("BEGIN IMMEDIATE");
|
|
221
|
+
try {
|
|
222
|
+
applySchema(db);
|
|
223
|
+
db.exec("COMMIT");
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
if (db.inTransaction)
|
|
227
|
+
db.exec("ROLLBACK");
|
|
228
|
+
throw error;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
function applySchema(db) {
|
|
209
232
|
db.exec(`
|
|
210
233
|
CREATE TABLE IF NOT EXISTS index_meta (
|
|
211
234
|
key TEXT PRIMARY KEY,
|
package/dist/workspace/assets.js
CHANGED
|
@@ -13,12 +13,11 @@ import { loadConfig } from "../core/config/config.js";
|
|
|
13
13
|
import { resolveRefOwner } from "../core/env-secret-ref.js";
|
|
14
14
|
import { UsageError } from "../core/errors.js";
|
|
15
15
|
import { includedTypes, includesProposed } from "../core/include.js";
|
|
16
|
-
import { getDbPath } from "../core/paths.js";
|
|
17
16
|
import { projectView, resolveProject } from "../core/projects.js";
|
|
17
|
+
import { readUsableIndex } from "../indexer/index-readiness.js";
|
|
18
18
|
import { resolveReadSources } from "../indexer/read-preflight.js";
|
|
19
19
|
import { countBrowseEntries } from "../indexer/search/db-search.js";
|
|
20
20
|
import { findSourceForPath, resolveSourceEntries } from "../indexer/search/search-source.js";
|
|
21
|
-
import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
22
21
|
import { getEntryIdByFilePath, getItemRefById } from "../storage/repositories/index-entries-repository.js";
|
|
23
22
|
import { countFeedbackBySignal, listFeedbackEventRows } from "../storage/repositories/metrics-repository.js";
|
|
24
23
|
import { assetImprove } from "./asset-improve.js";
|
|
@@ -81,16 +80,9 @@ export function projectOverview(input) {
|
|
|
81
80
|
? local.map((source) => ({ name: source.registryId ?? source.path, source }))
|
|
82
81
|
: [project.from].flat().map((from) => ({ name: from, source: narrowSourcesToName(local, [from])[0] }));
|
|
83
82
|
const readable = selected.flatMap((bundle) => (bundle.source ? [bundle.source] : []));
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
try {
|
|
88
|
-
counts = countBrowseEntries(db, readable, project.filters, (config.search?.defaultExcludeTypes ?? ["session"]).filter((type) => !includedTypes(project.include).includes(type)), includesProposed(project.include));
|
|
89
|
-
}
|
|
90
|
-
finally {
|
|
91
|
-
closeDatabase(db);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
83
|
+
const counts = readable.length > 0
|
|
84
|
+
? readUsableIndex((db) => countBrowseEntries(db, readable, project.filters, (config.search?.defaultExcludeTypes ?? ["session"]).filter((type) => !includedTypes(project.include).includes(type)), includesProposed(project.include)), undefined)
|
|
85
|
+
: undefined;
|
|
94
86
|
const writeName = project.write.bundle ?? config.defaultWriteTarget ?? automaticWriteTarget(config)?.name ?? null;
|
|
95
87
|
const writeSource = writeName === null ? undefined : local.find((source) => source.registryId === writeName);
|
|
96
88
|
return {
|
|
@@ -121,16 +113,10 @@ export function projectOverview(input) {
|
|
|
121
113
|
}
|
|
122
114
|
/** The durable `bundle//conceptId` of the indexed asset at `filePath`, the key feedback rows use. */
|
|
123
115
|
function durableRefForPath(filePath) {
|
|
124
|
-
|
|
125
|
-
if (!db)
|
|
126
|
-
return null;
|
|
127
|
-
try {
|
|
116
|
+
return readUsableIndex((db) => {
|
|
128
117
|
const id = getEntryIdByFilePath(db, filePath);
|
|
129
118
|
return id === undefined ? null : getItemRefById(db, id);
|
|
130
|
-
}
|
|
131
|
-
finally {
|
|
132
|
-
closeDatabase(db);
|
|
133
|
-
}
|
|
119
|
+
}, null);
|
|
134
120
|
}
|
|
135
121
|
function feedbackFor(durableRef) {
|
|
136
122
|
const empty = { counts: {}, history: [] };
|
|
@@ -174,32 +160,26 @@ export async function show(input) {
|
|
|
174
160
|
const links = {};
|
|
175
161
|
if (shown.path && durableRef) {
|
|
176
162
|
const owner = parseBundleRef(durableRef).bundle;
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
try {
|
|
188
|
-
decoded = decodeURIComponent(file);
|
|
189
|
-
}
|
|
190
|
-
catch {
|
|
191
|
-
continue;
|
|
192
|
-
}
|
|
193
|
-
const id = getEntryIdByFilePath(db, path.resolve(path.dirname(shown.path), decoded));
|
|
194
|
-
const target = id === undefined ? undefined : getItemRefById(db, id);
|
|
195
|
-
if (target && parseBundleRef(target).bundle === owner)
|
|
196
|
-
links[raw] = target + (fragment ? `#${fragment}` : "");
|
|
163
|
+
readUsableIndex((db) => {
|
|
164
|
+
for (const raw of markdownLinkTargets(content, false)) {
|
|
165
|
+
if (/^(?:[a-z][a-z0-9+.-]*:|\/\/|#)/i.test(raw))
|
|
166
|
+
continue;
|
|
167
|
+
const [file, fragment] = raw.split("#", 2);
|
|
168
|
+
if (!file?.toLowerCase().endsWith(".md") || file.startsWith("/"))
|
|
169
|
+
continue;
|
|
170
|
+
let decoded;
|
|
171
|
+
try {
|
|
172
|
+
decoded = decodeURIComponent(file);
|
|
197
173
|
}
|
|
174
|
+
catch {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const id = getEntryIdByFilePath(db, path.resolve(path.dirname(shown.path), decoded));
|
|
178
|
+
const target = id === undefined ? undefined : getItemRefById(db, id);
|
|
179
|
+
if (target && parseBundleRef(target).bundle === owner)
|
|
180
|
+
links[raw] = target + (fragment ? `#${fragment}` : "");
|
|
198
181
|
}
|
|
199
|
-
|
|
200
|
-
closeDatabase(db);
|
|
201
|
-
}
|
|
202
|
-
}
|
|
182
|
+
}, undefined);
|
|
203
183
|
}
|
|
204
184
|
return {
|
|
205
185
|
redacted: false,
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import fs from "node:fs";
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
import { isWithin, safeRealpath } from "../core/common.js";
|
|
6
|
+
import { isWithin, relativeToRoot, safeRealpath } from "../core/common.js";
|
|
7
7
|
import { NotFoundError, UsageError } from "../core/errors.js";
|
|
8
8
|
import { findSourceForPath, resolveSourceEntries } from "../indexer/search/search-source.js";
|
|
9
9
|
import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
@@ -14,7 +14,7 @@ export function bundleFileLocation(filePath) {
|
|
|
14
14
|
const source = findSourceForPath(filePath, resolveSourceEntries());
|
|
15
15
|
if (!source?.registryId || source.unresolved || !isWithin(filePath, source.path))
|
|
16
16
|
return null;
|
|
17
|
-
const relative = path.relative(source.path, filePath);
|
|
17
|
+
const relative = relativeToRoot(filePath, source.path) ?? path.relative(source.path, filePath);
|
|
18
18
|
return {
|
|
19
19
|
bundle: source.registryId,
|
|
20
20
|
path: portable(relative),
|
|
@@ -24,7 +24,7 @@ import { appendEvent } from "../core/events.js";
|
|
|
24
24
|
import { resolveBundleWriteTarget } from "../core/mutation-target.js";
|
|
25
25
|
import { warn } from "../core/warn.js";
|
|
26
26
|
import { assertAkmAssetWrite } from "../core/write-source.js";
|
|
27
|
-
import {
|
|
27
|
+
import { NO_INDEX_MESSAGE, readUsableIndex } from "../indexer/index-readiness.js";
|
|
28
28
|
import { getEntryIdByFilePath, getItemRefById } from "../storage/repositories/index-entries-repository.js";
|
|
29
29
|
import { interactive } from "./lease.js";
|
|
30
30
|
import { redactUrlsInText } from "./redact.js";
|
|
@@ -37,17 +37,12 @@ async function resolveWritableIndexedAsset(ref) {
|
|
|
37
37
|
const shown = await akmShowUnified({ ref, readOnly: true });
|
|
38
38
|
if (!shown.path)
|
|
39
39
|
throw new UsageError("The document has no local source file.");
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
finally {
|
|
48
|
-
closeDatabase(db);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
40
|
+
const canonical = readUsableIndex((db) => {
|
|
41
|
+
const id = getEntryIdByFilePath(db, shown.path);
|
|
42
|
+
return id === undefined ? undefined : (getItemRefById(db, id) ?? undefined);
|
|
43
|
+
}, null);
|
|
44
|
+
if (canonical === null)
|
|
45
|
+
throw new UsageError(NO_INDEX_MESSAGE, "MISSING_REQUIRED_ARGUMENT", "akm index");
|
|
51
46
|
if (!canonical)
|
|
52
47
|
throw new UsageError("Only indexed documents can be edited.");
|
|
53
48
|
const target = resolveBundleWriteTarget(loadConfig(), parseBundleRef(canonical).bundle);
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* The library's topics and near-duplicates, from the stored embeddings: what `assets.graph` reports so a host can say
|
|
6
|
+
* what the library holds, where it overlaps, and which assets say the same thing. Nothing is embedded and nothing is
|
|
7
|
+
* written; the work yields to the event loop as it goes and is kept for one index build.
|
|
8
|
+
*/
|
|
9
|
+
/** Fewest embedded assets topics are found for. */
|
|
10
|
+
const MIN_ASSETS = 8;
|
|
11
|
+
/** Most topics, however large the library. */
|
|
12
|
+
const MAX_TOPICS = 36;
|
|
13
|
+
/** Most vectors the topic centres are found from; the rest are only assigned to them. */
|
|
14
|
+
const SAMPLE = 4000;
|
|
15
|
+
const ROUNDS = 16;
|
|
16
|
+
/** Cosine at or above which two assets of one topic are reported as near-duplicates. */
|
|
17
|
+
export const DUPLICATE_COSINE = 0.96;
|
|
18
|
+
/** Most near-duplicate pairs reported, the most alike first. */
|
|
19
|
+
const MAX_PAIRS = 2000;
|
|
20
|
+
/** Topics each topic names as nearest. */
|
|
21
|
+
const RELATED_TOPICS = 3;
|
|
22
|
+
const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
|
|
23
|
+
/** How many topics for `n` embedded assets: about the square root of n/2, from 2 to {@link MAX_TOPICS}. */
|
|
24
|
+
export function topicCount(n) {
|
|
25
|
+
return Math.max(2, Math.min(MAX_TOPICS, Math.round(Math.sqrt(n / 2))));
|
|
26
|
+
}
|
|
27
|
+
let cache;
|
|
28
|
+
/**
|
|
29
|
+
* Groups the assets by meaning (spherical k-means on the unit vectors: centres from an evenly spaced sample of at most
|
|
30
|
+
* 4,000, seeded deterministically, so the same vectors give the same topics), names each topic by the words of its
|
|
31
|
+
* assets' paths and titles that are most particular to it, and finds near-duplicate pairs inside each topic. `names[i]`
|
|
32
|
+
* is asset i's path and title. With a `key` (the index's build), the result is reused until the key changes.
|
|
33
|
+
*/
|
|
34
|
+
export async function findTopics(vectors, names, key) {
|
|
35
|
+
if (key !== undefined && cache?.key === key && cache.result.topic.length === vectors.length)
|
|
36
|
+
return cache.result;
|
|
37
|
+
const empty = {
|
|
38
|
+
topics: [],
|
|
39
|
+
topic: vectors.map(() => -1),
|
|
40
|
+
duplicates: { threshold: DUPLICATE_COSINE, a: [], b: [], cosine: [], truncated: false },
|
|
41
|
+
};
|
|
42
|
+
const dims = vectors.find((v) => v)?.length ?? 0;
|
|
43
|
+
const usable = [];
|
|
44
|
+
for (let i = 0; i < vectors.length; i++)
|
|
45
|
+
if (vectors[i]?.length === dims)
|
|
46
|
+
usable.push(i);
|
|
47
|
+
if (!dims || usable.length < MIN_ASSETS)
|
|
48
|
+
return empty;
|
|
49
|
+
// Unit vectors, packed: cosine is then a dot product.
|
|
50
|
+
const unit = new Float32Array(usable.length * dims);
|
|
51
|
+
usable.forEach((i, row) => {
|
|
52
|
+
const v = vectors[i];
|
|
53
|
+
let norm = 0;
|
|
54
|
+
for (let d = 0; d < dims; d++)
|
|
55
|
+
norm += v[d] * v[d];
|
|
56
|
+
norm = Math.sqrt(norm) || 1;
|
|
57
|
+
for (let d = 0; d < dims; d++)
|
|
58
|
+
unit[row * dims + d] = v[d] / norm;
|
|
59
|
+
});
|
|
60
|
+
const dot = (row, centre, c) => {
|
|
61
|
+
let sum = 0;
|
|
62
|
+
for (let d = 0; d < dims; d++)
|
|
63
|
+
sum += unit[row * dims + d] * centre[c * dims + d];
|
|
64
|
+
return sum;
|
|
65
|
+
};
|
|
66
|
+
const k = topicCount(usable.length);
|
|
67
|
+
const step = Math.max(1, usable.length / SAMPLE);
|
|
68
|
+
const sample = [];
|
|
69
|
+
for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
|
|
70
|
+
sample.push(Math.floor(at));
|
|
71
|
+
// k-means++ from a fixed start, with a deterministic generator, so a library always gets the same topics.
|
|
72
|
+
let seed = 7;
|
|
73
|
+
const random = () => {
|
|
74
|
+
seed = (seed * 1664525 + 1013904223) >>> 0;
|
|
75
|
+
return seed / 2 ** 32;
|
|
76
|
+
};
|
|
77
|
+
const centres = new Float32Array(k * dims);
|
|
78
|
+
const setCentre = (c, row) => centres.set(unit.subarray(row * dims, row * dims + dims), c * dims);
|
|
79
|
+
setCentre(0, sample[0]);
|
|
80
|
+
const far = new Float64Array(sample.length).fill(Number.POSITIVE_INFINITY);
|
|
81
|
+
for (let c = 1; c < k; c++) {
|
|
82
|
+
let total = 0;
|
|
83
|
+
for (let s = 0; s < sample.length; s++) {
|
|
84
|
+
far[s] = Math.min(far[s], Math.max(0, 1 - dot(sample[s], centres, c - 1)));
|
|
85
|
+
total += far[s] * far[s];
|
|
86
|
+
}
|
|
87
|
+
let r = random() * total;
|
|
88
|
+
let chosen = sample.length - 1;
|
|
89
|
+
for (let s = 0; s < sample.length; s++) {
|
|
90
|
+
r -= far[s] * far[s];
|
|
91
|
+
if (r <= 0) {
|
|
92
|
+
chosen = s;
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
setCentre(c, sample[chosen]);
|
|
97
|
+
}
|
|
98
|
+
const nearest = (row) => {
|
|
99
|
+
let best = 0;
|
|
100
|
+
let bestDot = Number.NEGATIVE_INFINITY;
|
|
101
|
+
for (let c = 0; c < k; c++) {
|
|
102
|
+
const value = dot(row, centres, c);
|
|
103
|
+
if (value > bestDot) {
|
|
104
|
+
bestDot = value;
|
|
105
|
+
best = c;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return best;
|
|
109
|
+
};
|
|
110
|
+
for (let round = 0; round < ROUNDS; round++) {
|
|
111
|
+
const sums = new Float64Array(k * dims);
|
|
112
|
+
for (const row of sample) {
|
|
113
|
+
const c = nearest(row);
|
|
114
|
+
for (let d = 0; d < dims; d++)
|
|
115
|
+
sums[c * dims + d] += unit[row * dims + d];
|
|
116
|
+
}
|
|
117
|
+
for (let c = 0; c < k; c++) {
|
|
118
|
+
let norm = 0;
|
|
119
|
+
for (let d = 0; d < dims; d++)
|
|
120
|
+
norm += sums[c * dims + d] ** 2;
|
|
121
|
+
norm = Math.sqrt(norm);
|
|
122
|
+
// A centre that lost every member keeps its place.
|
|
123
|
+
if (norm > 0)
|
|
124
|
+
for (let d = 0; d < dims; d++)
|
|
125
|
+
centres[c * dims + d] = sums[c * dims + d] / norm;
|
|
126
|
+
}
|
|
127
|
+
await yieldToLoop();
|
|
128
|
+
}
|
|
129
|
+
// Every asset to its nearest centre; topics are renumbered largest first, and empty ones dropped.
|
|
130
|
+
const raw = new Int32Array(usable.length);
|
|
131
|
+
for (let row = 0; row < usable.length; row++) {
|
|
132
|
+
raw[row] = nearest(row);
|
|
133
|
+
if (row % 2000 === 1999)
|
|
134
|
+
await yieldToLoop();
|
|
135
|
+
}
|
|
136
|
+
const sizes = new Array(k).fill(0);
|
|
137
|
+
for (const c of raw)
|
|
138
|
+
sizes[c]++;
|
|
139
|
+
const order = sizes
|
|
140
|
+
.map((size, c) => ({ size, c }))
|
|
141
|
+
.filter((t) => t.size > 0)
|
|
142
|
+
.sort((x, y) => y.size - x.size || x.c - y.c);
|
|
143
|
+
const renumber = new Map(order.map((t, n) => [t.c, n]));
|
|
144
|
+
const topic = vectors.map(() => -1);
|
|
145
|
+
usable.forEach((i, row) => {
|
|
146
|
+
topic[i] = renumber.get(raw[row]);
|
|
147
|
+
});
|
|
148
|
+
const members = order.map(() => []);
|
|
149
|
+
usable.forEach((i, row) => {
|
|
150
|
+
members[topic[i]].push(row);
|
|
151
|
+
});
|
|
152
|
+
const centreOf = (n) => order[n].c;
|
|
153
|
+
const topics = order.map((t, n) => ({
|
|
154
|
+
...nameTopic(members[n].map((row) => names[usable[row]] ?? ""), names),
|
|
155
|
+
size: t.size,
|
|
156
|
+
related: order
|
|
157
|
+
.map((_, m) => {
|
|
158
|
+
let cosine = 0;
|
|
159
|
+
for (let d = 0; d < dims; d++)
|
|
160
|
+
cosine += centres[centreOf(n) * dims + d] * centres[centreOf(m) * dims + d];
|
|
161
|
+
return { topic: m, cosine: Math.round(cosine * 1000) / 1000 };
|
|
162
|
+
})
|
|
163
|
+
.filter((r) => r.topic !== n)
|
|
164
|
+
.sort((x, y) => y.cosine - x.cosine)
|
|
165
|
+
.slice(0, RELATED_TOPICS),
|
|
166
|
+
}));
|
|
167
|
+
// Near-duplicates: every pair inside a topic. Pairs across topics are not compared; two assets that alike almost
|
|
168
|
+
// always share a topic.
|
|
169
|
+
// Only the best MAX_PAIRS are kept, so a library full of copies does not hold every pair in memory: the list is cut
|
|
170
|
+
// back whenever it grows to eight times that, to the same top pairs a full sort would keep.
|
|
171
|
+
let pairs = [];
|
|
172
|
+
let pairCount = 0;
|
|
173
|
+
const byAlike = (x, y) => y[2] - x[2] || x[0] - y[0] || x[1] - y[1];
|
|
174
|
+
for (const rows of members) {
|
|
175
|
+
for (let x = 0; x < rows.length; x++) {
|
|
176
|
+
const p = rows[x] * dims;
|
|
177
|
+
for (let y = x + 1; y < rows.length; y++) {
|
|
178
|
+
const q = rows[y] * dims;
|
|
179
|
+
let cosine = 0;
|
|
180
|
+
for (let d = 0; d < dims; d++)
|
|
181
|
+
cosine += unit[p + d] * unit[q + d];
|
|
182
|
+
if (cosine >= DUPLICATE_COSINE) {
|
|
183
|
+
pairs.push([usable[rows[x]], usable[rows[y]], cosine]);
|
|
184
|
+
pairCount++;
|
|
185
|
+
if (pairs.length >= MAX_PAIRS * 8)
|
|
186
|
+
pairs = pairs.sort(byAlike).slice(0, MAX_PAIRS);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (x % 200 === 199)
|
|
190
|
+
await yieldToLoop();
|
|
191
|
+
}
|
|
192
|
+
await yieldToLoop();
|
|
193
|
+
}
|
|
194
|
+
const kept = pairs.sort(byAlike).slice(0, MAX_PAIRS);
|
|
195
|
+
const result = {
|
|
196
|
+
topics,
|
|
197
|
+
topic,
|
|
198
|
+
duplicates: {
|
|
199
|
+
threshold: DUPLICATE_COSINE,
|
|
200
|
+
a: kept.map((p) => p[0]),
|
|
201
|
+
b: kept.map((p) => p[1]),
|
|
202
|
+
cosine: kept.map((p) => Math.round(p[2] * 10000) / 10000),
|
|
203
|
+
truncated: pairCount > MAX_PAIRS,
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
if (key !== undefined)
|
|
207
|
+
cache = { key, result };
|
|
208
|
+
return result;
|
|
209
|
+
}
|
|
210
|
+
const STOP = new Set("the and for with from into this that what when how why are was not use using about your you our its md readme index skill skills knowledge memory memories command commands agent agents script scripts workflow workflows fact facts lesson lessons references reference assets asset src lib test tests docs".split(" "));
|
|
211
|
+
/** The words of a path or title worth naming a topic by: lower case, three letters or more, no digits, no stop words. */
|
|
212
|
+
export function words(text) {
|
|
213
|
+
return text
|
|
214
|
+
.toLowerCase()
|
|
215
|
+
.split(/[^a-z0-9]+/)
|
|
216
|
+
.filter((w) => w.length >= 3 && !/\d/.test(w) && !STOP.has(w));
|
|
217
|
+
}
|
|
218
|
+
let documentFrequency;
|
|
219
|
+
/**
|
|
220
|
+
* A topic's distinctive words: those most frequent among its assets relative to how many assets of the whole library
|
|
221
|
+
* use them (each asset counts a word once). The label is the top two.
|
|
222
|
+
*/
|
|
223
|
+
export function nameTopic(memberNames, allNames) {
|
|
224
|
+
if (documentFrequency?.names !== allNames) {
|
|
225
|
+
const counts = new Map();
|
|
226
|
+
for (const name of allNames)
|
|
227
|
+
for (const w of new Set(words(name)))
|
|
228
|
+
counts.set(w, (counts.get(w) ?? 0) + 1);
|
|
229
|
+
documentFrequency = { names: allNames, counts };
|
|
230
|
+
}
|
|
231
|
+
const inTopic = new Map();
|
|
232
|
+
for (const name of memberNames)
|
|
233
|
+
for (const w of new Set(words(name)))
|
|
234
|
+
inTopic.set(w, (inTopic.get(w) ?? 0) + 1);
|
|
235
|
+
const total = allNames.length;
|
|
236
|
+
const terms = [...inTopic]
|
|
237
|
+
// A word one asset uses names nothing.
|
|
238
|
+
.filter(([, n]) => n >= Math.min(2, memberNames.length))
|
|
239
|
+
.map(([w, n]) => ({
|
|
240
|
+
w,
|
|
241
|
+
score: (n / memberNames.length) * Math.log(total / (documentFrequency.counts.get(w) ?? 1)),
|
|
242
|
+
}))
|
|
243
|
+
// A word every asset of the library has says nothing about this topic.
|
|
244
|
+
.filter((t) => t.score > 0)
|
|
245
|
+
.sort((x, y) => y.score - x.score || (x.w < y.w ? -1 : 1))
|
|
246
|
+
.slice(0, 5)
|
|
247
|
+
.map((t) => t.w);
|
|
248
|
+
return { label: terms.slice(0, 2).join(" · ") || "unnamed", terms };
|
|
249
|
+
}
|
package/dist/workspace/graph.js
CHANGED
|
@@ -2,29 +2,50 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
/** The whole library as a graph: every indexed asset, every declared link between them, and how each is used. */
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import fs from "node:fs";
|
|
5
7
|
import { loadConfig } from "../core/config/config.js";
|
|
8
|
+
import { ConfigError } from "../core/errors.js";
|
|
6
9
|
import { getDbPath } from "../core/paths.js";
|
|
7
10
|
import { applyEntryFilters } from "../indexer/search/db-search.js";
|
|
8
11
|
import { resolveSourceEntries } from "../indexer/search/search-source.js";
|
|
9
|
-
import { countUseByRef, usageRetentionSince } from "../indexer/usage/usage-events.js";
|
|
12
|
+
import { countFeedbackSignalsByRef, countUseByRef, usageRetentionSince } from "../indexer/usage/usage-events.js";
|
|
10
13
|
import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
11
14
|
import { getAllEntries } from "../storage/repositories/index-entries-repository.js";
|
|
12
15
|
import { listDeclaredLinks } from "../storage/repositories/index-links-repository.js";
|
|
13
16
|
import { getMeta } from "../storage/repositories/index-meta-repository.js";
|
|
14
17
|
import { listCurrentEmbeddings } from "../storage/repositories/index-vec-repository.js";
|
|
18
|
+
import { findTopics } from "./graph-topics.js";
|
|
15
19
|
import { withReadonlyStateDb } from "./readers.js";
|
|
16
20
|
/** Types never drawn: protected files and session logs, as `assets.related` leaves them out. */
|
|
17
21
|
const HIDDEN_TYPES = ["secret", "env", "session"];
|
|
18
22
|
/**
|
|
19
23
|
* Every asset of the enabled bundles and the declared links between them, in column form: node `i` is `nodes.ref[i]`,
|
|
20
24
|
* and its type, bundle and belief state are indexes into `types`, `bundles` (largest first) and `beliefs` (-1: no state
|
|
21
|
-
* recorded). `shows
|
|
22
|
-
*
|
|
25
|
+
* recorded). `shows`, `searches`, `lastShownAt` and the feedback counts `positive` and `negative` are read live from the
|
|
26
|
+
* user-attributed usage events within retention (`use`); `modifiedAt` is the file's modification time. A link joins two
|
|
27
|
+
* nodes by index and names its kind by index into `kinds`. `topic` indexes `topics` (see {@link findTopics}), -1 for
|
|
28
|
+
* an asset with no stored embedding, and `duplicates` pairs assets of one topic whose embeddings are nearly the same. `meaning` places each node with a stored embedding in three dimensions (see
|
|
23
29
|
* {@link projectEmbeddings}); it is `null` when the index has no embeddings. Nothing is written. The projection yields
|
|
24
30
|
* to the event loop as it goes, so a host serving other requests keeps answering them.
|
|
25
31
|
*/
|
|
26
32
|
export async function graph() {
|
|
27
|
-
|
|
33
|
+
try {
|
|
34
|
+
return await readGraph();
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
// A damaged or unreadable index file is "no usable index" to the map, not a raw SQLite error.
|
|
38
|
+
if (error instanceof Error && error.name === "SQLiteError")
|
|
39
|
+
return {
|
|
40
|
+
available: false,
|
|
41
|
+
note: `The index could not be read (${error.message}). Rebuild it with 'akm index --full'.`,
|
|
42
|
+
...emptyGraph(),
|
|
43
|
+
};
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function emptyGraph() {
|
|
48
|
+
return {
|
|
28
49
|
types: [],
|
|
29
50
|
bundles: [],
|
|
30
51
|
beliefs: [],
|
|
@@ -42,15 +63,28 @@ export async function graph() {
|
|
|
42
63
|
belief: [],
|
|
43
64
|
shows: [],
|
|
44
65
|
searches: [],
|
|
66
|
+
lastShownAt: [],
|
|
67
|
+
modifiedAt: [],
|
|
68
|
+
positive: [],
|
|
69
|
+
negative: [],
|
|
70
|
+
topic: [],
|
|
45
71
|
},
|
|
46
72
|
links: { from: [], to: [], kind: [] },
|
|
47
73
|
meaning: null,
|
|
74
|
+
topics: [],
|
|
75
|
+
duplicates: null,
|
|
48
76
|
};
|
|
77
|
+
}
|
|
78
|
+
async function readGraph() {
|
|
79
|
+
const empty = emptyGraph();
|
|
49
80
|
const db = openReadonlyExistingDatabase();
|
|
50
81
|
if (!db)
|
|
51
82
|
return { available: false, note: "No index is available. Build the AKM index first.", ...empty };
|
|
52
83
|
let closed = false;
|
|
53
84
|
try {
|
|
85
|
+
// An index still being built for the first time has its meta table but not yet its entries.
|
|
86
|
+
if (!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'entries'").get())
|
|
87
|
+
return { available: false, note: "The index is still being built. Try again when it finishes.", ...empty };
|
|
54
88
|
const sources = resolveSourceEntries(undefined, loadConfig()).filter((source) => !source.unresolved);
|
|
55
89
|
const rows = applyEntryFilters(getAllEntries(db, undefined, HIDDEN_TYPES), {
|
|
56
90
|
db,
|
|
@@ -84,6 +118,7 @@ export async function graph() {
|
|
|
84
118
|
result.emptyBundles = sources.flatMap((source) => source.registryId && !perBundle.has(source.registryId) ? [source.registryId] : []);
|
|
85
119
|
const since = usageRetentionSince();
|
|
86
120
|
const use = withReadonlyStateDb((state) => countUseByRef(state, since), null);
|
|
121
|
+
const feedback = withReadonlyStateDb((state) => countFeedbackSignalsByRef(state, rows.map((row) => row.itemRef)), null);
|
|
87
122
|
result.use = { since, source: "user" };
|
|
88
123
|
for (const row of rows) {
|
|
89
124
|
const counts = use?.get(row.itemRef);
|
|
@@ -94,6 +129,11 @@ export async function graph() {
|
|
|
94
129
|
result.nodes.belief.push(row.entry.beliefState ? beliefCode(row.entry.beliefState) : -1);
|
|
95
130
|
result.nodes.shows.push(counts?.shows ?? 0);
|
|
96
131
|
result.nodes.searches.push(counts?.searches ?? 0);
|
|
132
|
+
result.nodes.lastShownAt.push(counts?.lastShownAt ?? null);
|
|
133
|
+
result.nodes.modifiedAt.push(modifiedAt(row.filePath));
|
|
134
|
+
const signals = feedback?.get(row.itemRef);
|
|
135
|
+
result.nodes.positive.push(signals?.pos ?? 0);
|
|
136
|
+
result.nodes.negative.push(signals?.neg ?? 0);
|
|
97
137
|
}
|
|
98
138
|
for (const link of listDeclaredLinks(db)) {
|
|
99
139
|
const from = index.get(link.from);
|
|
@@ -114,6 +154,15 @@ export async function graph() {
|
|
|
114
154
|
closed = true;
|
|
115
155
|
const placed = await projectEmbeddings(rows.map((row) => vectors.get(row.id)), build);
|
|
116
156
|
result.meaning = placed.embedded ? placed : null;
|
|
157
|
+
const libraryKey = createHash("sha1")
|
|
158
|
+
.update(rows.map((row) => row.id).join(","))
|
|
159
|
+
.digest("hex");
|
|
160
|
+
const found = await findTopics(rows.map((row) => vectors.get(row.id)), rows.map((row, i) => `${row.conceptId} ${result.nodes.title[i]}`),
|
|
161
|
+
// Topics are per asset: the same build read through a different set of enabled bundles is a different library.
|
|
162
|
+
`${build}|${libraryKey}`);
|
|
163
|
+
result.topics = found.topics;
|
|
164
|
+
result.nodes.topic = found.topic;
|
|
165
|
+
result.duplicates = found.topics.length ? found.duplicates : null;
|
|
117
166
|
const noted = [`${rows.length} assets and ${result.links.from.length} links of the enabled bundles.`];
|
|
118
167
|
if (result.unresolvedLinks)
|
|
119
168
|
noted.push(`${result.unresolvedLinks} declared links are not drawn (target missing, hidden or in a bundle that is not enabled).`);
|
|
@@ -129,11 +178,26 @@ export async function graph() {
|
|
|
129
178
|
...result,
|
|
130
179
|
};
|
|
131
180
|
}
|
|
181
|
+
catch (error) {
|
|
182
|
+
// A query that fails on a damaged index is no index; permission and newer-layout errors still raise.
|
|
183
|
+
if (closed || error instanceof ConfigError)
|
|
184
|
+
throw error;
|
|
185
|
+
return { available: false, note: "The index could not be read. Rebuild it with `akm index`.", ...empty };
|
|
186
|
+
}
|
|
132
187
|
finally {
|
|
133
188
|
if (!closed)
|
|
134
189
|
closeDatabase(db);
|
|
135
190
|
}
|
|
136
191
|
}
|
|
192
|
+
/** A file's modification time as an ISO timestamp, or null when it cannot be read. */
|
|
193
|
+
function modifiedAt(file) {
|
|
194
|
+
try {
|
|
195
|
+
return fs.statSync(file).mtime.toISOString();
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
137
201
|
/** Fewest vectors the principal components can be found from. */
|
|
138
202
|
const MIN_VECTORS = 4;
|
|
139
203
|
/** The note's sentence on how many assets have a position by meaning. */
|