engine-dj-mcp 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +169 -0
- package/dist/blobs/index.d.ts +337 -0
- package/dist/blobs/index.js +483 -0
- package/dist/blobs/qcompress.d.ts +44 -0
- package/dist/blobs/qcompress.js +146 -0
- package/dist/discovery.d.ts +36 -0
- package/dist/discovery.js +111 -0
- package/dist/errors.d.ts +30 -0
- package/dist/errors.js +49 -0
- package/dist/guard.d.ts +31 -0
- package/dist/guard.js +236 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +13 -0
- package/dist/library-select.d.ts +63 -0
- package/dist/library-select.js +97 -0
- package/dist/paths.d.ts +18 -0
- package/dist/paths.js +34 -0
- package/dist/probe.d.ts +7 -0
- package/dist/probe.js +20 -0
- package/dist/proc/query-client.d.ts +36 -0
- package/dist/proc/query-client.js +249 -0
- package/dist/proc/query-worker.d.ts +1 -0
- package/dist/proc/query-worker.js +72 -0
- package/dist/semantics.d.ts +43 -0
- package/dist/semantics.js +95 -0
- package/dist/server.d.ts +31 -0
- package/dist/server.js +439 -0
- package/dist/sidecar/build.d.ts +19 -0
- package/dist/sidecar/build.js +85 -0
- package/dist/sidecar/schema.d.ts +25 -0
- package/dist/sidecar/schema.js +36 -0
- package/dist/store/connections.d.ts +28 -0
- package/dist/store/connections.js +116 -0
- package/dist/store/index-manager.d.ts +29 -0
- package/dist/store/index-manager.js +187 -0
- package/dist/tools/audit.d.ts +15 -0
- package/dist/tools/audit.js +148 -0
- package/dist/tools/libraries.d.ts +40 -0
- package/dist/tools/libraries.js +30 -0
- package/dist/tools/performance.d.ts +15 -0
- package/dist/tools/performance.js +47 -0
- package/dist/tools/refresh.d.ts +8 -0
- package/dist/tools/refresh.js +3 -0
- package/dist/tools/search.d.ts +60 -0
- package/dist/tools/search.js +328 -0
- package/dist/tools/sql.d.ts +14 -0
- package/dist/tools/sql.js +21 -0
- package/dist/tools/tracks.d.ts +12 -0
- package/dist/tools/tracks.js +49 -0
- package/package.json +53 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import { existsSync, openSync, readSync, closeSync } from "node:fs";
|
|
3
|
+
import { registerFunctions } from "../semantics.js";
|
|
4
|
+
import { EngineErrorException, libraryNeedsRecovery } from "../errors.js";
|
|
5
|
+
/**
|
|
6
|
+
* SQLite's rollback-journal header magic (see aJournalMagic in sqlite3
|
|
7
|
+
* source). A journal only needs recovery — is "hot" — once this has actually
|
|
8
|
+
* been written to disk. A live writer whose whole transaction still fits in
|
|
9
|
+
* the page cache never reaches this: nothing is hot until a page is forced
|
|
10
|
+
* out to the journal, which is why an *active* transaction from a concurrent
|
|
11
|
+
* writer (see the "concurrent writer" test) is a completely different case
|
|
12
|
+
* from a *hot* journal left by a killed one.
|
|
13
|
+
*/
|
|
14
|
+
const HOT_JOURNAL_MAGIC = "d9d505f920a163d7";
|
|
15
|
+
/**
|
|
16
|
+
* True when `<mdbPath>-journal` exists and carries SQLite's real hot-journal
|
|
17
|
+
* magic, meaning a previous writer died mid-transaction with unflushed pages
|
|
18
|
+
* on disk. SQLite itself will refuse to open such a database read-only (it
|
|
19
|
+
* needs to roll the journal forward, which requires a write) and raises a
|
|
20
|
+
* raw, unhelpful "attempt to write a readonly database" error. This lets
|
|
21
|
+
* openQueryConnection detect the condition first and explain it instead.
|
|
22
|
+
*/
|
|
23
|
+
export function hasHotJournal(mdbPath) {
|
|
24
|
+
const journalPath = `${mdbPath}-journal`;
|
|
25
|
+
if (!existsSync(journalPath))
|
|
26
|
+
return false;
|
|
27
|
+
// A journal that exists but cannot be opened or read (a permissions
|
|
28
|
+
// problem on the sibling file, say) is "cannot determine", not "is hot".
|
|
29
|
+
// Folding that to false -- rather than throwing, or assuming the worst --
|
|
30
|
+
// is a deliberate choice: this function's only three callers use a true
|
|
31
|
+
// result to skip the real open/query attempt and report a specific,
|
|
32
|
+
// actionable claim ("launch Engine DJ to recover it") without ever
|
|
33
|
+
// checking the actual database. If the real problem is unrelated to a hot
|
|
34
|
+
// journal, that claim would be false, and a healthy library would be
|
|
35
|
+
// permanently unusable through this server for a reason relaunching
|
|
36
|
+
// Engine DJ cannot fix. Returning false instead lets the real attempt
|
|
37
|
+
// proceed and surface whatever is actually wrong through the paths that
|
|
38
|
+
// already handle it as a structured error (library_busy,
|
|
39
|
+
// query_process_crashed, ...).
|
|
40
|
+
let fd;
|
|
41
|
+
try {
|
|
42
|
+
fd = openSync(journalPath, "r");
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
try {
|
|
48
|
+
const buf = Buffer.alloc(8);
|
|
49
|
+
const n = readSync(fd, buf, 0, 8, 0);
|
|
50
|
+
return n === 8 && buf.toString("hex") === HOT_JOURNAL_MAGIC;
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
closeSync(fd);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* SQLite's "file:" URI filename syntax treats `#`, `?` and unescaped
|
|
61
|
+
* whitespace as delimiters, so a raw path can misparse even once it is no
|
|
62
|
+
* longer inside a SQL string literal. encodeURI leaves `#` alone (it is a
|
|
63
|
+
* legal URI character, just not inside our path segment), hence the extra
|
|
64
|
+
* replace.
|
|
65
|
+
*/
|
|
66
|
+
function toReadOnlyUri(p) {
|
|
67
|
+
return "file:" + encodeURI(p).replace(/#/g, "%23") + "?mode=ro";
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Connection A — the one the model reaches through.
|
|
71
|
+
*
|
|
72
|
+
* m.db is the MAIN database and is opened readOnly, so the restriction is a
|
|
73
|
+
* property of the file descriptor rather than a PRAGMA. The inverse layout
|
|
74
|
+
* (sidecar as main, m.db attached read-only) was rejected: SQL-level
|
|
75
|
+
* `ATTACH '<m.db>' AS rw` escapes it and can write to the user's library.
|
|
76
|
+
* PRAGMA query_only was rejected too, because SQL can turn it back off.
|
|
77
|
+
*
|
|
78
|
+
* The attach path is bound as a parameter, not interpolated into the SQL
|
|
79
|
+
* text: a library directory containing an apostrophe (e.g. `Rock 'n' Roll`)
|
|
80
|
+
* breaks out of a string literal built by concatenation.
|
|
81
|
+
*/
|
|
82
|
+
export function openQueryConnection(mdbPath, sidecar) {
|
|
83
|
+
if (hasHotJournal(mdbPath)) {
|
|
84
|
+
// Never open the database writable to roll the journal forward
|
|
85
|
+
// ourselves: writing to the user's library is the one thing this
|
|
86
|
+
// project promises never to do. v1 detects and explains instead.
|
|
87
|
+
//
|
|
88
|
+
// The structured error is carried by the exception rather than
|
|
89
|
+
// flattened to its message: this runs inside the forked worker, where
|
|
90
|
+
// returning is not an option, and the parent used to re-stat the disk to
|
|
91
|
+
// rediscover what this function already knew.
|
|
92
|
+
throw new EngineErrorException(libraryNeedsRecovery());
|
|
93
|
+
}
|
|
94
|
+
const db = new DatabaseSync(mdbPath, { readOnly: true });
|
|
95
|
+
db.exec("PRAGMA busy_timeout = 3000");
|
|
96
|
+
if (sidecar)
|
|
97
|
+
db.prepare("ATTACH DATABASE ? AS side").run(toReadOnlyUri(sidecar));
|
|
98
|
+
registerFunctions(db, mdbPath);
|
|
99
|
+
return db;
|
|
100
|
+
}
|
|
101
|
+
/** Connection B — used only by our own rebuild code, never exposed to the model. */
|
|
102
|
+
export function openSyncConnection(sidecar, mdbPath) {
|
|
103
|
+
const db = new DatabaseSync(sidecar);
|
|
104
|
+
db.exec("PRAGMA busy_timeout = 3000");
|
|
105
|
+
db.prepare("ATTACH DATABASE ? AS engine").run(toReadOnlyUri(mdbPath));
|
|
106
|
+
registerFunctions(db, mdbPath);
|
|
107
|
+
return db;
|
|
108
|
+
}
|
|
109
|
+
/** Re-attach the sidecar after an atomic swap; rename() alone is invisible. */
|
|
110
|
+
export function reattachSidecar(db, sidecar) {
|
|
111
|
+
try {
|
|
112
|
+
db.exec("DETACH DATABASE side");
|
|
113
|
+
}
|
|
114
|
+
catch { /* not attached yet */ }
|
|
115
|
+
db.prepare("ATTACH DATABASE ? AS side").run(toReadOnlyUri(sidecar));
|
|
116
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type EngineError } from "../errors.js";
|
|
2
|
+
import type { LibraryInfo } from "../discovery.js";
|
|
3
|
+
import type { QueryProcess } from "../proc/query-client.js";
|
|
4
|
+
export interface FreshResult {
|
|
5
|
+
rebuilt: boolean;
|
|
6
|
+
indexed: number | null;
|
|
7
|
+
elapsed_ms: number;
|
|
8
|
+
generation: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Owns the sidecar's whole lifecycle: staleness probe, rebuild, atomic swap
|
|
12
|
+
* and the re-attach handshake with the live query process.
|
|
13
|
+
*
|
|
14
|
+
* The staleness probe reads the SQLite header change counter, not mtime and
|
|
15
|
+
* size: a restore that preserves mtime (rsync -t, Dropbox, Time Machine,
|
|
16
|
+
* Engine DJ Cloud) changes the file's contents while mtime and size stay
|
|
17
|
+
* identical, and an mtime/size probe would then never see the library as
|
|
18
|
+
* stale again.
|
|
19
|
+
*/
|
|
20
|
+
export declare class IndexManager {
|
|
21
|
+
#private;
|
|
22
|
+
private readonly lib;
|
|
23
|
+
private readonly qp;
|
|
24
|
+
private readonly baseDir;
|
|
25
|
+
constructor(lib: LibraryInfo, qp: QueryProcess, baseDir?: string);
|
|
26
|
+
get generation(): number;
|
|
27
|
+
get path(): string;
|
|
28
|
+
ensureFresh(): Promise<FreshResult | EngineError>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, renameSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { DatabaseSync } from "node:sqlite";
|
|
4
|
+
import { readChangeCounter } from "../probe.js";
|
|
5
|
+
import { buildSidecar } from "../sidecar/build.js";
|
|
6
|
+
import { SIDECAR_FORMAT } from "../sidecar/schema.js";
|
|
7
|
+
import { sidecarDir } from "../paths.js";
|
|
8
|
+
import { err } from "../errors.js";
|
|
9
|
+
/**
|
|
10
|
+
* Owns the sidecar's whole lifecycle: staleness probe, rebuild, atomic swap
|
|
11
|
+
* and the re-attach handshake with the live query process.
|
|
12
|
+
*
|
|
13
|
+
* The staleness probe reads the SQLite header change counter, not mtime and
|
|
14
|
+
* size: a restore that preserves mtime (rsync -t, Dropbox, Time Machine,
|
|
15
|
+
* Engine DJ Cloud) changes the file's contents while mtime and size stay
|
|
16
|
+
* identical, and an mtime/size probe would then never see the library as
|
|
17
|
+
* stale again.
|
|
18
|
+
*/
|
|
19
|
+
export class IndexManager {
|
|
20
|
+
lib;
|
|
21
|
+
qp;
|
|
22
|
+
baseDir;
|
|
23
|
+
#generation = 0;
|
|
24
|
+
#attached = false;
|
|
25
|
+
constructor(lib, qp, baseDir = sidecarDir("")) {
|
|
26
|
+
this.lib = lib;
|
|
27
|
+
this.qp = qp;
|
|
28
|
+
this.baseDir = baseDir;
|
|
29
|
+
}
|
|
30
|
+
get generation() {
|
|
31
|
+
return this.#generation;
|
|
32
|
+
}
|
|
33
|
+
get path() {
|
|
34
|
+
return join(this.baseDir, this.lib.uuid, "index.db");
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* The change counter the sidecar on disk was built from, or null when
|
|
38
|
+
* there is no usable sidecar — which forces a rebuild.
|
|
39
|
+
*
|
|
40
|
+
* `index_format` is checked alongside it. The change counter alone answers
|
|
41
|
+
* "is the index's *content* current"; it cannot answer "does the index
|
|
42
|
+
* still mean what this code thinks it means". When a derived column's
|
|
43
|
+
* definition changes (has_cues did: from "a blob exists" to "a cue is
|
|
44
|
+
* set"), a library nobody has touched since keeps the same counter, and
|
|
45
|
+
* without this check the old meaning would be served from disk for as long
|
|
46
|
+
* as the user left their library alone. A sidecar predating the column
|
|
47
|
+
* fails the SELECT outright and lands in the same catch.
|
|
48
|
+
*/
|
|
49
|
+
#storedCounter() {
|
|
50
|
+
if (!existsSync(this.path))
|
|
51
|
+
return null;
|
|
52
|
+
try {
|
|
53
|
+
const db = new DatabaseSync(this.path, { readOnly: true });
|
|
54
|
+
try {
|
|
55
|
+
const row = db
|
|
56
|
+
.prepare("SELECT change_counter, generation, index_format FROM index_meta LIMIT 1")
|
|
57
|
+
.get();
|
|
58
|
+
if (row?.generation)
|
|
59
|
+
this.#generation = Number(row.generation);
|
|
60
|
+
if (!row || Number(row.index_format) !== SIDECAR_FORMAT)
|
|
61
|
+
return null;
|
|
62
|
+
return Number(row.change_counter);
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
db.close();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Hands the on-disk index to the live query process, unless it is already
|
|
74
|
+
* attached. A QueryProcess is constructed with no sidecar, so *every* path
|
|
75
|
+
* out of ensureFresh that leaves an existing index in service has to do
|
|
76
|
+
* this -- including the two that never rebuild anything (the index is
|
|
77
|
+
* already fresh; the library was busy and the previous index still
|
|
78
|
+
* serves). Without it those paths return a success/index_stale result
|
|
79
|
+
* while `side` is not attached at all, and the next tool query dies on
|
|
80
|
+
* "no such table: side.track_derived".
|
|
81
|
+
*
|
|
82
|
+
* The short-circuit also checks qp.hasSidecar, not just #attached: a
|
|
83
|
+
* worker respawn racing a failed setSidecar could otherwise leave
|
|
84
|
+
* #attached true while the live process actually has nothing attached,
|
|
85
|
+
* which would let this report success with nothing attached -- the same
|
|
86
|
+
* class of failure the comment above describes, just reached from the
|
|
87
|
+
* other side.
|
|
88
|
+
*/
|
|
89
|
+
async #attach() {
|
|
90
|
+
if (this.#attached && this.qp.hasSidecar)
|
|
91
|
+
return true;
|
|
92
|
+
this.#attached = await this.qp.setSidecar(this.path);
|
|
93
|
+
return this.#attached;
|
|
94
|
+
}
|
|
95
|
+
async ensureFresh() {
|
|
96
|
+
if (!this.lib.supported) {
|
|
97
|
+
return err("unsupported_schema", `Schema ${this.lib.schema.join(".")} is not supported`, {
|
|
98
|
+
detail: "Supported versions are 3.0.0, 3.0.1 and 3.0.2",
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
let current;
|
|
102
|
+
try {
|
|
103
|
+
current = readChangeCounter(this.lib.path);
|
|
104
|
+
}
|
|
105
|
+
catch (e) {
|
|
106
|
+
return err("library_not_found", "Could not read the library header", { detail: String(e) });
|
|
107
|
+
}
|
|
108
|
+
if (this.#storedCounter() === current) {
|
|
109
|
+
// A fresh index on disk is not the same as a fresh index in service:
|
|
110
|
+
// on the first call of a new process nothing is attached yet, and the
|
|
111
|
+
// index built by a previous run would otherwise never be reached.
|
|
112
|
+
if (!(await this.#attach())) {
|
|
113
|
+
return err("index_stale", "The existing index could not be attached", {
|
|
114
|
+
retry_after_ms: 5000,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
// null, not a count: nothing was rebuilt this call, so there is no
|
|
118
|
+
// "tracks indexed just now" number to report. -1 previously stood in
|
|
119
|
+
// here and could be read by a caller as "minus one track indexed".
|
|
120
|
+
return { rebuilt: false, indexed: null, elapsed_ms: 0, generation: this.#generation };
|
|
121
|
+
}
|
|
122
|
+
// Captured before the directory/build attempt: distinguishes "the
|
|
123
|
+
// library was locked and we had nothing to fall back on" from "the
|
|
124
|
+
// library was locked but the previous index is still serving fine".
|
|
125
|
+
const hadPrevious = existsSync(this.path);
|
|
126
|
+
try {
|
|
127
|
+
mkdirSync(join(this.baseDir, this.lib.uuid), { recursive: true });
|
|
128
|
+
}
|
|
129
|
+
catch (e) {
|
|
130
|
+
// A permissions failure or full disk here must not throw out of
|
|
131
|
+
// ensureFresh: errors at this layer are structured returns, not
|
|
132
|
+
// exceptions.
|
|
133
|
+
return err("library_busy", "Could not create the sidecar directory", {
|
|
134
|
+
detail: String(e.message),
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
const tmp = `${this.path}.tmp`;
|
|
138
|
+
let built;
|
|
139
|
+
try {
|
|
140
|
+
built = buildSidecar({
|
|
141
|
+
mdbPath: this.lib.path,
|
|
142
|
+
outPath: tmp,
|
|
143
|
+
uuid: this.lib.uuid,
|
|
144
|
+
schema: this.lib.schema.join("."),
|
|
145
|
+
generation: this.#generation + 1,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
catch (e) {
|
|
149
|
+
const message = String(e.message);
|
|
150
|
+
if (!/locked|busy/i.test(message)) {
|
|
151
|
+
return err("library_busy", "Could not rebuild the index", { detail: message });
|
|
152
|
+
}
|
|
153
|
+
// Serving the previous index beats refusing to answer: search while
|
|
154
|
+
// Engine DJ is open should keep working on a slightly old index. But
|
|
155
|
+
// say so honestly, and only after the previous index is genuinely in
|
|
156
|
+
// service -- on the very first build there is nothing to fall back on,
|
|
157
|
+
// and "the previous index is still in use" was previously reported
|
|
158
|
+
// even when nothing had ever been attached.
|
|
159
|
+
const serving = hadPrevious && (await this.#attach());
|
|
160
|
+
return err("index_stale", serving
|
|
161
|
+
? "The library was busy; the previous index is still in use"
|
|
162
|
+
: "The library was busy; the index could not be built yet", { retry_after_ms: 5000 });
|
|
163
|
+
}
|
|
164
|
+
// rename() is invisible to an already-open connection: the query
|
|
165
|
+
// process keeps reading the old inode unless told to re-attach.
|
|
166
|
+
try {
|
|
167
|
+
renameSync(tmp, this.path);
|
|
168
|
+
}
|
|
169
|
+
catch (e) {
|
|
170
|
+
// A cross-device rename or a filesystem failure here must not throw:
|
|
171
|
+
// the freshly built file just never becomes the active index.
|
|
172
|
+
const serving = hadPrevious && (await this.#attach());
|
|
173
|
+
return err("index_stale", serving
|
|
174
|
+
? "The rebuilt index could not be swapped in; the previous index is still in use"
|
|
175
|
+
: "The rebuilt index could not be swapped in; no index is available yet", { detail: String(e.message), retry_after_ms: 5000 });
|
|
176
|
+
}
|
|
177
|
+
this.#generation += 1;
|
|
178
|
+
// The swap replaced the inode, so an already-attached connection is
|
|
179
|
+
// still reading the old file: force the re-attach, do not short-circuit
|
|
180
|
+
// on #attached.
|
|
181
|
+
this.#attached = await this.qp.setSidecar(this.path);
|
|
182
|
+
if (!this.#attached) {
|
|
183
|
+
return err("index_stale", "The rebuilt index could not be attached", { retry_after_ms: 5000 });
|
|
184
|
+
}
|
|
185
|
+
return { rebuilt: true, generation: this.#generation, ...built };
|
|
186
|
+
}
|
|
187
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { type EngineError } from "../errors.js";
|
|
3
|
+
import type { QueryProcess } from "../proc/query-client.js";
|
|
4
|
+
export declare const AUDIT_CHECKS: readonly ["missing_files", "unavailable", "unanalyzed", "no_cues", "no_beatgrid", "missing_key", "suspicious_bpm", "duplicates", "empty_metadata", "orphan_entries"];
|
|
5
|
+
export declare const AuditInput: z.ZodObject<{
|
|
6
|
+
checks: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
7
|
+
}, z.core.$strip>;
|
|
8
|
+
export type AuditInput = z.input<typeof AuditInput>;
|
|
9
|
+
export declare function auditLibrary(qp: QueryProcess, mdbPath: string, raw: AuditInput): Promise<{
|
|
10
|
+
checks: {
|
|
11
|
+
name: string;
|
|
12
|
+
count: number;
|
|
13
|
+
sample_ids: number[];
|
|
14
|
+
}[];
|
|
15
|
+
} | EngineError>;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// src/tools/audit.ts
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { err, isEngineError } from "../errors.js";
|
|
5
|
+
import { absTrackPath } from "../paths.js";
|
|
6
|
+
export const AUDIT_CHECKS = [
|
|
7
|
+
"missing_files",
|
|
8
|
+
"unavailable",
|
|
9
|
+
"unanalyzed",
|
|
10
|
+
"no_cues",
|
|
11
|
+
"no_beatgrid",
|
|
12
|
+
"missing_key",
|
|
13
|
+
"suspicious_bpm",
|
|
14
|
+
"duplicates",
|
|
15
|
+
"empty_metadata",
|
|
16
|
+
"orphan_entries",
|
|
17
|
+
];
|
|
18
|
+
export const AuditInput = z.object({ checks: z.array(z.string()).optional() });
|
|
19
|
+
/**
|
|
20
|
+
* Counts plus a small sample, never rows: this result lands in an LLM's
|
|
21
|
+
* context. A library with thousands of unanalysed tracks would blow the
|
|
22
|
+
* context window if every offending id came back, and the model only needs
|
|
23
|
+
* to know there are thousands and see a handful — detail is fetched
|
|
24
|
+
* separately (get_tracks) once someone wants it.
|
|
25
|
+
*/
|
|
26
|
+
const SAMPLE = 10;
|
|
27
|
+
const SQL_CHECKS = {
|
|
28
|
+
unavailable: { id: "t.id", body: `FROM Track t WHERE t.isAvailable = 0` },
|
|
29
|
+
unanalyzed: {
|
|
30
|
+
id: "t.id",
|
|
31
|
+
body: `FROM Track t WHERE t.isAnalyzed = 0 OR t.isAnalyzed IS NULL`,
|
|
32
|
+
},
|
|
33
|
+
// "No cue is set", not "no quickCues blob" -- and therefore read from the
|
|
34
|
+
// sidecar rather than computed here.
|
|
35
|
+
//
|
|
36
|
+
// SQL cannot decode a blob, and Engine writes a full eight-slot quickCues
|
|
37
|
+
// blob to every analysed track, so `length(quickCues) = 0` reported zero
|
|
38
|
+
// offenders on a reference library where 255 of 257 tracks have no cue at
|
|
39
|
+
// all. The check named the right thing and structurally could not report
|
|
40
|
+
// it -- the one shape of wrong answer this tool exists to avoid.
|
|
41
|
+
// side.track_derived.has_cues is decoded during the rebuild (see
|
|
42
|
+
// sidecar/build.ts), which is our code and not limited to SQL, so the
|
|
43
|
+
// check reads it from there.
|
|
44
|
+
//
|
|
45
|
+
// Every Track gets a track_derived row, so this still counts tracks,
|
|
46
|
+
// exactly as the LEFT JOIN version did. The sidecar is attached by the
|
|
47
|
+
// time any tool runs: every gated tool goes through IndexManager
|
|
48
|
+
// .ensureFresh first, and search_tracks has always read the same table.
|
|
49
|
+
no_cues: {
|
|
50
|
+
id: "d.track_id",
|
|
51
|
+
body: `FROM side.track_derived d WHERE d.has_cues = 0`,
|
|
52
|
+
},
|
|
53
|
+
// Still "empty OR NULL": a zero-length blob is not a beatgrid. Unlike
|
|
54
|
+
// quickCues, beatData has no "written but empty" state to see through --
|
|
55
|
+
// all 281 blobs in the reference library decode to a real two-marker grid
|
|
56
|
+
// whose implied tempo matches Track.bpmAnalyzed, so presence and a usable
|
|
57
|
+
// grid have not once disagreed. sidecar/build.ts records why decoding it
|
|
58
|
+
// as well would buy nothing.
|
|
59
|
+
no_beatgrid: {
|
|
60
|
+
id: "t.id",
|
|
61
|
+
body: `FROM Track t LEFT JOIN PerformanceData p ON p.trackId = t.id
|
|
62
|
+
WHERE COALESCE(length(p.beatData), 0) = 0`,
|
|
63
|
+
},
|
|
64
|
+
missing_key: { id: "t.id", body: `FROM Track t WHERE t.key = -1 OR t.key IS NULL` },
|
|
65
|
+
// bpm is stored at face value (not times 100, as rekordbox does).
|
|
66
|
+
suspicious_bpm: {
|
|
67
|
+
id: "t.id",
|
|
68
|
+
body: `FROM Track t
|
|
69
|
+
WHERE (t.bpmAnalyzed IS NOT NULL AND t.bpm IS NOT NULL
|
|
70
|
+
AND ABS(t.bpmAnalyzed - t.bpm) > 1.0)
|
|
71
|
+
OR COALESCE(t.bpmAnalyzed, t.bpm) NOT BETWEEN 60 AND 200`,
|
|
72
|
+
},
|
|
73
|
+
empty_metadata: {
|
|
74
|
+
id: "t.id",
|
|
75
|
+
body: `FROM Track t
|
|
76
|
+
WHERE t.title IS NULL OR TRIM(t.title) = ''
|
|
77
|
+
OR t.artist IS NULL OR TRIM(t.artist) = ''`,
|
|
78
|
+
},
|
|
79
|
+
duplicates: {
|
|
80
|
+
id: "t.id",
|
|
81
|
+
body: `FROM Track t WHERE LOWER(TRIM(t.artist)) || '|' || LOWER(TRIM(t.title)) IN (
|
|
82
|
+
SELECT LOWER(TRIM(artist)) || '|' || LOWER(TRIM(title)) FROM Track
|
|
83
|
+
WHERE artist IS NOT NULL AND title IS NOT NULL
|
|
84
|
+
GROUP BY 1 HAVING COUNT(*) > 1)`,
|
|
85
|
+
},
|
|
86
|
+
orphan_entries: {
|
|
87
|
+
id: "e.id",
|
|
88
|
+
body: `FROM PlaylistEntity e LEFT JOIN Track t ON t.id = e.trackId WHERE t.id IS NULL`,
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
export async function auditLibrary(qp, mdbPath, raw) {
|
|
92
|
+
const parsed = AuditInput.safeParse(raw);
|
|
93
|
+
if (!parsed.success) {
|
|
94
|
+
return err("invalid_argument", "checks must be an array of check names");
|
|
95
|
+
}
|
|
96
|
+
// Distinguish "omitted" from "explicitly empty" before defaulting:
|
|
97
|
+
// omitting checks already means "run everything", so an empty array
|
|
98
|
+
// carries no coherent second meaning, and returning { checks: [] } would
|
|
99
|
+
// read to a model exactly like a clean bill of health on a library nobody
|
|
100
|
+
// examined.
|
|
101
|
+
if (parsed.data.checks && parsed.data.checks.length === 0) {
|
|
102
|
+
return err("invalid_argument", "checks cannot be an empty array", {
|
|
103
|
+
detail: "An empty list has no meaningful result; omit checks entirely to run every check.",
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
const requested = parsed.data.checks ?? [...AUDIT_CHECKS];
|
|
107
|
+
// A check name is not a value that flows into SQL — it only ever selects
|
|
108
|
+
// which fixed query text runs — but an unrecognised one must still be
|
|
109
|
+
// rejected outright rather than silently ignored, naming what was wrong.
|
|
110
|
+
const unknown = requested.filter((c) => !AUDIT_CHECKS.includes(c));
|
|
111
|
+
if (unknown.length) {
|
|
112
|
+
return err("invalid_argument", `Unknown audit checks: ${unknown.join(", ")}`, {
|
|
113
|
+
detail: `Known checks: ${AUDIT_CHECKS.join(", ")}`,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
const out = [];
|
|
117
|
+
for (const name of requested) {
|
|
118
|
+
if (name === "missing_files") {
|
|
119
|
+
const res = await qp.run(`SELECT id, path FROM Track WHERE path IS NOT NULL`);
|
|
120
|
+
if (isEngineError(res))
|
|
121
|
+
return res;
|
|
122
|
+
const missing = [];
|
|
123
|
+
for (const row of res.rows) {
|
|
124
|
+
if (!existsSync(absTrackPath(mdbPath, String(row[1]))))
|
|
125
|
+
missing.push(Number(row[0]));
|
|
126
|
+
}
|
|
127
|
+
out.push({ name, count: missing.length, sample_ids: missing.slice(0, SAMPLE) });
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
const check = SQL_CHECKS[name];
|
|
131
|
+
const counted = await qp.run(`SELECT COUNT(*) AS c ${check.body}`);
|
|
132
|
+
if (isEngineError(counted))
|
|
133
|
+
return counted;
|
|
134
|
+
// ORDER BY the id keeps the sample stable between calls; every id
|
|
135
|
+
// expression here is a primary key, so this is not an extra sort.
|
|
136
|
+
// SAMPLE is a module constant, never caller input, but bind it anyway
|
|
137
|
+
// rather than making an exception to "every value is a parameter".
|
|
138
|
+
const sampled = await qp.run(`SELECT ${check.id} AS id ${check.body} ORDER BY ${check.id} LIMIT ?`, [SAMPLE]);
|
|
139
|
+
if (isEngineError(sampled))
|
|
140
|
+
return sampled;
|
|
141
|
+
out.push({
|
|
142
|
+
name,
|
|
143
|
+
count: Number(counted.rows[0]?.[0] ?? 0),
|
|
144
|
+
sample_ids: sampled.rows.map((r) => Number(r[0])),
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
return { checks: out };
|
|
148
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { type LibraryInfo } from "../discovery.js";
|
|
2
|
+
import type { EngineError } from "../errors.js";
|
|
3
|
+
/**
|
|
4
|
+
* A library entry as reported to a caller. `unreadable` is set when the most
|
|
5
|
+
* recent scan could not actually read this candidate (e.g. Engine DJ holds a
|
|
6
|
+
* write lock on it right now); the rest of the fields then still carry the
|
|
7
|
+
* last known-good read, the same way `supported: false` carries a library's
|
|
8
|
+
* version rather than hiding it -- so a caller can tell a broken server from
|
|
9
|
+
* a missing library, and now "present but unreadable" from "present and
|
|
10
|
+
* fine" too. Plain LibraryInfo (no `unreadable`) is always a valid entry.
|
|
11
|
+
*/
|
|
12
|
+
export type LibraryEntry = LibraryInfo & {
|
|
13
|
+
unreadable?: EngineError;
|
|
14
|
+
};
|
|
15
|
+
export interface LibraryReport {
|
|
16
|
+
path: string;
|
|
17
|
+
uuid: string;
|
|
18
|
+
schema: string;
|
|
19
|
+
supported: boolean;
|
|
20
|
+
track_count: number | null;
|
|
21
|
+
index_generation: number | null;
|
|
22
|
+
status: "ok" | "unreadable";
|
|
23
|
+
error: EngineError | null;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Always succeeds, including for unsupported schemas: a user staring at an
|
|
27
|
+
* empty list cannot tell a broken server from a missing library. A library
|
|
28
|
+
* that was discoverable before must not silently disappear because it is
|
|
29
|
+
* momentarily unreadable either -- see `status`/`error` below and
|
|
30
|
+
* LibraryEntry's `unreadable`.
|
|
31
|
+
*
|
|
32
|
+
* `path` is the absolute location of `m.db`, which carries the user's
|
|
33
|
+
* account name (see src/paths.ts's `redactPath`) -- unlike Track.path
|
|
34
|
+
* elsewhere in this codebase, this one really is routinely absolute, so
|
|
35
|
+
* redaction here is not defence in depth, it is the primary case.
|
|
36
|
+
*/
|
|
37
|
+
export declare function listLibraries(generations?: Map<string, number>, libs?: LibraryEntry[]): {
|
|
38
|
+
libraries: LibraryReport[];
|
|
39
|
+
supported_schemas: string[];
|
|
40
|
+
};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// src/tools/libraries.ts
|
|
2
|
+
import { discoverLibraries, SUPPORTED_SCHEMAS } from "../discovery.js";
|
|
3
|
+
import { redactPath } from "../paths.js";
|
|
4
|
+
/**
|
|
5
|
+
* Always succeeds, including for unsupported schemas: a user staring at an
|
|
6
|
+
* empty list cannot tell a broken server from a missing library. A library
|
|
7
|
+
* that was discoverable before must not silently disappear because it is
|
|
8
|
+
* momentarily unreadable either -- see `status`/`error` below and
|
|
9
|
+
* LibraryEntry's `unreadable`.
|
|
10
|
+
*
|
|
11
|
+
* `path` is the absolute location of `m.db`, which carries the user's
|
|
12
|
+
* account name (see src/paths.ts's `redactPath`) -- unlike Track.path
|
|
13
|
+
* elsewhere in this codebase, this one really is routinely absolute, so
|
|
14
|
+
* redaction here is not defence in depth, it is the primary case.
|
|
15
|
+
*/
|
|
16
|
+
export function listLibraries(generations = new Map(), libs = discoverLibraries()) {
|
|
17
|
+
return {
|
|
18
|
+
libraries: libs.map((l) => ({
|
|
19
|
+
path: redactPath(l.path),
|
|
20
|
+
uuid: l.uuid,
|
|
21
|
+
schema: l.schema.join("."),
|
|
22
|
+
supported: l.supported,
|
|
23
|
+
track_count: l.trackCount,
|
|
24
|
+
index_generation: generations.get(l.uuid) ?? null,
|
|
25
|
+
status: l.unreadable ? "unreadable" : "ok",
|
|
26
|
+
error: l.unreadable ?? null,
|
|
27
|
+
})),
|
|
28
|
+
supported_schemas: [...SUPPORTED_SCHEMAS],
|
|
29
|
+
};
|
|
30
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { type EngineError } from "../errors.js";
|
|
3
|
+
import type { QueryProcess } from "../proc/query-client.js";
|
|
4
|
+
export declare const PerformanceInput: z.ZodObject<{
|
|
5
|
+
id: z.ZodNumber;
|
|
6
|
+
}, z.core.$strip>;
|
|
7
|
+
export type PerformanceInput = z.input<typeof PerformanceInput>;
|
|
8
|
+
export declare function getTrackPerformance(qp: QueryProcess, raw: PerformanceInput): Promise<EngineError | {
|
|
9
|
+
sample_rate: number | null;
|
|
10
|
+
cues: import("../blobs/index.js").CuesResult;
|
|
11
|
+
loops: import("../blobs/index.js").LoopsResult;
|
|
12
|
+
beatgrid: import("../blobs/index.js").BeatgridResult;
|
|
13
|
+
waveform_summary: import("../blobs/index.js").WaveformSummary;
|
|
14
|
+
track_id: number;
|
|
15
|
+
}>;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { err, isEngineError } from "../errors.js";
|
|
3
|
+
import { decodePerformance } from "../blobs/index.js";
|
|
4
|
+
export const PerformanceInput = z.object({ id: z.number().int().positive() });
|
|
5
|
+
function asBuffer(v) {
|
|
6
|
+
if (v === null || v === undefined)
|
|
7
|
+
return null;
|
|
8
|
+
if (Buffer.isBuffer(v))
|
|
9
|
+
return v;
|
|
10
|
+
if (v instanceof Uint8Array)
|
|
11
|
+
return Buffer.from(v);
|
|
12
|
+
return null;
|
|
13
|
+
}
|
|
14
|
+
export async function getTrackPerformance(qp, raw) {
|
|
15
|
+
const parsed = PerformanceInput.safeParse(raw);
|
|
16
|
+
if (!parsed.success)
|
|
17
|
+
return err("invalid_argument", "id must be a positive integer");
|
|
18
|
+
const { id } = parsed.data;
|
|
19
|
+
// Track.length carries the duration the waveform summary reports. The
|
|
20
|
+
// waveform blob does carry its own point spacing, and that spacing times
|
|
21
|
+
// the point count matches this column to within a second on all 281 real
|
|
22
|
+
// tracks measured — but the blob has no sample rate of its own, so
|
|
23
|
+
// deriving seconds from it alone would still be a guess. LEFT JOIN so a
|
|
24
|
+
// PerformanceData row whose Track is missing still decodes.
|
|
25
|
+
const res = await qp.run(`SELECT p.quickCues, p.loops, p.beatData, p.overviewWaveFormData, t.length
|
|
26
|
+
FROM PerformanceData p LEFT JOIN Track t ON t.id = p.trackId
|
|
27
|
+
WHERE p.trackId = ?`, [id]);
|
|
28
|
+
if (isEngineError(res))
|
|
29
|
+
return res;
|
|
30
|
+
if (!res.rows.length) {
|
|
31
|
+
return err("decode_failed", `No performance data for track ${id}`, {
|
|
32
|
+
detail: "The track may not exist, or Engine has not analysed it yet",
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
const row = res.rows[0];
|
|
36
|
+
const [quickCues, loops, beatData, overviewWaveFormData, length] = row;
|
|
37
|
+
return {
|
|
38
|
+
track_id: id,
|
|
39
|
+
...decodePerformance({
|
|
40
|
+
quickCues: asBuffer(quickCues),
|
|
41
|
+
loops: asBuffer(loops),
|
|
42
|
+
beatData: asBuffer(beatData),
|
|
43
|
+
overviewWaveFormData: asBuffer(overviewWaveFormData),
|
|
44
|
+
durationSeconds: typeof length === "number" ? length : null,
|
|
45
|
+
}),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { IndexManager } from "../store/index-manager.js";
|
|
2
|
+
import type { EngineError } from "../errors.js";
|
|
3
|
+
export declare function refreshIndex(mgr: IndexManager): Promise<{
|
|
4
|
+
rebuilt: boolean;
|
|
5
|
+
indexed: number | null;
|
|
6
|
+
elapsed_ms: number;
|
|
7
|
+
generation: number;
|
|
8
|
+
} | EngineError>;
|