funoteka 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/.env.example +90 -0
  2. package/CHANGELOG.md +104 -0
  3. package/DEPLOY.md +375 -0
  4. package/LICENSE +21 -0
  5. package/README.md +138 -0
  6. package/deploy/answers.schema.json +151 -0
  7. package/deploy/build.mjs +88 -0
  8. package/deploy/check-publishable.mjs +246 -0
  9. package/deploy/smoke.mjs +681 -0
  10. package/deploy/systemd/funoteka.env.example +41 -0
  11. package/deploy/systemd/funoteka.service +77 -0
  12. package/deploy/windows/funoteka-service.xml +58 -0
  13. package/deploy/windows/install-service.ps1 +158 -0
  14. package/dist/annotation/store.js +67 -0
  15. package/dist/api/admin-export.js +227 -0
  16. package/dist/api/admin-guard.js +181 -0
  17. package/dist/api/admin-library.js +113 -0
  18. package/dist/api/admin.js +1162 -0
  19. package/dist/api/annotation.js +193 -0
  20. package/dist/api/artistinfo.js +140 -0
  21. package/dist/api/audit.js +32 -0
  22. package/dist/api/auth.js +145 -0
  23. package/dist/api/bookmark.js +108 -0
  24. package/dist/api/browse.js +1241 -0
  25. package/dist/api/config-file.js +202 -0
  26. package/dist/api/config.js +285 -0
  27. package/dist/api/cover.js +304 -0
  28. package/dist/api/download.js +112 -0
  29. package/dist/api/envelope.js +162 -0
  30. package/dist/api/extensions.js +43 -0
  31. package/dist/api/genre.js +81 -0
  32. package/dist/api/health.js +35 -0
  33. package/dist/api/history.js +374 -0
  34. package/dist/api/idempotency.js +66 -0
  35. package/dist/api/keys.js +90 -0
  36. package/dist/api/log-file.js +65 -0
  37. package/dist/api/meta.js +1410 -0
  38. package/dist/api/playlist.js +311 -0
  39. package/dist/api/rescan.js +196 -0
  40. package/dist/api/roots.js +161 -0
  41. package/dist/api/router.js +133 -0
  42. package/dist/api/scanner.js +163 -0
  43. package/dist/api/search.js +82 -0
  44. package/dist/api/server.js +643 -0
  45. package/dist/api/settings.js +67 -0
  46. package/dist/api/stream.js +442 -0
  47. package/dist/api/stubs.js +183 -0
  48. package/dist/api/transcode.js +678 -0
  49. package/dist/api/user.js +73 -0
  50. package/dist/api/virtual.js +362 -0
  51. package/dist/api/visibility.js +21 -0
  52. package/dist/artist/apply.js +752 -0
  53. package/dist/artist/credit.js +95 -0
  54. package/dist/artist/folder.js +65 -0
  55. package/dist/artist/name.js +205 -0
  56. package/dist/artist/nfo.js +101 -0
  57. package/dist/artist/translit.js +79 -0
  58. package/dist/bookmark/store.js +54 -0
  59. package/dist/classify/classify.js +405 -0
  60. package/dist/classify/collision-name.js +109 -0
  61. package/dist/classify/folder-name.js +466 -0
  62. package/dist/classify/roles.js +277 -0
  63. package/dist/classify/shelf-name.js +178 -0
  64. package/dist/classify/tree.js +43 -0
  65. package/dist/cli/args.js +169 -0
  66. package/dist/cli/daemon.js +192 -0
  67. package/dist/cli/entry.js +25 -0
  68. package/dist/cli/keys.js +52 -0
  69. package/dist/cli.js +638 -0
  70. package/dist/cover/pick.js +58 -0
  71. package/dist/cover/picture.js +111 -0
  72. package/dist/cue/engine.js +759 -0
  73. package/dist/cue/match.js +142 -0
  74. package/dist/cue/parse.js +182 -0
  75. package/dist/cue/plan.js +286 -0
  76. package/dist/cue/track-name.js +120 -0
  77. package/dist/db/index.js +163 -0
  78. package/dist/db/issue.js +73 -0
  79. package/dist/db/ledger.js +84 -0
  80. package/dist/db/migrations/001_init.sql +196 -0
  81. package/dist/db/migrations/002_cue_track_flexible.sql +35 -0
  82. package/dist/db/migrations/003_cue_metadata.sql +14 -0
  83. package/dist/db/migrations/004_cue_performer.sql +12 -0
  84. package/dist/db/migrations/005_incremental.sql +23 -0
  85. package/dist/db/migrations/006_tags.sql +57 -0
  86. package/dist/db/migrations/007_artist_credit.sql +35 -0
  87. package/dist/db/migrations/008_issue_stage.sql +39 -0
  88. package/dist/db/migrations/009_cue_title.sql +20 -0
  89. package/dist/db/migrations/010_duration_source.sql +31 -0
  90. package/dist/db/migrations/011_track_title_source.sql +28 -0
  91. package/dist/db/migrations/012_file_container.sql +25 -0
  92. package/dist/db/migrations/013_cover_art.sql +28 -0
  93. package/dist/db/migrations/014_album_year.sql +17 -0
  94. package/dist/db/migrations/015_probe_method.sql +14 -0
  95. package/dist/db/migrations/016_tags_method.sql +20 -0
  96. package/dist/db/migrations/017_cover_indirect.sql +23 -0
  97. package/dist/db/migrations/018_release_year.sql +14 -0
  98. package/dist/db/migrations/019_file_tag_name_file_value.sql +20 -0
  99. package/dist/db/migrations/020_analyze_after_index.sql +16 -0
  100. package/dist/db/migrations/021_cue_index_files.sql +15 -0
  101. package/dist/db/migrations/022_sidecar_text.sql +19 -0
  102. package/dist/db/migrations/023_track_credit.sql +23 -0
  103. package/dist/db/migrations/024_playlists.sql +63 -0
  104. package/dist/db/migrations/025_playlist_sequence.sql +27 -0
  105. package/dist/db/migrations/026_playlist_source_file.sql +19 -0
  106. package/dist/db/migrations/027_lookup_indexes.sql +30 -0
  107. package/dist/db/migrations/028_annotations.sql +47 -0
  108. package/dist/db/migrations/029_scan_state_root_run.sql +28 -0
  109. package/dist/db/migrations/030_history.sql +108 -0
  110. package/dist/db/migrations/031_queue_position_and_plays.sql +46 -0
  111. package/dist/db/migrations/032_bookmarks.sql +27 -0
  112. package/dist/db/migrations/033_bookmark_order.sql +14 -0
  113. package/dist/db/migrations/034_junk.sql +33 -0
  114. package/dist/db/migrations/035_api_key.sql +38 -0
  115. package/dist/db/migrations/036_file_tag_first.sql +88 -0
  116. package/dist/db/migrations/037_admin_idempotency.sql +31 -0
  117. package/dist/db/migrations/038_cue_audio_index.sql +24 -0
  118. package/dist/db/sweep.js +109 -0
  119. package/dist/history/store.js +151 -0
  120. package/dist/inventory/inventory.js +294 -0
  121. package/dist/junk/marks.js +143 -0
  122. package/dist/junk/rule.js +93 -0
  123. package/dist/mcp/client.js +48 -0
  124. package/dist/mcp/server.js +175 -0
  125. package/dist/mcp/tools.js +348 -0
  126. package/dist/playlist/files.js +116 -0
  127. package/dist/playlist/import.js +260 -0
  128. package/dist/playlist/store.js +240 -0
  129. package/dist/probe/ffprobe.js +179 -0
  130. package/dist/run.js +120 -0
  131. package/dist/scan/kinds.js +49 -0
  132. package/dist/scan/scan.js +503 -0
  133. package/dist/scan/schedule.js +132 -0
  134. package/dist/scan/settle.js +172 -0
  135. package/dist/scan/walk.js +132 -0
  136. package/dist/search/index.js +41 -0
  137. package/dist/search/query.js +28 -0
  138. package/dist/stream/flac.js +541 -0
  139. package/dist/stream/mpeg.js +179 -0
  140. package/dist/stream/recode.js +393 -0
  141. package/dist/stream/rewrite.js +163 -0
  142. package/dist/stream/segment.js +624 -0
  143. package/dist/tags/apply.js +703 -0
  144. package/dist/tags/encode.js +164 -0
  145. package/dist/tags/first.js +130 -0
  146. package/dist/tags/flac.js +196 -0
  147. package/dist/tags/genres.js +123 -0
  148. package/dist/tags/id3v1.js +209 -0
  149. package/dist/tags/id3v2.js +754 -0
  150. package/dist/tags/mp4.js +534 -0
  151. package/dist/tags/mpeg.js +355 -0
  152. package/dist/tags/ogg.js +463 -0
  153. package/dist/tags/picture.js +71 -0
  154. package/dist/tags/read.js +269 -0
  155. package/dist/tags/types.js +51 -0
  156. package/dist/tags/vorbis-comment.js +155 -0
  157. package/dist/text/encoding.js +384 -0
  158. package/dist/text/site-name.js +37 -0
  159. package/dist/util/child.js +21 -0
  160. package/dist/util/names.js +97 -0
  161. package/funoteka.json.example +27 -0
  162. package/package.json +55 -0
@@ -0,0 +1,503 @@
1
+ import { realpathSync } from 'node:fs';
2
+ import { resolve, sep } from 'node:path';
3
+ import { withTransaction } from "../db/index.js";
4
+ import { clearIssues, recordStageFailure } from "../db/issue.js";
5
+ import { movedSince } from "../db/ledger.js";
6
+ import { prepareSweep, SWEPT_BY_SCAN } from "../db/sweep.js";
7
+ import { stillMoving } from "./settle.js";
8
+ import { walkRoot } from "./walk.js";
9
+ /** This stage's name in `issue.stage`. Bound to the insert and to the clear. */
10
+ const STAGE = 'scan';
11
+ const ALL_KINDS = ['audio', 'cue', 'image', 'nfo', 'log', 'playlist', 'other'];
12
+ /** Directory holding `relPath`, in forward-slash terms; '' for a top-level entry. */
13
+ function parentOf(relPath) {
14
+ const cut = relPath.lastIndexOf('/');
15
+ return cut === -1 ? '' : relPath.slice(0, cut);
16
+ }
17
+ /**
18
+ * Is `inner` a strict subdirectory of `outer`?
19
+ *
20
+ * Both are canonical keys from `rootKey`, so case and separators are already
21
+ * settled the platform's way. The separator is not a nicety: a bare prefix test
22
+ * reads `C:\Music Hall` as living under `C:\Music`, which is the difference
23
+ * between the finding this exists for and a false alarm on every sibling.
24
+ */
25
+ function below(outer, inner) {
26
+ return inner.startsWith(outer.endsWith(sep) ? outer : outer + sep);
27
+ }
28
+ /**
29
+ * The filesystem's own answer for a root, so that one directory is one root.
30
+ *
31
+ * `path.resolve` alone would settle separators and `.`/`..` but stop there:
32
+ * `C:/x` and `C:\x` name one directory, `C:\x` and `c:\x` name one directory on
33
+ * a filesystem that folds case, and a junction names a directory that may
34
+ * already be configured. `realpath` settles all three, which is why root
35
+ * identity asks the filesystem instead of trusting the typed string.
36
+ *
37
+ * A root that is not on disk cannot be resolved at all. That degrades to the
38
+ * resolved path and lets the walk report the miss, exactly as it did before.
39
+ */
40
+ function canonicalRoot(rawPath) {
41
+ const absolute = resolve(rawPath);
42
+ try {
43
+ return realpathSync.native(absolute);
44
+ }
45
+ catch {
46
+ return absolute;
47
+ }
48
+ }
49
+ /**
50
+ * Comparison key for a stored root path, canonicalisation included.
51
+ *
52
+ * Applied to a path already in the table, this answers "is this the same
53
+ * directory?" for rows written back when the typed string was the identity —
54
+ * a trailing separator, a `.` segment, a junction, a different case. Asking
55
+ * the filesystem of the stored path is the point: those rows have no other way
56
+ * of being recognised as the same place.
57
+ *
58
+ * Case is folded on top only where the filesystem folds it, for the paths
59
+ * `realpath` could not answer for: on a case-sensitive volume `/music` and
60
+ * `/Music` really are two directories, and merging them would destroy the
61
+ * distinction the contract exists to protect.
62
+ */
63
+ export function rootKey(rootPath) {
64
+ const canonical = canonicalRoot(rootPath);
65
+ return process.platform === 'win32' ? canonical.toLowerCase() : canonical;
66
+ }
67
+ /**
68
+ * The id of `path`'s root, created if it is new.
69
+ *
70
+ * A root is the identity every album hangs off, so this is the one place that
71
+ * has to be certain two spellings of one directory never become two rows.
72
+ * Rows written before canonicalisation are healed in place: the surviving twin
73
+ * is the one already holding the canonical path, or the oldest rewritten to
74
+ * it, and the rest are dropped. Their derived rows go with them under
75
+ * `ON DELETE CASCADE`, and the walk that follows puts them back under the
76
+ * surviving root — safe precisely because everything below `root` is derived
77
+ * from the filesystem.
78
+ *
79
+ * A root that merely *contains* another is a different case and is not healed:
80
+ * the two are different directories, so both stay, and the doubling they cause
81
+ * is reported instead. Collapsing them would delete a root the operator
82
+ * configured.
83
+ */
84
+ function upsertRoot(db, path, nowIso, nestedReported) {
85
+ const rows = db.prepare('SELECT id, path FROM root ORDER BY id').all();
86
+ const key = rootKey(path);
87
+ const twins = rows.filter((row) => rootKey(row.path) === key);
88
+ // One directory read twice. The contract holds two roots apart on purpose —
89
+ // two copies of an album are two albums, and acceptance checks exactly that —
90
+ // and a root sitting *inside* another is not two copies. Nothing downstream
91
+ // can see it: both albums have tracks, so `unaccounted` reads the collection
92
+ // as healthy and only the file count is quietly larger than the disk. The
93
+ // pair is reported and both roots stay: collapsing them would delete a root
94
+ // the operator configured (task:2709).
95
+ //
96
+ // Both roots stay, which is why the pair is found twice — once from each side
97
+ // — and why `nestedReported` exists. On a second scan both are already in the
98
+ // table, and without it the reader would be told the same fact about the same
99
+ // two directories twice. The set is the run's, like the cue engine's sets for
100
+ // a report that is about a record rather than a file.
101
+ const nested = [];
102
+ for (const row of rows) {
103
+ const other = rootKey(row.path);
104
+ // The same directory is the twin case below, and it is reported there.
105
+ if (other === key)
106
+ continue;
107
+ // Which of the two is the inner one. Read from both ends, so the finding
108
+ // does not depend on which root the scan met first — and named in the same
109
+ // order whichever end raised it, or the two discoveries would describe one
110
+ // relationship as two different ones.
111
+ const isInner = below(other, key);
112
+ const isOuter = below(key, other);
113
+ if (!isInner && !isOuter)
114
+ continue;
115
+ const outerKey = isInner ? other : key;
116
+ const innerKey = isInner ? key : other;
117
+ const pair = `${outerKey}\u0000${innerKey}`;
118
+ if (nestedReported.has(pair))
119
+ continue;
120
+ nestedReported.add(pair);
121
+ const outer = isInner ? row.path : path;
122
+ const inner = isInner ? path : row.path;
123
+ nested.push({
124
+ kind: 'root_nested',
125
+ severity: 'warn',
126
+ detail: `${inner} is inside ${outer}, which is also a root — everything below it is read twice, once under each`,
127
+ });
128
+ }
129
+ // A row already holding the canonical path is the one to keep: it needs no
130
+ // rewrite, and its derived rows are the ones this scan is about to touch
131
+ // anyway. Finding it is not a reason to stop — a twin beside it is still the
132
+ // same directory, and leaving that one in place is the doubling this exists
133
+ // to prevent.
134
+ const survivor = rows.find((row) => row.path === path) ?? twins[0];
135
+ if (survivor === undefined) {
136
+ const inserted = db
137
+ .prepare('INSERT INTO root (path, created_at) VALUES (?, ?)')
138
+ .run(path, nowIso);
139
+ return { id: Number(inserted.lastInsertRowid), issues: nested };
140
+ }
141
+ const issues = [...nested];
142
+ if (survivor.path !== path) {
143
+ db.prepare('UPDATE root SET path = ? WHERE id = ?').run(path, survivor.id);
144
+ issues.push({
145
+ kind: 'root_path_canonicalised',
146
+ severity: 'info',
147
+ detail: `${survivor.path} → ${path}`,
148
+ });
149
+ }
150
+ // The twins' own diagnostics describe this same directory, so they move to
151
+ // the survivor rather than blocking the delete on the foreign key.
152
+ const repoint = db.prepare('UPDATE issue SET root_id = ? WHERE root_id = ?');
153
+ const drop = db.prepare('DELETE FROM root WHERE id = ?');
154
+ for (const redundant of twins) {
155
+ if (redundant.id === survivor.id)
156
+ continue;
157
+ repoint.run(survivor.id, redundant.id);
158
+ drop.run(redundant.id);
159
+ issues.push({
160
+ kind: 'root_duplicate_collapsed',
161
+ severity: 'warn',
162
+ detail: `${redundant.path} was the same directory as ${path}; its rows were dropped and re-read by this scan`,
163
+ });
164
+ }
165
+ return { id: survivor.id, issues };
166
+ }
167
+ /**
168
+ * How many rows one transaction carries.
169
+ *
170
+ * Measured, because this number is the whole of what the stage promises: on the
171
+ * live collection a batch of five hundred holds the write lock **12 ms at the
172
+ * median and 96 ms at its worst**, across the thirty batches of one run. The
173
+ * budget it is held to is the 250 ms a listener's save waits with
174
+ * (`busy_timeout`, `db/index.ts`) — so a save can lose the race for one batch
175
+ * and still get in on the next.
176
+ *
177
+ * Rows and not files: one file writes two of them (`insertFile` and the ledger
178
+ * row beside it), so five hundred rows is two hundred and fifty files.
179
+ */
180
+ const ROWS_PER_TRANSACTION = 500;
181
+ /**
182
+ * Write `rows`, a transaction at a time.
183
+ *
184
+ * Each batch is committed before the next begins, which is the whole of what
185
+ * this is for: the lock is held for a batch rather than for the stage, so a
186
+ * listener's save waiting on it gets in between them.
187
+ *
188
+ * A batch that throws unwinds only itself — what came before it stays, and the
189
+ * run is marked `failed` with its reason filed. That is the recovery the stages
190
+ * already promise (`run.ts`): a rerun after a failure reaches the state a clean
191
+ * run would.
192
+ */
193
+ function inBatches(db, rows, write, onBatch) {
194
+ for (let start = 0; start < rows.length; start += ROWS_PER_TRANSACTION) {
195
+ const batch = rows.slice(start, start + ROWS_PER_TRANSACTION);
196
+ withTransaction(db, () => {
197
+ for (const row of batch)
198
+ write(row);
199
+ });
200
+ onBatch?.(batch.length);
201
+ }
202
+ }
203
+ /**
204
+ * Read the given roots into the meta layer.
205
+ *
206
+ * This stage records what is on disk and nothing more — no tag reading, no cue
207
+ * parsing, no classification. It is a faithful inventory, and the classifier
208
+ * layers on top of it.
209
+ *
210
+ * Files are matched by (root, relative path), so rescanning updates rows in
211
+ * place and keeps `first_seen_run_id` pointing at the run that found them.
212
+ */
213
+ export function scan(db, rootPaths, options = {}) {
214
+ const walk = options.walk ?? walkRoot;
215
+ const now = options.now ?? (() => new Date());
216
+ // One directory is one root: the same directory named twice in one run —
217
+ // by a second spelling or a second argument — is walked once.
218
+ const roots = [...new Set(rootPaths.map(canonicalRoot))];
219
+ const startedAt = now().toISOString();
220
+ const run = db
221
+ .prepare('INSERT INTO scan_run (started_at, status, roots_json) VALUES (?, ?, ?)')
222
+ .run(startedAt, 'running', JSON.stringify(roots));
223
+ const scanRunId = Number(run.lastInsertRowid);
224
+ // The ledger is emptied first, so that every file the walk meets is one
225
+ // nothing was observed about — which is the same state as a file that has
226
+ // just appeared, and is what makes `full` reach every stage downstream
227
+ // without any of them knowing this option exists. Before the walk and not
228
+ // after: a run that dies halfway leaves part of the library due for a
229
+ // re-read, and a re-read is the safe direction to be wrong in.
230
+ if (options.full === true)
231
+ db.exec('DELETE FROM scan_state');
232
+ // One finding per pair of overlapping roots, whatever order the scan met them
233
+ // in — see `upsertRoot`.
234
+ const nestedReported = new Set();
235
+ const byKind = Object.fromEntries(ALL_KINDS.map((k) => [k, 0]));
236
+ let folders = 0;
237
+ let files = 0;
238
+ const perRoot = [];
239
+ let audioFiles = 0;
240
+ let skipped = 0;
241
+ let ignored = 0;
242
+ let ignoredAudio = 0;
243
+ let issues = 0;
244
+ let changed = 0;
245
+ let unchanged = 0;
246
+ const finish = (status) => {
247
+ db.prepare('UPDATE scan_run SET finished_at = ?, status = ? WHERE id = ?').run(now().toISOString(), status, scanRunId);
248
+ };
249
+ // Everything that can throw lives inside the guard, statement preparation
250
+ // included. A prepare compiles against the live schema, so a table it cannot
251
+ // find throws there — and it used to run before the `try`, which left
252
+ // `scan_run` saying `running` with nothing to correct it: the row is inserted
253
+ // before the transaction, so no rollback reaches it, and the stages after this
254
+ // one never run to file the failure either. `recordStageFailure` is what
255
+ // writes it down; the run id it needs is the one this function returns, which
256
+ // is exactly what a caller does not have when this throws.
257
+ try {
258
+ // **Written in batches, and the batching is the point of this stage's shape.**
259
+ //
260
+ // The scan and the server are two connections to one file, and SQLite gives
261
+ // one writer at a time. This stage used to open a single transaction around
262
+ // everything — every root, every folder, every file — and hold the write
263
+ // lock from its first row to its last: measured on the live collection,
264
+ // **1854 ms** for 11 541 files, against a `busy_timeout` of 250 ms
265
+ // (`db/index.ts`). A listener's save arriving anywhere in those 1854 ms was
266
+ // refused, and the save that matters most arrives exactly then: people
267
+ // listen to music while a scan runs.
268
+ //
269
+ // Half of that window was not writing at all. `walk` reads the filesystem —
270
+ // 929 ms of the 1854 — and reading a disk needs no write lock, so the walk
271
+ // now happens with nothing open. What is left to write goes in batches, each
272
+ // its own `BEGIN IMMEDIATE` (`db/index.ts` has why that word is
273
+ // load-bearing), so the lock is let go between them and a save waiting on it
274
+ // gets in.
275
+ //
276
+ // **The run is no longer one transaction, and that is the trade.**
277
+ //
278
+ // A failure part-way through a root leaves the batches before it committed
279
+ // instead of unwinding the root. What it does not leave is a root half-swept:
280
+ // `sweep` runs last, in a transaction of its own, so a run that dies before
281
+ // reaching it has taken nothing away. Recovery is the one the stages already
282
+ // promise — `run.ts`: "a rerun after a failure reaches the state a clean run
283
+ // would" — and the run is marked `failed` with its reason filed.
284
+ //
285
+ // The other half of the trade is what a *reader* sees. Nothing this stage
286
+ // wrote used to be visible until its single `COMMIT`, so a client browsing
287
+ // during a run browsed the library as it was before the run. Now each batch
288
+ // is visible as it commits, so the same client sees the root being walked in
289
+ // its new state and the roots after it in the old one — a library that is
290
+ // part-way, which is a true description of it. The sweep staying whole per
291
+ // root is what keeps that from being a library that is *wrong*: no row is
292
+ // taken away until everything under that root has been written.
293
+ // The stamp is the whole point of the upsert: a folder seen this run has to
294
+ // say so, or the sweep below deletes it as something that is no longer there.
295
+ const insertFolder = db.prepare(`INSERT INTO folder (root_id, rel_path, parent_rel_path, last_seen_run_id)
296
+ VALUES (?, ?, ?, ?)
297
+ ON CONFLICT (root_id, rel_path) DO UPDATE SET last_seen_run_id = excluded.last_seen_run_id`);
298
+ // Letting go of what vanished, and the predicate that decides it, live in
299
+ // `db/sweep.ts` — the scan only says which tables are its to empty.
300
+ const sweep = prepareSweep(db, SWEPT_BY_SCAN);
301
+ // first_seen_run_id is deliberately absent from the UPDATE branch: it is
302
+ // what makes "when did this file first appear" answerable after a rescan.
303
+ const insertFile = db.prepare(`INSERT INTO file (root_id, rel_path, folder_rel_path, name, kind, ext, size, mtime_ms,
304
+ first_seen_run_id, last_seen_run_id)
305
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
306
+ ON CONFLICT (root_id, rel_path) DO UPDATE SET
307
+ folder_rel_path = excluded.folder_rel_path,
308
+ name = excluded.name,
309
+ kind = excluded.kind,
310
+ ext = excluded.ext,
311
+ size = excluded.size,
312
+ mtime_ms = excluded.mtime_ms,
313
+ last_seen_run_id = excluded.last_seen_run_id`);
314
+ const selectScanState = db.prepare('SELECT rel_path, size, mtime_ms FROM scan_state WHERE root_id = ?');
315
+ const upsertScanState = db.prepare(`INSERT INTO scan_state (root_id, rel_path, size, mtime_ms, last_seen_run_id, changed)
316
+ VALUES (?, ?, ?, ?, ?, ?)
317
+ ON CONFLICT (root_id, rel_path) DO UPDATE SET
318
+ size = excluded.size,
319
+ mtime_ms = excluded.mtime_ms,
320
+ last_seen_run_id = excluded.last_seen_run_id,
321
+ changed = excluded.changed`);
322
+ const insertIssue = db.prepare(`INSERT INTO issue (scan_run_id, stage, root_id, rel_path, kind, severity, detail)
323
+ VALUES (?, ?, ?, ?, ?, ?, ?)`);
324
+ // Only the derived kinds, because this stage files both sorts of row. The
325
+ // root-settlement issues are events and stay — a healed spelling, a twin
326
+ // that was dropped, both of which happen once — while a skipped walk, an
327
+ // ignored directory and a root that sits inside another are read again
328
+ // every run, from the filesystem and from the configured root set, and
329
+ // would otherwise leave one more row behind on every scan. `db/issue.ts`
330
+ // has the rule.
331
+ withTransaction(db, () => {
332
+ clearIssues(db, STAGE, { kind: 'walk_skipped' });
333
+ clearIssues(db, STAGE, { kind: 'walk_ignored_audio' });
334
+ clearIssues(db, STAGE, { kind: 'walk_ignored' });
335
+ clearIssues(db, STAGE, { kind: 'root_nested' });
336
+ });
337
+ for (const rootPath of roots) {
338
+ // The root and whatever registering it found. Small, quick, and has to be
339
+ // in place before any row below can name it.
340
+ const root = withTransaction(db, () => {
341
+ const registered = upsertRoot(db, rootPath, startedAt, nestedReported);
342
+ for (const rootIssue of registered.issues) {
343
+ insertIssue.run(scanRunId, STAGE, registered.id, null, rootIssue.kind, rootIssue.severity, rootIssue.detail);
344
+ issues += 1;
345
+ }
346
+ return registered;
347
+ });
348
+ // With nothing open. This is the filesystem, and the write lock is not for
349
+ // the filesystem — it is the longest part of the stage and it now holds up
350
+ // nobody.
351
+ //
352
+ // **Read before the walk and handed to the gate, which is the walk's own
353
+ // clock and not this line's.** The gate asks whether a file was quiet when
354
+ // the walk met it, and a walk of a real collection takes seconds — so the
355
+ // reading taken *after* it would make the first file met the oldest in the
356
+ // run rather than the freshest, and a file a copy was halfway through when
357
+ // the walk passed it would be recorded at the size it had reached by then.
358
+ const walkedAtMs = now().getTime();
359
+ const result = walk(rootPath);
360
+ // **The half-write gate.** A file the disk has not been quiet about for
361
+ // five seconds is not a file yet, so this run does not record it. The rule
362
+ // and the number are `scan/settle.ts`'s, which are also the watcher's — the
363
+ // same clause of the contract, held the same way on both paths.
364
+ //
365
+ // **Reported as skipped and held back from the sweep, and those are two
366
+ // different statements.** Skipped is what the issue is recorded from, so
367
+ // `GET /issues` says why the file is not in the library this run. Held is
368
+ // what the sweep is told, and it is the *opposite* of the amnesty a skipped
369
+ // path otherwise gets: a file being rewritten in place keeps the rows the
370
+ // previous run gave it, and does not come out of the sweep claiming this
371
+ // run stands behind it — because it does not, and every stage selects its
372
+ // work by exactly that claim. Handing it over as `kept` was the bug this
373
+ // shape fixes: the file was read, and the tags of half a song landed in
374
+ // `file_tag`. A file that was not there before has no rows to keep, so it
375
+ // simply arrives on a later scan, once it has stopped moving.
376
+ const held = new Set(stillMoving(rootPath, result.files, walkedAtMs, options.settleMs ?? 0));
377
+ for (const relPath of held) {
378
+ result.skipped.push({
379
+ relPath,
380
+ reason: 'still being written: its size or mtime moved between two looks',
381
+ });
382
+ }
383
+ const walked = held.size === 0 ? result.files : result.files.filter((f) => !held.has(f.relPath));
384
+ // The amnesty is what the walk met and could not look into; what the gate
385
+ // held back is not it. Split here rather than filtered at the sweep, so
386
+ // that a path cannot be in both lists and neither list can grow a second
387
+ // meaning later.
388
+ const amnesty = result.skipped.filter((skip) => !held.has(skip.relPath));
389
+ const withheld = result.skipped.filter((skip) => held.has(skip.relPath));
390
+ // **The count is what this run recorded, not what the walk handed over.**
391
+ // The two differ only when the gate held something, and that difference is
392
+ // the whole point — but it is legible in `skipped` and in the issue, not
393
+ // here, so this reads as "0 files" for a root where everything was held
394
+ // exactly as it does for an empty directory.
395
+ perRoot.push({ path: rootPath, folders: result.folders.length, files: walked.length });
396
+ // The ledger as the previous run left it. Reading it before this run
397
+ // writes its own is the only moment the comparison still means anything:
398
+ // afterwards the file row and the ledger hold the same observation.
399
+ const previous = new Map(selectScanState.all(root.id).map((row) => [row.rel_path, row]));
400
+ inBatches(db, result.folders, (folder) => {
401
+ insertFolder.run(root.id, folder, parentOf(folder), scanRunId);
402
+ folders += 1;
403
+ }, options.onBatch);
404
+ inBatches(db, walked, (file) => {
405
+ const moved = movedSince(previous.get(file.relPath), file);
406
+ if (moved)
407
+ changed += 1;
408
+ else
409
+ unchanged += 1;
410
+ insertFile.run(root.id, file.relPath, file.folderRelPath, file.name, file.kind, file.ext, file.size, file.mtimeMs, scanRunId, scanRunId);
411
+ upsertScanState.run(root.id, file.relPath, file.size, file.mtimeMs, scanRunId, moved ? 1 : 0);
412
+ files += 1;
413
+ byKind[file.kind] += 1;
414
+ if (file.kind === 'audio')
415
+ audioFiles += 1;
416
+ }, options.onBatch);
417
+ // What the walk met but did not walk, and the end of the root — the last
418
+ // transaction of this root, with the sweep in it. `sweep` is the stage's
419
+ // only destructive step, and it goes in the same commit as the rows it
420
+ // judges rather than after them; the reason is in the comment at the top
421
+ // of this function.
422
+ withTransaction(db, () => {
423
+ for (const skip of result.skipped) {
424
+ insertIssue.run(scanRunId, STAGE, root.id, skip.relPath, 'walk_skipped', 'warn', skip.reason);
425
+ skipped += 1;
426
+ issues += 1;
427
+ }
428
+ // An ignored directory is a guess made from a name, and it stays ignored:
429
+ // a Synology thumbnail tree is not a collection, and walking one into the
430
+ // result invents a phantom album per track. What may not stay silent is
431
+ // that the guess was made, or that it was made about music — `#recycle`
432
+ // holds whatever was deleted. Both are counted either way, so `files N`
433
+ // can be reconciled with the filesystem.
434
+ for (const dir of result.ignored) {
435
+ ignored += 1;
436
+ ignoredAudio += dir.audio;
437
+ if (dir.audio === 0)
438
+ continue;
439
+ insertIssue.run(scanRunId, STAGE, root.id, dir.relPath, 'walk_ignored_audio', 'warn', `ignored by name, but it holds ${dir.audio} audio file(s) — they are not in the collection`);
440
+ issues += 1;
441
+ }
442
+ // The count itself, once for the root. Each directory above says what it
443
+ // is only when the guess was wrong about music; the guess being made at
444
+ // all is what a reader reconciling `files N` with the filesystem needs,
445
+ // and it was on stdout alone — gone by the time anyone opens the database.
446
+ // Aggregated rather than one row per directory because the tree a NAS
447
+ // writes holds hundreds of entries, and the report is about one root.
448
+ if (result.ignored.length > 0) {
449
+ const dirs = result.ignored.length;
450
+ const beneath = dirs === 1 ? 'it' : 'them';
451
+ const audio = result.ignored.reduce((sum, dir) => sum + dir.audio, 0);
452
+ insertIssue.run(scanRunId, STAGE, root.id, null, 'walk_ignored', 'info', `ignored ${dirs} ${dirs === 1 ? 'directory' : 'directories'} by name; ${audio === 0
453
+ ? `no audio beneath ${beneath}`
454
+ : `${audio} audio file(s) beneath ${beneath} are not in the collection`}`);
455
+ issues += 1;
456
+ }
457
+ // A root nobody could read reports no folders and no files — which reads
458
+ // exactly like an empty collection, with the run still calling itself ok.
459
+ // The walk says so as a skip at '', and a path of '' would keep nothing:
460
+ // what has to survive is the whole root, and no path below it was seen to
461
+ // name. So the root is left alone entirely rather than handed a keep-list.
462
+ const rootUnreadable = result.skipped.some((skip) => skip.relPath === '');
463
+ // Only for the roots actually walked: a root nobody scanned has not been
464
+ // seen to lose anything, and its rows are not ours to drop.
465
+ if (!rootUnreadable) {
466
+ sweep.run(root.id, scanRunId, amnesty.map((skip) => skip.relPath), withheld.map((skip) => skip.relPath));
467
+ }
468
+ });
469
+ }
470
+ }
471
+ catch (err) {
472
+ // Nothing to unwind here, and that is worth saying rather than guarding: a
473
+ // batch that threw was already rolled back by `withTransaction`, which is
474
+ // the only thing in this function that opens a transaction. A rollback here
475
+ // would be a second one, and a rollback with nothing to roll back throws.
476
+ try {
477
+ recordStageFailure(db, scanRunId, STAGE, err);
478
+ }
479
+ catch {
480
+ // The failure is already on its way to the caller. A trace that cannot be
481
+ // written must not replace the reason it was being written for.
482
+ }
483
+ finish('failed');
484
+ throw err;
485
+ }
486
+ finish('ok');
487
+ return {
488
+ scanRunId,
489
+ roots: roots.length,
490
+ rootPaths: roots,
491
+ perRoot,
492
+ changed,
493
+ unchanged,
494
+ folders,
495
+ files,
496
+ audioFiles,
497
+ byKind,
498
+ skipped,
499
+ ignored,
500
+ ignoredAudio,
501
+ issues,
502
+ };
503
+ }
@@ -0,0 +1,132 @@
1
+ import { readFileSync } from 'node:fs';
2
+ import { PROBE_METHOD } from "../probe/ffprobe.js";
3
+ import { TAGS_METHOD } from "../tags/read.js";
4
+ /**
5
+ * The instant a scan may next start, or nothing when this server does not scan
6
+ * by itself at all.
7
+ *
8
+ * Two rules and they compose in one direction: the interval says *when* the
9
+ * reading is old enough, and the quiet hours say *when* it is allowed to happen.
10
+ * A due time that lands inside the quiet window is pushed to the end of it — not
11
+ * skipped, because skipping would mean a deployment that is always in a quiet
12
+ * window never scans again.
13
+ *
14
+ * The window may cross midnight (`23` to `6`), which is the shape it has for
15
+ * anybody who works late, and `from === to` is an empty window rather than a
16
+ * whole-day one.
17
+ */
18
+ export function dueAt(lastStartedAt, now, schedule) {
19
+ if (schedule.intervalMinutes <= 0)
20
+ return null;
21
+ const last = lastStartedAt === null ? null : Date.parse(lastStartedAt);
22
+ const base = last === null || Number.isNaN(last) ? now.getTime() : last + schedule.intervalMinutes * 60_000;
23
+ const due = new Date(Math.max(base, now.getTime()));
24
+ return quietUntil(due, schedule);
25
+ }
26
+ /** The same instant, moved out of the quiet hours when it lands inside them. */
27
+ export function quietUntil(when, schedule) {
28
+ if (!quiet(when, schedule))
29
+ return when;
30
+ const end = new Date(when);
31
+ end.setMinutes(0, 0, 0);
32
+ end.setHours(schedule.quietTo);
33
+ // A window that crosses midnight is entered before it ends, so the end is
34
+ // tomorrow's — and an hour of `to` that is *behind* the hour of `when` is
35
+ // exactly that case and not a window that has already passed.
36
+ if (end.getTime() <= when.getTime())
37
+ end.setDate(end.getDate() + 1);
38
+ return end;
39
+ }
40
+ /** Whether an instant falls in the quiet hours. */
41
+ export function quiet(when, schedule) {
42
+ const hour = when.getHours();
43
+ const { quietFrom: from, quietTo: to } = schedule;
44
+ if (from === to)
45
+ return false;
46
+ // `23` to `6`: the hours that are quiet are the ones at or after `from`, or
47
+ // before `to` — the window is the union rather than the interval between.
48
+ if (from > to)
49
+ return hour >= from || hour < to;
50
+ return hour >= from && hour < to;
51
+ }
52
+ /**
53
+ * Whether watching this directory for changes is worth trying at all.
54
+ *
55
+ * **The honest answer is "not always", so it is asked rather than assumed.**
56
+ * Filesystem notifications on a network share are the classic thing that appears
57
+ * to work and does not: inotify has no way to hear about a change made on
58
+ * another machine, and on Windows a UNC path has no change journal this process
59
+ * can subscribe to. A watcher that is silent there is worse than no watcher,
60
+ * because the deployment believes it has one — so a share is recognised and the
61
+ * interval takes over, which is what the contract asks for and what the log
62
+ * says happened.
63
+ *
64
+ * What this cannot answer is a *mapped* drive letter on Windows pointing at a
65
+ * share: telling that apart from a local disk needs a syscall Node does not
66
+ * expose. The runtime covers that case instead — `fs.watch` on such a path
67
+ * either throws or delivers nothing, and the engine falls back with a line
68
+ * saying so.
69
+ */
70
+ export function watchable(rootPath) {
71
+ if (/^[\\/]{2}/.test(rootPath)) {
72
+ return { ok: false, why: 'a network path: nothing this process can subscribe to reports changes made elsewhere' };
73
+ }
74
+ // Linux says what a mount is; the others do not have a file that answers it.
75
+ if (process.platform === 'linux') {
76
+ const mounted = mountTypeOf(rootPath);
77
+ if (mounted !== null && NETWORK_FILESYSTEMS.some((kind) => mounted.startsWith(kind))) {
78
+ return { ok: false, why: `a ${mounted} mount: no inotify event arrives for a change made on another machine` };
79
+ }
80
+ }
81
+ return { ok: true };
82
+ }
83
+ /** The filesystems whose notifications do not cross a network. */
84
+ const NETWORK_FILESYSTEMS = ['nfs', 'cifs', 'smbfs', 'smb3', 'fuse.sshfs', 'fuse.rclone', '9p'];
85
+ /**
86
+ * The filesystem type a path is on, as `/proc/mounts` describes it.
87
+ *
88
+ * The longest matching mount point wins, because `/` and `/mnt/media` are both
89
+ * mounts and a file under the second is not on the first. Nothing here throws: a
90
+ * machine without `/proc` (a container, another platform) simply does not
91
+ * answer, and the fallback above is the interval either way.
92
+ */
93
+ function mountTypeOf(path) {
94
+ let mounts;
95
+ try {
96
+ mounts = readFileSync('/proc/mounts', 'utf8');
97
+ }
98
+ catch {
99
+ return null;
100
+ }
101
+ const normal = path.replace(/\/+$/, '');
102
+ let best = null;
103
+ for (const line of mounts.split('\n')) {
104
+ const [device, point, type] = line.split(' ');
105
+ if (point === undefined || type === undefined)
106
+ continue;
107
+ const decoded = point.replace(/\\040/g, ' ');
108
+ if (normal === decoded || normal.startsWith(`${decoded.replace(/\/+$/, '')}/`)) {
109
+ if (best === null || decoded.length > best.point.length)
110
+ best = { point: decoded, type };
111
+ }
112
+ }
113
+ return best?.type ?? null;
114
+ }
115
+ /**
116
+ * How much of the library is read by a method older than this build's.
117
+ *
118
+ * **This is the classifier-version trigger, and it is a question rather than a
119
+ * mechanism.** The stages already re-read a file whose stamp is behind
120
+ * (`tags/apply.ts` selects `f.tags_method <> ?` among the reasons a file is due),
121
+ * so a changed reader is picked up by the next scan without anyone arranging it
122
+ * — what was missing is anybody *knowing* that the next scan is not an ordinary
123
+ * one. An operator sees a number here, and the engine uses it to scan now
124
+ * instead of in six hours.
125
+ */
126
+ export function staleReadings(db) {
127
+ const count = (sql, method) => db.prepare(sql).get(method).n;
128
+ return {
129
+ tags: count("SELECT COUNT(*) AS n FROM file WHERE kind = 'audio' AND tags_method <> ?", TAGS_METHOD),
130
+ probes: count('SELECT COUNT(*) AS n FROM audio_probe WHERE probe_method <> ?', PROBE_METHOD),
131
+ };
132
+ }