nixamp 0.24.2 → 0.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +38 -0
  2. package/dist/captions.d.ts +12 -0
  3. package/dist/captions.js +29 -2
  4. package/dist/hash.d.ts +12 -0
  5. package/dist/hash.js +139 -0
  6. package/dist/main.js +27 -3
  7. package/dist/mcp.d.ts +3 -0
  8. package/dist/mcp.js +49 -1
  9. package/dist/media-index.d.ts +67 -0
  10. package/dist/media-index.js +114 -0
  11. package/dist/media-local.d.ts +43 -0
  12. package/dist/media-local.js +88 -0
  13. package/dist/media-page.d.ts +18 -0
  14. package/dist/media-page.js +146 -0
  15. package/dist/media.d.ts +145 -0
  16. package/dist/media.js +364 -0
  17. package/dist/server.d.ts +3 -0
  18. package/dist/server.js +186 -1
  19. package/dist/transcribe.js +20 -0
  20. package/dist/transcript-client.d.ts +19 -0
  21. package/dist/transcript-client.js +24 -0
  22. package/package.json +1 -1
  23. package/src/captions.ts +32 -2
  24. package/src/hash.ts +139 -0
  25. package/src/main.ts +27 -3
  26. package/src/mcp.ts +49 -1
  27. package/src/media-index.ts +145 -0
  28. package/src/media-local.ts +117 -0
  29. package/src/media-page.ts +159 -0
  30. package/src/media.ts +433 -0
  31. package/src/server.ts +178 -1
  32. package/src/transcribe.ts +17 -0
  33. package/src/transcript-client.ts +34 -0
  34. package/web/dist/assets/{hls-3VKVEQE3-B3PGV4wK.js → hls-3VKVEQE3-GlzSYi0P.js} +1 -1
  35. package/web/dist/assets/index-DJHHB_63.js +1 -0
  36. package/web/dist/assets/{mpegts-CtxeYR5-.js → mpegts-KIbyW_RL.js} +1 -1
  37. package/web/dist/assets/{mpegts-LO6RVLD6-CBHUeoME.js → mpegts-LO6RVLD6-Fs_vbGcB.js} +1 -1
  38. package/web/dist/index.html +1 -1
  39. package/web/dist/sw.js +5 -5
  40. package/web/dist/assets/index-DOOjAq1w.js +0 -1
@@ -51,6 +51,30 @@ export async function translateTexts(signed, texts, from, to, fetcher = fetch) {
51
51
  return unreachable(signed.site, error);
52
52
  }
53
53
  }
54
+ /** Keep what is known about a file, by its SHA-256 hex. */
55
+ export async function keepMedia(signed, id, ask, fetcher = fetch) {
56
+ try {
57
+ const response = await fetcher(`${base(signed.site)}/api/v1/media/${encodeURIComponent(id)}`, {
58
+ method: "PUT",
59
+ headers: { authorization: `Bearer ${signed.token}`, "content-type": "application/json" },
60
+ body: JSON.stringify(ask),
61
+ });
62
+ return await asJson(response);
63
+ }
64
+ catch (error) {
65
+ return unreachable(signed.site, error);
66
+ }
67
+ }
68
+ /** The record of a file, by id, fingerprint or transcript id. Public: no token needed. */
69
+ export async function fetchMedia(site, id, fetcher = fetch) {
70
+ try {
71
+ const response = await fetcher(`${base(site)}/api/v1/media/${encodeURIComponent(id)}`, { headers: { accept: "application/json" } });
72
+ return await asJson(response);
73
+ }
74
+ catch (error) {
75
+ return unreachable(site, error);
76
+ }
77
+ }
54
78
  /** Forget some media's transcripts. Only whoever stored them may. */
55
79
  export async function forgetTranscript(signed, id, fetcher = fetch) {
56
80
  try {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "nixamp",
3
- "version": "0.24.2",
3
+ "version": "0.25.1",
4
4
  "description": "It really whips the terminal's ass. A Winamp-shaped audio player for your terminal.",
5
5
  "license": "MIT",
6
6
  "type": "module",
package/src/captions.ts CHANGED
@@ -33,7 +33,7 @@
33
33
  import { spawn } from "node:child_process";
34
34
  import type { Listener } from "./channels.ts";
35
35
  import { RATE } from "./speech.ts";
36
- import { fetchTranscript, keepLines, translateTexts } from "./transcript-client.ts";
36
+ import { fetchTranscript, keepLines, keepMedia, translateTexts, type MediaToKeep } from "./transcript-client.ts";
37
37
  import { covered, lineAt, transcriptIdOf, type TranscriptLine } from "./transcripts.ts";
38
38
 
39
39
  export interface CaptionLine {
@@ -69,6 +69,12 @@ export interface ChannelMedia {
69
69
  startedAt: number;
70
70
  /** How many seconds behind the live edge a new listener's sound starts. */
71
71
  backlog: number;
72
+ /**
73
+ * The file itself, described for the record at nixamp.com/hash/<id>:
74
+ * its SHA-256 and what is known about it. Absent for anything that is
75
+ * not a file on this machine. Asked once when the captioner starts.
76
+ */
77
+ describe?: () => Promise<{ id: string; keep: MediaToKeep } | null>;
72
78
  }
73
79
 
74
80
  export interface CaptionsOptions {
@@ -220,6 +226,8 @@ export interface CaptionStatus {
220
226
  known: number;
221
227
  /** The languages lines are being given in besides the original. */
222
228
  languages: string[];
229
+ /** The file's SHA-256, once it has been described for nixamp.com/hash/<id>; "" until then, or for a live. */
230
+ hash: string;
223
231
  }
224
232
 
225
233
  class Captioner {
@@ -232,6 +240,8 @@ class Captioner {
232
240
  error = "";
233
241
  /** What the ear says the sound is in, or the store said it was; "" until one of them has. */
234
242
  language = "";
243
+ /** The file's SHA-256, once described. */
244
+ hash = "";
235
245
  private model = "";
236
246
  private decoder: Decoder | null = null;
237
247
  private detach: (() => void) | null = null;
@@ -288,9 +298,28 @@ class Captioner {
288
298
  return false;
289
299
  }
290
300
  void this.consult("");
301
+ void this.record();
291
302
  return true;
292
303
  }
293
304
 
305
+ /** Tell nixamp.com about the file itself, once: its hash and what this server knows of it. */
306
+ private async record(): Promise<void> {
307
+ const describe = this.media?.describe;
308
+ if (!describe) return;
309
+ const session = this.options.session();
310
+ if (session === null) return;
311
+ try {
312
+ const described = await describe();
313
+ if (!described || this.stopped) return;
314
+ this.hash = described.id;
315
+ const got = await keepMedia(session, described.id, described.keep, this.options.fetcher ?? fetch);
316
+ if (!got.ok) this.complain(`the record was not kept: ${got.error}`);
317
+ else this.options.onEvent?.(`captions for "${this.id}": nixamp.com/hash/${described.id.slice(0, 12)}… knows this file`);
318
+ } catch (error) {
319
+ this.complain(`could not describe the file: ${(error as Error).message}`);
320
+ }
321
+ }
322
+
294
323
  /** Ask the store what it already knows of this media in a language, once. */
295
324
  private async consult(language: string): Promise<void> {
296
325
  if (!this.transcriptId || this.asked.has(language)) return;
@@ -512,6 +541,7 @@ class Captioner {
512
541
  language: this.language,
513
542
  known: this.known.get("")?.length ?? 0,
514
543
  languages: [...this.linesBy.keys()],
544
+ hash: this.hash,
515
545
  };
516
546
  }
517
547
 
@@ -572,7 +602,7 @@ export class Captions {
572
602
  /** Whether a channel is being captioned, and what last went wrong if the lines are not coming. */
573
603
  status(id: string): CaptionStatus {
574
604
  const captioner = this.running.get(id);
575
- return captioner ? captioner.status() : { on: false, lines: 0, error: "", language: "", known: 0, languages: [] };
605
+ return captioner ? captioner.status() : { on: false, lines: 0, error: "", language: "", known: 0, languages: [], hash: "" };
576
606
  }
577
607
 
578
608
  stopAll(): void {
package/src/hash.ts ADDED
@@ -0,0 +1,139 @@
1
+ /**
2
+ * `nixamp hash` -- one address for a file: nixamp.com/hash/<sha256>.
3
+ *
4
+ * Hashes every byte, says the address, and unless told not to tells
5
+ * nixamp.com what this machine knows about the file (media-local.ts) so
6
+ * the address answers with it: what is inside, what nichedb says it is,
7
+ * and, once `nixamp transcribe` or a server has been at it, the
8
+ * transcript in every language. The file goes into this machine's index
9
+ * too, so the daemon looks at it again on a schedule and a changed file
10
+ * gets a new address that points back at this one.
11
+ */
12
+ import { detectTools } from "./audio.ts";
13
+ import { Enricher } from "./enrich.ts";
14
+ import { describeFile, keepFile, type Described } from "./media-local.ts";
15
+ import { readSession, type Session } from "./session.ts";
16
+ import { fetchMedia } from "./transcript-client.ts";
17
+
18
+ const HELP = `nixamp hash — one address for a file.
19
+
20
+ nixamp hash FILE [FILE...] sha256, and nixamp.com/hash/<sha256>, kept there with what is known
21
+ nixamp hash FILE --no-keep the hash and the address only; nothing sent
22
+ nixamp hash FILE --json the record as nixamp.com keeps it
23
+ nixamp hash --get ID what nixamp.com knows about a file, by hash or fingerprint
24
+
25
+ The address is the SHA-256 of the file's bytes, the way OpenFile
26
+ (logicsrc.com/docs/openfile) names a file, so the same file on two
27
+ machines is one page. What is kept: the size, the type, when the file last
28
+ changed, what ffprobe found inside, what nichedb.dev says it is, where it
29
+ has been carried, and its transcripts. Keeping needs a sign-in
30
+ (\`nixamp login\`); the hash needs nothing.
31
+ `;
32
+
33
+ export interface HashDeps {
34
+ fetcher?: typeof fetch;
35
+ session?: Pick<Session, "site" | "token"> | null;
36
+ describe?: (path: string) => Promise<Described>;
37
+ index?: string;
38
+ now?: () => number;
39
+ }
40
+
41
+ function flag(argv: string[], name: string): string | undefined {
42
+ const at = argv.indexOf(name);
43
+ return at === -1 ? undefined : argv[at + 1];
44
+ }
45
+
46
+ /** The site an address is printed for: the session's, or nixamp.com. */
47
+ function siteOf(session: Pick<Session, "site"> | null): string {
48
+ return (session?.site ?? "https://nixamp.com").replace(/\/+$/, "");
49
+ }
50
+
51
+ export async function hash(argv: string[], deps: HashDeps = {}): Promise<number> {
52
+ if (argv.length === 0 || argv.includes("--help") || argv.includes("-h") || argv[0] === "help") {
53
+ console.log(HELP);
54
+ return argv.length === 0 ? 64 : 0;
55
+ }
56
+ const session = deps.session === undefined ? readSession() : deps.session;
57
+ const fetcher = deps.fetcher ?? fetch;
58
+ const asJson = argv.includes("--json");
59
+ const site = siteOf(session);
60
+
61
+ const get = flag(argv, "--get");
62
+ if (argv.includes("--get")) {
63
+ if (!get) {
64
+ console.error("nixamp: --get needs the file's hash, or its fingerprint.");
65
+ return 64;
66
+ }
67
+ const got = await fetchMedia(site, get, fetcher);
68
+ if (!got.ok) {
69
+ console.error(`nixamp: ${got.error}`);
70
+ return 1;
71
+ }
72
+ if (asJson) console.log(JSON.stringify(got.body, null, 2));
73
+ else console.log(describeRecord(got.body));
74
+ return 0;
75
+ }
76
+
77
+ const files = argv.filter((one, at) => !one.startsWith("-") && !(at > 0 && argv[at - 1] === "--get"));
78
+ if (files.length === 0) {
79
+ console.error("nixamp: which file? `nixamp hash film.mkv`.");
80
+ return 64;
81
+ }
82
+ const keep = !argv.includes("--no-keep");
83
+ if (keep && session === null) console.error("nixamp: not signed in, so nothing is kept; `nixamp login` to keep the record. The hash still is:");
84
+ let tools: ReturnType<typeof detectTools> | null = null;
85
+ let enricher: Enricher | null = null;
86
+ const describe = deps.describe ?? (async (path: string) => {
87
+ tools ??= detectTools();
88
+ enricher ??= new Enricher();
89
+ return describeFile(path, { tools, enricher, now: deps.now });
90
+ });
91
+ let failed = 0;
92
+ const records: unknown[] = [];
93
+ for (const file of files) {
94
+ let described: Described;
95
+ try {
96
+ described = await describe(file);
97
+ } catch (error) {
98
+ console.error(`nixamp: ${file}: ${(error as Error).message}`);
99
+ failed += 1;
100
+ continue;
101
+ }
102
+ let record: Record<string, unknown> | null = null;
103
+ if (keep && session) {
104
+ const refused = await keepFile({ site: session.site, token: session.token }, file, described, { fetcher, ...(deps.index ? { index: deps.index } : {}), ...(deps.now ? { now: deps.now } : {}) });
105
+ if (refused) {
106
+ console.error(`nixamp: ${file}: not kept: ${refused}`);
107
+ failed += 1;
108
+ } else if (asJson) {
109
+ const got = await fetchMedia(site, described.id, fetcher);
110
+ record = got.ok ? got.body : null;
111
+ }
112
+ }
113
+ if (asJson) {
114
+ records.push(record ?? { id: `sha256:${described.id}`, name: described.keep.name, url: `${site}/hash/${described.id}`, nixamp: described.keep.facts });
115
+ } else {
116
+ console.log(`sha256:${described.id} ${described.keep.name ?? file}\n ${site}/hash/${described.id}${keep && session ? "" : " (not kept)"}`);
117
+ }
118
+ }
119
+ if (asJson) console.log(JSON.stringify(records.length === 1 ? records[0] : records, null, 2));
120
+ return failed > 0 ? 1 : 0;
121
+ }
122
+
123
+ /** A record as the terminal prints it. */
124
+ export function describeRecord(record: Record<string, unknown>): string {
125
+ const nixamp = (record["nixamp"] ?? {}) as Record<string, unknown>;
126
+ const lines = [`${record["name"] ?? ""} ${record["id"] ?? ""}`, ` ${record["url"] ?? ""}`];
127
+ if (typeof record["size"] === "number") lines.push(` size: ${record["size"]} bytes${record["contentType"] ? `, ${record["contentType"]}` : ""}`);
128
+ if (typeof nixamp["duration"] === "number") lines.push(` length: ${Math.round(nixamp["duration"] as number)} s`);
129
+ const enrichment = nixamp["enrichment"] as { title?: string; year?: number | null } | undefined;
130
+ if (enrichment?.title) lines.push(` nichedb: ${enrichment.title}${enrichment.year ? ` (${enrichment.year})` : ""}`);
131
+ if (record["updated"]) lines.push(` file changed: ${record["updated"]}`);
132
+ if (nixamp["checkAfter"]) lines.push(` next check: ${nixamp["checkAfter"]}`);
133
+ const holders = (record["holders"] ?? []) as { url: string; channel?: string; seenAt: string }[];
134
+ for (const holder of holders) lines.push(` carried by ${holder.url}${holder.channel ? ` as ${holder.channel}` : ""} at ${holder.seenAt}`);
135
+ const transcripts = (nixamp["transcripts"] ?? []) as { language: string; translatedFrom: string | null; lines: number; complete: boolean; url: string }[];
136
+ for (const one of transcripts) lines.push(` transcript ${one.language || "as spoken"}${one.translatedFrom ? ` (from ${one.translatedFrom})` : ""}: ${one.lines} lines${one.complete ? "" : " so far"} ${one.url}`);
137
+ if (nixamp["supersededBy"]) lines.push(` became: ${nixamp["supersededBy"]}`);
138
+ return lines.join("\n");
139
+ }
package/src/main.ts CHANGED
@@ -87,6 +87,7 @@ const HELP = `nixamp — it really whips the terminal's ass.
87
87
  nixamp transcribe FILE [--translate sv] a recording or a whole film written down, kept, and in other languages
88
88
  nixamp transcript --channel ID [--follow] what a channel is saying, as it says it; --kept for what nixamp.com keeps
89
89
  nixamp translate --to sv TEXT say it in another language
90
+ nixamp hash FILE one address for a file: nixamp.com/hash/<sha256>, with all that is known
90
91
  nixamp profile [--handle H] [--voice V] [--profile URL] who the rooms know you as
91
92
  nixamp voices the voices a line is read in on the phone
92
93
  nixamp opendir list|add|remove folders found on the web, published for everyone
@@ -274,9 +275,10 @@ posted straight into a trollbox), say a line in a room (trollbox_say), read
274
275
  a room (trollbox_read), read what a channel is saying (transcript_read, in
275
276
  any language). The transcript tools: a kept transcript by its id or its
276
277
  media (transcript_get), what this account has had written down
277
- (transcripts_list), and text in another language (translate_text). And who
278
- you are in the rooms and how you sound on the phone (profile_get,
279
- profile_set, voices_list). It acts as whoever this machine is signed in as,
278
+ (transcripts_list), and text in another language (translate_text). The file
279
+ tools: one address for a file (media_hash) and what nixamp.com knows about
280
+ one (media_get). And who you are in the rooms and how you sound on the
281
+ phone (profile_get, profile_set, voices_list). It acts as whoever this machine is signed in as,
280
282
  so \`nixamp login\` (or NIXAMP_TOKEN) comes first.
281
283
 
282
284
  Point an MCP client at it as a stdio server running \`nixamp mcp\`. The same
@@ -309,6 +311,23 @@ with an open-source model (OPUS-MT), and kept beside the original.
309
311
 
310
312
  The same ear is behind the microphone button in every nixamp.com trollbox,
311
313
  and behind the transcribe_audio tool of \`nixamp mcp\`.
314
+ `,
315
+ hash: `nixamp hash — one address for a file.
316
+
317
+ nixamp hash FILE [FILE...] sha256, and nixamp.com/hash/<sha256>, kept there with what is known
318
+ nixamp hash FILE --no-keep the hash and the address only; nothing sent
319
+ nixamp hash FILE --json the record as nixamp.com keeps it
320
+ nixamp hash --get ID what nixamp.com knows about a file, by hash or fingerprint
321
+
322
+ The address is the SHA-256 of the file's bytes, the way OpenFile
323
+ (logicsrc.com/docs/openfile) names a file, so the same file on two machines
324
+ is one page. The page, and /hash/<id>.openfile.json beside it, carry the
325
+ size, the type, when the file last changed, what ffprobe found inside, what
326
+ nichedb.dev says it is, which servers have carried it, and its transcripts
327
+ in every language, as .srt, .vtt or text. \`nixamp transcribe\` and a server
328
+ captioning a file keep the same record. The daemon looks at a kept file
329
+ again on a schedule set by how recently it changed, and a changed file
330
+ gets a new address that points back at the old one.
312
331
  `,
313
332
  translate: `nixamp translate — say it in another language.
314
333
 
@@ -588,6 +607,11 @@ export async function main(): Promise<void> {
588
607
  process.exitCode = await translate(rest);
589
608
  return;
590
609
  }
610
+ if (first === "hash") {
611
+ const { hash } = await import("./hash.ts");
612
+ process.exitCode = await hash(rest);
613
+ return;
614
+ }
591
615
  if (first === "token" || first === "tokens") {
592
616
  const { tokens } = await import("./session.ts");
593
617
  process.exitCode = await tokens(rest);
package/src/mcp.ts CHANGED
@@ -23,7 +23,11 @@ import { clock, type PartyRow } from "./party.ts";
23
23
  import { readSession } from "./session.ts";
24
24
  import { askToHear, awaitTranscript, hearWhole, rendered, wavOf, type Window } from "./transcribe.ts";
25
25
  import { readTranscript } from "./transcript.ts";
26
- import { fetchTranscript, listTranscripts, translateTexts, type StoredTranscript } from "./transcript-client.ts";
26
+ import { fetchMedia, fetchTranscript, listTranscripts, translateTexts, type StoredTranscript } from "./transcript-client.ts";
27
+ import { detectTools } from "./audio.ts";
28
+ import { Enricher } from "./enrich.ts";
29
+ import { describeRecord } from "./hash.ts";
30
+ import { describeFile, keepFile, type Described } from "./media-local.ts";
27
31
  import { fileFingerprint, idFrom, languageCode, mediaOfUrl, transcriptIdOf } from "./transcripts.ts";
28
32
  import { personaLines, readPersona, readVoices, writePersona } from "./profile.ts";
29
33
 
@@ -134,6 +138,24 @@ export const TOOLS: ToolDefinition[] = [
134
138
  required: ["media"],
135
139
  },
136
140
  },
141
+ {
142
+ name: "media_hash",
143
+ description:
144
+ "One address for a file on this machine: its SHA-256 and nixamp.com/hash/<sha256>, kept there with what this machine knows (size, type, when it changed, what is inside, what nichedb says it is). The same file anywhere is the same page.",
145
+ inputSchema: {
146
+ type: "object",
147
+ properties: {
148
+ path: { ...STRING, description: "The file's path on this machine." },
149
+ keep: { type: "boolean", description: "Tell nixamp.com about it (default true)." },
150
+ },
151
+ required: ["path"],
152
+ },
153
+ },
154
+ {
155
+ name: "media_get",
156
+ description: "What nixamp.com knows about a file, by its SHA-256 (with or without sha256:), its transcript fingerprint, or a transcript id: the OpenFile record with nixamp's facts, holders and transcripts.",
157
+ inputSchema: { type: "object", properties: { id: { ...STRING, description: "The hash, fingerprint or transcript id." } }, required: ["id"] },
158
+ },
137
159
  {
138
160
  name: "transcripts_list",
139
161
  description: "What this account has had written down on nixamp.com: each transcript's id, what it is, its language, how many lines, and when.",
@@ -245,6 +267,8 @@ export interface McpOptions {
245
267
  sleep?: (ms: number) => Promise<void>;
246
268
  /** How many times a translation is asked about before answering with its progress. */
247
269
  polls?: number;
270
+ /** How a file is described for its record; the tests hand in a fake. */
271
+ describe?: (path: string) => Promise<Described>;
248
272
  say?: (line: string) => void;
249
273
  }
250
274
 
@@ -439,6 +463,30 @@ export async function callTool(name: string, args: Record<string, unknown>, opti
439
463
  return text(transcriptText(got.body, typeof args["format"] === "string" ? args["format"] : "lines"));
440
464
  }
441
465
 
466
+ if (name === "media_hash") {
467
+ const path = typeof args["path"] === "string" ? args["path"] : "";
468
+ if (!path) return failed("Which file? Pass its path.");
469
+ let described: Described;
470
+ try {
471
+ described = await (options.describe ?? ((file: string) => describeFile(file, { tools: detectTools(), enricher: new Enricher() })))(path);
472
+ } catch (error) {
473
+ return failed(`${path}: ${(error as Error).message}`);
474
+ }
475
+ if (args["keep"] !== false) {
476
+ const refused = await keepFile({ site, token: session.token }, path, described, { fetcher: send });
477
+ if (refused) return failed(`hashed, but not kept: ${refused}`);
478
+ }
479
+ return text(`sha256:${described.id}\n${site}/hash/${described.id}${args["keep"] === false ? "\n(not kept)" : ""}`);
480
+ }
481
+
482
+ if (name === "media_get") {
483
+ const id = typeof args["id"] === "string" ? args["id"].trim() : "";
484
+ if (!id) return failed("Which file? Pass its hash, fingerprint or transcript id.");
485
+ const got = await fetchMedia(site, id, send);
486
+ if (!got.ok) return failed(got.error);
487
+ return text(describeRecord(got.body));
488
+ }
489
+
442
490
  if (name === "transcripts_list") {
443
491
  const got = await listTranscripts({ site, token: session.token }, send);
444
492
  if (!got.ok) return failed(got.error);
@@ -0,0 +1,145 @@
1
+ /**
2
+ * The files this machine has told nixamp.com about, and when to look again.
3
+ *
4
+ * nixamp.com keeps the record (media.ts); the bytes stay here. So whether a
5
+ * file has changed is a question only this machine can answer, and it
6
+ * answers it on a schedule that follows the file: a quarter of the time
7
+ * since it last changed, never more often than every quarter of an hour
8
+ * and never less often than monthly (see checkInterval). Looking is one
9
+ * stat. Only a file whose size or modification time moved is hashed
10
+ * again, and a new hash is a new record on nixamp.com, with the old one
11
+ * told what it became.
12
+ *
13
+ * The index is a JSON file beside the daemon's other state. The CLI adds
14
+ * to it when it hashes or transcribes a file, the server when it captions
15
+ * one, and the server's watcher reads it every few minutes for what is
16
+ * due. Written whole and renamed into place, so two of them writing at
17
+ * once lose an entry at worst, never the file.
18
+ */
19
+ import { mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
20
+ import { dirname, join } from "node:path";
21
+ import { stateDir } from "./daemon.ts";
22
+ import { checkInterval } from "./media.ts";
23
+
24
+ export interface IndexedFile {
25
+ /** The SHA-256 hex nixamp.com knows it by. */
26
+ id: string;
27
+ fingerprint: string;
28
+ size: number;
29
+ mtimeMs: number;
30
+ checkedAt: number;
31
+ checkAfter: number;
32
+ }
33
+
34
+ export interface IndexFile {
35
+ version: 1;
36
+ files: Record<string, IndexedFile>;
37
+ }
38
+
39
+ export function indexPath(): string {
40
+ return process.env["NIXAMP_MEDIA_INDEX"] ?? join(stateDir(), "media.json");
41
+ }
42
+
43
+ export function readIndex(path = indexPath()): IndexFile {
44
+ try {
45
+ const parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<IndexFile>;
46
+ if (parsed && parsed.version === 1 && parsed.files && typeof parsed.files === "object") return { version: 1, files: parsed.files };
47
+ } catch {
48
+ // No index yet, or one this cannot read: start again.
49
+ }
50
+ return { version: 1, files: {} };
51
+ }
52
+
53
+ export function writeIndex(index: IndexFile, path = indexPath()): void {
54
+ mkdirSync(dirname(path), { recursive: true });
55
+ const temp = `${path}.${process.pid}.tmp`;
56
+ writeFileSync(temp, JSON.stringify(index, null, 2));
57
+ renameSync(temp, path);
58
+ }
59
+
60
+ /** Note a file this machine has told nixamp.com about, with when to look at it next. */
61
+ export function remember(file: string, entry: { id: string; fingerprint: string; size: number; mtimeMs: number }, now = Date.now(), path = indexPath()): IndexedFile {
62
+ const index = readIndex(path);
63
+ const kept: IndexedFile = { ...entry, checkedAt: now, checkAfter: now + checkInterval(entry.mtimeMs, now) };
64
+ index.files[file] = kept;
65
+ writeIndex(index, path);
66
+ return kept;
67
+ }
68
+
69
+ /** The files whose turn it is. */
70
+ export function due(index: IndexFile, now = Date.now()): string[] {
71
+ return Object.entries(index.files).filter(([, one]) => one.checkAfter <= now).map(([file]) => file);
72
+ }
73
+
74
+ export type Looked =
75
+ | { state: "same"; entry: IndexedFile }
76
+ | { state: "changed"; entry: IndexedFile; size: number; mtimeMs: number }
77
+ | { state: "gone" };
78
+
79
+ /**
80
+ * One look at a file: the same as it was, changed, or gone. Nothing is
81
+ * hashed here; the caller hashes a changed file, since that is the part
82
+ * that costs, and then remembers the new record.
83
+ */
84
+ export function look(file: string, entry: IndexedFile, now = Date.now()): Looked {
85
+ let stat: { size: number; mtimeMs: number };
86
+ try {
87
+ stat = statSync(file);
88
+ } catch {
89
+ return { state: "gone" };
90
+ }
91
+ if (stat.size === entry.size && Math.floor(stat.mtimeMs) === Math.floor(entry.mtimeMs)) {
92
+ return { state: "same", entry: { ...entry, checkedAt: now, checkAfter: now + checkInterval(entry.mtimeMs, now) } };
93
+ }
94
+ return { state: "changed", entry, size: stat.size, mtimeMs: stat.mtimeMs };
95
+ }
96
+
97
+ /** How often the watcher reads the index for what is due. */
98
+ export const WATCH_EVERY_MS = 10 * 60 * 1000;
99
+
100
+ export interface WatchOptions {
101
+ path?: string;
102
+ now?: () => number;
103
+ /** Hash the changed file, tell nixamp.com, and answer the new entry; null when it could not. */
104
+ refresh: (file: string, before: IndexedFile, stat: { size: number; mtimeMs: number }) => Promise<{ id: string; fingerprint: string } | null>;
105
+ onEvent?: (message: string) => void;
106
+ }
107
+
108
+ /**
109
+ * One pass over what is due. Files that are the same are pushed out to
110
+ * their next check; changed ones are refreshed; gone ones are forgotten.
111
+ * Answers how many of each, for the log.
112
+ */
113
+ export async function watchOnce(options: WatchOptions): Promise<{ same: number; changed: number; gone: number }> {
114
+ const now = options.now ?? Date.now;
115
+ const path = options.path ?? indexPath();
116
+ const index = readIndex(path);
117
+ const counts = { same: 0, changed: 0, gone: 0 };
118
+ for (const file of due(index, now())) {
119
+ const entry = index.files[file];
120
+ if (!entry) continue;
121
+ const looked = look(file, entry, now());
122
+ if (looked.state === "gone") {
123
+ delete index.files[file];
124
+ counts.gone += 1;
125
+ continue;
126
+ }
127
+ if (looked.state === "same") {
128
+ index.files[file] = looked.entry;
129
+ counts.same += 1;
130
+ continue;
131
+ }
132
+ const fresh = await options.refresh(file, entry, { size: looked.size, mtimeMs: looked.mtimeMs });
133
+ const at = now();
134
+ if (fresh) {
135
+ index.files[file] = { ...fresh, size: looked.size, mtimeMs: looked.mtimeMs, checkedAt: at, checkAfter: at + checkInterval(looked.mtimeMs, at) };
136
+ counts.changed += 1;
137
+ options.onEvent?.(`${file} changed: now sha256:${fresh.id.slice(0, 12)} (was ${entry.id.slice(0, 12)})`);
138
+ } else {
139
+ // Could not say; ask again in a quarter of an hour rather than never.
140
+ index.files[file] = { ...entry, checkedAt: at, checkAfter: at + 15 * 60 * 1000 };
141
+ }
142
+ }
143
+ writeIndex(index, path);
144
+ return counts;
145
+ }
@@ -0,0 +1,117 @@
1
+ /**
2
+ * A file on this machine, described for nixamp.com's record of it.
3
+ *
4
+ * The same few steps whoever meets the file: hash every byte (the OpenFile
5
+ * id), take the quick fingerprint (the transcript key), ask ffprobe what is
6
+ * inside, ask nichedb what it is, note where it is being carried, and say
7
+ * when this machine will look at it again. `nixamp hash` does it for a
8
+ * file you name, `nixamp transcribe` for the one it is writing down, and a
9
+ * server for a file it puts on the air. A file that changed since the
10
+ * machine last looked is described again as a new record, and the two
11
+ * records point at each other.
12
+ */
13
+ import { basename, extname } from "node:path";
14
+ import { codecsOf, type Tools } from "./audio.ts";
15
+ import type { Enricher } from "./enrich.ts";
16
+ import { checkInterval, contentHash, factsFrom, fileFacts, type Holder, type MediaFacts } from "./media.ts";
17
+ import { remember, type IndexedFile } from "./media-index.ts";
18
+ import { keepMedia, type MediaToKeep, type Signed } from "./transcript-client.ts";
19
+ import { fileFingerprint } from "./transcripts.ts";
20
+
21
+ export interface Described {
22
+ id: string;
23
+ fingerprint: string;
24
+ size: number;
25
+ mtimeMs: number;
26
+ keep: MediaToKeep;
27
+ }
28
+
29
+ export interface DescribeDeps {
30
+ /** ffprobe, for what is inside. Absent: only what the filesystem says. */
31
+ tools?: Tools | null;
32
+ enricher?: Pick<Enricher, "lookup"> | null;
33
+ holder?: Holder | null;
34
+ now?: () => number;
35
+ /** The tests hand in fakes for the two things that cost. */
36
+ hash?: (path: string) => Promise<string>;
37
+ fingerprint?: (path: string) => string;
38
+ }
39
+
40
+ /** Everything this machine can say about a file, ready to keep. */
41
+ export async function describeFile(path: string, deps: DescribeDeps = {}): Promise<Described> {
42
+ const now = deps.now ?? Date.now;
43
+ const file = fileFacts(path);
44
+ const id = await (deps.hash ?? contentHash)(path);
45
+ const fingerprint = (deps.fingerprint ?? fileFingerprint)(path);
46
+ const codecs = deps.tools && deps.tools.carries !== false ? await codecsOf(deps.tools, path) : null;
47
+ const name = codecs?.tags?.title || basename(path, extname(path));
48
+ let enriched = null;
49
+ try {
50
+ enriched = deps.enricher ? await deps.enricher.lookup(name) : null;
51
+ } catch {
52
+ enriched = null;
53
+ }
54
+ const at = now();
55
+ const facts: MediaFacts = {
56
+ ...factsFrom(codecs, enriched),
57
+ fingerprint,
58
+ checkedAt: new Date(at).toISOString(),
59
+ checkAfter: new Date(at + checkInterval(file.mtimeMs, at)).toISOString(),
60
+ };
61
+ return {
62
+ id,
63
+ fingerprint,
64
+ size: file.size,
65
+ mtimeMs: file.mtimeMs,
66
+ keep: {
67
+ name: file.name,
68
+ size: file.size,
69
+ contentType: file.contentType,
70
+ updated: file.updated,
71
+ facts: facts as Record<string, unknown>,
72
+ ...(deps.holder ? { holder: deps.holder } : {}),
73
+ },
74
+ };
75
+ }
76
+
77
+ /** Keep a described file on nixamp.com and in this machine's index. Answers the store's sentence when it refused. */
78
+ export async function keepFile(
79
+ signed: Signed,
80
+ path: string,
81
+ described: Described,
82
+ deps: { fetcher?: typeof fetch; index?: string; now?: () => number } = {},
83
+ ): Promise<string | null> {
84
+ const got = await keepMedia(signed, described.id, described.keep, deps.fetcher ?? fetch);
85
+ if (!got.ok) return got.error;
86
+ try {
87
+ remember(path, { id: described.id, fingerprint: described.fingerprint, size: described.size, mtimeMs: described.mtimeMs }, (deps.now ?? Date.now)(), deps.index);
88
+ } catch {
89
+ // The index is a convenience; the record is what matters.
90
+ }
91
+ return null;
92
+ }
93
+
94
+ /**
95
+ * A file that changed since it was last described: described again as a
96
+ * new record that says what it was, while the old record says what it
97
+ * became. Answers the new identity, or null when nixamp.com would not
98
+ * keep it.
99
+ */
100
+ export async function refreshChanged(
101
+ signed: Signed,
102
+ path: string,
103
+ before: Pick<IndexedFile, "id">,
104
+ deps: DescribeDeps & { fetcher?: typeof fetch; index?: string } = {},
105
+ ): Promise<{ id: string; fingerprint: string } | null> {
106
+ const described = await describeFile(path, deps);
107
+ if (described.id === before.id) {
108
+ // Touched but not changed: the same bytes, a new modification time.
109
+ const failed = await keepFile(signed, path, described, deps);
110
+ return failed ? null : { id: described.id, fingerprint: described.fingerprint };
111
+ }
112
+ described.keep.facts = { ...(described.keep.facts ?? {}), supersedes: `sha256:${before.id}` };
113
+ const failed = await keepFile(signed, path, described, deps);
114
+ if (failed) return null;
115
+ await keepMedia(signed, before.id, { facts: { supersededBy: `sha256:${described.id}` } }, deps.fetcher ?? fetch);
116
+ return { id: described.id, fingerprint: described.fingerprint };
117
+ }