@openparachute/vault 0.2.3 → 0.3.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +8 -0
- package/CHANGELOG.md +70 -0
- package/CLAUDE.md +17 -7
- package/README.md +169 -136
- package/core/src/core.test.ts +603 -19
- package/core/src/indexed-fields.test.ts +285 -0
- package/core/src/indexed-fields.ts +238 -0
- package/core/src/mcp.ts +127 -6
- package/core/src/notes.ts +157 -11
- package/core/src/query-operators.ts +174 -0
- package/core/src/schema.ts +69 -2
- package/core/src/store.ts +92 -0
- package/core/src/tag-schemas.ts +5 -0
- package/core/src/types.ts +29 -1
- package/docs/HTTP_API.md +105 -1
- package/package/package.json +32 -0
- package/package.json +2 -2
- package/src/auth.test.ts +83 -114
- package/src/auth.ts +68 -6
- package/src/backup-launchd.ts +1 -1
- package/src/backup.test.ts +1 -1
- package/src/backup.ts +18 -17
- package/src/cli.ts +179 -121
- package/src/config-triggers.test.ts +49 -0
- package/src/config.test.ts +317 -2
- package/src/config.ts +420 -40
- package/src/context.test.ts +136 -0
- package/src/context.ts +115 -0
- package/src/daemon.ts +17 -16
- package/src/doctor.test.ts +9 -7
- package/src/launchd.test.ts +1 -1
- package/src/launchd.ts +6 -6
- package/src/mcp-http.ts +75 -21
- package/src/mcp-install.test.ts +125 -0
- package/src/mcp-install.ts +60 -0
- package/src/mcp-tools.ts +34 -96
- package/src/module-config.ts +109 -0
- package/src/oauth.test.ts +345 -57
- package/src/oauth.ts +155 -35
- package/src/published.test.ts +2 -2
- package/src/routes.ts +209 -33
- package/src/routing.test.ts +817 -300
- package/src/routing.ts +204 -202
- package/src/scopes.test.ts +136 -0
- package/src/scopes.ts +105 -0
- package/src/scribe-env.test.ts +49 -0
- package/src/scribe-env.ts +33 -0
- package/src/server.ts +57 -5
- package/src/services-manifest.test.ts +140 -0
- package/src/services-manifest.ts +99 -0
- package/src/systemd.ts +3 -3
- package/src/token-store.ts +42 -9
- package/src/transcription-worker.test.ts +583 -0
- package/src/transcription-worker.ts +346 -0
- package/src/triggers.test.ts +191 -1
- package/src/triggers.ts +17 -2
- package/src/vault.test.ts +693 -77
- package/src/version.test.ts +1 -1
package/src/token-store.ts
CHANGED
|
@@ -16,6 +16,11 @@
|
|
|
16
16
|
import { Database } from "bun:sqlite";
|
|
17
17
|
import crypto from "node:crypto";
|
|
18
18
|
import { hashKey } from "./config.ts";
|
|
19
|
+
import { legacyPermissionToScopes, parseScopes, serializeScopes } from "./scopes.ts";
|
|
20
|
+
|
|
21
|
+
function scopesForMigratedPermission(permission: string): string {
|
|
22
|
+
return serializeScopes(legacyPermissionToScopes(permission));
|
|
23
|
+
}
|
|
19
24
|
|
|
20
25
|
// ---------------------------------------------------------------------------
|
|
21
26
|
// Types
|
|
@@ -47,6 +52,15 @@ export interface Token {
|
|
|
47
52
|
|
|
48
53
|
export interface ResolvedToken {
|
|
49
54
|
permission: TokenPermission;
|
|
55
|
+
/**
|
|
56
|
+
* Granted scopes, parsed from the token row's `scopes` column. Pre-v12
|
|
57
|
+
* tokens (where the column is NULL) fall back to the legacy permission
|
|
58
|
+
* → scopes mapping and `legacyDerived` is set true so callers can log
|
|
59
|
+
* a deprecation warning on first use.
|
|
60
|
+
*/
|
|
61
|
+
scopes: string[];
|
|
62
|
+
/** True iff `scopes` was derived from the legacy `permission` column. */
|
|
63
|
+
legacyDerived: boolean;
|
|
50
64
|
}
|
|
51
65
|
|
|
52
66
|
// ---------------------------------------------------------------------------
|
|
@@ -65,6 +79,12 @@ export function createToken(
|
|
|
65
79
|
opts: {
|
|
66
80
|
label: string;
|
|
67
81
|
permission?: TokenPermission;
|
|
82
|
+
/**
|
|
83
|
+
* Explicit OAuth-standard scopes to persist. If omitted, derived from
|
|
84
|
+
* `permission` (read → [vault:read], anything else → [vault:read,
|
|
85
|
+
* vault:write, vault:admin]). Written as a whitespace-separated string.
|
|
86
|
+
*/
|
|
87
|
+
scopes?: string[];
|
|
68
88
|
/** @deprecated Written to DB but not enforced at runtime. */
|
|
69
89
|
scope_tag?: string | null;
|
|
70
90
|
/** @deprecated Written to DB but not enforced at runtime. */
|
|
@@ -75,14 +95,17 @@ export function createToken(
|
|
|
75
95
|
const tokenHash = hashKey(fullToken);
|
|
76
96
|
const now = new Date().toISOString();
|
|
77
97
|
const permission = opts.permission ?? "full";
|
|
98
|
+
const scopes = opts.scopes ?? legacyPermissionToScopes(permission);
|
|
99
|
+
const scopesStr = serializeScopes(scopes);
|
|
78
100
|
|
|
79
101
|
db.prepare(`
|
|
80
|
-
INSERT INTO tokens (token_hash, label, permission, scope_tag, scope_path_prefix, expires_at, created_at)
|
|
81
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
102
|
+
INSERT INTO tokens (token_hash, label, permission, scopes, scope_tag, scope_path_prefix, expires_at, created_at)
|
|
103
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
82
104
|
`).run(
|
|
83
105
|
tokenHash,
|
|
84
106
|
opts.label,
|
|
85
107
|
permission,
|
|
108
|
+
scopesStr,
|
|
86
109
|
opts.scope_tag ?? null,
|
|
87
110
|
opts.scope_path_prefix ?? null,
|
|
88
111
|
opts.expires_at ?? null,
|
|
@@ -112,11 +135,12 @@ export function resolveToken(db: Database, providedToken: string): ResolvedToken
|
|
|
112
135
|
const candidateHash = hashKey(providedToken);
|
|
113
136
|
|
|
114
137
|
const row = db.prepare(`
|
|
115
|
-
SELECT token_hash, permission, expires_at
|
|
138
|
+
SELECT token_hash, permission, scopes, expires_at
|
|
116
139
|
FROM tokens WHERE token_hash = ?
|
|
117
140
|
`).get(candidateHash) as {
|
|
118
141
|
token_hash: string;
|
|
119
142
|
permission: string;
|
|
143
|
+
scopes: string | null;
|
|
120
144
|
expires_at: string | null;
|
|
121
145
|
} | null;
|
|
122
146
|
|
|
@@ -131,7 +155,13 @@ export function resolveToken(db: Database, providedToken: string): ResolvedToken
|
|
|
131
155
|
db.prepare("UPDATE tokens SET last_used_at = ? WHERE token_hash = ?")
|
|
132
156
|
.run(new Date().toISOString(), row.token_hash);
|
|
133
157
|
|
|
134
|
-
|
|
158
|
+
const permission = normalizePermission(row.permission);
|
|
159
|
+
const parsed = parseScopes(row.scopes);
|
|
160
|
+
const hasVaultScope = parsed.some((s) => s.startsWith("vault:"));
|
|
161
|
+
const scopes = hasVaultScope ? parsed : legacyPermissionToScopes(permission);
|
|
162
|
+
const legacyDerived = !hasVaultScope;
|
|
163
|
+
|
|
164
|
+
return { permission, scopes, legacyDerived };
|
|
135
165
|
}
|
|
136
166
|
|
|
137
167
|
/**
|
|
@@ -203,13 +233,15 @@ export function migrateVaultKeys(
|
|
|
203
233
|
for (const key of vaultKeys) {
|
|
204
234
|
const exists = db.prepare("SELECT 1 FROM tokens WHERE token_hash = ?").get(key.key_hash);
|
|
205
235
|
if (!exists) {
|
|
236
|
+
const permission = key.scope === "read" ? "read" : "full";
|
|
206
237
|
db.prepare(`
|
|
207
|
-
INSERT INTO tokens (token_hash, label, permission, created_at, last_used_at)
|
|
208
|
-
VALUES (?, ?, ?, ?, ?)
|
|
238
|
+
INSERT INTO tokens (token_hash, label, permission, scopes, created_at, last_used_at)
|
|
239
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
209
240
|
`).run(
|
|
210
241
|
key.key_hash,
|
|
211
242
|
key.label,
|
|
212
|
-
|
|
243
|
+
permission,
|
|
244
|
+
scopesForMigratedPermission(permission),
|
|
213
245
|
key.created_at,
|
|
214
246
|
key.last_used_at ?? null,
|
|
215
247
|
);
|
|
@@ -223,12 +255,13 @@ export function migrateVaultKeys(
|
|
|
223
255
|
const exists = db.prepare("SELECT 1 FROM tokens WHERE token_hash = ?").get(key.key_hash);
|
|
224
256
|
if (!exists) {
|
|
225
257
|
db.prepare(`
|
|
226
|
-
INSERT INTO tokens (token_hash, label, permission, created_at, last_used_at)
|
|
227
|
-
VALUES (?, ?, ?, ?, ?)
|
|
258
|
+
INSERT INTO tokens (token_hash, label, permission, scopes, created_at, last_used_at)
|
|
259
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
228
260
|
`).run(
|
|
229
261
|
key.key_hash,
|
|
230
262
|
key.label,
|
|
231
263
|
"full",
|
|
264
|
+
scopesForMigratedPermission("full"),
|
|
232
265
|
key.created_at,
|
|
233
266
|
key.last_used_at ?? null,
|
|
234
267
|
);
|
|
@@ -0,0 +1,583 @@
|
|
|
1
|
+
import { describe, test, expect, beforeEach, afterEach } from "bun:test";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
import { mkdirSync, rmSync, writeFileSync, existsSync } from "fs";
|
|
4
|
+
import { join } from "path";
|
|
5
|
+
import { tmpdir } from "os";
|
|
6
|
+
import { BunStore } from "./vault-store.ts";
|
|
7
|
+
import { startTranscriptionWorker } from "./transcription-worker.ts";
|
|
8
|
+
import type { Store } from "../core/src/types.ts";
|
|
9
|
+
|
|
10
|
+
let db: Database;
|
|
11
|
+
let store: BunStore;
|
|
12
|
+
let tmpDir: string;
|
|
13
|
+
let assetsRoot: string;
|
|
14
|
+
|
|
15
|
+
const silentLogger = { error: () => {}, info: () => {} };
|
|
16
|
+
|
|
17
|
+
beforeEach(() => {
|
|
18
|
+
tmpDir = join(tmpdir(), `transcribe-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
19
|
+
mkdirSync(tmpDir, { recursive: true });
|
|
20
|
+
assetsRoot = join(tmpDir, "assets");
|
|
21
|
+
mkdirSync(assetsRoot, { recursive: true });
|
|
22
|
+
db = new Database(join(tmpDir, "test.db"));
|
|
23
|
+
store = new BunStore(db);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
afterEach(() => {
|
|
27
|
+
db.close();
|
|
28
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
function mkFetchMock(responses: Array<{ text: string } | { error: string; status?: number }>): typeof fetch {
|
|
32
|
+
let i = 0;
|
|
33
|
+
return (async (_url: RequestInfo | URL, _init?: RequestInit) => {
|
|
34
|
+
const r = responses[Math.min(i, responses.length - 1)];
|
|
35
|
+
i++;
|
|
36
|
+
if ("error" in r) {
|
|
37
|
+
return new Response(r.error, { status: r.status ?? 500 });
|
|
38
|
+
}
|
|
39
|
+
return new Response(JSON.stringify({ text: r.text }), {
|
|
40
|
+
status: 200,
|
|
41
|
+
headers: { "content-type": "application/json" },
|
|
42
|
+
});
|
|
43
|
+
}) as typeof fetch;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function seedAudio(relPath: string): string {
|
|
47
|
+
const full = join(assetsRoot, relPath);
|
|
48
|
+
mkdirSync(join(full, "..").toString(), { recursive: true });
|
|
49
|
+
writeFileSync(full, Buffer.from([1, 2, 3, 4]));
|
|
50
|
+
return full;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function makeWorker(opts: {
|
|
54
|
+
fetchImpl: typeof fetch;
|
|
55
|
+
retention?: "keep" | "until_transcribed" | "never";
|
|
56
|
+
maxAttempts?: number;
|
|
57
|
+
}) {
|
|
58
|
+
return startTranscriptionWorker({
|
|
59
|
+
vaultList: () => ["default"],
|
|
60
|
+
getStore: () => store as unknown as Store,
|
|
61
|
+
scribeUrl: "http://scribe.test",
|
|
62
|
+
resolveAssetsDir: () => assetsRoot,
|
|
63
|
+
getAudioRetention: () => opts.retention ?? "keep",
|
|
64
|
+
pollIntervalMs: 10_000_000, // never auto-fire; tests drive ticks manually
|
|
65
|
+
maxAttempts: opts.maxAttempts ?? 3,
|
|
66
|
+
fetchImpl: opts.fetchImpl,
|
|
67
|
+
logger: silentLogger,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
describe("transcription worker", () => {
|
|
72
|
+
test("happy path: replaces _Transcript pending._ and clears stub marker", async () => {
|
|
73
|
+
const note = await store.createNote(
|
|
74
|
+
"# 🎙️ Voice memo\n\n_Transcript pending._\n",
|
|
75
|
+
{ id: "n1", metadata: { transcribe_stub: true } },
|
|
76
|
+
);
|
|
77
|
+
seedAudio("memos/a.webm");
|
|
78
|
+
await store.addAttachment(note.id, "memos/a.webm", "audio/webm", {
|
|
79
|
+
transcribe_status: "pending",
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
const worker = makeWorker({
|
|
83
|
+
fetchImpl: mkFetchMock([{ text: "hello world transcript" }]),
|
|
84
|
+
});
|
|
85
|
+
try {
|
|
86
|
+
const processed = await worker.tick();
|
|
87
|
+
expect(processed).toBe(1);
|
|
88
|
+
} finally {
|
|
89
|
+
await worker.stop();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const updated = await store.getNote("n1");
|
|
93
|
+
expect(updated!.content).toBe("# 🎙️ Voice memo\n\nhello world transcript\n");
|
|
94
|
+
expect((updated!.metadata as any)?.transcribe_stub).toBeUndefined();
|
|
95
|
+
|
|
96
|
+
const [att] = await store.getAttachments("n1");
|
|
97
|
+
expect(att.metadata?.transcribe_status).toBe("done");
|
|
98
|
+
expect(att.metadata?.transcript).toBe("hello world transcript");
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("no-clobber: stub flag absent → does not touch note content", async () => {
|
|
102
|
+
await store.createNote("my own edit", { id: "n2" });
|
|
103
|
+
seedAudio("memos/b.webm");
|
|
104
|
+
await store.addAttachment("n2", "memos/b.webm", "audio/webm", {
|
|
105
|
+
transcribe_status: "pending",
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
const worker = makeWorker({
|
|
109
|
+
fetchImpl: mkFetchMock([{ text: "would clobber" }]),
|
|
110
|
+
});
|
|
111
|
+
try {
|
|
112
|
+
await worker.tick();
|
|
113
|
+
} finally {
|
|
114
|
+
await worker.stop();
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const updated = await store.getNote("n2");
|
|
118
|
+
expect(updated!.content).toBe("my own edit");
|
|
119
|
+
const [att] = await store.getAttachments("n2");
|
|
120
|
+
// Transcript still captured on the attachment — we don't throw work away,
|
|
121
|
+
// we just don't overwrite the note the user explicitly edited.
|
|
122
|
+
expect(att.metadata?.transcribe_status).toBe("done");
|
|
123
|
+
expect(att.metadata?.transcript).toBe("would clobber");
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("no placeholder: replaces full body when stub is set", async () => {
|
|
127
|
+
await store.createNote("", {
|
|
128
|
+
id: "n3",
|
|
129
|
+
metadata: { transcribe_stub: true },
|
|
130
|
+
});
|
|
131
|
+
seedAudio("memos/c.webm");
|
|
132
|
+
await store.addAttachment("n3", "memos/c.webm", "audio/webm", {
|
|
133
|
+
transcribe_status: "pending",
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const worker = makeWorker({
|
|
137
|
+
fetchImpl: mkFetchMock([{ text: "bare transcript" }]),
|
|
138
|
+
});
|
|
139
|
+
try {
|
|
140
|
+
await worker.tick();
|
|
141
|
+
} finally {
|
|
142
|
+
await worker.stop();
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const updated = await store.getNote("n3");
|
|
146
|
+
expect(updated!.content).toBe("bare transcript");
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("retry on failure: status stays pending with backoff + attempts bumped", async () => {
|
|
150
|
+
await store.createNote("stub", {
|
|
151
|
+
id: "n4",
|
|
152
|
+
metadata: { transcribe_stub: true },
|
|
153
|
+
});
|
|
154
|
+
seedAudio("memos/d.webm");
|
|
155
|
+
await store.addAttachment("n4", "memos/d.webm", "audio/webm", {
|
|
156
|
+
transcribe_status: "pending",
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
const worker = makeWorker({
|
|
160
|
+
fetchImpl: mkFetchMock([{ error: "scribe down", status: 503 }]),
|
|
161
|
+
maxAttempts: 3,
|
|
162
|
+
});
|
|
163
|
+
try {
|
|
164
|
+
await worker.tick();
|
|
165
|
+
} finally {
|
|
166
|
+
await worker.stop();
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const [att] = await store.getAttachments("n4");
|
|
170
|
+
expect(att.metadata?.transcribe_status).toBe("pending");
|
|
171
|
+
expect(att.metadata?.transcribe_attempts).toBe(1);
|
|
172
|
+
expect(att.metadata?.transcribe_backoff_until).toBeTruthy();
|
|
173
|
+
expect(att.metadata?.transcribe_error).toContain("503");
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test("gives up after maxAttempts → status failed", async () => {
|
|
177
|
+
await store.createNote("stub", {
|
|
178
|
+
id: "n5",
|
|
179
|
+
metadata: { transcribe_stub: true },
|
|
180
|
+
});
|
|
181
|
+
seedAudio("memos/e.webm");
|
|
182
|
+
// Simulate already 2 attempts done — one more failure flips to failed
|
|
183
|
+
// when maxAttempts=3.
|
|
184
|
+
await store.addAttachment("n5", "memos/e.webm", "audio/webm", {
|
|
185
|
+
transcribe_status: "pending",
|
|
186
|
+
transcribe_attempts: 2,
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
const worker = makeWorker({
|
|
190
|
+
fetchImpl: mkFetchMock([{ error: "boom", status: 500 }]),
|
|
191
|
+
maxAttempts: 3,
|
|
192
|
+
});
|
|
193
|
+
try {
|
|
194
|
+
await worker.tick();
|
|
195
|
+
} finally {
|
|
196
|
+
await worker.stop();
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const [att] = await store.getAttachments("n5");
|
|
200
|
+
expect(att.metadata?.transcribe_status).toBe("failed");
|
|
201
|
+
expect(att.metadata?.transcribe_attempts).toBe(3);
|
|
202
|
+
expect(att.metadata?.transcribe_error).toContain("boom");
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("FIFO: oldest pending is processed first", async () => {
|
|
206
|
+
await store.createNote("s", { id: "f1", metadata: { transcribe_stub: true } });
|
|
207
|
+
await store.createNote("s", { id: "f2", metadata: { transcribe_stub: true } });
|
|
208
|
+
seedAudio("memos/first.webm");
|
|
209
|
+
seedAudio("memos/second.webm");
|
|
210
|
+
await store.addAttachment("f1", "memos/first.webm", "audio/webm", {
|
|
211
|
+
transcribe_status: "pending",
|
|
212
|
+
});
|
|
213
|
+
// Ensure a distinct created_at — bun:sqlite stores ISO timestamps at ms
|
|
214
|
+
// granularity, so sleep briefly.
|
|
215
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
216
|
+
await store.addAttachment("f2", "memos/second.webm", "audio/webm", {
|
|
217
|
+
transcribe_status: "pending",
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
const calls: string[] = [];
|
|
221
|
+
const worker = makeWorker({
|
|
222
|
+
fetchImpl: (async () => {
|
|
223
|
+
calls.push("call");
|
|
224
|
+
return new Response(JSON.stringify({ text: `t${calls.length}` }), {
|
|
225
|
+
status: 200,
|
|
226
|
+
headers: { "content-type": "application/json" },
|
|
227
|
+
});
|
|
228
|
+
}) as typeof fetch,
|
|
229
|
+
});
|
|
230
|
+
try {
|
|
231
|
+
await worker.tick();
|
|
232
|
+
} finally {
|
|
233
|
+
await worker.stop();
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const n1 = await store.getNote("f1");
|
|
237
|
+
const n2 = await store.getNote("f2");
|
|
238
|
+
expect(n1!.content).toBe("t1");
|
|
239
|
+
expect(n2!.content).toBe("t2");
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
test("backoff gate skips attachments whose backoff has not elapsed", async () => {
|
|
243
|
+
await store.createNote("s", { id: "b1", metadata: { transcribe_stub: true } });
|
|
244
|
+
seedAudio("memos/b1.webm");
|
|
245
|
+
const future = new Date(Date.now() + 60_000).toISOString();
|
|
246
|
+
await store.addAttachment("b1", "memos/b1.webm", "audio/webm", {
|
|
247
|
+
transcribe_status: "pending",
|
|
248
|
+
transcribe_attempts: 1,
|
|
249
|
+
transcribe_backoff_until: future,
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
let called = 0;
|
|
253
|
+
const worker = makeWorker({
|
|
254
|
+
fetchImpl: (async () => {
|
|
255
|
+
called++;
|
|
256
|
+
return new Response(JSON.stringify({ text: "x" }), { status: 200 });
|
|
257
|
+
}) as typeof fetch,
|
|
258
|
+
});
|
|
259
|
+
try {
|
|
260
|
+
await worker.tick();
|
|
261
|
+
} finally {
|
|
262
|
+
await worker.stop();
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
expect(called).toBe(0);
|
|
266
|
+
const [att] = await store.getAttachments("b1");
|
|
267
|
+
expect(att.metadata?.transcribe_status).toBe("pending");
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test("retention=until_transcribed unlinks the audio file after success", async () => {
|
|
271
|
+
await store.createNote("s", { id: "r1", metadata: { transcribe_stub: true } });
|
|
272
|
+
const full = seedAudio("memos/r1.webm");
|
|
273
|
+
await store.addAttachment("r1", "memos/r1.webm", "audio/webm", {
|
|
274
|
+
transcribe_status: "pending",
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
const worker = makeWorker({
|
|
278
|
+
fetchImpl: mkFetchMock([{ text: "t" }]),
|
|
279
|
+
retention: "until_transcribed",
|
|
280
|
+
});
|
|
281
|
+
try {
|
|
282
|
+
await worker.tick();
|
|
283
|
+
} finally {
|
|
284
|
+
await worker.stop();
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
expect(existsSync(full)).toBe(false);
|
|
288
|
+
// Attachment row preserved — transcript still addressable.
|
|
289
|
+
const [att] = await store.getAttachments("r1");
|
|
290
|
+
expect(att.metadata?.transcribe_status).toBe("done");
|
|
291
|
+
expect(att.metadata?.transcript).toBe("t");
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
test("retention=never unlinks the audio file after success", async () => {
|
|
295
|
+
await store.createNote("s", { id: "rn1", metadata: { transcribe_stub: true } });
|
|
296
|
+
const full = seedAudio("memos/rn1.webm");
|
|
297
|
+
await store.addAttachment("rn1", "memos/rn1.webm", "audio/webm", {
|
|
298
|
+
transcribe_status: "pending",
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
const worker = makeWorker({
|
|
302
|
+
fetchImpl: mkFetchMock([{ text: "t" }]),
|
|
303
|
+
retention: "never",
|
|
304
|
+
});
|
|
305
|
+
try {
|
|
306
|
+
await worker.tick();
|
|
307
|
+
} finally {
|
|
308
|
+
await worker.stop();
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
expect(existsSync(full)).toBe(false);
|
|
312
|
+
const [att] = await store.getAttachments("rn1");
|
|
313
|
+
expect(att.metadata?.transcribe_status).toBe("done");
|
|
314
|
+
expect(att.metadata?.transcript).toBe("t");
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
test("retention=never unlinks the audio file after terminal failure", async () => {
|
|
318
|
+
await store.createNote("s", { id: "rn2", metadata: { transcribe_stub: true } });
|
|
319
|
+
const full = seedAudio("memos/rn2.webm");
|
|
320
|
+
// Pre-seed attempts=2 so a single tick with maxAttempts=3 is terminal.
|
|
321
|
+
await store.addAttachment("rn2", "memos/rn2.webm", "audio/webm", {
|
|
322
|
+
transcribe_status: "pending",
|
|
323
|
+
transcribe_attempts: 2,
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
const worker = makeWorker({
|
|
327
|
+
fetchImpl: mkFetchMock([{ error: "boom", status: 500 }]),
|
|
328
|
+
retention: "never",
|
|
329
|
+
maxAttempts: 3,
|
|
330
|
+
});
|
|
331
|
+
try {
|
|
332
|
+
await worker.tick();
|
|
333
|
+
} finally {
|
|
334
|
+
await worker.stop();
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const [att] = await store.getAttachments("rn2");
|
|
338
|
+
expect(att.metadata?.transcribe_status).toBe("failed");
|
|
339
|
+
// The whole point of "never": audio gone even when transcription failed.
|
|
340
|
+
expect(existsSync(full)).toBe(false);
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
test("retention=never keeps the audio file during non-terminal retry", async () => {
|
|
344
|
+
await store.createNote("s", { id: "rn3", metadata: { transcribe_stub: true } });
|
|
345
|
+
const full = seedAudio("memos/rn3.webm");
|
|
346
|
+
// attempts=0 so a single failure is retry-pending, not terminal.
|
|
347
|
+
await store.addAttachment("rn3", "memos/rn3.webm", "audio/webm", {
|
|
348
|
+
transcribe_status: "pending",
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
const worker = makeWorker({
|
|
352
|
+
fetchImpl: mkFetchMock([{ error: "transient", status: 503 }]),
|
|
353
|
+
retention: "never",
|
|
354
|
+
maxAttempts: 3,
|
|
355
|
+
});
|
|
356
|
+
try {
|
|
357
|
+
await worker.tick();
|
|
358
|
+
} finally {
|
|
359
|
+
await worker.stop();
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const [att] = await store.getAttachments("rn3");
|
|
363
|
+
expect(att.metadata?.transcribe_status).toBe("pending");
|
|
364
|
+
// File must remain for the retry to have something to send.
|
|
365
|
+
expect(existsSync(full)).toBe(true);
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
test("retention=keep leaves the audio file in place after success", async () => {
|
|
369
|
+
await store.createNote("s", { id: "k1", metadata: { transcribe_stub: true } });
|
|
370
|
+
const full = seedAudio("memos/k1.webm");
|
|
371
|
+
await store.addAttachment("k1", "memos/k1.webm", "audio/webm", {
|
|
372
|
+
transcribe_status: "pending",
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
const worker = makeWorker({
|
|
376
|
+
fetchImpl: mkFetchMock([{ text: "t" }]),
|
|
377
|
+
retention: "keep",
|
|
378
|
+
});
|
|
379
|
+
try {
|
|
380
|
+
await worker.tick();
|
|
381
|
+
} finally {
|
|
382
|
+
await worker.stop();
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
expect(existsSync(full)).toBe(true);
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
test("missing audio file → flips to failed, no infinite retry", async () => {
|
|
389
|
+
await store.createNote("s", { id: "m1", metadata: { transcribe_stub: true } });
|
|
390
|
+
await store.addAttachment("m1", "memos/not-there.webm", "audio/webm", {
|
|
391
|
+
transcribe_status: "pending",
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
let called = 0;
|
|
395
|
+
const worker = makeWorker({
|
|
396
|
+
fetchImpl: (async () => {
|
|
397
|
+
called++;
|
|
398
|
+
return new Response("x", { status: 200 });
|
|
399
|
+
}) as typeof fetch,
|
|
400
|
+
});
|
|
401
|
+
try {
|
|
402
|
+
await worker.tick();
|
|
403
|
+
} finally {
|
|
404
|
+
await worker.stop();
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
expect(called).toBe(0);
|
|
408
|
+
const [att] = await store.getAttachments("m1");
|
|
409
|
+
expect(att.metadata?.transcribe_status).toBe("failed");
|
|
410
|
+
expect(att.metadata?.transcribe_error).toContain("audio file not found");
|
|
411
|
+
});
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
describe("transcription worker — auth + context", () => {
|
|
415
|
+
test("attaches multipart context part when getContextPredicates returns entries", async () => {
|
|
416
|
+
await store.createNote("stub", { id: "ctx1", metadata: { transcribe_stub: true } });
|
|
417
|
+
seedAudio("memos/ctx1.webm");
|
|
418
|
+
await store.addAttachment("ctx1", "memos/ctx1.webm", "audio/webm", {
|
|
419
|
+
transcribe_status: "pending",
|
|
420
|
+
});
|
|
421
|
+
// Seed a context note the worker will fetch via queryNotes.
|
|
422
|
+
await store.createNote("", {
|
|
423
|
+
id: "p1",
|
|
424
|
+
path: "People/Aaron.md",
|
|
425
|
+
tags: ["person"],
|
|
426
|
+
metadata: { summary: "founder", aliases: ["AG"] },
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
let captured: { headers: Headers; form: FormData } | null = null;
|
|
430
|
+
const fetchImpl = (async (_url: RequestInfo | URL, init?: RequestInit) => {
|
|
431
|
+
const form = init?.body as unknown as FormData;
|
|
432
|
+
captured = { headers: new Headers(init?.headers as HeadersInit), form };
|
|
433
|
+
return new Response(JSON.stringify({ text: "ok" }), {
|
|
434
|
+
status: 200,
|
|
435
|
+
headers: { "content-type": "application/json" },
|
|
436
|
+
});
|
|
437
|
+
}) as typeof fetch;
|
|
438
|
+
|
|
439
|
+
const worker = startTranscriptionWorker({
|
|
440
|
+
vaultList: () => ["default"],
|
|
441
|
+
getStore: () => store as unknown as Store,
|
|
442
|
+
scribeUrl: "http://scribe.test",
|
|
443
|
+
resolveAssetsDir: () => assetsRoot,
|
|
444
|
+
getContextPredicates: () => [
|
|
445
|
+
{ tag: "person", include_metadata: ["summary", "aliases"] },
|
|
446
|
+
],
|
|
447
|
+
pollIntervalMs: 10_000_000,
|
|
448
|
+
fetchImpl,
|
|
449
|
+
logger: silentLogger,
|
|
450
|
+
});
|
|
451
|
+
try {
|
|
452
|
+
await worker.tick();
|
|
453
|
+
} finally {
|
|
454
|
+
await worker.stop();
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
expect(captured).not.toBeNull();
|
|
458
|
+
const part = captured!.form.get("context");
|
|
459
|
+
expect(part).toBeInstanceOf(Blob);
|
|
460
|
+
const body = JSON.parse(await (part as Blob).text());
|
|
461
|
+
expect(body.entries).toEqual([
|
|
462
|
+
{ name: "Aaron", summary: "founder", aliases: ["AG"] },
|
|
463
|
+
]);
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
test("sends Bearer header when scribeToken is set", async () => {
|
|
467
|
+
await store.createNote("stub", { id: "auth1", metadata: { transcribe_stub: true } });
|
|
468
|
+
seedAudio("memos/auth1.webm");
|
|
469
|
+
await store.addAttachment("auth1", "memos/auth1.webm", "audio/webm", {
|
|
470
|
+
transcribe_status: "pending",
|
|
471
|
+
});
|
|
472
|
+
|
|
473
|
+
let capturedAuth: string | null = null;
|
|
474
|
+
const fetchImpl = (async (_url: RequestInfo | URL, init?: RequestInit) => {
|
|
475
|
+
capturedAuth = new Headers(init?.headers as HeadersInit).get("authorization");
|
|
476
|
+
return new Response(JSON.stringify({ text: "ok" }), { status: 200 });
|
|
477
|
+
}) as typeof fetch;
|
|
478
|
+
|
|
479
|
+
const worker = startTranscriptionWorker({
|
|
480
|
+
vaultList: () => ["default"],
|
|
481
|
+
getStore: () => store as unknown as Store,
|
|
482
|
+
scribeUrl: "http://scribe.test",
|
|
483
|
+
scribeToken: "shh-secret",
|
|
484
|
+
resolveAssetsDir: () => assetsRoot,
|
|
485
|
+
pollIntervalMs: 10_000_000,
|
|
486
|
+
fetchImpl,
|
|
487
|
+
logger: silentLogger,
|
|
488
|
+
});
|
|
489
|
+
try {
|
|
490
|
+
await worker.tick();
|
|
491
|
+
} finally {
|
|
492
|
+
await worker.stop();
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
expect(capturedAuth).toBe("Bearer shh-secret");
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
test("omits Authorization header when scribeToken is unset (loopback back-compat)", async () => {
|
|
499
|
+
await store.createNote("stub", { id: "auth2", metadata: { transcribe_stub: true } });
|
|
500
|
+
seedAudio("memos/auth2.webm");
|
|
501
|
+
await store.addAttachment("auth2", "memos/auth2.webm", "audio/webm", {
|
|
502
|
+
transcribe_status: "pending",
|
|
503
|
+
});
|
|
504
|
+
|
|
505
|
+
let capturedAuth: string | null | undefined = undefined;
|
|
506
|
+
const fetchImpl = (async (_url: RequestInfo | URL, init?: RequestInit) => {
|
|
507
|
+
capturedAuth = new Headers(init?.headers as HeadersInit).get("authorization");
|
|
508
|
+
return new Response(JSON.stringify({ text: "ok" }), { status: 200 });
|
|
509
|
+
}) as typeof fetch;
|
|
510
|
+
|
|
511
|
+
const worker = startTranscriptionWorker({
|
|
512
|
+
vaultList: () => ["default"],
|
|
513
|
+
getStore: () => store as unknown as Store,
|
|
514
|
+
scribeUrl: "http://scribe.test",
|
|
515
|
+
resolveAssetsDir: () => assetsRoot,
|
|
516
|
+
pollIntervalMs: 10_000_000,
|
|
517
|
+
fetchImpl,
|
|
518
|
+
logger: silentLogger,
|
|
519
|
+
});
|
|
520
|
+
try {
|
|
521
|
+
await worker.tick();
|
|
522
|
+
} finally {
|
|
523
|
+
await worker.stop();
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// Headers#get returns null when absent — this is how we confirm no header was set.
|
|
527
|
+
expect(capturedAuth).toBeNull();
|
|
528
|
+
});
|
|
529
|
+
|
|
530
|
+
test("no context attached when getContextPredicates is undefined (no regression)", async () => {
|
|
531
|
+
await store.createNote("stub", { id: "np1", metadata: { transcribe_stub: true } });
|
|
532
|
+
seedAudio("memos/np1.webm");
|
|
533
|
+
await store.addAttachment("np1", "memos/np1.webm", "audio/webm", {
|
|
534
|
+
transcribe_status: "pending",
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
let capturedForm: FormData | null = null;
|
|
538
|
+
const fetchImpl = (async (_url: RequestInfo | URL, init?: RequestInit) => {
|
|
539
|
+
capturedForm = init?.body as unknown as FormData;
|
|
540
|
+
return new Response(JSON.stringify({ text: "ok" }), { status: 200 });
|
|
541
|
+
}) as typeof fetch;
|
|
542
|
+
|
|
543
|
+
const worker = startTranscriptionWorker({
|
|
544
|
+
vaultList: () => ["default"],
|
|
545
|
+
getStore: () => store as unknown as Store,
|
|
546
|
+
scribeUrl: "http://scribe.test",
|
|
547
|
+
resolveAssetsDir: () => assetsRoot,
|
|
548
|
+
pollIntervalMs: 10_000_000,
|
|
549
|
+
fetchImpl,
|
|
550
|
+
logger: silentLogger,
|
|
551
|
+
});
|
|
552
|
+
try {
|
|
553
|
+
await worker.tick();
|
|
554
|
+
} finally {
|
|
555
|
+
await worker.stop();
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
expect(capturedForm).not.toBeNull();
|
|
559
|
+
expect(capturedForm!.get("context")).toBeNull();
|
|
560
|
+
expect(capturedForm!.get("file")).not.toBeNull();
|
|
561
|
+
});
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
describe("store.listAttachmentsByTranscribeStatus", () => {
|
|
565
|
+
test("returns only matching status, oldest first", async () => {
|
|
566
|
+
await store.createNote("s", { id: "q1" });
|
|
567
|
+
await store.addAttachment("q1", "a.webm", "audio/webm", { transcribe_status: "done" });
|
|
568
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
569
|
+
await store.addAttachment("q1", "b.webm", "audio/webm", { transcribe_status: "pending" });
|
|
570
|
+
await new Promise((r) => setTimeout(r, 5));
|
|
571
|
+
await store.addAttachment("q1", "c.webm", "audio/webm", { transcribe_status: "pending" });
|
|
572
|
+
await store.addAttachment("q1", "d.webm", "audio/webm"); // no status
|
|
573
|
+
|
|
574
|
+
const pending = await store.listAttachmentsByTranscribeStatus("pending");
|
|
575
|
+
expect(pending).toHaveLength(2);
|
|
576
|
+
expect(pending[0]!.path).toBe("b.webm");
|
|
577
|
+
expect(pending[1]!.path).toBe("c.webm");
|
|
578
|
+
|
|
579
|
+
const done = await store.listAttachmentsByTranscribeStatus("done");
|
|
580
|
+
expect(done).toHaveLength(1);
|
|
581
|
+
expect(done[0]!.path).toBe("a.webm");
|
|
582
|
+
});
|
|
583
|
+
});
|