@openparachute/vault 0.6.5-rc.2 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.parachute/module.json +1 -0
- package/core/src/core.test.ts +7 -7
- package/core/src/do-param-cap.test.ts +161 -0
- package/core/src/links.ts +8 -13
- package/core/src/mcp.ts +1 -1
- package/core/src/notes.ts +33 -15
- package/core/src/onboarding.ts +14 -296
- package/core/src/schema.ts +5 -5
- package/core/src/seed-packs.test.ts +357 -0
- package/core/src/seed-packs.ts +823 -0
- package/core/src/sql-in.test.ts +58 -0
- package/core/src/sql-in.ts +76 -0
- package/core/src/tag-hierarchy.ts +1 -1
- package/core/src/tag-schemas.ts +2 -2
- package/core/src/transcription/provider.ts +141 -0
- package/core/src/types.ts +1 -1
- package/core/src/vault-projection.ts +1 -1
- package/core/src/wikilinks.ts +10 -4
- package/package.json +1 -1
- package/src/add-pack.test.ts +142 -0
- package/src/admin-spa.ts +2 -1
- package/src/auth.ts +1 -1
- package/src/cli.ts +806 -7
- package/src/export-watch.ts +1 -1
- package/src/live-frame-parity.test.ts +201 -0
- package/src/module-manifest.ts +8 -0
- package/src/onboarding-seed.test.ts +188 -40
- package/src/onboarding-seed.ts +41 -46
- package/src/routes.ts +20 -3
- package/src/routing.test.ts +2 -2
- package/src/routing.ts +18 -6
- package/src/self-register.test.ts +19 -0
- package/src/self-register.ts +6 -1
- package/src/server.ts +133 -31
- package/src/services-manifest.ts +8 -0
- package/src/subscriptions.ts +100 -42
- package/src/tag-scope.ts +3 -3
- package/src/test-support/live-frame-corpus.ts +72 -0
- package/src/token-store.ts +2 -2
- package/src/transcription/build.test.ts +305 -0
- package/src/transcription/build.ts +332 -0
- package/src/transcription/capability.test.ts +118 -0
- package/src/transcription/capability.ts +96 -0
- package/src/transcription/download.test.ts +101 -0
- package/src/transcription/download.ts +71 -0
- package/src/transcription/install-python.test.ts +366 -0
- package/src/transcription/install-python.ts +471 -0
- package/src/transcription/install.test.ts +167 -0
- package/src/transcription/install.ts +296 -0
- package/src/transcription/providers/onnx-asr.test.ts +229 -0
- package/src/transcription/providers/onnx-asr.ts +239 -0
- package/src/transcription/providers/parakeet-mlx.test.ts +290 -0
- package/src/transcription/providers/parakeet-mlx.ts +242 -0
- package/src/transcription/providers/scribe-http.test.ts +195 -0
- package/src/transcription/providers/scribe-http.ts +144 -0
- package/src/transcription/providers/transcribe-cpp.test.ts +314 -0
- package/src/transcription/providers/transcribe-cpp.ts +293 -0
- package/src/transcription/select.test.ts +299 -0
- package/src/transcription/select.ts +397 -0
- package/src/transcription/tiers.test.ts +197 -0
- package/src/transcription/tiers.ts +184 -0
- package/src/transcription-worker.test.ts +44 -0
- package/src/transcription-worker.ts +57 -122
- package/src/vault-create.test.ts +43 -10
- package/src/vault.test.ts +49 -1
- package/src/ws-server.ts +408 -0
- package/src/ws-subscribe.test.ts +474 -0
- package/src/ws-subscribe.ts +242 -0
- package/src/subscribe.test.ts +0 -609
- package/src/subscribe.ts +0 -248
package/src/routes.ts
CHANGED
|
@@ -29,6 +29,10 @@ import * as linkOps from "../core/src/links.ts";
|
|
|
29
29
|
import * as tagSchemaOps from "../core/src/tag-schemas.ts";
|
|
30
30
|
import { IndexedFieldError } from "../core/src/indexed-fields.ts";
|
|
31
31
|
import { buildVaultProjection, resolveTagInheritance } from "../core/src/vault-projection.ts";
|
|
32
|
+
import {
|
|
33
|
+
resolveTranscriptionCapability,
|
|
34
|
+
type TranscriptionCapability,
|
|
35
|
+
} from "./transcription/capability.ts";
|
|
32
36
|
import { loadSchemaConfig } from "../core/src/schema-defaults.ts";
|
|
33
37
|
import {
|
|
34
38
|
buildExpandVisibility,
|
|
@@ -2085,7 +2089,7 @@ export async function handleTags(
|
|
|
2085
2089
|
|
|
2086
2090
|
// PUT /tags/:name — upsert tag identity row. Body accepts any combination
|
|
2087
2091
|
// of { description, fields, relationships, parent_names }; omitted keys
|
|
2088
|
-
// are preserved, explicit null clears. See
|
|
2092
|
+
// are preserved, explicit null clears. See docs/contracts/tag-data-model.md.
|
|
2089
2093
|
if (req.method === "PUT") {
|
|
2090
2094
|
// Canonical-bare-tag guard (vault#XXX): normalize the upserted tag NAME so
|
|
2091
2095
|
// the existing-field merge read (store.getTagSchema below) and the upsert
|
|
@@ -2110,7 +2114,7 @@ export async function handleTags(
|
|
|
2110
2114
|
* the MCP `update-tag` tool relies on (omitted keys preserved). The
|
|
2111
2115
|
* Schema editor (vault#283) sends the full map + `replace_fields: true`
|
|
2112
2116
|
* so removing a field row actually deletes the field. See
|
|
2113
|
-
*
|
|
2117
|
+
* docs/contracts/tag-data-model.md.
|
|
2114
2118
|
*/
|
|
2115
2119
|
replace_fields?: unknown;
|
|
2116
2120
|
};
|
|
@@ -2207,7 +2211,7 @@ export async function handleTags(
|
|
|
2207
2211
|
// Tag-scoped tokens reference root tags by name; deleting a referenced
|
|
2208
2212
|
// tag would silently orphan the token's allowlist. Fail closed (409)
|
|
2209
2213
|
// and name the offending tokens so the operator can revoke or re-mint
|
|
2210
|
-
// before retrying.
|
|
2214
|
+
// before retrying. docs/contracts/tag-scoped-tokens.md §Dependencies.
|
|
2211
2215
|
const referenced_by = findTokensReferencingTag(store.db, tagName);
|
|
2212
2216
|
if (referenced_by.length > 0) {
|
|
2213
2217
|
return json(
|
|
@@ -2328,11 +2332,24 @@ export async function handleVault(
|
|
|
2328
2332
|
store: Store,
|
|
2329
2333
|
vaultConfig: VaultConfigLike,
|
|
2330
2334
|
persist?: () => void,
|
|
2335
|
+
/**
|
|
2336
|
+
* Injection seam (scribe-fold Phase 1) for the transcription capability
|
|
2337
|
+
* probe on the GET landing. Production omits it and resolves the live
|
|
2338
|
+
* default (scribe-http) provider's availability; tests inject a deterministic
|
|
2339
|
+
* resolver. Kept separate from the `auto_transcribe` POLICY toggle: this
|
|
2340
|
+
* reports whether transcription is actually POSSIBLE (a provider is
|
|
2341
|
+
* configured AND available), which is what a surface gates its mic on.
|
|
2342
|
+
*/
|
|
2343
|
+
resolveCapability: () => Promise<TranscriptionCapability> = resolveTranscriptionCapability,
|
|
2331
2344
|
): Promise<Response> {
|
|
2332
2345
|
const url = new URL(req.url);
|
|
2333
2346
|
|
|
2334
2347
|
if (req.method === "GET") {
|
|
2335
2348
|
const result: Record<string, unknown> = vaultResponse(vaultConfig);
|
|
2349
|
+
// Transcription capability flag — `enabled` iff a provider is configured
|
|
2350
|
+
// and available. `minutes_remaining` is omitted (cloud/plan concern;
|
|
2351
|
+
// self-host is unmetered). This is the field Notes gates the mic on.
|
|
2352
|
+
result.transcription = await resolveCapability();
|
|
2336
2353
|
if (parseBool(parseQuery(url, "include_stats"), false)) {
|
|
2337
2354
|
result.stats = await store.getVaultStats();
|
|
2338
2355
|
}
|
package/src/routing.test.ts
CHANGED
|
@@ -1631,7 +1631,7 @@ describe("scope enforcement on /api/*", () => {
|
|
|
1631
1631
|
expect(res.status).toBe(401);
|
|
1632
1632
|
});
|
|
1633
1633
|
|
|
1634
|
-
// ----- tag-scoped tokens (
|
|
1634
|
+
// ----- tag-scoped tokens (docs/contracts/tag-scoped-tokens.md) -----------------
|
|
1635
1635
|
|
|
1636
1636
|
/**
|
|
1637
1637
|
* Mint a tag-scoped token. Mirrors `mintToken` above but threads
|
|
@@ -1745,7 +1745,7 @@ describe("scope enforcement on /api/*", () => {
|
|
|
1745
1745
|
});
|
|
1746
1746
|
|
|
1747
1747
|
// ----- Q6: orphan-sub-tag fail-open ------------------------------------
|
|
1748
|
-
//
|
|
1748
|
+
// docs/contracts/tag-scoped-tokens.md §Storage: when a sub-tag has no declared
|
|
1749
1749
|
// schema, the string-form root authorizes. Token allowlisted for `health`
|
|
1750
1750
|
// must see `#health/food` even when no `_tags/health/food` schema exists.
|
|
1751
1751
|
|
package/src/routing.ts
CHANGED
|
@@ -73,7 +73,6 @@ import {
|
|
|
73
73
|
type TagScopeCtx,
|
|
74
74
|
type WriteCtx,
|
|
75
75
|
} from "./routes.ts";
|
|
76
|
-
import { handleSubscribe } from "./subscribe.ts";
|
|
77
76
|
import { handleTriggers } from "./triggers-api.ts";
|
|
78
77
|
import { expandTokenTagScope } from "./tag-scope.ts";
|
|
79
78
|
import {
|
|
@@ -856,7 +855,7 @@ export async function route(
|
|
|
856
855
|
|
|
857
856
|
const apiPath = apiMatch[1] ?? "";
|
|
858
857
|
|
|
859
|
-
// Tag-scoped tokens (
|
|
858
|
+
// Tag-scoped tokens (docs/contracts/tag-scoped-tokens.md): expand the token's
|
|
860
859
|
// root-tag allowlist into `{root} ∪ descendants(root)` once per request,
|
|
861
860
|
// so handlers can intersect against the note's tag set without re-walking
|
|
862
861
|
// the `_tags/<name>` hierarchy on every check. `tagScope.allowed` is null
|
|
@@ -883,10 +882,23 @@ export async function route(
|
|
|
883
882
|
};
|
|
884
883
|
|
|
885
884
|
if (apiPath.startsWith("/notes")) return handleNotes(req, store, apiPath.slice(6), vaultName, tagScope, writeCtx);
|
|
886
|
-
// Live-query SSE
|
|
887
|
-
//
|
|
888
|
-
//
|
|
889
|
-
|
|
885
|
+
// Live-query SSE transport REMOVED (WS-hibernation migration Phase 5). The
|
|
886
|
+
// WebSocket binding (an `Upgrade: websocket` GET /subscribe, handled in
|
|
887
|
+
// server.ts before this pipeline) is now the SOLE live transport; polling
|
|
888
|
+
// GET /notes is the client floor beneath it. A non-WS GET /subscribe — a
|
|
889
|
+
// straggler on a cached pre-WS notes-ui bundle — gets a clean 410 Gone
|
|
890
|
+
// pointing at the WS binding so its consumer degrades to polling gracefully;
|
|
891
|
+
// NEVER a 500 or an unhandled path.
|
|
892
|
+
if (apiPath === "/subscribe") {
|
|
893
|
+
return Response.json(
|
|
894
|
+
{
|
|
895
|
+
error:
|
|
896
|
+
"The live-query SSE transport has been removed. Reconnect with an `Upgrade: websocket` request to GET /api/subscribe, or poll GET /notes.",
|
|
897
|
+
code: "SSE_TRANSPORT_REMOVED",
|
|
898
|
+
},
|
|
899
|
+
{ status: 410 },
|
|
900
|
+
);
|
|
901
|
+
}
|
|
890
902
|
if (apiPath.startsWith("/tags")) return handleTags(req, store, apiPath.slice(5), tagScope);
|
|
891
903
|
if (apiPath === "/find-path") return handleFindPath(req, store, tagScope);
|
|
892
904
|
if (apiPath === "/vault") {
|
|
@@ -382,6 +382,25 @@ describe("self-register", () => {
|
|
|
382
382
|
});
|
|
383
383
|
});
|
|
384
384
|
|
|
385
|
+
test("websocket capability flows from manifest to row when set (hub ws-bridge gate)", () => {
|
|
386
|
+
withParachuteHome((home) => {
|
|
387
|
+
const { log, warn } = captureLogs();
|
|
388
|
+
selfRegister({
|
|
389
|
+
version: "0.4.8-rc.3",
|
|
390
|
+
log,
|
|
391
|
+
warn,
|
|
392
|
+
readManifest: () => ({ ...TEST_MANIFEST, websocket: true }),
|
|
393
|
+
resolvePackageRoot: () => "/fake/install/dir",
|
|
394
|
+
listVaults: () => [],
|
|
395
|
+
readGlobalConfig: () => ({ port: 1940 }),
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
const raw = readFileSync(join(home, "services.json"), "utf8");
|
|
399
|
+
const parsed = JSON.parse(raw) as { services: Record<string, unknown>[] };
|
|
400
|
+
expect(parsed.services[0]!.websocket).toBe(true);
|
|
401
|
+
});
|
|
402
|
+
});
|
|
403
|
+
|
|
385
404
|
test("changed=true when installDir differs from prior row", () => {
|
|
386
405
|
withParachuteHome(() => {
|
|
387
406
|
const { log, warn } = captureLogs();
|
package/src/self-register.ts
CHANGED
|
@@ -191,6 +191,7 @@ export function selfRegister(deps: SelfRegisterDeps): SelfRegisterResult {
|
|
|
191
191
|
displayName?: string;
|
|
192
192
|
tagline?: string;
|
|
193
193
|
stripPrefix?: boolean;
|
|
194
|
+
websocket?: boolean;
|
|
194
195
|
} = {
|
|
195
196
|
name: manifest.manifestName,
|
|
196
197
|
port,
|
|
@@ -202,6 +203,9 @@ export function selfRegister(deps: SelfRegisterDeps): SelfRegisterResult {
|
|
|
202
203
|
if (manifest.displayName !== undefined) entry.displayName = manifest.displayName;
|
|
203
204
|
if (manifest.tagline !== undefined) entry.tagline = manifest.tagline;
|
|
204
205
|
if (manifest.stripPrefix !== undefined) entry.stripPrefix = manifest.stripPrefix;
|
|
206
|
+
// Carry the live-query WS capability onto the row so the hub ws-bridge admits
|
|
207
|
+
// upgrades without having to read module.json (the row is its first source).
|
|
208
|
+
if (manifest.websocket !== undefined) entry.websocket = manifest.websocket;
|
|
205
209
|
|
|
206
210
|
// Detect whether the existing row already matches (no-op idempotency
|
|
207
211
|
// signal). We don't gate the write on this — `upsertService` itself is
|
|
@@ -232,7 +236,8 @@ export function selfRegister(deps: SelfRegisterDeps): SelfRegisterResult {
|
|
|
232
236
|
priorRow.health !== health ||
|
|
233
237
|
(priorRow as { displayName?: string }).displayName !== manifest.displayName ||
|
|
234
238
|
(priorRow as { tagline?: string }).tagline !== manifest.tagline ||
|
|
235
|
-
(priorRow as { stripPrefix?: boolean }).stripPrefix !== manifest.stripPrefix
|
|
239
|
+
(priorRow as { stripPrefix?: boolean }).stripPrefix !== manifest.stripPrefix ||
|
|
240
|
+
(priorRow as { websocket?: boolean }).websocket !== manifest.websocket;
|
|
236
241
|
|
|
237
242
|
try {
|
|
238
243
|
upsertImpl(entry);
|
package/src/server.ts
CHANGED
|
@@ -30,6 +30,20 @@ import { setTranscriptionWorker } from "./transcription-registry.ts";
|
|
|
30
30
|
import { assetsDir } from "./routes.ts";
|
|
31
31
|
import { resolveScribeAuthToken, ensureScribeBearer } from "./scribe-env.ts";
|
|
32
32
|
import { getCachedScribeUrl } from "./scribe-discovery.ts";
|
|
33
|
+
import { TranscribeCppProvider } from "./transcription/providers/transcribe-cpp.ts";
|
|
34
|
+
import { ParakeetMlxProvider } from "./transcription/providers/parakeet-mlx.ts";
|
|
35
|
+
import { OnnxAsrProvider } from "./transcription/providers/onnx-asr.ts";
|
|
36
|
+
import {
|
|
37
|
+
resolveTranscriptionProviderName,
|
|
38
|
+
resolveTranscribeCppPaths,
|
|
39
|
+
transcribeCppInstalled,
|
|
40
|
+
resolveParakeetMlxBin,
|
|
41
|
+
resolveParakeetMlxModel,
|
|
42
|
+
resolveOnnxAsrBin,
|
|
43
|
+
resolveOnnxAsrModel,
|
|
44
|
+
parakeetMlxInstalled,
|
|
45
|
+
onnxAsrInstalled,
|
|
46
|
+
} from "./transcription/select.ts";
|
|
33
47
|
import { readEnvFile, setEnvVar } from "./config.ts";
|
|
34
48
|
import { resolveBindHostname } from "./bind.ts";
|
|
35
49
|
import { MirrorManager } from "./mirror-manager.ts";
|
|
@@ -47,6 +61,7 @@ import {
|
|
|
47
61
|
} from "./mirror-config.ts";
|
|
48
62
|
import { GLOBAL_CONFIG_PATH } from "./config.ts";
|
|
49
63
|
import { selfRegister } from "./self-register.ts";
|
|
64
|
+
import { createSubscribeWsBinding, isWebSocketUpgrade } from "./ws-server.ts";
|
|
50
65
|
import { warnLegacyGlobalApiKeys } from "./auth.ts";
|
|
51
66
|
import pkg from "../package.json" with { type: "json" };
|
|
52
67
|
|
|
@@ -107,40 +122,102 @@ registerConfiguredTriggers();
|
|
|
107
122
|
* Authorization header on a 401 and transcription fails with a friendly
|
|
108
123
|
* error captured on the transcript note.
|
|
109
124
|
*/
|
|
110
|
-
const scribeUrl = getCachedScribeUrl();
|
|
111
125
|
let transcriptionWorker: TranscriptionWorker | null = null;
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
// pick up the just-written value without restart.
|
|
117
|
-
const { created, token } = ensureScribeBearer(readEnvFile, setEnvVar);
|
|
118
|
-
if (created) {
|
|
119
|
-
process.env.SCRIBE_AUTH_TOKEN = token;
|
|
120
|
-
console.log("[transcribe] generated SCRIBE_AUTH_TOKEN (32 bytes, base64url) — mirror this value into scribe's config");
|
|
121
|
-
}
|
|
122
|
-
transcriptionWorker = startTranscriptionWorker({
|
|
123
|
-
vaultList: () => listVaults(),
|
|
124
|
-
getStore: (name) => getVaultStore(name),
|
|
125
|
-
scribeUrl,
|
|
126
|
-
scribeToken: resolveScribeAuthToken(),
|
|
127
|
-
resolveAssetsDir: (vault) => assetsDir(vault),
|
|
128
|
-
getAudioRetention: (vault) => readVaultConfig(vault)?.audio_retention ?? "keep",
|
|
129
|
-
getContextPredicates: (vault) => readVaultConfig(vault)?.transcription?.context,
|
|
130
|
-
});
|
|
131
|
-
// Event-driven hot path — the `attachment:created` hook fires the worker
|
|
132
|
-
// in a microtask instead of waiting for the 30s sweep.
|
|
126
|
+
|
|
127
|
+
// Shared worker wiring: the event-driven `attachment:created` hot path + the
|
|
128
|
+
// REST-retry registry. Same for whichever provider the worker was built with.
|
|
129
|
+
function wireTranscriptionWorker(worker: TranscriptionWorker): void {
|
|
133
130
|
registerTranscriptionHook(
|
|
134
131
|
defaultHookRegistry,
|
|
135
|
-
|
|
132
|
+
worker,
|
|
136
133
|
(store) => getVaultNameForStore(store as never),
|
|
137
134
|
);
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
135
|
+
setTranscriptionWorker(worker);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// Fields the worker needs regardless of provider (queue/retention/context).
|
|
139
|
+
const commonWorkerOpts = {
|
|
140
|
+
vaultList: () => listVaults(),
|
|
141
|
+
getStore: (name: string) => getVaultStore(name),
|
|
142
|
+
resolveAssetsDir: (vault: string) => assetsDir(vault),
|
|
143
|
+
getAudioRetention: (vault: string) => readVaultConfig(vault)?.audio_retention ?? "keep",
|
|
144
|
+
getContextPredicates: (vault: string) => readVaultConfig(vault)?.transcription?.context,
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
// Provider selection (scribe-fold Phase 2a). Default is `scribe-http` — unset
|
|
148
|
+
// TRANSCRIPTION_PROVIDER means the existing scribe-http path runs unchanged.
|
|
149
|
+
const providerName = resolveTranscriptionProviderName();
|
|
150
|
+
if (providerName === "transcribe-cpp") {
|
|
151
|
+
// Local, no-Python provider: subprocess a transcribe-cli. Only start the
|
|
152
|
+
// worker when a runnable CLI + model are actually present — otherwise every
|
|
153
|
+
// pending item would terminal-fail with `missing_provider`. (v0.1.1 ships a
|
|
154
|
+
// library, not a CLI, so this gate stays closed until TRANSCRIBE_CPP_BIN or a
|
|
155
|
+
// build-from-source CLI exists.) `transcribeCppInstalled` is a cheap
|
|
156
|
+
// existsSync check (no spawn), matching `available()`.
|
|
157
|
+
const tcPaths = resolveTranscribeCppPaths();
|
|
158
|
+
if (transcribeCppInstalled(tcPaths)) {
|
|
159
|
+
transcriptionWorker = startTranscriptionWorker({
|
|
160
|
+
...commonWorkerOpts,
|
|
161
|
+
provider: new TranscribeCppProvider({ binPath: tcPaths.binPath, modelPath: tcPaths.modelPath }),
|
|
162
|
+
});
|
|
163
|
+
wireTranscriptionWorker(transcriptionWorker);
|
|
164
|
+
console.log(`[transcribe] worker started → transcribe-cpp (${tcPaths.modelPath})`);
|
|
165
|
+
} else {
|
|
166
|
+
console.log(
|
|
167
|
+
"[transcribe] TRANSCRIPTION_PROVIDER=transcribe-cpp but no runnable transcribe-cli + model installed — see `parachute-vault transcription status` (v0.1.1 ships a library, not a CLI; set TRANSCRIBE_CPP_BIN or build one)",
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
} else if (providerName === "parakeet-mlx" || providerName === "onnx-asr") {
|
|
171
|
+
// Python-based local providers (scribe-fold Phase 2b): subprocess the
|
|
172
|
+
// parakeet-mlx / onnx-asr CLI. Only start the worker when a runnable binary
|
|
173
|
+
// resolves (env override → managed venv → PATH) — otherwise every pending
|
|
174
|
+
// item would terminal-fail with `missing_provider`. Cheap existsSync check
|
|
175
|
+
// (no spawn), matching the providers' `available()`.
|
|
176
|
+
const installed = providerName === "parakeet-mlx" ? parakeetMlxInstalled() : onnxAsrInstalled();
|
|
177
|
+
if (installed) {
|
|
178
|
+
const provider =
|
|
179
|
+
providerName === "parakeet-mlx"
|
|
180
|
+
? new ParakeetMlxProvider({
|
|
181
|
+
binPath: resolveParakeetMlxBin(),
|
|
182
|
+
model: resolveParakeetMlxModel(),
|
|
183
|
+
})
|
|
184
|
+
: new OnnxAsrProvider({ binPath: resolveOnnxAsrBin(), model: resolveOnnxAsrModel() });
|
|
185
|
+
transcriptionWorker = startTranscriptionWorker({ ...commonWorkerOpts, provider });
|
|
186
|
+
wireTranscriptionWorker(transcriptionWorker);
|
|
187
|
+
console.log(`[transcribe] worker started → ${providerName}`);
|
|
188
|
+
} else {
|
|
189
|
+
console.log(
|
|
190
|
+
`[transcribe] TRANSCRIPTION_PROVIDER=${providerName} but no runnable ${providerName} binary found — run \`parachute-vault transcription install\` (see \`transcription status\`)`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
142
193
|
} else {
|
|
143
|
-
|
|
194
|
+
// Default scribe-http path — behavior-preserving (Phase 1). Start the worker
|
|
195
|
+
// if scribe is discoverable; otherwise transcription is a no-op.
|
|
196
|
+
//
|
|
197
|
+
// Scribe URL resolution order (per `scribe-discovery.ts`): `SCRIBE_URL` env
|
|
198
|
+
// var, then `~/.parachute/services.json` `parachute-scribe` entry, then none.
|
|
199
|
+
//
|
|
200
|
+
// Bearer generation (vault#353): if neither `SCRIBE_AUTH_TOKEN` nor the
|
|
201
|
+
// legacy `SCRIBE_TOKEN` is set, generate a fresh 32-byte base64url bearer
|
|
202
|
+
// and persist it to vault's `.env` so subsequent restarts use the same
|
|
203
|
+
// value. The operator (or hub install) mirrors this bearer to scribe.
|
|
204
|
+
const scribeUrl = getCachedScribeUrl();
|
|
205
|
+
if (scribeUrl) {
|
|
206
|
+
const { created, token } = ensureScribeBearer(readEnvFile, setEnvVar);
|
|
207
|
+
if (created) {
|
|
208
|
+
process.env.SCRIBE_AUTH_TOKEN = token;
|
|
209
|
+
console.log("[transcribe] generated SCRIBE_AUTH_TOKEN (32 bytes, base64url) — mirror this value into scribe's config");
|
|
210
|
+
}
|
|
211
|
+
transcriptionWorker = startTranscriptionWorker({
|
|
212
|
+
...commonWorkerOpts,
|
|
213
|
+
scribeUrl,
|
|
214
|
+
scribeToken: resolveScribeAuthToken(),
|
|
215
|
+
});
|
|
216
|
+
wireTranscriptionWorker(transcriptionWorker);
|
|
217
|
+
console.log(`[transcribe] worker started → ${scribeUrl}`);
|
|
218
|
+
} else {
|
|
219
|
+
console.log("[transcribe] worker disabled (no scribe in services.json and SCRIBE_URL unset)");
|
|
220
|
+
}
|
|
144
221
|
}
|
|
145
222
|
|
|
146
223
|
if (process.env.VAULT_AUTH_TOKEN?.trim()) {
|
|
@@ -206,9 +283,10 @@ if (listVaults().length === 0) {
|
|
|
206
283
|
}
|
|
207
284
|
writeGlobalConfig(globalConfig);
|
|
208
285
|
console.log(`Auto-created vault "${vaultName}" (API key: ${fullKey})`);
|
|
209
|
-
// Seed the
|
|
210
|
-
//
|
|
211
|
-
//
|
|
286
|
+
// Seed the default packs (welcome web + capture tag, Getting Started
|
|
287
|
+
// guide) so a connected AI can self-orient and help set the vault up —
|
|
288
|
+
// same as the `create`/`init` CLI path. Idempotent + best-effort (never
|
|
289
|
+
// fails first boot). Mirrors createVault() in cli.ts.
|
|
212
290
|
await seedOnboardingNotesBestEffort(getVaultStore(vaultName));
|
|
213
291
|
} else if (firstBoot.source === "env-invalid") {
|
|
214
292
|
// PARACHUTE_VAULT_NAME was set but failed validation — do NOT silently
|
|
@@ -413,10 +491,18 @@ const hostname = resolveBindHostname();
|
|
|
413
491
|
}
|
|
414
492
|
}
|
|
415
493
|
|
|
494
|
+
// Live-query WebSocket binding (WS-hibernation migration Phase 3). Same
|
|
495
|
+
// subscription contract as the SSE route, WebSocket transport. Handlers +
|
|
496
|
+
// upgrade decision live in ws-server.ts; the per-vault WS cap is closure state
|
|
497
|
+
// there. Advertised via `"websocket": true` in `.parachute/module.json` so the
|
|
498
|
+
// hub ws-bridge admits the upgrade.
|
|
499
|
+
const subscribeWs = createSubscribeWsBinding();
|
|
500
|
+
|
|
416
501
|
const server = Bun.serve({
|
|
417
502
|
port,
|
|
418
503
|
hostname,
|
|
419
504
|
idleTimeout: 120, // seconds — webhook triggers can take a while
|
|
505
|
+
websocket: subscribeWs.handlers,
|
|
420
506
|
async fetch(req, server) {
|
|
421
507
|
const url = new URL(req.url);
|
|
422
508
|
const path = url.pathname;
|
|
@@ -440,6 +526,22 @@ const server = Bun.serve({
|
|
|
440
526
|
return new Response(null, { status: 204, headers: corsHeaders });
|
|
441
527
|
}
|
|
442
528
|
|
|
529
|
+
// Live-query WebSocket upgrade — detected BEFORE the fetch pipeline because
|
|
530
|
+
// WS upgrades don't traverse `route()`. Non-upgrade requests (incl. a
|
|
531
|
+
// straggler non-WS GET /subscribe, which now gets a 410 Gone in routing.ts —
|
|
532
|
+
// the SSE transport was removed in Phase 5) fall straight through unchanged.
|
|
533
|
+
if (isWebSocketUpgrade(req)) {
|
|
534
|
+
const verdict = subscribeWs.tryUpgrade(req, server, path);
|
|
535
|
+
if (verdict.kind === "upgraded") return undefined; // Bun owns the socket now
|
|
536
|
+
if (verdict.kind === "response") {
|
|
537
|
+
for (const [k, v] of Object.entries(corsHeaders)) {
|
|
538
|
+
verdict.response.headers.set(k, v);
|
|
539
|
+
}
|
|
540
|
+
return verdict.response;
|
|
541
|
+
}
|
|
542
|
+
// "pass" → not a subscribe WS upgrade; fall through to the pipeline.
|
|
543
|
+
}
|
|
544
|
+
|
|
443
545
|
try {
|
|
444
546
|
const start = Date.now();
|
|
445
547
|
const response = await route(req, path);
|
package/src/services-manifest.ts
CHANGED
|
@@ -15,6 +15,14 @@ export interface ServiceEntry {
|
|
|
15
15
|
paths: string[];
|
|
16
16
|
health: string;
|
|
17
17
|
version: string;
|
|
18
|
+
/**
|
|
19
|
+
* When `true`, the hub's ws-bridge forwards `Upgrade: websocket` requests on
|
|
20
|
+
* this module's mounts to it (deny-by-default). Carried from the module's
|
|
21
|
+
* `.parachute/module.json` `websocket` field by self-registration; the hub
|
|
22
|
+
* honors either source. Optional + free-riding — extra fields survive the
|
|
23
|
+
* read/merge via the `...e` spread in `validateEntry` / `upsertService`.
|
|
24
|
+
*/
|
|
25
|
+
websocket?: boolean;
|
|
18
26
|
}
|
|
19
27
|
|
|
20
28
|
export interface ServicesManifest {
|
package/src/subscriptions.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Live-query subscription registry
|
|
3
|
-
*
|
|
2
|
+
* Live-query subscription registry — fans out post-commit note events to live
|
|
3
|
+
* WebSocket subscriptions (originally an SSE fan-out — design
|
|
4
|
+
* `design/2026-06-08-live-query-sse.md`; the SSE transport was removed in
|
|
5
|
+
* Phase 5 of the WS-hibernation migration).
|
|
4
6
|
*
|
|
5
7
|
* A second consumer of the post-commit hook dispatcher (`core/src/hooks.ts`),
|
|
6
8
|
* alongside the durable webhook-trigger sink. Where a trigger survives across
|
|
@@ -8,6 +10,24 @@
|
|
|
8
10
|
* connection-scoped, driving a live UI. Same event source, same predicate,
|
|
9
11
|
* different durability.
|
|
10
12
|
*
|
|
13
|
+
* ## The live transport: WebSocket (WS-hibernation migration)
|
|
14
|
+
*
|
|
15
|
+
* The manager is transport-agnostic: it emits abstract `(event, data)` tuples
|
|
16
|
+
* and a {@link SubscriptionSink} renders them for its wire.
|
|
17
|
+
* - {@link WsSink} renders a WebSocket message `{ type: event, ...data }`. The
|
|
18
|
+
* inner payload (`note` / `id` / `notes`) is pinned byte-for-byte against the
|
|
19
|
+
* shared frame-corpus fixture and is congruent with the cloud door
|
|
20
|
+
* (`parachute-cloud/workers/vault/src/live/subscriptions.ts`). See
|
|
21
|
+
* `parachute-cloud/workers/vault/docs/live-query-ws.md`.
|
|
22
|
+
*
|
|
23
|
+
* WebSocket is the SOLE live transport as of Phase 5 — the earlier SSE binding
|
|
24
|
+
* was removed once cached notes-ui bundles converged, and polling (client-side)
|
|
25
|
+
* is the floor beneath live. All subscriptions share ONE process-wide
|
|
26
|
+
* {@link SubscriptionManager}. Self-host has no hibernation (a self-run Bun box
|
|
27
|
+
* has no per-connection duration bill) — the WS binding gives bidirectionality +
|
|
28
|
+
* contract-congruence with cloud, nothing more. The Bun.serve WS integration
|
|
29
|
+
* lives in `ws-server.ts` + `ws-subscribe.ts`.
|
|
30
|
+
*
|
|
11
31
|
* ## How fan-out works
|
|
12
32
|
*
|
|
13
33
|
* All vault stores share the process-wide `defaultHookRegistry`
|
|
@@ -54,16 +74,19 @@ export const DEFAULT_MAX_SUBSCRIPTIONS_PER_VAULT = 100;
|
|
|
54
74
|
* Default bound on a single subscription's pending (unflushed) event buffer.
|
|
55
75
|
* If a slow client lets the buffer grow past this, the stream is closed — it
|
|
56
76
|
* reconnects and re-snapshots rather than the server growing memory unbounded.
|
|
77
|
+
* Only enforced for flush-tracking sinks (SSE); the WS transport delegates
|
|
78
|
+
* outgoing-buffer bounds to the Bun runtime.
|
|
57
79
|
*/
|
|
58
80
|
export const DEFAULT_MAX_BUFFERED_EVENTS = 1000;
|
|
59
81
|
|
|
60
|
-
/**
|
|
61
|
-
|
|
62
|
-
|
|
82
|
+
/**
|
|
83
|
+
* The abstract sink an event is rendered to. The manager calls `send(event,
|
|
84
|
+
* data)`; each transport formats for its wire. Returns `false` when the sink is
|
|
85
|
+
* gone (the manager then drops the subscription).
|
|
86
|
+
*/
|
|
63
87
|
export interface SubscriptionSink {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
/** Close the underlying stream (teardown). Idempotent. */
|
|
88
|
+
send(event: string, data: unknown): boolean;
|
|
89
|
+
/** Close the underlying stream/socket (teardown). Idempotent. */
|
|
67
90
|
close(): void;
|
|
68
91
|
}
|
|
69
92
|
|
|
@@ -75,14 +98,56 @@ interface Subscription {
|
|
|
75
98
|
/** Raw root-tag allowlist (null = unscoped) — `noteWithinTagScope` arg. */
|
|
76
99
|
readonly tagScopeRaw: string[] | null;
|
|
77
100
|
readonly sink: SubscriptionSink;
|
|
78
|
-
/**
|
|
101
|
+
/** SSE tracks unflushed frames to bound memory; WS delegates to the runtime. */
|
|
102
|
+
readonly tracksFlush: boolean;
|
|
103
|
+
/** Whether this sub counts against the per-vault manager cap (SSE) — the WS
|
|
104
|
+
* transport enforces its own cap via the live-socket count at upgrade. */
|
|
105
|
+
readonly countsTowardCap: boolean;
|
|
106
|
+
/** Pending unflushed frame count (backpressure bound; SSE only). */
|
|
79
107
|
buffered: number;
|
|
80
108
|
readonly maxBuffered: number;
|
|
81
109
|
closed: boolean;
|
|
82
110
|
}
|
|
83
111
|
|
|
84
|
-
|
|
85
|
-
|
|
112
|
+
/** WebSocket message: `{ type: <event>, ...data }` — the event name becomes the
|
|
113
|
+
* `type` discriminator and the inner payload (`note` / `id` / `notes`)
|
|
114
|
+
* serializes as-is. The ONE place WS bytes are formatted. */
|
|
115
|
+
export function wsFrame(event: string, data: unknown): string {
|
|
116
|
+
const body = data && typeof data === "object" && !Array.isArray(data) ? (data as Record<string, unknown>) : {};
|
|
117
|
+
return JSON.stringify({ type: event, ...body });
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Minimal structural shape of a Bun `ServerWebSocket` (or a test double) that
|
|
121
|
+
* {@link WsSink} needs — keeps this module free of a hard Bun-server import. */
|
|
122
|
+
export interface WsLike {
|
|
123
|
+
send(data: string): unknown;
|
|
124
|
+
close(code?: number, reason?: string): unknown;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** WebSocket sink: renders `(event, data)` to a `{ type, ...data }` message and
|
|
128
|
+
* sends it on the socket. `sendRaw` is for pre-formatted frames (the chunked
|
|
129
|
+
* snapshot the WS binding builds directly). A send on a dead socket returns
|
|
130
|
+
* false → the manager drops the sub. */
|
|
131
|
+
export class WsSink implements SubscriptionSink {
|
|
132
|
+
constructor(private readonly ws: WsLike) {}
|
|
133
|
+
send(event: string, data: unknown): boolean {
|
|
134
|
+
return this.sendRaw(wsFrame(event, data));
|
|
135
|
+
}
|
|
136
|
+
sendRaw(frame: string): boolean {
|
|
137
|
+
try {
|
|
138
|
+
this.ws.send(frame);
|
|
139
|
+
return true;
|
|
140
|
+
} catch {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
close(): void {
|
|
145
|
+
try {
|
|
146
|
+
this.ws.close(1011, "subscription closed");
|
|
147
|
+
} catch {
|
|
148
|
+
/* already closing/closed */
|
|
149
|
+
}
|
|
150
|
+
}
|
|
86
151
|
}
|
|
87
152
|
|
|
88
153
|
export class SubscriptionManager {
|
|
@@ -140,9 +205,15 @@ export class SubscriptionManager {
|
|
|
140
205
|
tagScopeRaw: string[] | null;
|
|
141
206
|
sink: SubscriptionSink;
|
|
142
207
|
maxBuffered?: number;
|
|
208
|
+
/** SSE (default true) tracks unflushed frames; WS passes false. */
|
|
209
|
+
tracksFlush?: boolean;
|
|
210
|
+
/** SSE (default true) counts against the per-vault manager cap; WS passes
|
|
211
|
+
* false — the WS transport caps via the live-socket count at upgrade. */
|
|
212
|
+
countsTowardCap?: boolean;
|
|
143
213
|
}): SubscriptionHandle | null {
|
|
214
|
+
const countsTowardCap = args.countsTowardCap ?? true;
|
|
144
215
|
const current = this.countForVault(args.vaultName);
|
|
145
|
-
if (current >= this.maxPerVault) return null;
|
|
216
|
+
if (countsTowardCap && current >= this.maxPerVault) return null;
|
|
146
217
|
|
|
147
218
|
this.ensureHooks();
|
|
148
219
|
|
|
@@ -152,12 +223,14 @@ export class SubscriptionManager {
|
|
|
152
223
|
tagScopeAllowed: args.tagScopeAllowed,
|
|
153
224
|
tagScopeRaw: args.tagScopeRaw,
|
|
154
225
|
sink: args.sink,
|
|
226
|
+
tracksFlush: args.tracksFlush ?? true,
|
|
227
|
+
countsTowardCap,
|
|
155
228
|
buffered: 0,
|
|
156
229
|
maxBuffered: args.maxBuffered ?? DEFAULT_MAX_BUFFERED_EVENTS,
|
|
157
230
|
closed: false,
|
|
158
231
|
};
|
|
159
232
|
this.subs.add(sub);
|
|
160
|
-
this.perVaultCount.set(args.vaultName, current + 1);
|
|
233
|
+
if (countsTowardCap) this.perVaultCount.set(args.vaultName, current + 1);
|
|
161
234
|
|
|
162
235
|
return {
|
|
163
236
|
/** Note that a previously-buffered frame flushed to the wire. */
|
|
@@ -172,9 +245,11 @@ export class SubscriptionManager {
|
|
|
172
245
|
if (sub.closed) return;
|
|
173
246
|
sub.closed = true;
|
|
174
247
|
this.subs.delete(sub);
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
248
|
+
if (sub.countsTowardCap) {
|
|
249
|
+
const n = this.perVaultCount.get(sub.vaultName) ?? 0;
|
|
250
|
+
if (n <= 1) this.perVaultCount.delete(sub.vaultName);
|
|
251
|
+
else this.perVaultCount.set(sub.vaultName, n - 1);
|
|
252
|
+
}
|
|
178
253
|
try {
|
|
179
254
|
sub.sink.close();
|
|
180
255
|
} catch {
|
|
@@ -182,33 +257,21 @@ export class SubscriptionManager {
|
|
|
182
257
|
}
|
|
183
258
|
}
|
|
184
259
|
|
|
185
|
-
/**
|
|
186
|
-
*
|
|
187
|
-
|
|
188
|
-
* so it never counts against the buffer bound.
|
|
189
|
-
*/
|
|
190
|
-
keepaliveAll(): void {
|
|
191
|
-
for (const sub of this.subs) {
|
|
192
|
-
if (sub.closed) continue;
|
|
193
|
-
const ok = sub.sink.write(":\n\n");
|
|
194
|
-
if (!ok) this.remove(sub);
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
/** Emit a frame to one subscription, enforcing the buffer bound. */
|
|
199
|
-
private emit(sub: Subscription, frame: SseFrame): void {
|
|
260
|
+
/** Emit an `(event, data)` tuple to one subscription, enforcing the buffer
|
|
261
|
+
* bound for flush-tracking (SSE) sinks. */
|
|
262
|
+
private emit(sub: Subscription, event: string, data: unknown): void {
|
|
200
263
|
if (sub.closed) return;
|
|
201
|
-
if (sub.buffered >= sub.maxBuffered) {
|
|
264
|
+
if (sub.tracksFlush && sub.buffered >= sub.maxBuffered) {
|
|
202
265
|
// Backpressure: client can't keep up. Close → it reconnects + re-snapshots.
|
|
203
266
|
this.remove(sub);
|
|
204
267
|
return;
|
|
205
268
|
}
|
|
206
|
-
const ok = sub.sink.
|
|
269
|
+
const ok = sub.sink.send(event, data);
|
|
207
270
|
if (!ok) {
|
|
208
271
|
this.remove(sub);
|
|
209
272
|
return;
|
|
210
273
|
}
|
|
211
|
-
sub.buffered++;
|
|
274
|
+
if (sub.tracksFlush) sub.buffered++;
|
|
212
275
|
}
|
|
213
276
|
|
|
214
277
|
/** Lazily register the three broad hooks (once per manager). */
|
|
@@ -244,7 +307,7 @@ export class SubscriptionManager {
|
|
|
244
307
|
// the client ignores ids it never held. Documented low-sensitivity
|
|
245
308
|
// existence leak (see design §scope-intersection).
|
|
246
309
|
const ref = payload as DeletedNoteRef;
|
|
247
|
-
this.emit(sub,
|
|
310
|
+
this.emit(sub, "remove", { id: ref.id });
|
|
248
311
|
continue;
|
|
249
312
|
}
|
|
250
313
|
|
|
@@ -257,14 +320,14 @@ export class SubscriptionManager {
|
|
|
257
320
|
const matches = sub.matcher.match(note) && inScope;
|
|
258
321
|
|
|
259
322
|
if (matches) {
|
|
260
|
-
this.emit(sub,
|
|
323
|
+
this.emit(sub, "upsert", { note });
|
|
261
324
|
} else if (event === "updated" && inScope) {
|
|
262
325
|
// Left the set (predicate no longer true) BUT still within this
|
|
263
326
|
// token's scope, so the sub could have held it — idempotent remove
|
|
264
327
|
// (client drops the id if held, no-op otherwise). When the note is
|
|
265
328
|
// OUT of scope the sub never had it; emitting would leak its id, so
|
|
266
329
|
// we stay silent.
|
|
267
|
-
this.emit(sub,
|
|
330
|
+
this.emit(sub, "remove", { id: note.id });
|
|
268
331
|
}
|
|
269
332
|
// created that doesn't match → nothing (it was never in the set).
|
|
270
333
|
}
|
|
@@ -286,10 +349,5 @@ export interface SubscriptionHandle {
|
|
|
286
349
|
close: () => void;
|
|
287
350
|
}
|
|
288
351
|
|
|
289
|
-
/** Serialize a snapshot frame (exported for the route + tests). */
|
|
290
|
-
export function snapshotFrame(notes: Note[]): SseFrame {
|
|
291
|
-
return sseEvent("snapshot", { notes });
|
|
292
|
-
}
|
|
293
|
-
|
|
294
352
|
/** Process-wide manager — shared like `defaultHookRegistry`. */
|
|
295
353
|
export const subscriptionManager = new SubscriptionManager();
|