@openparachute/vault 0.6.4 → 0.6.5-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/__fixtures__/golden-vault.ts +125 -0
- package/core/src/__fixtures__/portable-export-golden.json +10 -0
- package/core/src/do-param-cap.test.ts +161 -0
- package/core/src/links.ts +8 -13
- package/core/src/mcp.ts +306 -314
- package/core/src/notes.ts +54 -62
- package/core/src/onboarding.ts +14 -296
- package/core/src/portable-md-batching.test.ts +67 -0
- package/core/src/portable-md-golden.test.ts +55 -0
- package/core/src/portable-md.ts +579 -435
- package/core/src/schema.ts +21 -43
- package/core/src/seed-packs.test.ts +191 -0
- package/core/src/seed-packs.ts +559 -0
- package/core/src/sql-in.test.ts +58 -0
- package/core/src/sql-in.ts +76 -0
- package/core/src/store.ts +14 -9
- package/core/src/transcription/provider.ts +141 -0
- package/core/src/txn.test.ts +229 -0
- package/core/src/txn.ts +105 -0
- package/core/src/types.ts +9 -0
- package/core/src/vault-projection.ts +1 -1
- package/core/src/wikilinks.ts +10 -4
- package/package.json +1 -1
- package/src/add-pack.test.ts +142 -0
- package/src/cli.ts +778 -7
- package/src/onboarding-seed.test.ts +126 -40
- package/src/onboarding-seed.ts +41 -46
- package/src/routes.ts +25 -7
- package/src/server.ts +108 -31
- package/src/transcription/build.test.ts +224 -0
- package/src/transcription/build.ts +252 -0
- package/src/transcription/capability.test.ts +118 -0
- package/src/transcription/capability.ts +96 -0
- package/src/transcription/install-python.test.ts +366 -0
- package/src/transcription/install-python.ts +471 -0
- package/src/transcription/install.test.ts +167 -0
- package/src/transcription/install.ts +296 -0
- package/src/transcription/providers/onnx-asr.test.ts +229 -0
- package/src/transcription/providers/onnx-asr.ts +239 -0
- package/src/transcription/providers/parakeet-mlx.test.ts +290 -0
- package/src/transcription/providers/parakeet-mlx.ts +242 -0
- package/src/transcription/providers/scribe-http.test.ts +195 -0
- package/src/transcription/providers/scribe-http.ts +144 -0
- package/src/transcription/providers/transcribe-cpp.test.ts +314 -0
- package/src/transcription/providers/transcribe-cpp.ts +293 -0
- package/src/transcription/select.test.ts +259 -0
- package/src/transcription/select.ts +334 -0
- package/src/transcription/tiers.test.ts +197 -0
- package/src/transcription/tiers.ts +184 -0
- package/src/transcription-worker.test.ts +44 -0
- package/src/transcription-worker.ts +57 -122
- package/src/vault-create.test.ts +38 -10
- package/src/vault.test.ts +48 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQL `IN`-list helpers sized for the smallest backing SQLite we run on.
|
|
3
|
+
*
|
|
4
|
+
* ## The DO 100-bound-parameter cap
|
|
5
|
+
*
|
|
6
|
+
* Cloudflare Durable Object SQLite (the cloud vault's backing store) rejects
|
|
7
|
+
* any statement carrying **more than 100 bound parameters** with
|
|
8
|
+
* `too many SQL variables`. Standard SQLite — including bun:sqlite (self-host)
|
|
9
|
+
* — tolerates 999 on old builds and 32766 on new ones, which is why an
|
|
10
|
+
* `IN (?, ?, …)` list sized for the 999 floor shipped fine on self-host and
|
|
11
|
+
* only 500'd on cloud vaults: a >100-note export page passed >100 ids in one
|
|
12
|
+
* statement and DO SQLite rejected it. See vault export-500 diagnosis.
|
|
13
|
+
*
|
|
14
|
+
* Two DO-safe shapes, depending on whether the id-list is the *only* thing
|
|
15
|
+
* in the statement:
|
|
16
|
+
*
|
|
17
|
+
* 1. **Standalone id-list** (its own `SELECT … WHERE id IN (…)`, no other
|
|
18
|
+
* params, no pagination): chunk at {@link IN_PARAM_CHUNK} and union the
|
|
19
|
+
* results in JS. Use {@link chunkForInClause}.
|
|
20
|
+
*
|
|
21
|
+
* 2. **Embedded id-set** (an `IN` filter inside a larger statement that also
|
|
22
|
+
* carries `ORDER BY` / `LIMIT` / `OFFSET` / other params — chunking would
|
|
23
|
+
* break the shared pagination window): bind the whole set as a SINGLE JSON
|
|
24
|
+
* array param via {@link IN_VIA_JSON_EACH} + {@link jsonEachParam}. One
|
|
25
|
+
* param regardless of set size, so it can never trip the cap. Requires the
|
|
26
|
+
* JSON1 extension, which both bun:sqlite and DO SQLite ship (the vault
|
|
27
|
+
* already relies on `json_extract` on DO for metadata queries).
|
|
28
|
+
*
|
|
29
|
+
* ## Future refinement — `maxBoundParams`
|
|
30
|
+
*
|
|
31
|
+
* The clean long-term shape is for the Store / DB shim to advertise a
|
|
32
|
+
* `maxBoundParams` (DO = 100, bun = 999) and chunk to
|
|
33
|
+
* `min(IN_PARAM_CHUNK, maxBoundParams - reservedParams)`, so self-host keeps
|
|
34
|
+
* large-batch efficiency instead of paying the DO tax. Until then a global
|
|
35
|
+
* 90 is SAFE on bun — it just issues more cheap, fully-indexed IN queries —
|
|
36
|
+
* and is the conservative floor everywhere.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Chunk size for standalone `IN (?, ?, …)` id-lists. 90 leaves headroom
|
|
41
|
+
* under the DO 100-param cap for statements whose only bound params are the
|
|
42
|
+
* IN-list (the extra 10 covers the rare embedded extra param). Never raise
|
|
43
|
+
* this above 100 without teaching every embedded callsite the DO cap — see
|
|
44
|
+
* the module header.
|
|
45
|
+
*/
|
|
46
|
+
export const IN_PARAM_CHUNK = 90;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Split `items` into consecutive chunks of at most `size` (default
|
|
50
|
+
* {@link IN_PARAM_CHUNK}) for feeding to a chunked `IN (?, …)` query. Empty
|
|
51
|
+
* input yields no chunks. This is the single place that decides IN-list chunk
|
|
52
|
+
* boundaries, so a regression that raises the size is caught by one test.
|
|
53
|
+
*/
|
|
54
|
+
export function chunkForInClause<T>(items: readonly T[], size = IN_PARAM_CHUNK): T[][] {
|
|
55
|
+
const chunks: T[][] = [];
|
|
56
|
+
for (let i = 0; i < items.length; i += size) {
|
|
57
|
+
chunks.push(items.slice(i, i + size));
|
|
58
|
+
}
|
|
59
|
+
return chunks;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* SQL fragment for a single-param `IN` filter over an id-set embedded in a
|
|
64
|
+
* larger statement: `WHERE col IN ${IN_VIA_JSON_EACH}`. Bind the matching
|
|
65
|
+
* `?` with {@link jsonEachParam}. json_each expands the bound JSON array into
|
|
66
|
+
* a one-column table, so the whole set costs exactly ONE bound parameter and
|
|
67
|
+
* can never exceed the DO cap. Callers must guarantee the set is non-empty
|
|
68
|
+
* (`json_each('[]')` yields no rows — correct for `IN`, but an empty set is
|
|
69
|
+
* usually short-circuited to a `0 = 1` no-match upstream to skip the query).
|
|
70
|
+
*/
|
|
71
|
+
export const IN_VIA_JSON_EACH = "(SELECT value FROM json_each(?))";
|
|
72
|
+
|
|
73
|
+
/** Serialize an id-set for the single {@link IN_VIA_JSON_EACH} bound param. */
|
|
74
|
+
export function jsonEachParam(values: readonly (string | number)[]): string {
|
|
75
|
+
return JSON.stringify(values);
|
|
76
|
+
}
|
package/core/src/store.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
} from "./indexed-fields.js";
|
|
13
13
|
import { syncWikilinks, resolveUnresolvedWikilinks } from "./wikilinks.js";
|
|
14
14
|
import { pathTitle } from "./paths.js";
|
|
15
|
+
import { transaction } from "./txn.js";
|
|
15
16
|
import { HookRegistry } from "./hooks.js";
|
|
16
17
|
import {
|
|
17
18
|
loadTagHierarchy,
|
|
@@ -55,6 +56,15 @@ export class BunSqliteStore implements Store {
|
|
|
55
56
|
this.hooks = opts?.hooks ?? new HookRegistry();
|
|
56
57
|
}
|
|
57
58
|
|
|
59
|
+
/**
|
|
60
|
+
* The transaction seam (see core/src/txn.ts). bun backs it with
|
|
61
|
+
* `BEGIN IMMEDIATE … COMMIT`; a DO-backed Store overrides this with
|
|
62
|
+
* `ctx.storage.transactionSync`. Synchronous — `fn` must not await.
|
|
63
|
+
*/
|
|
64
|
+
transaction<T>(fn: () => T): T {
|
|
65
|
+
return transaction(this.db, fn);
|
|
66
|
+
}
|
|
67
|
+
|
|
58
68
|
/**
|
|
59
69
|
* Lazy accessor for the `_tags/*` config-note hierarchy. First call after
|
|
60
70
|
* boot or after an invalidation does the scan; subsequent calls hit the
|
|
@@ -716,10 +726,8 @@ export class BunSqliteStore implements Store {
|
|
|
716
726
|
// mismatch only detectable once the existing declarer set is consulted),
|
|
717
727
|
// the whole write rolls back — the schema never ends up claiming an index
|
|
718
728
|
// that doesn't exist. vault#478 transactional fix.
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
try {
|
|
722
|
-
result = tagSchemaOps.upsertTagRecord(this.db, tag, patch);
|
|
729
|
+
const result = this.transaction(() => {
|
|
730
|
+
const record = tagSchemaOps.upsertTagRecord(this.db, tag, patch);
|
|
723
731
|
|
|
724
732
|
if (patch.fields !== undefined) {
|
|
725
733
|
for (const fieldName of nextIndexed) {
|
|
@@ -734,11 +742,8 @@ export class BunSqliteStore implements Store {
|
|
|
734
742
|
}
|
|
735
743
|
}
|
|
736
744
|
}
|
|
737
|
-
|
|
738
|
-
}
|
|
739
|
-
this.db.exec("ROLLBACK");
|
|
740
|
-
throw err;
|
|
741
|
-
}
|
|
745
|
+
return record;
|
|
746
|
+
});
|
|
742
747
|
|
|
743
748
|
if (patch.parent_names !== undefined) {
|
|
744
749
|
// parent_names drives both query expansion (tag hierarchy) AND, post
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transcription provider seam (scribe-fold Phase 1).
|
|
3
|
+
*
|
|
4
|
+
* A `TranscriptionProvider` is the single, runtime-agnostic contract for
|
|
5
|
+
* turning an audio blob into text. It exists so vault's transcription worker
|
|
6
|
+
* doesn't hard-code "POST to scribe" — the worker resolves a provider and
|
|
7
|
+
* calls `transcribe()`, and the provider owns HOW the bytes become text.
|
|
8
|
+
*
|
|
9
|
+
* ## Why this lives in core (not src)
|
|
10
|
+
*
|
|
11
|
+
* The interface must be importable from BOTH runtimes without dragging in
|
|
12
|
+
* Bun- or Node-only APIs:
|
|
13
|
+
*
|
|
14
|
+
* - the Bun vault (`src/transcription/providers/scribe-http.ts` — the
|
|
15
|
+
* whisper-compatible remote provider that reproduces today's scribe call);
|
|
16
|
+
* - the cloud vault (a future Cloudflare Workers-AI provider implemented
|
|
17
|
+
* against THIS interface for the metered voice build).
|
|
18
|
+
*
|
|
19
|
+
* So it is pure types + one dependency-free error class — no `fs`, no
|
|
20
|
+
* `FormData` serialization, no `fetch`. `audio` arrives as a `Uint8Array`
|
|
21
|
+
* (both runtimes have it); a provider decides how to ship it. Core has no
|
|
22
|
+
* package `exports` map, so consumers import the src subpath directly
|
|
23
|
+
* (`../core/src/transcription/provider.ts`), the same convention as
|
|
24
|
+
* `core/src/seed-packs.ts`.
|
|
25
|
+
*
|
|
26
|
+
* ## The fold (context)
|
|
27
|
+
*
|
|
28
|
+
* Ratified 2026-07-03 (Aaron): scribe folds into vault as a built-in
|
|
29
|
+
* transcription feature. Phase 1 (this seam) is behavior-preserving — the
|
|
30
|
+
* default `scribe-http` provider does EXACTLY what today's `callScribe` did,
|
|
31
|
+
* so existing scribe installs keep working unchanged. Phase 2 moves local-ASR
|
|
32
|
+
* backends behind the same interface. See
|
|
33
|
+
* `design/2026-07-03-transcription-provider-seam.md`.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* A single context entry attached alongside the audio (person/project notes,
|
|
38
|
+
* etc.). Structurally identical to `src/context.ts`'s `ContextEntry` — the
|
|
39
|
+
* same shape scribe already consumes over the `context` multipart part — so a
|
|
40
|
+
* `ContextPayload` produced there is assignable here without conversion. The
|
|
41
|
+
* interface stays runtime-agnostic: a provider decides whether/how to
|
|
42
|
+
* serialize it.
|
|
43
|
+
*/
|
|
44
|
+
export interface TranscriptionContextEntry {
|
|
45
|
+
/** Note path basename, or note id if no path. */
|
|
46
|
+
name: string;
|
|
47
|
+
/** Whitelisted metadata fields carried through from the vault query. */
|
|
48
|
+
[key: string]: unknown;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** The `{ entries: [...] }` context blob a provider may forward with the audio. */
|
|
52
|
+
export interface TranscriptionContext {
|
|
53
|
+
entries: TranscriptionContextEntry[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The audio + descriptors handed to a provider for one transcription. */
|
|
57
|
+
export interface TranscribeInput {
|
|
58
|
+
/** Raw audio bytes. Both runtimes expose `Uint8Array`; a `Buffer` is one. */
|
|
59
|
+
audio: Uint8Array;
|
|
60
|
+
/** Original filename (used for the multipart `file` part / provider hints). */
|
|
61
|
+
filename: string;
|
|
62
|
+
/** MIME type of the audio (e.g. `audio/webm`). */
|
|
63
|
+
mimeType: string;
|
|
64
|
+
/**
|
|
65
|
+
* Optional vault-provided context (person/project notes) to forward with the
|
|
66
|
+
* audio so the provider/model can use it. `null`/`undefined` means none.
|
|
67
|
+
*/
|
|
68
|
+
context?: TranscriptionContext | null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** What a provider returns on a successful transcription. */
|
|
72
|
+
export interface TranscribeResult {
|
|
73
|
+
/** The transcript text. */
|
|
74
|
+
text: string;
|
|
75
|
+
/**
|
|
76
|
+
* Duration of the transcribed AUDIO in seconds, when the provider reports it.
|
|
77
|
+
* This is a metered/plan concern (the cloud voice build will decrement a
|
|
78
|
+
* minutes balance by it); self-host providers that don't measure it leave it
|
|
79
|
+
* unset. NOT the wall-clock request time — the worker measures that itself.
|
|
80
|
+
*/
|
|
81
|
+
audioSeconds?: number;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Provider readiness probe result. */
|
|
85
|
+
export interface ProviderAvailability {
|
|
86
|
+
/** True when the provider can service `transcribe()` calls right now. */
|
|
87
|
+
ok: boolean;
|
|
88
|
+
/** Human-readable cause when `ok` is false (e.g. "no scribe URL configured"). */
|
|
89
|
+
reason?: string;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* The transcription contract. Implementations own the audio→text step; the
|
|
94
|
+
* worker owns queueing, backoff/retry, transcript-note materialization, and
|
|
95
|
+
* retention — none of which belong here.
|
|
96
|
+
*/
|
|
97
|
+
export interface TranscriptionProvider {
|
|
98
|
+
/** Stable provider identifier, surfaced on the capability flag (e.g. "scribe-http"). */
|
|
99
|
+
readonly name: string;
|
|
100
|
+
/** Turn audio into text (or throw — see `TranscriptionError` for the retry contract). */
|
|
101
|
+
transcribe(input: TranscribeInput): Promise<TranscribeResult>;
|
|
102
|
+
/**
|
|
103
|
+
* Cheap, side-effect-free readiness check. MUST NOT perform the actual
|
|
104
|
+
* transcription or a network round-trip unless that is genuinely cheap —
|
|
105
|
+
* it gates the capability flag on the vault landing, called on config reads.
|
|
106
|
+
*/
|
|
107
|
+
available(): Promise<ProviderAvailability>;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Structured, provider-agnostic transcription failure.
|
|
112
|
+
*
|
|
113
|
+
* The worker's terminal-vs-retry decision keys off `retriable`: a
|
|
114
|
+
* non-retriable error (a 4xx the operator must fix — no provider configured,
|
|
115
|
+
* bad auth) fails immediately with no backoff; a retriable one (5xx, transient
|
|
116
|
+
* upstream) is retried with exponential backoff. `code` is a stable machine
|
|
117
|
+
* string (e.g. `missing_provider`) surfaced on the transcript note's
|
|
118
|
+
* frontmatter and the attachment's `transcribe_error_code` so callers branch
|
|
119
|
+
* on it instead of regex-matching message text.
|
|
120
|
+
*
|
|
121
|
+
* Providers throw this for structured failures; a plain `Error` (network drop,
|
|
122
|
+
* timeout, malformed response) is treated as retriable by the worker.
|
|
123
|
+
*/
|
|
124
|
+
export class TranscriptionError extends Error {
|
|
125
|
+
/** Stable machine code (e.g. "missing_provider"), when the provider supplied one. */
|
|
126
|
+
readonly code?: string;
|
|
127
|
+
/** Upstream HTTP status, when the failure came from an HTTP provider. */
|
|
128
|
+
readonly httpStatus?: number;
|
|
129
|
+
/** Whether retrying the same audio could plausibly succeed. */
|
|
130
|
+
readonly retriable: boolean;
|
|
131
|
+
constructor(
|
|
132
|
+
message: string,
|
|
133
|
+
opts: { code?: string; httpStatus?: number; retriable: boolean },
|
|
134
|
+
) {
|
|
135
|
+
super(message);
|
|
136
|
+
this.name = "TranscriptionError";
|
|
137
|
+
this.code = opts.code;
|
|
138
|
+
this.httpStatus = opts.httpStatus;
|
|
139
|
+
this.retriable = opts.retriable;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Contract tests for the transaction seam (core/src/txn.ts).
|
|
3
|
+
*
|
|
4
|
+
* Pins the behavior every migrated call site now depends on: commit returns
|
|
5
|
+
* the callback's value, a throw rolls back and re-throws the ORIGINAL error,
|
|
6
|
+
* and a ROLLBACK that itself fails (the "COMMIT already resolved the txn"
|
|
7
|
+
* case) is swallowed so the original error still propagates. The swallow /
|
|
8
|
+
* ordering cases use a structural fake `TxnCapableDb` so we can force a
|
|
9
|
+
* COMMIT/ROLLBACK failure deterministically; the happy + real-rollback cases
|
|
10
|
+
* run against a live bun:sqlite connection.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from "bun:test";
|
|
14
|
+
import { Database } from "bun:sqlite";
|
|
15
|
+
import { transaction, transactionAsync, type TxnCapableDb } from "./txn.js";
|
|
16
|
+
|
|
17
|
+
/** A fake `TxnCapableDb` that records exec calls and can be told to throw on
|
|
18
|
+
* COMMIT and/or ROLLBACK — lets us drive the failure branches exactly. */
|
|
19
|
+
function fakeDb(opts: { commitThrows?: Error; rollbackThrows?: Error } = {}): {
|
|
20
|
+
db: TxnCapableDb;
|
|
21
|
+
calls: string[];
|
|
22
|
+
} {
|
|
23
|
+
const calls: string[] = [];
|
|
24
|
+
const db: TxnCapableDb = {
|
|
25
|
+
exec(sql: string): void {
|
|
26
|
+
calls.push(sql);
|
|
27
|
+
if (sql === "COMMIT" && opts.commitThrows) throw opts.commitThrows;
|
|
28
|
+
if (sql === "ROLLBACK" && opts.rollbackThrows) throw opts.rollbackThrows;
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
return { db, calls };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function freshDb(): Database {
|
|
35
|
+
const db = new Database(":memory:");
|
|
36
|
+
db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)");
|
|
37
|
+
return db;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
describe("transaction (sync)", () => {
|
|
41
|
+
it("commits and returns the callback's value", () => {
|
|
42
|
+
const db = freshDb();
|
|
43
|
+
const result = transaction(db, () => {
|
|
44
|
+
db.exec("INSERT INTO t (id, v) VALUES (1, 'a')");
|
|
45
|
+
return { count: 1 };
|
|
46
|
+
});
|
|
47
|
+
expect(result).toEqual({ count: 1 });
|
|
48
|
+
// Write is durable after commit.
|
|
49
|
+
expect((db.prepare("SELECT COUNT(*) AS c FROM t").get() as { c: number }).c).toBe(1);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("wraps in BEGIN IMMEDIATE … COMMIT", () => {
|
|
53
|
+
const { db, calls } = fakeDb();
|
|
54
|
+
transaction(db, () => "ok");
|
|
55
|
+
expect(calls).toEqual(["BEGIN IMMEDIATE", "COMMIT"]);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("rolls back and re-throws the ORIGINAL error when the callback throws", () => {
|
|
59
|
+
const db = freshDb();
|
|
60
|
+
const sentinel = new Error("boom");
|
|
61
|
+
expect(() =>
|
|
62
|
+
transaction(db, () => {
|
|
63
|
+
db.exec("INSERT INTO t (id, v) VALUES (1, 'a')");
|
|
64
|
+
throw sentinel;
|
|
65
|
+
}),
|
|
66
|
+
).toThrow(sentinel);
|
|
67
|
+
// The inner INSERT was rolled back.
|
|
68
|
+
expect((db.prepare("SELECT COUNT(*) AS c FROM t").get() as { c: number }).c).toBe(0);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("issues ROLLBACK (not COMMIT) on a callback throw", () => {
|
|
72
|
+
const { db, calls } = fakeDb();
|
|
73
|
+
expect(() => transaction(db, () => { throw new Error("x"); })).toThrow("x");
|
|
74
|
+
expect(calls).toEqual(["BEGIN IMMEDIATE", "ROLLBACK"]);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("swallows a ROLLBACK failure after a COMMIT throw and propagates the ORIGINAL (commit) error", () => {
|
|
78
|
+
const commitErr = new Error("commit failed");
|
|
79
|
+
const rollbackErr = new Error("no transaction is active");
|
|
80
|
+
const { db, calls } = fakeDb({ commitThrows: commitErr, rollbackThrows: rollbackErr });
|
|
81
|
+
let thrown: unknown;
|
|
82
|
+
try {
|
|
83
|
+
transaction(db, () => "value");
|
|
84
|
+
} catch (e) {
|
|
85
|
+
thrown = e;
|
|
86
|
+
}
|
|
87
|
+
// The COMMIT error is the original that propagates; the ROLLBACK error is swallowed.
|
|
88
|
+
expect(thrown).toBe(commitErr);
|
|
89
|
+
expect(calls).toEqual(["BEGIN IMMEDIATE", "COMMIT", "ROLLBACK"]);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it("preserves the callback error even when ROLLBACK also fails", () => {
|
|
93
|
+
const callbackErr = new Error("callback boom");
|
|
94
|
+
const rollbackErr = new Error("rollback boom");
|
|
95
|
+
const { db } = fakeDb({ rollbackThrows: rollbackErr });
|
|
96
|
+
let thrown: unknown;
|
|
97
|
+
try {
|
|
98
|
+
transaction(db, () => { throw callbackErr; });
|
|
99
|
+
} catch (e) {
|
|
100
|
+
thrown = e;
|
|
101
|
+
}
|
|
102
|
+
expect(thrown).toBe(callbackErr);
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe("transaction — native transactionSync preference (DO backend)", () => {
|
|
107
|
+
/** A db that exposes a native `transactionSync` (standing in for a Durable
|
|
108
|
+
* Object shim delegating to `ctx.storage.transactionSync`). Here it's backed
|
|
109
|
+
* by bun's own `.transaction()` so rollback-on-throw is real, while the raw
|
|
110
|
+
* BEGIN/COMMIT it issues internally never flow through our `exec` — letting
|
|
111
|
+
* us assert the seam issued no explicit transaction SQL. */
|
|
112
|
+
function nativeDb(): { db: TxnCapableDb; execs: string[]; bun: Database } {
|
|
113
|
+
const bun = freshDb();
|
|
114
|
+
const execs: string[] = [];
|
|
115
|
+
const db: TxnCapableDb = {
|
|
116
|
+
exec(sql: string): void {
|
|
117
|
+
execs.push(sql);
|
|
118
|
+
bun.exec(sql);
|
|
119
|
+
},
|
|
120
|
+
transactionSync<T>(fn: () => T): T {
|
|
121
|
+
return bun.transaction(fn)();
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
return { db, execs, bun };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
it("prefers transactionSync over BEGIN IMMEDIATE — delegates once, issues no raw txn SQL", () => {
|
|
128
|
+
const calls: string[] = [];
|
|
129
|
+
const state = { txnCalls: 0 };
|
|
130
|
+
const db: TxnCapableDb = {
|
|
131
|
+
exec(sql: string): void { calls.push(sql); },
|
|
132
|
+
transactionSync<T>(fn: () => T): T { state.txnCalls++; return fn(); },
|
|
133
|
+
};
|
|
134
|
+
const result = transaction(db, () => { db.exec("WORK"); return "ok"; });
|
|
135
|
+
expect(result).toBe("ok");
|
|
136
|
+
expect(state.txnCalls).toBe(1);
|
|
137
|
+
// Only the inner write — never BEGIN IMMEDIATE / COMMIT / ROLLBACK.
|
|
138
|
+
expect(calls).toEqual(["WORK"]);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("commits via native transactionSync and returns the callback's value", () => {
|
|
142
|
+
const { db, bun } = nativeDb();
|
|
143
|
+
const result = transaction(db, () => {
|
|
144
|
+
db.exec("INSERT INTO t (id, v) VALUES (1, 'a')");
|
|
145
|
+
return 42;
|
|
146
|
+
});
|
|
147
|
+
expect(result).toBe(42);
|
|
148
|
+
expect((bun.prepare("SELECT COUNT(*) AS c FROM t").get() as { c: number }).c).toBe(1);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it("rolls back through the native transactionSync on a throw, re-throwing the ORIGINAL error", () => {
|
|
152
|
+
const { db, execs, bun } = nativeDb();
|
|
153
|
+
const sentinel = new Error("native boom");
|
|
154
|
+
expect(() =>
|
|
155
|
+
transaction(db, () => {
|
|
156
|
+
db.exec("INSERT INTO t (id, v) VALUES (1, 'a')");
|
|
157
|
+
throw sentinel;
|
|
158
|
+
}),
|
|
159
|
+
).toThrow(sentinel);
|
|
160
|
+
// Real rollback — the write did not persist.
|
|
161
|
+
expect((bun.prepare("SELECT COUNT(*) AS c FROM t").get() as { c: number }).c).toBe(0);
|
|
162
|
+
// The seam issued NO raw transaction SQL of its own (the native primitive
|
|
163
|
+
// owns BEGIN/COMMIT/ROLLBACK internally); only the inner INSERT flowed through.
|
|
164
|
+
expect(execs).toEqual(["INSERT INTO t (id, v) VALUES (1, 'a')"]);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
it("bun path unchanged when the db has no transactionSync (BEGIN IMMEDIATE … COMMIT)", () => {
|
|
168
|
+
const { db, calls } = fakeDb(); // no transactionSync
|
|
169
|
+
transaction(db, () => "ok");
|
|
170
|
+
expect(calls).toEqual(["BEGIN IMMEDIATE", "COMMIT"]);
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
describe("transactionAsync", () => {
|
|
175
|
+
it("commits and returns the callback's resolved value", async () => {
|
|
176
|
+
const db = freshDb();
|
|
177
|
+
const result = await transactionAsync(db, async () => {
|
|
178
|
+
db.exec("INSERT INTO t (id, v) VALUES (1, 'a')");
|
|
179
|
+
return 7;
|
|
180
|
+
});
|
|
181
|
+
expect(result).toBe(7);
|
|
182
|
+
expect((db.prepare("SELECT COUNT(*) AS c FROM t").get() as { c: number }).c).toBe(1);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it("wraps in BEGIN IMMEDIATE … COMMIT", async () => {
|
|
186
|
+
const { db, calls } = fakeDb();
|
|
187
|
+
await transactionAsync(db, async () => "ok");
|
|
188
|
+
expect(calls).toEqual(["BEGIN IMMEDIATE", "COMMIT"]);
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
it("rolls back and re-throws the ORIGINAL error when the callback rejects", async () => {
|
|
192
|
+
const db = freshDb();
|
|
193
|
+
const sentinel = new Error("async boom");
|
|
194
|
+
await expect(
|
|
195
|
+
transactionAsync(db, async () => {
|
|
196
|
+
db.exec("INSERT INTO t (id, v) VALUES (1, 'a')");
|
|
197
|
+
throw sentinel;
|
|
198
|
+
}),
|
|
199
|
+
).rejects.toBe(sentinel);
|
|
200
|
+
expect((db.prepare("SELECT COUNT(*) AS c FROM t").get() as { c: number }).c).toBe(0);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it("swallows a ROLLBACK failure after a COMMIT throw and propagates the ORIGINAL (commit) error", async () => {
|
|
204
|
+
const commitErr = new Error("commit failed");
|
|
205
|
+
const rollbackErr = new Error("no transaction is active");
|
|
206
|
+
const { db, calls } = fakeDb({ commitThrows: commitErr, rollbackThrows: rollbackErr });
|
|
207
|
+
let thrown: unknown;
|
|
208
|
+
try {
|
|
209
|
+
await transactionAsync(db, async () => "value");
|
|
210
|
+
} catch (e) {
|
|
211
|
+
thrown = e;
|
|
212
|
+
}
|
|
213
|
+
expect(thrown).toBe(commitErr);
|
|
214
|
+
expect(calls).toEqual(["BEGIN IMMEDIATE", "COMMIT", "ROLLBACK"]);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it("preserves the callback error even when ROLLBACK also fails", async () => {
|
|
218
|
+
const callbackErr = new Error("callback boom");
|
|
219
|
+
const rollbackErr = new Error("rollback boom");
|
|
220
|
+
const { db } = fakeDb({ rollbackThrows: rollbackErr });
|
|
221
|
+
let thrown: unknown;
|
|
222
|
+
try {
|
|
223
|
+
await transactionAsync(db, async () => { throw callbackErr; });
|
|
224
|
+
} catch (e) {
|
|
225
|
+
thrown = e;
|
|
226
|
+
}
|
|
227
|
+
expect(thrown).toBe(callbackErr);
|
|
228
|
+
});
|
|
229
|
+
});
|
package/core/src/txn.ts
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The transaction seam (Phase-1 shared-core refactor, vault cloud design §4).
|
|
3
|
+
*
|
|
4
|
+
* Core code must never emit raw `BEGIN`/`COMMIT`/`ROLLBACK` — that's the one
|
|
5
|
+
* SQLite construct DO's `sql.exec` blocks, so it can't be shimmed like the
|
|
6
|
+
* rest of the `Database` surface. Instead every atomic block routes through
|
|
7
|
+
* `transaction` / `transactionAsync` here. The bun backend implements them
|
|
8
|
+
* as `BEGIN IMMEDIATE … COMMIT` with rollback-on-throw; a future Durable-
|
|
9
|
+
* Object backend implements the same contract with `ctx.storage.transactionSync`
|
|
10
|
+
* (and `Store.transaction`, see types.ts, is the object-level entry point).
|
|
11
|
+
*
|
|
12
|
+
* `BEGIN IMMEDIATE` (write lock up front) matches the pre-seam behavior of
|
|
13
|
+
* the 13 core blocks this replaced — several used a bare `BEGIN`, but every
|
|
14
|
+
* one of them only ever wrote, so acquiring the write lock eagerly is a
|
|
15
|
+
* strict improvement (no lazy busy-upgrade) and never a semantic change.
|
|
16
|
+
*
|
|
17
|
+
* **Nesting is unsupported**, matching the pre-seam code exactly: SQLite
|
|
18
|
+
* throws "cannot start a transaction within a transaction" on a nested
|
|
19
|
+
* `BEGIN`, and none of the migrated call sites nest (a batch wraps individual
|
|
20
|
+
* `createNote`s, none of which open their own transaction). If a nested
|
|
21
|
+
* caller ever appears, the fix is SAVEPOINT-based re-entrancy *in the bun
|
|
22
|
+
* implementation here* — the call sites stay untouched.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** The minimal DB surface the transaction seam needs — kept structural (no
|
|
26
|
+
* `bun:sqlite` import) so a non-bun backend satisfies it too.
|
|
27
|
+
*
|
|
28
|
+
* `transactionSync` is OPTIONAL: a backend whose engine blocks explicit
|
|
29
|
+
* `BEGIN`/`COMMIT` (Cloudflare Durable Objects — `sql.exec` throws on them)
|
|
30
|
+
* exposes a native synchronous transaction primitive instead, and
|
|
31
|
+
* {@link transaction} delegates to it. bun's `Database` has no such method, so
|
|
32
|
+
* it takes the `BEGIN IMMEDIATE` path — the check is a duck-type, never a
|
|
33
|
+
* behavior change for the bun backend. */
|
|
34
|
+
export interface TxnCapableDb {
|
|
35
|
+
exec(sql: string): void;
|
|
36
|
+
/** Native commit-on-return / rollback-on-throw transaction, when the backend
|
|
37
|
+
* provides one (e.g. a DO shim delegating to `ctx.storage.transactionSync`). */
|
|
38
|
+
transactionSync?<T>(fn: () => T): T;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Run `fn` inside a single write transaction, committing its result or
|
|
43
|
+
* rolling back on throw. Synchronous — the callback must not await (see
|
|
44
|
+
* `transactionAsync` for the batch paths that legitimately await the async
|
|
45
|
+
* Store facade).
|
|
46
|
+
*
|
|
47
|
+
* When the backing db exposes a native `transactionSync` (a backend whose
|
|
48
|
+
* engine blocks raw `BEGIN`/`COMMIT` — Durable Objects), we delegate to it: a
|
|
49
|
+
* real transaction with the same commit-on-return / rollback-on-throw contract,
|
|
50
|
+
* without ever issuing the explicit transaction SQL that backend rejects. bun's
|
|
51
|
+
* `Database` has no `transactionSync`, so it takes the `BEGIN IMMEDIATE` path
|
|
52
|
+
* below unchanged.
|
|
53
|
+
*/
|
|
54
|
+
export function transaction<T>(db: TxnCapableDb, fn: () => T): T {
|
|
55
|
+
if (typeof db.transactionSync === "function") {
|
|
56
|
+
return db.transactionSync(fn);
|
|
57
|
+
}
|
|
58
|
+
db.exec("BEGIN IMMEDIATE");
|
|
59
|
+
try {
|
|
60
|
+
const result = fn();
|
|
61
|
+
db.exec("COMMIT");
|
|
62
|
+
return result;
|
|
63
|
+
} catch (err) {
|
|
64
|
+
// Best-effort rollback — if the COMMIT itself threw the transaction is
|
|
65
|
+
// already resolved and ROLLBACK would throw "no transaction is active";
|
|
66
|
+
// swallow that so the original error is the one that propagates.
|
|
67
|
+
try {
|
|
68
|
+
db.exec("ROLLBACK");
|
|
69
|
+
} catch {
|
|
70
|
+
// no active transaction to roll back
|
|
71
|
+
}
|
|
72
|
+
throw err;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Async sibling of {@link transaction} for the batch executors, which run
|
|
78
|
+
* their body through the async `Store` facade (`await store.createNote(...)`
|
|
79
|
+
* etc.). The transaction spans the awaits on the shared connection — the
|
|
80
|
+
* same shape as the pre-seam `if (batched) db.exec("BEGIN")` blocks in
|
|
81
|
+
* mcp.ts / routes.ts. A DO backend can't map this onto `transactionSync`
|
|
82
|
+
* (which forbids awaiting); porting the batch path is a Phase-2 concern.
|
|
83
|
+
*
|
|
84
|
+
* Deliberately NOT given the `db.transactionSync` fast-path that {@link transaction}
|
|
85
|
+
* has: `transactionSync` runs its callback synchronously and forbids awaiting,
|
|
86
|
+
* so it cannot wrap an `async` body that awaits the Store facade between writes.
|
|
87
|
+
* A DO backend that needs an atomic async batch has to reshape the batch as a
|
|
88
|
+
* synchronous unit (or use an async DO transaction primitive) — out of scope
|
|
89
|
+
* for this seam; the raw `BEGIN IMMEDIATE` path stays the single implementation.
|
|
90
|
+
*/
|
|
91
|
+
export async function transactionAsync<T>(db: TxnCapableDb, fn: () => Promise<T>): Promise<T> {
|
|
92
|
+
db.exec("BEGIN IMMEDIATE");
|
|
93
|
+
try {
|
|
94
|
+
const result = await fn();
|
|
95
|
+
db.exec("COMMIT");
|
|
96
|
+
return result;
|
|
97
|
+
} catch (err) {
|
|
98
|
+
try {
|
|
99
|
+
db.exec("ROLLBACK");
|
|
100
|
+
} catch {
|
|
101
|
+
// no active transaction to roll back
|
|
102
|
+
}
|
|
103
|
+
throw err;
|
|
104
|
+
}
|
|
105
|
+
}
|
package/core/src/types.ts
CHANGED
|
@@ -263,6 +263,15 @@ export interface Store {
|
|
|
263
263
|
*/
|
|
264
264
|
readonly db: Database;
|
|
265
265
|
|
|
266
|
+
/**
|
|
267
|
+
* Run `fn` inside a single atomic write transaction (commit on return,
|
|
268
|
+
* rollback on throw). The transaction seam (see core/src/txn.ts): the
|
|
269
|
+
* bun backend implements it as `BEGIN IMMEDIATE … COMMIT`, a future
|
|
270
|
+
* Durable-Object backend as `ctx.storage.transactionSync`. Synchronous —
|
|
271
|
+
* `fn` must not await. Nesting is unsupported (matches raw-SQLite BEGIN).
|
|
272
|
+
*/
|
|
273
|
+
transaction<T>(fn: () => T): T;
|
|
274
|
+
|
|
266
275
|
// Notes. `actor` / `via` carry write-attribution (vault#298) — the
|
|
267
276
|
// principal + interface stamped onto created_by/created_via (and mirrored
|
|
268
277
|
// into the last_updated_* pair on create). Omitted → attribution NULL.
|
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
import { DEFAULT_TAG_NAME } from "./tag-hierarchy.ts";
|
|
34
34
|
import * as noteOps from "./notes.ts";
|
|
35
35
|
import type { VaultStats } from "./types.ts";
|
|
36
|
-
import { GETTING_STARTED_PATH } from "./
|
|
36
|
+
import { GETTING_STARTED_PATH } from "./seed-packs.ts";
|
|
37
37
|
|
|
38
38
|
/**
|
|
39
39
|
* Does a note live at the seeded onboarding path? Used to gate the "Start here"
|
package/core/src/wikilinks.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite";
|
|
2
2
|
import * as linkOps from "./links.js";
|
|
3
|
+
import { chunkForInClause } from "./sql-in.js";
|
|
3
4
|
|
|
4
5
|
// ---------------------------------------------------------------------------
|
|
5
6
|
// Parser — extract [[wikilinks]] from markdown content
|
|
@@ -272,10 +273,15 @@ export function listUnresolvedWikilinks(db: Database, limit = 50): { unresolved:
|
|
|
272
273
|
if (rows.length === 0) return { unresolved: [], count: total };
|
|
273
274
|
|
|
274
275
|
const sourceIds = [...new Set(rows.map((r) => r.source_id))];
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
276
|
+
// Chunk under the DO 100-bound-param cap (see sql-in.ts) — with a large
|
|
277
|
+
// `limit` this hydrates >100 source ids in one IN-list otherwise.
|
|
278
|
+
const pathRows: { id: string; path: string | null }[] = [];
|
|
279
|
+
for (const chunk of chunkForInClause(sourceIds)) {
|
|
280
|
+
const placeholders = chunk.map(() => "?").join(", ");
|
|
281
|
+
pathRows.push(...db.prepare(
|
|
282
|
+
`SELECT id, path FROM notes WHERE id IN (${placeholders})`,
|
|
283
|
+
).all(...chunk) as { id: string; path: string | null }[]);
|
|
284
|
+
}
|
|
279
285
|
const pathMap = new Map(pathRows.map((r) => [r.id, r.path]));
|
|
280
286
|
|
|
281
287
|
const unresolved: UnresolvedWikilink[] = rows.map((r) => ({
|