@gmickel/gno 1.37.1 → 1.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/browser-extension/artifacts/{gno-browser-clipper-v1.37.1.zip → gno-browser-clipper-v1.38.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.38.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +35 -15
- package/src/cli/commands/cleanup.ts +8 -2
- package/src/cli/commands/collection/clear-embeddings.ts +6 -1
- package/src/cli/commands/doctor-activation.ts +5 -1
- package/src/cli/commands/doctor.ts +72 -2
- package/src/cli/commands/embed.ts +227 -194
- package/src/cli/commands/index-cmd.ts +74 -50
- package/src/cli/commands/init.ts +5 -1
- package/src/cli/commands/profile-apply.ts +5 -1
- package/src/cli/commands/setup-activation.ts +2 -1
- package/src/cli/commands/setup.ts +2 -1
- package/src/cli/commands/shared.ts +5 -1
- package/src/cli/commands/status.ts +5 -1
- package/src/cli/commands/tags.ts +18 -3
- package/src/cli/commands/update.ts +34 -27
- package/src/cli/commands/vec.ts +13 -4
- package/src/cli/errors.ts +3 -2
- package/src/cli/program.ts +345 -194
- package/src/config/defaults.ts +2 -0
- package/src/config/index.ts +3 -0
- package/src/config/types.ts +32 -1
- package/src/core/file-lock.ts +16 -4
- package/src/core/write-lease.ts +354 -0
- package/src/embed/backlog.ts +9 -1
- package/src/embed/retry.ts +116 -3
- package/src/sdk/client.ts +3 -1
- package/src/sdk/embed.ts +8 -3
- package/src/sdk/types.ts +2 -0
- package/src/serve/embed-scheduler.ts +8 -0
- package/src/serve/resident-runtime.ts +5 -1
- package/src/store/sqlite/adapter.ts +28 -4
- package/src/store/sqlite/scoped-index.ts +5 -1
- package/src/store/vector/sqlite-vec.ts +2 -1
- package/browser-extension/artifacts/gno-browser-clipper-v1.37.1.zip.sha256 +0 -1
package/src/config/types.ts
CHANGED
|
@@ -48,6 +48,17 @@ export type FtsTokenizer = (typeof FTS_TOKENIZERS)[number];
|
|
|
48
48
|
/** Default FTS tokenizer - snowball english for multilingual stemming */
|
|
49
49
|
export const DEFAULT_FTS_TOKENIZER: FtsTokenizer = "snowball english";
|
|
50
50
|
|
|
51
|
+
/**
|
|
52
|
+
* SQLite `busy_timeout` in milliseconds. Writers wait this long for a lock
|
|
53
|
+
* before failing with SQLITE_BUSY. Matched to real embedding-pass duration
|
|
54
|
+
* rather than a fail-fast 5s floor.
|
|
55
|
+
*/
|
|
56
|
+
export const DEFAULT_BUSY_TIMEOUT_MS = 60_000;
|
|
57
|
+
export const MIN_BUSY_TIMEOUT_MS = 1_000;
|
|
58
|
+
export const MAX_BUSY_TIMEOUT_MS = 600_000;
|
|
59
|
+
const BUSY_TIMEOUT_RANGE_MESSAGE =
|
|
60
|
+
"busyTimeoutMs must be an integer between 1000 and 600000";
|
|
61
|
+
|
|
51
62
|
/** Collection-owned boundary for where indexed content may travel. */
|
|
52
63
|
export const EGRESS_POLICIES = ["local_only", "lan", "remote"] as const;
|
|
53
64
|
export const EgressPolicySchema = z.enum(EGRESS_POLICIES);
|
|
@@ -502,6 +513,21 @@ export const ConfigSchema = z.object({
|
|
|
502
513
|
/** FTS tokenizer (immutable after init) */
|
|
503
514
|
ftsTokenizer: z.enum(FTS_TOKENIZERS).default(DEFAULT_FTS_TOKENIZER),
|
|
504
515
|
|
|
516
|
+
/**
|
|
517
|
+
* SQLite busy_timeout in milliseconds. Default 60000. Range 1000-600000.
|
|
518
|
+
* Raise for long embedding passes on slow disks.
|
|
519
|
+
*/
|
|
520
|
+
busyTimeoutMs: z
|
|
521
|
+
.number()
|
|
522
|
+
.refine(
|
|
523
|
+
(value) =>
|
|
524
|
+
Number.isInteger(value) &&
|
|
525
|
+
value >= MIN_BUSY_TIMEOUT_MS &&
|
|
526
|
+
value <= MAX_BUSY_TIMEOUT_MS,
|
|
527
|
+
{ message: BUSY_TIMEOUT_RANGE_MESSAGE }
|
|
528
|
+
)
|
|
529
|
+
.default(DEFAULT_BUSY_TIMEOUT_MS),
|
|
530
|
+
|
|
505
531
|
/** Optional terminal hyperlink editor URI template */
|
|
506
532
|
editorUriTemplate: z.string().min(1).optional(),
|
|
507
533
|
|
|
@@ -546,8 +572,13 @@ export const ConfigSchema = z.object({
|
|
|
546
572
|
.optional(),
|
|
547
573
|
});
|
|
548
574
|
|
|
549
|
-
export type Config = Omit<
|
|
575
|
+
export type Config = Omit<
|
|
576
|
+
z.infer<typeof ConfigSchema>,
|
|
577
|
+
"contentTypes" | "busyTimeoutMs"
|
|
578
|
+
> & {
|
|
550
579
|
contentTypes?: ContentTypeConfig[];
|
|
580
|
+
/** Present after schema parse; omitted on hand-built Config objects. */
|
|
581
|
+
busyTimeoutMs?: number;
|
|
551
582
|
};
|
|
552
583
|
|
|
553
584
|
// ─────────────────────────────────────────────────────────────────────────────
|
package/src/core/file-lock.ts
CHANGED
|
@@ -12,7 +12,6 @@ import { dirname } from "node:path";
|
|
|
12
12
|
|
|
13
13
|
import { MCP_ERRORS } from "./errors";
|
|
14
14
|
const DEFAULT_TIMEOUT_MS = 5000;
|
|
15
|
-
const HOLD_SECONDS = 60 * 60 * 24 * 365;
|
|
16
15
|
const READY_TOKEN = "READY";
|
|
17
16
|
const SQLITE_LOCK_SUFFIX = ".sqlite";
|
|
18
17
|
const MAX_BUSY_TIMEOUT_MS = 60_000;
|
|
@@ -67,7 +66,9 @@ function resolveLockCommand(): LockCommand | null {
|
|
|
67
66
|
}
|
|
68
67
|
|
|
69
68
|
function buildHoldCommand(): string {
|
|
70
|
-
|
|
69
|
+
// Block on stdin so parent death (including SIGKILL) closes the pipe,
|
|
70
|
+
// the holder exits, and the OS advisory lock is released.
|
|
71
|
+
return `printf '${READY_TOKEN}\\n'; read _`;
|
|
71
72
|
}
|
|
72
73
|
|
|
73
74
|
async function waitForReady(
|
|
@@ -121,7 +122,7 @@ function normalizedBusyTimeout(timeoutMs: number): number {
|
|
|
121
122
|
return Math.min(Math.max(0, Math.floor(timeoutMs)), MAX_BUSY_TIMEOUT_MS);
|
|
122
123
|
}
|
|
123
124
|
|
|
124
|
-
function isSqliteLockContention(cause: unknown): boolean {
|
|
125
|
+
export function isSqliteLockContention(cause: unknown): boolean {
|
|
125
126
|
if (cause === null || typeof cause !== "object") {
|
|
126
127
|
return false;
|
|
127
128
|
}
|
|
@@ -181,10 +182,12 @@ export async function acquireWriteLock(
|
|
|
181
182
|
[cmd.path, ...cmd.args(lockPath, timeoutSeconds, holdCommand)],
|
|
182
183
|
{
|
|
183
184
|
detached: true,
|
|
185
|
+
stdin: "pipe",
|
|
184
186
|
stdout: "pipe",
|
|
185
187
|
stderr: "pipe",
|
|
186
188
|
}
|
|
187
189
|
);
|
|
190
|
+
const stdin = proc.stdin;
|
|
188
191
|
|
|
189
192
|
const ready = await waitForReady(proc);
|
|
190
193
|
if (!ready) {
|
|
@@ -193,7 +196,16 @@ export async function acquireWriteLock(
|
|
|
193
196
|
}
|
|
194
197
|
|
|
195
198
|
return {
|
|
196
|
-
release: () =>
|
|
199
|
+
release: async () => {
|
|
200
|
+
if (stdin && typeof stdin !== "number") {
|
|
201
|
+
try {
|
|
202
|
+
await stdin.end();
|
|
203
|
+
} catch {
|
|
204
|
+
// Ignore: terminateLockProcess still reaps the holder.
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
await terminateLockProcess(proc);
|
|
208
|
+
},
|
|
197
209
|
};
|
|
198
210
|
}
|
|
199
211
|
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI write-lease wrapper around the shared `.mcp-write.lock` namespace.
|
|
3
|
+
*
|
|
4
|
+
* @module src/core/write-lease
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
// node:fs/promises unlink — filesystem structure op, no Bun equivalent.
|
|
8
|
+
import { unlink } from "node:fs/promises";
|
|
9
|
+
// node:path for dirname/join (no Bun path utils)
|
|
10
|
+
import { dirname, join } from "node:path";
|
|
11
|
+
|
|
12
|
+
import { getIndexDbPath } from "../app/constants";
|
|
13
|
+
import { acquireWriteLock } from "./file-lock";
|
|
14
|
+
|
|
15
|
+
export const DEFAULT_LOCK_WAIT_MS = 120_000;
|
|
16
|
+
export const WRITE_LEASE_BUSY_MESSAGE =
|
|
17
|
+
"index is busy -- another write is in progress";
|
|
18
|
+
|
|
19
|
+
const LOCK_FILE_NAME = ".mcp-write.lock";
|
|
20
|
+
const HOLDER_SIDECAR_SUFFIX = ".holder.json";
|
|
21
|
+
const LOCK_SLICE_MS = 5_000;
|
|
22
|
+
const WAIT_PROGRESS_INTERVAL_MS = 15_000;
|
|
23
|
+
const LOCK_WAIT_PATTERN = /^(?<value>\d+)(?<unit>s|m)?$/;
|
|
24
|
+
/** Ceiling on any lease wait: a longer value is a misconfiguration, not a queue. */
|
|
25
|
+
export const MAX_LOCK_WAIT_MS = 86_400_000;
|
|
26
|
+
|
|
27
|
+
export type WriteLeaseResult =
|
|
28
|
+
| { ok: true; release: () => Promise<void> }
|
|
29
|
+
| { ok: false; timedOut: true; waitedMs: number; holder: string | null };
|
|
30
|
+
|
|
31
|
+
export interface WriteLeaseContention {
|
|
32
|
+
outcome: "lock_timeout";
|
|
33
|
+
waitedMs: number;
|
|
34
|
+
holder: string | null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface WriteLeaseBusyFailure {
|
|
38
|
+
success: false;
|
|
39
|
+
error: string;
|
|
40
|
+
contention: WriteLeaseContention;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface AcquireCliWriteLeaseOptions {
|
|
44
|
+
dbPath: string;
|
|
45
|
+
waitMs: number;
|
|
46
|
+
noWait?: boolean;
|
|
47
|
+
/** Override the sidecar command string. Defaults to `gno ` + argv subcommand. */
|
|
48
|
+
command?: string;
|
|
49
|
+
onWaitProgress?: (info: { waitedMs: number; holder: string | null }) => void;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface CliWriteLeaseOptions {
|
|
53
|
+
indexName?: string;
|
|
54
|
+
lockWaitMs?: number;
|
|
55
|
+
noWait?: boolean;
|
|
56
|
+
skipWriteLease?: boolean;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
interface HolderSidecar {
|
|
60
|
+
pid: number;
|
|
61
|
+
command: string;
|
|
62
|
+
startedAtIso: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Lock file path used by MCP tools and JobManager: `<dbDir>/.mcp-write.lock`.
|
|
67
|
+
*/
|
|
68
|
+
export function writeLeasePath(dbPath: string): string {
|
|
69
|
+
return join(dirname(dbPath), LOCK_FILE_NAME);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function holderSidecarPath(lockPath: string): string {
|
|
73
|
+
return `${lockPath}${HOLDER_SIDECAR_SUFFIX}`;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Parse `--lock-wait` into milliseconds.
|
|
78
|
+
* Accepts plain seconds ("120"), "120s", or "2m". Returns null when invalid.
|
|
79
|
+
*/
|
|
80
|
+
export function parseLockWaitMs(raw: unknown): number | null {
|
|
81
|
+
if (typeof raw !== "string" && typeof raw !== "number") {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
const text = String(raw).trim();
|
|
85
|
+
const match = LOCK_WAIT_PATTERN.exec(text);
|
|
86
|
+
const valueText = match?.groups?.value;
|
|
87
|
+
if (!valueText) {
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
const value = Number(valueText);
|
|
91
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
const unit = match.groups?.unit ?? "s";
|
|
95
|
+
const ms = unit === "m" ? value * 60_000 : value * 1_000;
|
|
96
|
+
if (!Number.isSafeInteger(ms) || ms > MAX_LOCK_WAIT_MS) {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
return ms;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function formatWriteLeaseBusyMessage(contention: {
|
|
103
|
+
waitedMs: number;
|
|
104
|
+
holder: string | null;
|
|
105
|
+
}): string {
|
|
106
|
+
const holder = contention.holder ?? "unknown (no holder metadata)";
|
|
107
|
+
const waitedS = Math.max(0, Math.floor(contention.waitedMs / 1000));
|
|
108
|
+
return [
|
|
109
|
+
`gno: ${WRITE_LEASE_BUSY_MESSAGE}`,
|
|
110
|
+
` held by: ${holder}`,
|
|
111
|
+
` waited: ${waitedS}s (--lock-wait)`,
|
|
112
|
+
" This is contention, not corruption. Reads are unaffected. Retry when it finishes,",
|
|
113
|
+
" or raise the wait with --lock-wait.",
|
|
114
|
+
].join("\n");
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function formatWriteLeaseBusyJson(contention: {
|
|
118
|
+
waitedMs: number;
|
|
119
|
+
holder: string | null;
|
|
120
|
+
}): WriteLeaseBusyFailure {
|
|
121
|
+
return {
|
|
122
|
+
success: false,
|
|
123
|
+
error: WRITE_LEASE_BUSY_MESSAGE,
|
|
124
|
+
contention: {
|
|
125
|
+
outcome: "lock_timeout",
|
|
126
|
+
waitedMs: contention.waitedMs,
|
|
127
|
+
holder: contention.holder,
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export function isWriteLeaseBusyResult(result: {
|
|
133
|
+
success: boolean;
|
|
134
|
+
contention?: unknown;
|
|
135
|
+
}): result is WriteLeaseBusyFailure {
|
|
136
|
+
if (result.success !== false) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
if (result.contention === null || typeof result.contention !== "object") {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
return (
|
|
143
|
+
"outcome" in result.contention &&
|
|
144
|
+
result.contention.outcome === "lock_timeout"
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export async function acquireCliWriteLease(
|
|
149
|
+
options: AcquireCliWriteLeaseOptions
|
|
150
|
+
): Promise<WriteLeaseResult> {
|
|
151
|
+
const lockPath = writeLeasePath(options.dbPath);
|
|
152
|
+
// A non-finite or out-of-range wait must never loop unbounded (fn-127 review).
|
|
153
|
+
const requestedWaitMs = Number.isFinite(options.waitMs)
|
|
154
|
+
? Math.min(Math.max(0, options.waitMs), MAX_LOCK_WAIT_MS)
|
|
155
|
+
: DEFAULT_LOCK_WAIT_MS;
|
|
156
|
+
const waitMs = options.noWait ? 0 : requestedWaitMs;
|
|
157
|
+
const startedAt = Date.now();
|
|
158
|
+
let lastProgressAt = -WAIT_PROGRESS_INTERVAL_MS;
|
|
159
|
+
|
|
160
|
+
while (true) {
|
|
161
|
+
const elapsed = Date.now() - startedAt;
|
|
162
|
+
const remaining = Math.max(0, waitMs - elapsed);
|
|
163
|
+
const sliceMs = options.noWait ? 0 : Math.min(LOCK_SLICE_MS, remaining);
|
|
164
|
+
const handle = await acquireWriteLock(lockPath, sliceMs);
|
|
165
|
+
if (handle) {
|
|
166
|
+
await writeHolderSidecar(lockPath, options.command);
|
|
167
|
+
return {
|
|
168
|
+
ok: true,
|
|
169
|
+
release: async () => {
|
|
170
|
+
await deleteHolderSidecar(lockPath);
|
|
171
|
+
await handle.release();
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const waitedMs = Date.now() - startedAt;
|
|
177
|
+
const holder = await readHolderDescription(lockPath);
|
|
178
|
+
if (options.noWait || waitedMs >= waitMs) {
|
|
179
|
+
return { ok: false, timedOut: true, waitedMs, holder };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (waitedMs - lastProgressAt >= WAIT_PROGRESS_INTERVAL_MS) {
|
|
183
|
+
options.onWaitProgress?.({ waitedMs, holder });
|
|
184
|
+
lastProgressAt = waitedMs;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Acquire the shared write lease, run `fn`, and release on every path.
|
|
191
|
+
* Nested callers pass `skipWriteLease` to avoid self-deadlock (index → embed).
|
|
192
|
+
*/
|
|
193
|
+
export async function withCliWriteLease<T>(
|
|
194
|
+
options: CliWriteLeaseOptions,
|
|
195
|
+
fn: () => Promise<T>
|
|
196
|
+
): Promise<T | WriteLeaseBusyFailure> {
|
|
197
|
+
if (options.skipWriteLease) {
|
|
198
|
+
return await fn();
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const result = await acquireCliWriteLease({
|
|
202
|
+
dbPath: getIndexDbPath(options.indexName),
|
|
203
|
+
waitMs: options.lockWaitMs ?? DEFAULT_LOCK_WAIT_MS,
|
|
204
|
+
noWait: options.noWait,
|
|
205
|
+
onWaitProgress: writeWaitProgress,
|
|
206
|
+
});
|
|
207
|
+
if (!result.ok) {
|
|
208
|
+
return formatWriteLeaseBusyJson(result);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
try {
|
|
212
|
+
return await fn();
|
|
213
|
+
} finally {
|
|
214
|
+
try {
|
|
215
|
+
await result.release();
|
|
216
|
+
} catch (cause) {
|
|
217
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
218
|
+
process.stderr.write(`gno: failed to release write lease: ${message}\n`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function writeWaitProgress(info: {
|
|
224
|
+
waitedMs: number;
|
|
225
|
+
holder: string | null;
|
|
226
|
+
}): void {
|
|
227
|
+
const holder = info.holder ?? "unknown";
|
|
228
|
+
const waitedS = Math.max(0, Math.floor(info.waitedMs / 1000));
|
|
229
|
+
process.stderr.write(
|
|
230
|
+
`gno: waiting for index write lease (held by ${holder}, waited ${waitedS}s)\n`
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function resolveHolderCommand(explicit?: string): string {
|
|
235
|
+
if (explicit && explicit.trim().length > 0) {
|
|
236
|
+
return explicit.trim();
|
|
237
|
+
}
|
|
238
|
+
const tokens: string[] = [];
|
|
239
|
+
for (const token of process.argv.slice(2)) {
|
|
240
|
+
if (token.startsWith("-")) {
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
if (token.includes("/") || token.includes("\\")) {
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
tokens.push(token);
|
|
247
|
+
}
|
|
248
|
+
return tokens.length > 0 ? `gno ${tokens.join(" ")}` : "gno";
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function isPidAlive(pid: number): boolean {
|
|
252
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
253
|
+
return false;
|
|
254
|
+
}
|
|
255
|
+
try {
|
|
256
|
+
process.kill(pid, 0);
|
|
257
|
+
return true;
|
|
258
|
+
} catch (error) {
|
|
259
|
+
return (
|
|
260
|
+
error !== null &&
|
|
261
|
+
typeof error === "object" &&
|
|
262
|
+
"code" in error &&
|
|
263
|
+
error.code === "EPERM"
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function asHolderSidecar(value: unknown): HolderSidecar | null {
|
|
269
|
+
if (value === null || typeof value !== "object") {
|
|
270
|
+
return null;
|
|
271
|
+
}
|
|
272
|
+
if (
|
|
273
|
+
!("pid" in value) ||
|
|
274
|
+
!("command" in value) ||
|
|
275
|
+
!("startedAtIso" in value)
|
|
276
|
+
) {
|
|
277
|
+
return null;
|
|
278
|
+
}
|
|
279
|
+
if (typeof value.pid !== "number" || !Number.isInteger(value.pid)) {
|
|
280
|
+
return null;
|
|
281
|
+
}
|
|
282
|
+
if (typeof value.command !== "string" || value.command.length === 0) {
|
|
283
|
+
return null;
|
|
284
|
+
}
|
|
285
|
+
if (typeof value.startedAtIso !== "string") {
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
return {
|
|
289
|
+
pid: value.pid,
|
|
290
|
+
command: value.command,
|
|
291
|
+
startedAtIso: value.startedAtIso,
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function formatElapsed(ms: number): string {
|
|
296
|
+
const totalSeconds = Math.max(0, Math.floor(ms / 1000));
|
|
297
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
298
|
+
const seconds = totalSeconds % 60;
|
|
299
|
+
if (minutes === 0) {
|
|
300
|
+
return `${seconds}s`;
|
|
301
|
+
}
|
|
302
|
+
return `${minutes}m${String(seconds).padStart(2, "0")}s`;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function formatHolder(sidecar: HolderSidecar): string {
|
|
306
|
+
const startedAt = Date.parse(sidecar.startedAtIso);
|
|
307
|
+
const elapsedMs = Number.isFinite(startedAt)
|
|
308
|
+
? Math.max(0, Date.now() - startedAt)
|
|
309
|
+
: 0;
|
|
310
|
+
return `${sidecar.command} (pid ${sidecar.pid}), running ${formatElapsed(elapsedMs)}`;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
async function writeHolderSidecar(
|
|
314
|
+
lockPath: string,
|
|
315
|
+
command?: string
|
|
316
|
+
): Promise<void> {
|
|
317
|
+
try {
|
|
318
|
+
const payload: HolderSidecar = {
|
|
319
|
+
pid: process.pid,
|
|
320
|
+
command: resolveHolderCommand(command),
|
|
321
|
+
startedAtIso: new Date().toISOString(),
|
|
322
|
+
};
|
|
323
|
+
await Bun.write(
|
|
324
|
+
holderSidecarPath(lockPath),
|
|
325
|
+
`${JSON.stringify(payload)}\n`
|
|
326
|
+
);
|
|
327
|
+
} catch {
|
|
328
|
+
// Identification is best-effort and must never fail acquisition.
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
async function deleteHolderSidecar(lockPath: string): Promise<void> {
|
|
333
|
+
try {
|
|
334
|
+
await unlink(holderSidecarPath(lockPath));
|
|
335
|
+
} catch {
|
|
336
|
+
// Sidecar cleanup must never fail release.
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
async function readHolderDescription(lockPath: string): Promise<string | null> {
|
|
341
|
+
try {
|
|
342
|
+
const sidecarFile = Bun.file(holderSidecarPath(lockPath));
|
|
343
|
+
if (!(await sidecarFile.exists())) {
|
|
344
|
+
return null;
|
|
345
|
+
}
|
|
346
|
+
const sidecar = asHolderSidecar(await sidecarFile.json());
|
|
347
|
+
if (!sidecar || !isPidAlive(sidecar.pid)) {
|
|
348
|
+
return null;
|
|
349
|
+
}
|
|
350
|
+
return formatHolder(sidecar);
|
|
351
|
+
} catch {
|
|
352
|
+
return null;
|
|
353
|
+
}
|
|
354
|
+
}
|
package/src/embed/backlog.ts
CHANGED
|
@@ -37,6 +37,11 @@ export interface EmbedBacklogDeps {
|
|
|
37
37
|
export interface EmbedBacklogResult {
|
|
38
38
|
embedded: number;
|
|
39
39
|
errors: number;
|
|
40
|
+
/**
|
|
41
|
+
* Chunks whose persistence failed after SQLITE_BUSY/SQLITE_LOCKED retries.
|
|
42
|
+
* Distinct from `errors` (embedding-provider failures). Default 0.
|
|
43
|
+
*/
|
|
44
|
+
contentionErrors?: number;
|
|
40
45
|
/** Error message if vec index sync failed (embeddings stored, but search may be stale) */
|
|
41
46
|
syncError?: string;
|
|
42
47
|
}
|
|
@@ -66,6 +71,7 @@ export async function embedBacklog(
|
|
|
66
71
|
|
|
67
72
|
let embedded = 0;
|
|
68
73
|
let errors = 0;
|
|
74
|
+
let contentionErrors = 0;
|
|
69
75
|
let cursor: Cursor | undefined;
|
|
70
76
|
const retryQueue = new Map<string, { item: BacklogItem; attempts: number }>();
|
|
71
77
|
|
|
@@ -107,6 +113,7 @@ export async function embedBacklog(
|
|
|
107
113
|
|
|
108
114
|
embedded += retryResult.embedded;
|
|
109
115
|
errors += retryResult.errors;
|
|
116
|
+
contentionErrors += retryResult.contentionErrors;
|
|
110
117
|
retryEmbedded += retryResult.embedded;
|
|
111
118
|
|
|
112
119
|
const retryByKey = new Set(
|
|
@@ -165,6 +172,7 @@ export async function embedBacklog(
|
|
|
165
172
|
});
|
|
166
173
|
embedded += batchStoreResult.embedded;
|
|
167
174
|
errors += batchStoreResult.errors;
|
|
175
|
+
contentionErrors += batchStoreResult.contentionErrors;
|
|
168
176
|
enqueueRetryItems(batchStoreResult.retryItems, 1);
|
|
169
177
|
|
|
170
178
|
if (embedded > beforeEmbedded) {
|
|
@@ -190,7 +198,7 @@ export async function embedBacklog(
|
|
|
190
198
|
}
|
|
191
199
|
}
|
|
192
200
|
|
|
193
|
-
return ok({ embedded, errors, syncError });
|
|
201
|
+
return ok({ embedded, errors, contentionErrors, syncError });
|
|
194
202
|
} catch (e) {
|
|
195
203
|
return err(
|
|
196
204
|
"INTERNAL",
|
package/src/embed/retry.ts
CHANGED
|
@@ -1,15 +1,36 @@
|
|
|
1
1
|
import type { EmbeddingPort } from "../llm/types";
|
|
2
|
+
import type { StoreResult } from "../store/types";
|
|
2
3
|
import type { BacklogItem, VectorIndexPort, VectorRow } from "../store/vector";
|
|
3
4
|
|
|
5
|
+
import { isSqliteLockContention } from "../core/file-lock";
|
|
4
6
|
import { formatDocForEmbedding } from "../pipeline/contextual";
|
|
5
7
|
import { embedTextsWithRecovery } from "./batch";
|
|
6
8
|
|
|
7
9
|
export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
|
|
8
10
|
export const MAX_EMBED_FAILURE_SAMPLES = 5;
|
|
9
11
|
|
|
12
|
+
/** Total upsert attempts (initial + retries) when persistence hits SQLITE_BUSY/LOCKED. */
|
|
13
|
+
export const UPSERT_CONTENTION_MAX_ATTEMPTS = 5;
|
|
14
|
+
export const UPSERT_CONTENTION_BASE_DELAY_MS = 250;
|
|
15
|
+
export const UPSERT_CONTENTION_BACKOFF_FACTOR = 2;
|
|
16
|
+
export const UPSERT_CONTENTION_JITTER_RATIO = 0.25;
|
|
17
|
+
export const UPSERT_CONTENTION_MAX_DELAY_MS = 5_000;
|
|
18
|
+
|
|
19
|
+
export const UPSERT_CONTENTION_ERROR_SAMPLE =
|
|
20
|
+
"index is busy (SQLITE_BUSY) — another writer holds the database; rerun `gno embed`";
|
|
21
|
+
|
|
22
|
+
export const STORE_WRITE_FAILURE_SUGGESTION =
|
|
23
|
+
"Store write failed. Rerun `gno embed` once more; if it repeats, run `gno doctor` and `gno vec sync`.";
|
|
24
|
+
|
|
10
25
|
export interface EmbedStoreBatchResult {
|
|
11
26
|
embedded: number;
|
|
12
27
|
errors: number;
|
|
28
|
+
/**
|
|
29
|
+
* Chunks whose persistence failed after SQLITE_BUSY/SQLITE_LOCKED retries.
|
|
30
|
+
* Distinct from `errors` (embedding-provider / non-contention store failures).
|
|
31
|
+
* Default 0.
|
|
32
|
+
*/
|
|
33
|
+
contentionErrors: number;
|
|
13
34
|
retryItems: BacklogItem[];
|
|
14
35
|
errorSamples: string[];
|
|
15
36
|
suggestion?: string;
|
|
@@ -17,6 +38,10 @@ export interface EmbedStoreBatchResult {
|
|
|
17
38
|
batchError?: string;
|
|
18
39
|
}
|
|
19
40
|
|
|
41
|
+
// fn-127 integration: CLI consumers (src/cli/commands/embed.ts,
|
|
42
|
+
// src/cli/commands/index-cmd.ts) must surface contentionErrors in their
|
|
43
|
+
// summary and exit non-zero — the integrator wires that.
|
|
44
|
+
|
|
20
45
|
export function chunkRetryKey(item: Pick<BacklogItem, "mirrorHash" | "seq">) {
|
|
21
46
|
return `${item.mirrorHash}\0${item.seq}`;
|
|
22
47
|
}
|
|
@@ -52,12 +77,81 @@ export function formatLlmFailure(
|
|
|
52
77
|
: error.message;
|
|
53
78
|
}
|
|
54
79
|
|
|
80
|
+
/**
|
|
81
|
+
* Classify a store-layer failure as SQLite lock contention.
|
|
82
|
+
* Matches SQLITE_BUSY/LOCKED on the error itself (stubs) or its `cause`
|
|
83
|
+
* (upsertVectors preserves the original SQLite error there).
|
|
84
|
+
*/
|
|
85
|
+
export function isUpsertLockContention(error: {
|
|
86
|
+
code?: unknown;
|
|
87
|
+
cause?: unknown;
|
|
88
|
+
}): boolean {
|
|
89
|
+
return isSqliteLockContention(error) || isSqliteLockContention(error.cause);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Delay before the next upsert retry after `failedAttemptIndex` (0-based)
|
|
94
|
+
* contention failures. Formula: base * factor^attempt ± jitter, capped.
|
|
95
|
+
*/
|
|
96
|
+
export function upsertContentionDelayMs(
|
|
97
|
+
failedAttemptIndex: number,
|
|
98
|
+
random: () => number = Math.random
|
|
99
|
+
): number {
|
|
100
|
+
const exponential =
|
|
101
|
+
UPSERT_CONTENTION_BASE_DELAY_MS *
|
|
102
|
+
UPSERT_CONTENTION_BACKOFF_FACTOR ** failedAttemptIndex;
|
|
103
|
+
const jitterMultiplier =
|
|
104
|
+
1 + (random() * 2 - 1) * UPSERT_CONTENTION_JITTER_RATIO;
|
|
105
|
+
return Math.min(
|
|
106
|
+
UPSERT_CONTENTION_MAX_DELAY_MS,
|
|
107
|
+
Math.max(0, exponential * jitterMultiplier)
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function resolveContentionDelayMs(
|
|
112
|
+
failedAttemptIndex: number,
|
|
113
|
+
delays: number[] | undefined
|
|
114
|
+
): number {
|
|
115
|
+
if (delays) {
|
|
116
|
+
return delays[failedAttemptIndex] ?? 0;
|
|
117
|
+
}
|
|
118
|
+
return upsertContentionDelayMs(failedAttemptIndex);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Retry `upsertVectors` on SQLITE_BUSY/SQLITE_LOCKED with exponential backoff.
|
|
123
|
+
* `delays` is a test seam that replaces the computed schedule (missing slots = 0).
|
|
124
|
+
*/
|
|
125
|
+
export async function upsertVectorsWithContentionRetry(
|
|
126
|
+
vectorIndex: Pick<VectorIndexPort, "upsertVectors">,
|
|
127
|
+
vectors: VectorRow[],
|
|
128
|
+
delays?: number[]
|
|
129
|
+
): Promise<StoreResult<void>> {
|
|
130
|
+
let storeResult = await vectorIndex.upsertVectors(vectors);
|
|
131
|
+
let attempts = 1;
|
|
132
|
+
while (
|
|
133
|
+
!storeResult.ok &&
|
|
134
|
+
isUpsertLockContention(storeResult.error) &&
|
|
135
|
+
attempts < UPSERT_CONTENTION_MAX_ATTEMPTS
|
|
136
|
+
) {
|
|
137
|
+
const delayMs = resolveContentionDelayMs(attempts - 1, delays);
|
|
138
|
+
if (delayMs > 0) {
|
|
139
|
+
await Bun.sleep(delayMs);
|
|
140
|
+
}
|
|
141
|
+
storeResult = await vectorIndex.upsertVectors(vectors);
|
|
142
|
+
attempts += 1;
|
|
143
|
+
}
|
|
144
|
+
return storeResult;
|
|
145
|
+
}
|
|
146
|
+
|
|
55
147
|
export async function embedAndStoreBatch(params: {
|
|
56
148
|
embedPort: EmbeddingPort;
|
|
57
149
|
vectorIndex: VectorIndexPort;
|
|
58
150
|
items: BacklogItem[];
|
|
59
151
|
modelUri: string;
|
|
60
152
|
embedFingerprint: string;
|
|
153
|
+
/** Test seam: override contention-retry delays in milliseconds. */
|
|
154
|
+
delays?: number[];
|
|
61
155
|
}): Promise<EmbedStoreBatchResult> {
|
|
62
156
|
const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
|
|
63
157
|
const embedResult = await embedTextsWithRecovery(
|
|
@@ -72,6 +166,7 @@ export async function embedAndStoreBatch(params: {
|
|
|
72
166
|
return {
|
|
73
167
|
embedded: 0,
|
|
74
168
|
errors: embedResult.error.retryable ? 0 : items.length,
|
|
169
|
+
contentionErrors: 0,
|
|
75
170
|
retryItems: embedResult.error.retryable ? items : [],
|
|
76
171
|
errorSamples: [formattedError],
|
|
77
172
|
suggestion: embedResult.error.retryable
|
|
@@ -103,6 +198,7 @@ export async function embedAndStoreBatch(params: {
|
|
|
103
198
|
return {
|
|
104
199
|
embedded: 0,
|
|
105
200
|
errors: 0,
|
|
201
|
+
contentionErrors: 0,
|
|
106
202
|
retryItems,
|
|
107
203
|
errorSamples: embedResult.value.failureSamples,
|
|
108
204
|
suggestion: embedResult.value.retrySuggestion,
|
|
@@ -111,15 +207,31 @@ export async function embedAndStoreBatch(params: {
|
|
|
111
207
|
};
|
|
112
208
|
}
|
|
113
209
|
|
|
114
|
-
const storeResult = await
|
|
210
|
+
const storeResult = await upsertVectorsWithContentionRetry(
|
|
211
|
+
vectorIndex,
|
|
212
|
+
vectors,
|
|
213
|
+
params.delays
|
|
214
|
+
);
|
|
115
215
|
if (!storeResult.ok) {
|
|
216
|
+
if (isUpsertLockContention(storeResult.error)) {
|
|
217
|
+
return {
|
|
218
|
+
embedded: 0,
|
|
219
|
+
errors: 0,
|
|
220
|
+
contentionErrors: vectors.length,
|
|
221
|
+
retryItems,
|
|
222
|
+
errorSamples: [UPSERT_CONTENTION_ERROR_SAMPLE],
|
|
223
|
+
suggestion: UPSERT_CONTENTION_ERROR_SAMPLE,
|
|
224
|
+
batchFailed: embedResult.value.batchFailed,
|
|
225
|
+
batchError: embedResult.value.batchError,
|
|
226
|
+
};
|
|
227
|
+
}
|
|
116
228
|
return {
|
|
117
229
|
embedded: 0,
|
|
118
230
|
errors: vectors.length,
|
|
231
|
+
contentionErrors: 0,
|
|
119
232
|
retryItems,
|
|
120
233
|
errorSamples: [storeResult.error.message],
|
|
121
|
-
suggestion:
|
|
122
|
-
"Store write failed. Rerun `gno embed` once more; if it repeats, run `gno doctor` and `gno vec sync`.",
|
|
234
|
+
suggestion: STORE_WRITE_FAILURE_SUGGESTION,
|
|
123
235
|
batchFailed: embedResult.value.batchFailed,
|
|
124
236
|
batchError: embedResult.value.batchError,
|
|
125
237
|
};
|
|
@@ -128,6 +240,7 @@ export async function embedAndStoreBatch(params: {
|
|
|
128
240
|
return {
|
|
129
241
|
embedded: vectors.length,
|
|
130
242
|
errors: 0,
|
|
243
|
+
contentionErrors: 0,
|
|
131
244
|
retryItems,
|
|
132
245
|
errorSamples: embedResult.value.failureSamples,
|
|
133
246
|
suggestion: embedResult.value.retrySuggestion,
|
package/src/sdk/client.ts
CHANGED
|
@@ -276,7 +276,9 @@ async function resolveClientState(
|
|
|
276
276
|
|
|
277
277
|
const store = new SqliteAdapter();
|
|
278
278
|
store.setConfigPath(configPath ?? "<inline-config>");
|
|
279
|
-
unwrapStore(
|
|
279
|
+
unwrapStore(
|
|
280
|
+
await store.open(dbPath, config.ftsTokenizer, config.busyTimeoutMs)
|
|
281
|
+
);
|
|
280
282
|
unwrapStore(await store.syncCollections(config.collections));
|
|
281
283
|
unwrapStore(await store.syncContexts(config.contexts ?? []));
|
|
282
284
|
|