@gmickel/gno 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/skill/SKILL.md +5 -3
- package/assets/skill/cli-reference.md +9 -2
- package/assets/skill/mcp-reference.md +2 -1
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.6.0.zip → gno-browser-clipper-v2.7.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.7.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +109 -18
- package/spec/mcp.md +36 -2
- package/spec/output-schemas/ask.schema.json +1 -1
- package/spec/output-schemas/capture-receipt.schema.json +1 -1
- package/spec/output-schemas/doctor.schema.json +88 -0
- package/spec/output-schemas/error.schema.json +11 -2
- package/spec/output-schemas/get.schema.json +1 -1
- package/spec/output-schemas/mcp-capture-result.schema.json +1 -2
- package/spec/output-schemas/memory-remember.schema.json +2 -2
- package/spec/output-schemas/multi-get.schema.json +4 -1
- package/spec/output-schemas/peek.schema.json +2 -9
- package/spec/output-schemas/resident-status.schema.json +22 -0
- package/spec/output-schemas/search-result.schema.json +1 -1
- package/spec/output-schemas/search-results.schema.json +1 -1
- package/spec/output-schemas/status.schema.json +98 -0
- package/src/cli/commands/doctor.ts +54 -20
- package/src/cli/commands/embed.ts +41 -3
- package/src/cli/commands/query.ts +5 -0
- package/src/cli/commands/status.ts +63 -5
- package/src/cli/commands/vec.ts +54 -0
- package/src/cli/detach.ts +29 -1
- package/src/cli/errors.ts +13 -9
- package/src/cli/program.ts +53 -1
- package/src/core/capture-sync.ts +9 -2
- package/src/core/host-paths.ts +31 -0
- package/src/core/memory-remember.ts +4 -3
- package/src/core/shutdown-budget.ts +6 -0
- package/src/core/vector-partition-status.ts +52 -0
- package/src/embed/backlog.ts +124 -18
- package/src/embed/fingerprint.ts +6 -3
- package/src/embed/retry.ts +66 -27
- package/src/embed/variant-backlog.ts +15 -10
- package/src/embed/variant-retry.ts +31 -22
- package/src/index.ts +21 -2
- package/src/llm/native-worker/dispatcher.ts +2 -0
- package/src/llm/native-worker/embedding-identity.ts +42 -0
- package/src/llm/native-worker/protocol.ts +1 -0
- package/src/llm/types.ts +3 -0
- package/src/mcp/context.ts +9 -0
- package/src/mcp/resources/index.ts +6 -5
- package/src/mcp/tool-descriptions-core.ts +1 -1
- package/src/mcp/tools/capture.ts +1 -3
- package/src/mcp/tools/index.ts +11 -4
- package/src/mcp/tools/memory-remember.ts +1 -1
- package/src/mcp/tools/status.ts +4 -0
- package/src/pipeline/hybrid.ts +37 -7
- package/src/pipeline/vsearch.ts +14 -2
- package/src/serve/embed-scheduler.ts +133 -19
- package/src/serve/host-path-redaction.ts +79 -0
- package/src/serve/public/components/sessions/SessionSearch.tsx +2 -2
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/hooks/use-api.ts +17 -2
- package/src/serve/public/lib/request-intent.ts +8 -0
- package/src/serve/public/{components/sessions → lib}/snippet.tsx +2 -3
- package/src/serve/public/pages/Dashboard.tsx +12 -9
- package/src/serve/public/pages/DocView.tsx +10 -5
- package/src/serve/public/pages/DocumentEditor.tsx +91 -14
- package/src/serve/public/pages/Search.tsx +1 -41
- package/src/serve/resident-runtime.ts +26 -1
- package/src/serve/resident-status.ts +13 -1
- package/src/serve/server.ts +10 -9
- package/src/serve/status-model.ts +16 -0
- package/src/serve/status.ts +2 -0
- package/src/serve/watch-reconciliation-shared.ts +3 -0
- package/src/serve/watch-service-events.ts +3 -2
- package/src/serve/watch-service-run-flush.ts +35 -2
- package/src/serve/watch-service.ts +5 -0
- package/src/store/migrations/031-runtime-independent-vectors.ts +29 -0
- package/src/store/migrations/032-vector-runtime-callers.ts +17 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +22 -1
- package/src/store/types.ts +11 -1
- package/src/store/vector/lazy.ts +46 -43
- package/src/store/vector/runtime-compat.ts +651 -0
- package/src/store/vector/sqlite-vec.ts +20 -2
- package/src/store/vector/status.ts +276 -35
- package/src/store/vector/types.ts +2 -0
- package/src/store/vector/variant-search.ts +71 -23
- package/src/store/vector/variants.ts +49 -14
- package/browser-extension/artifacts/gno-browser-clipper-v2.6.0.zip.sha256 +0 -1
package/src/cli/program.ts
CHANGED
|
@@ -41,7 +41,11 @@ import {
|
|
|
41
41
|
type GlobalOptions,
|
|
42
42
|
parseGlobalOptions,
|
|
43
43
|
} from "./context";
|
|
44
|
-
import {
|
|
44
|
+
import {
|
|
45
|
+
DETACHED_CHILD_FLAG,
|
|
46
|
+
formatBackgroundIssue,
|
|
47
|
+
residentIssues,
|
|
48
|
+
} from "./detach";
|
|
45
49
|
import { CliError } from "./errors";
|
|
46
50
|
import {
|
|
47
51
|
assertFormatSupported,
|
|
@@ -3544,6 +3548,10 @@ function wireManagementCommands(program: Command): void {
|
|
|
3544
3548
|
.option("--model <uri>", "embedding model URI")
|
|
3545
3549
|
.option("--batch-size <num>", "batch size", "32")
|
|
3546
3550
|
.option("--force", "regenerate all embeddings")
|
|
3551
|
+
.option(
|
|
3552
|
+
"--new-partition",
|
|
3553
|
+
"confirm building a separate vector partition for an incompatible runtime"
|
|
3554
|
+
)
|
|
3547
3555
|
.option("--dry-run", "show what would be done")
|
|
3548
3556
|
.option("--json", "JSON output")
|
|
3549
3557
|
).action(
|
|
@@ -3567,6 +3575,7 @@ function wireManagementCommands(program: Command): void {
|
|
|
3567
3575
|
force: Boolean(cmdOpts.force),
|
|
3568
3576
|
dryRun: Boolean(cmdOpts.dryRun),
|
|
3569
3577
|
yes: globals.yes,
|
|
3578
|
+
newPartition: Boolean(cmdOpts.newPartition),
|
|
3570
3579
|
json: format === "json",
|
|
3571
3580
|
verbose: globals.verbose,
|
|
3572
3581
|
offline: globals.offline,
|
|
@@ -3675,6 +3684,43 @@ function wireVecCommands(program: Command): void {
|
|
|
3675
3684
|
);
|
|
3676
3685
|
});
|
|
3677
3686
|
|
|
3687
|
+
// vec drop
|
|
3688
|
+
addWriteLeaseFlags(
|
|
3689
|
+
vecCmd
|
|
3690
|
+
.command("drop <partition>")
|
|
3691
|
+
.description(
|
|
3692
|
+
"Drop an abandoned shadow vector partition (id prefix from gno status)"
|
|
3693
|
+
)
|
|
3694
|
+
.option("--json", "JSON output")
|
|
3695
|
+
).action(async (partition: string, cmdOpts: Record<string, unknown>) => {
|
|
3696
|
+
const format = getFormat(cmdOpts);
|
|
3697
|
+
const globals = getGlobals();
|
|
3698
|
+
|
|
3699
|
+
const { vecDrop, formatVecDrop } = await import("./commands/vec");
|
|
3700
|
+
const lease = parseWriteLeaseFlags(cmdOpts);
|
|
3701
|
+
const result = await withCliWriteLease(
|
|
3702
|
+
{
|
|
3703
|
+
indexName: globals.index,
|
|
3704
|
+
lockWaitMs: lease.lockWaitMs,
|
|
3705
|
+
noWait: lease.noWait,
|
|
3706
|
+
},
|
|
3707
|
+
() =>
|
|
3708
|
+
vecDrop(partition, {
|
|
3709
|
+
configPath: globals.config,
|
|
3710
|
+
indexName: globals.index,
|
|
3711
|
+
})
|
|
3712
|
+
);
|
|
3713
|
+
throwIfWriteLeaseBusy(result, format === "json");
|
|
3714
|
+
|
|
3715
|
+
if (!result.success) {
|
|
3716
|
+
throw new CliError("VALIDATION", result.error);
|
|
3717
|
+
}
|
|
3718
|
+
|
|
3719
|
+
process.stdout.write(
|
|
3720
|
+
`${formatVecDrop(result, { json: format === "json" })}\n`
|
|
3721
|
+
);
|
|
3722
|
+
});
|
|
3723
|
+
|
|
3678
3724
|
// vec rebuild
|
|
3679
3725
|
addWriteLeaseFlags(
|
|
3680
3726
|
vecCmd
|
|
@@ -5017,6 +5063,9 @@ async function runDaemonStatus(deps: DaemonStatusDeps): Promise<void> {
|
|
|
5017
5063
|
} else {
|
|
5018
5064
|
process.stdout.write(` (${status.log_size_bytes} bytes)\n`);
|
|
5019
5065
|
}
|
|
5066
|
+
for (const issue of residentIssues(status)) {
|
|
5067
|
+
process.stdout.write(` issue ${formatBackgroundIssue(issue)}\n`);
|
|
5068
|
+
}
|
|
5020
5069
|
if (findings) {
|
|
5021
5070
|
process.stdout.write(
|
|
5022
5071
|
` findings ${formatFindingsRunStatusLine(findings)}\n`
|
|
@@ -5372,6 +5421,9 @@ async function runServeStatus(deps: ServeStatusDeps): Promise<void> {
|
|
|
5372
5421
|
} else {
|
|
5373
5422
|
process.stdout.write(` (${status.log_size_bytes} bytes)\n`);
|
|
5374
5423
|
}
|
|
5424
|
+
for (const issue of residentIssues(status)) {
|
|
5425
|
+
process.stdout.write(` issue ${formatBackgroundIssue(issue)}\n`);
|
|
5426
|
+
}
|
|
5375
5427
|
|
|
5376
5428
|
if (foreign) {
|
|
5377
5429
|
// Terminal mode: emit the operator-facing warning on stderr. JSON
|
package/src/core/capture-sync.ts
CHANGED
|
@@ -10,6 +10,7 @@ import type { Collection, Config } from "../config/types";
|
|
|
10
10
|
import type { StorePort } from "../store/types";
|
|
11
11
|
import type { CaptureIndexStatus } from "./capture";
|
|
12
12
|
|
|
13
|
+
import { buildUri } from "../app/constants";
|
|
13
14
|
import {
|
|
14
15
|
type CollectionSyncResult,
|
|
15
16
|
defaultSyncService,
|
|
@@ -30,9 +31,14 @@ export class CaptureSyncError extends Error {
|
|
|
30
31
|
/** The sync failure itself, without the write half of the message. */
|
|
31
32
|
readonly syncError: string;
|
|
32
33
|
|
|
33
|
-
constructor(input: {
|
|
34
|
+
constructor(input: {
|
|
35
|
+
absPath: string;
|
|
36
|
+
relPath: string;
|
|
37
|
+
uri: string;
|
|
38
|
+
cause: string;
|
|
39
|
+
}) {
|
|
34
40
|
super(
|
|
35
|
-
`Capture written to ${input.
|
|
41
|
+
`Capture written to ${input.uri} but lexical sync failed: ${input.cause}. Run gno update to retry indexing.`
|
|
36
42
|
);
|
|
37
43
|
this.name = "CaptureSyncError";
|
|
38
44
|
this.absPath = input.absPath;
|
|
@@ -82,6 +88,7 @@ export async function syncCapturedFile(
|
|
|
82
88
|
throw new CaptureSyncError({
|
|
83
89
|
absPath: input.absPath,
|
|
84
90
|
relPath: input.relPath,
|
|
91
|
+
uri: buildUri(input.collection.name, input.relPath),
|
|
85
92
|
cause,
|
|
86
93
|
});
|
|
87
94
|
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host path redaction for callers that are not on the owner's machine.
|
|
3
|
+
*
|
|
4
|
+
* Result payloads name a source file's host location in `absPath`
|
|
5
|
+
* (`source.absPath` on search/get results, top-level on capture, memory and
|
|
6
|
+
* peek receipts). A remote caller identifies documents by `uri` and
|
|
7
|
+
* collection-relative `relPath` instead, so every `absPath` key is removed
|
|
8
|
+
* before a payload leaves a remote-reachable surface.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const HOST_PATH_FIELD = "absPath";
|
|
12
|
+
|
|
13
|
+
const isPlainObject = (value: object): boolean => {
|
|
14
|
+
const proto = Object.getPrototypeOf(value) as object | null;
|
|
15
|
+
return proto === Object.prototype || proto === null;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/** Deep copy of a JSON-shaped value with every `absPath` key removed. */
|
|
19
|
+
export function withoutHostPaths<T>(value: T): T {
|
|
20
|
+
if (Array.isArray(value)) {
|
|
21
|
+
return value.map((item: unknown) => withoutHostPaths(item)) as T;
|
|
22
|
+
}
|
|
23
|
+
if (value === null || typeof value !== "object" || !isPlainObject(value)) {
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
26
|
+
const copy: Record<string, unknown> = {};
|
|
27
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
28
|
+
if (key !== HOST_PATH_FIELD) copy[key] = withoutHostPaths(entry);
|
|
29
|
+
}
|
|
30
|
+
return copy as T;
|
|
31
|
+
}
|
|
@@ -24,6 +24,7 @@ import type {
|
|
|
24
24
|
RememberResult,
|
|
25
25
|
} from "./memory-types";
|
|
26
26
|
|
|
27
|
+
import { buildUri } from "../app/constants";
|
|
27
28
|
import { defaultSyncService, withContentTypeRules } from "../ingestion";
|
|
28
29
|
import { withWriteLock } from "./file-lock";
|
|
29
30
|
import { atomicCreate } from "./file-ops";
|
|
@@ -399,7 +400,7 @@ export async function rememberFact(
|
|
|
399
400
|
if (sync.status === "failed") {
|
|
400
401
|
throw new MemoryError(
|
|
401
402
|
"MEMORY_SYNC_FAILED",
|
|
402
|
-
`Memory record written to ${
|
|
403
|
+
`Memory record written to ${buildUri(collection.name, relPath)} but lexical sync failed: ${sync.error}. Run gno update to retry indexing.`
|
|
403
404
|
);
|
|
404
405
|
}
|
|
405
406
|
const written = (doc as { value: DocumentRow }).value;
|
|
@@ -415,13 +416,13 @@ export async function rememberFact(
|
|
|
415
416
|
if (!projected) {
|
|
416
417
|
throw new MemoryError(
|
|
417
418
|
"MEMORY_SUPERSEDE_PROJECTION_FAILED",
|
|
418
|
-
`Successor written to ${
|
|
419
|
+
`Successor written to ${buildUri(collection.name, relPath)} but its supersedes edge did not project${projectionErrors ? ` (${projectionErrors})` : ""}; the predecessor still reads as current. Run gno update to retry the projection.`
|
|
419
420
|
);
|
|
420
421
|
}
|
|
421
422
|
} else if (projectionErrors.length > 0) {
|
|
422
423
|
throw new MemoryError(
|
|
423
424
|
"MEMORY_SYNC_FAILED",
|
|
424
|
-
`Memory record written to ${
|
|
425
|
+
`Memory record written to ${buildUri(collection.name, relPath)} but typed-edge projection failed: ${projectionErrors}. Run gno update to retry indexing.`
|
|
425
426
|
);
|
|
426
427
|
}
|
|
427
428
|
const record: MemoryFact = {
|
|
@@ -4,6 +4,12 @@ export const SHUTDOWN_ABORT_MS = 5_000;
|
|
|
4
4
|
export const SHUTDOWN_EXIT_MS = 1_000;
|
|
5
5
|
// The detached parent must not be killed before it can reap its native child.
|
|
6
6
|
export const RESIDENT_STOP_GRACE_MS = 12_000;
|
|
7
|
+
/**
|
|
8
|
+
* Longest synchronous SQLite busy wait a resident allows itself. A signal is
|
|
9
|
+
* handled only after the wait in progress ends, so this plus the shutdown
|
|
10
|
+
* clock must fit inside the stop grace (asserted in shutdown-budget tests).
|
|
11
|
+
*/
|
|
12
|
+
export const RESIDENT_BUSY_TIMEOUT_MS = 500;
|
|
7
13
|
|
|
8
14
|
/** Observe settlement without abandoning rejection handling or retaining a timer. */
|
|
9
15
|
export async function settlesBy(
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
VectorPartitionStatus,
|
|
3
|
+
VectorRuntimeStatus,
|
|
4
|
+
} from "../store/vector/status";
|
|
5
|
+
|
|
6
|
+
const shortId = (id: string): string => id.slice(0, 12);
|
|
7
|
+
|
|
8
|
+
function runtimeLine(runtime: VectorRuntimeStatus): string {
|
|
9
|
+
const label = runtime.label ? ` (${runtime.label})` : "";
|
|
10
|
+
if (runtime.state === "vectors")
|
|
11
|
+
return ` This runtime${label} reads ${shortId(runtime.partition ?? "")}`;
|
|
12
|
+
if (runtime.state === "unavailable")
|
|
13
|
+
return ` This runtime${label} uses lexical retrieval only: ${runtime.reason}`;
|
|
14
|
+
return " This runtime has not resolved a partition yet; run a query or `gno embed`";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The caller's runtime, then one line per partition. The healthy case (one
|
|
19
|
+
* current partition this runtime reads) prints nothing, so default status
|
|
20
|
+
* output is unchanged.
|
|
21
|
+
*/
|
|
22
|
+
export function formatVectorPartitionLines(
|
|
23
|
+
partitions?: VectorPartitionStatus[],
|
|
24
|
+
runtime?: VectorRuntimeStatus
|
|
25
|
+
): string[] {
|
|
26
|
+
if (!partitions?.length) return [];
|
|
27
|
+
const [only] = partitions;
|
|
28
|
+
if (
|
|
29
|
+
partitions.length === 1 &&
|
|
30
|
+
only?.retrieval &&
|
|
31
|
+
!only.legacy &&
|
|
32
|
+
!only.incompatibleRuntimes.length
|
|
33
|
+
)
|
|
34
|
+
return [];
|
|
35
|
+
const lines = ["Vector partitions:"];
|
|
36
|
+
if (runtime) lines.push(runtimeLine(runtime));
|
|
37
|
+
for (const p of partitions) {
|
|
38
|
+
const role = p.retrieval
|
|
39
|
+
? " (used by this runtime's retrieval)"
|
|
40
|
+
: p.droppable
|
|
41
|
+
? ` (drop with: gno vec drop ${shortId(p.id)})`
|
|
42
|
+
: "";
|
|
43
|
+
lines.push(
|
|
44
|
+
` ${p.retrieval ? "*" : " "} ${shortId(p.id)} ${p.state}${p.legacy ? " legacy" : ""}, ${p.owners} chunks, ${p.provenance}${role}`
|
|
45
|
+
);
|
|
46
|
+
if (p.compatibleRuntimes.length)
|
|
47
|
+
lines.push(` read by: ${p.compatibleRuntimes.join("; ")}`);
|
|
48
|
+
if (p.incompatibleRuntimes.length)
|
|
49
|
+
lines.push(` incompatible: ${p.incompatibleRuntimes.join("; ")}`);
|
|
50
|
+
}
|
|
51
|
+
return lines;
|
|
52
|
+
}
|
package/src/embed/backlog.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite";
|
|
2
|
+
|
|
1
3
|
import type { EmbeddingPort } from "../llm/types";
|
|
2
4
|
/**
|
|
3
5
|
* Shared embedding backlog processor.
|
|
@@ -12,21 +14,26 @@ import type {
|
|
|
12
14
|
VectorStatsPort,
|
|
13
15
|
} from "../store/vector";
|
|
14
16
|
import type { VectorVariantStore } from "../store/vector/variants";
|
|
17
|
+
import type { AcquireWriteTurn } from "./retry";
|
|
15
18
|
|
|
16
19
|
import {
|
|
17
20
|
assertInferenceActive,
|
|
18
21
|
isBackgroundInference,
|
|
19
22
|
} from "../llm/inference-scope";
|
|
23
|
+
import { formatDocForEmbedding } from "../pipeline/contextual";
|
|
20
24
|
import { err, ok } from "../store/types";
|
|
25
|
+
import {
|
|
26
|
+
embeddingPartitionIdentity,
|
|
27
|
+
recordReferenceRuntime,
|
|
28
|
+
resolveRuntimePartition,
|
|
29
|
+
} from "../store/vector/runtime-compat";
|
|
21
30
|
import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
22
31
|
import { createVectorVariantStore } from "../store/vector/variants";
|
|
23
|
-
import {
|
|
24
|
-
getEmbeddingFingerprint,
|
|
25
|
-
getVariantModelFingerprint,
|
|
26
|
-
} from "./fingerprint";
|
|
32
|
+
import { getEmbeddingFingerprint } from "./fingerprint";
|
|
27
33
|
import {
|
|
28
34
|
chunkRetryKey,
|
|
29
35
|
embedAndStoreBatch,
|
|
36
|
+
inWriteTurn,
|
|
30
37
|
MAX_EMBED_CHUNK_ATTEMPTS,
|
|
31
38
|
} from "./retry";
|
|
32
39
|
import { embedVariantBacklog } from "./variant-backlog";
|
|
@@ -47,6 +54,14 @@ export interface EmbedBacklogDeps {
|
|
|
47
54
|
variantStore?: VectorVariantStore;
|
|
48
55
|
/** Recheck the effective runtime identity after asynchronous inference. */
|
|
49
56
|
identityStillCurrent?: () => boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Write gate for callers that do not already hold the shared writer lease
|
|
59
|
+
* (the resident scheduler): taken around each page's writes and released
|
|
60
|
+
* after. null means another writer holds it; the pass stops as deferred.
|
|
61
|
+
*/
|
|
62
|
+
acquireWriteTurn?: AcquireWriteTurn;
|
|
63
|
+
/** Explicit confirmation to build a separate vector partition (never implied by --yes). */
|
|
64
|
+
allowNewPartition?: boolean;
|
|
50
65
|
}
|
|
51
66
|
|
|
52
67
|
export interface EmbedBacklogResult {
|
|
@@ -59,6 +74,8 @@ export interface EmbedBacklogResult {
|
|
|
59
74
|
contentionErrors?: number;
|
|
60
75
|
/** Error message if vec index sync failed (embeddings stored, but search may be stale) */
|
|
61
76
|
syncError?: string;
|
|
77
|
+
/** The pass stopped early because another writer held the write gate. */
|
|
78
|
+
deferred?: boolean;
|
|
62
79
|
}
|
|
63
80
|
|
|
64
81
|
interface Cursor {
|
|
@@ -78,7 +95,18 @@ export async function embedBacklog(
|
|
|
78
95
|
deps: EmbedBacklogDeps
|
|
79
96
|
): Promise<StoreResult<EmbedBacklogResult>> {
|
|
80
97
|
assertInferenceActive();
|
|
81
|
-
|
|
98
|
+
if (deps.acquireWriteTurn && !deps.variantStore) {
|
|
99
|
+
// Model loading stays outside the write turn; preparation's partition
|
|
100
|
+
// writes below take one like every other background write.
|
|
101
|
+
const initialized = await deps.embedPort.init();
|
|
102
|
+
if (!initialized.ok) return err("INTERNAL", initialized.error.message);
|
|
103
|
+
}
|
|
104
|
+
const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
|
|
105
|
+
prepareEmbeddingBacklog(deps)
|
|
106
|
+
);
|
|
107
|
+
if (turn.deferred)
|
|
108
|
+
return ok({ embedded: 0, errors: 0, contentionErrors: 0, deferred: true });
|
|
109
|
+
const prepared = turn.value;
|
|
82
110
|
if (!prepared.ok) return prepared;
|
|
83
111
|
deps = prepared.value;
|
|
84
112
|
if (deps.variantStore) return embedVariantBacklog(deps, deps.variantStore);
|
|
@@ -201,7 +229,10 @@ export async function embedBacklog(
|
|
|
201
229
|
embedFingerprint,
|
|
202
230
|
identityStillCurrent: deps.identityStillCurrent,
|
|
203
231
|
statsPort,
|
|
232
|
+
acquireWriteTurn: deps.acquireWriteTurn,
|
|
204
233
|
});
|
|
234
|
+
if (batchStoreResult.deferred)
|
|
235
|
+
return ok({ embedded, errors, contentionErrors, deferred: true });
|
|
205
236
|
embedded += batchStoreResult.embedded;
|
|
206
237
|
errors += batchStoreResult.errors;
|
|
207
238
|
contentionErrors += batchStoreResult.contentionErrors;
|
|
@@ -225,7 +256,12 @@ export async function embedBacklog(
|
|
|
225
256
|
// Sync vec index once at end if any vec0 writes failed
|
|
226
257
|
let syncError: string | undefined;
|
|
227
258
|
if (vectorIndex.vecDirty) {
|
|
228
|
-
const
|
|
259
|
+
const turn = await inWriteTurn(deps.acquireWriteTurn, () =>
|
|
260
|
+
vectorIndex.syncVecIndex()
|
|
261
|
+
);
|
|
262
|
+
if (turn.deferred)
|
|
263
|
+
return ok({ embedded, errors, contentionErrors, deferred: true });
|
|
264
|
+
const syncResult = turn.value;
|
|
229
265
|
if (syncResult.ok) {
|
|
230
266
|
const { added, removed } = syncResult.value;
|
|
231
267
|
if (added > 0 || removed > 0) {
|
|
@@ -248,6 +284,59 @@ export async function embedBacklog(
|
|
|
248
284
|
}
|
|
249
285
|
}
|
|
250
286
|
|
|
287
|
+
function formatEstimate(ms: number): string {
|
|
288
|
+
const seconds = Math.max(1, Math.round(ms / 1000));
|
|
289
|
+
if (seconds < 90) return `about ${seconds} s`;
|
|
290
|
+
const minutes = Math.round(seconds / 60);
|
|
291
|
+
return minutes < 90
|
|
292
|
+
? `about ${minutes} min`
|
|
293
|
+
: `about ${(minutes / 60).toFixed(1)} h`;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** Time this port on a few current chunks when no compatibility sample ran. */
|
|
297
|
+
async function measureEmbedRate(
|
|
298
|
+
db: Database,
|
|
299
|
+
port: EmbeddingPort
|
|
300
|
+
): Promise<number | undefined> {
|
|
301
|
+
const inputs = db
|
|
302
|
+
.query<{ text: string; title: string | null }, []>(`
|
|
303
|
+
SELECT c.text, d.title FROM documents d
|
|
304
|
+
JOIN content_chunks c ON c.mirror_hash = d.mirror_hash
|
|
305
|
+
WHERE d.active = 1 ORDER BY d.id, c.seq LIMIT 8
|
|
306
|
+
`)
|
|
307
|
+
.all()
|
|
308
|
+
.map((row) =>
|
|
309
|
+
formatDocForEmbedding(row.text, row.title ?? undefined, port.modelUri)
|
|
310
|
+
);
|
|
311
|
+
if (!inputs.length) return undefined;
|
|
312
|
+
const startedAt = performance.now();
|
|
313
|
+
const result = await port.embedBatch(inputs);
|
|
314
|
+
return result.ok
|
|
315
|
+
? (performance.now() - startedAt) / inputs.length
|
|
316
|
+
: undefined;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** R3: a fork names itself, its full size and a measured estimate before it runs. */
|
|
320
|
+
async function separatePartitionMessage(
|
|
321
|
+
db: Database,
|
|
322
|
+
port: EmbeddingPort,
|
|
323
|
+
reason: string,
|
|
324
|
+
msPerChunk: number | undefined
|
|
325
|
+
): Promise<string> {
|
|
326
|
+
const chunks = db
|
|
327
|
+
.query<{ count: number }, []>(`
|
|
328
|
+
SELECT count(*) AS count FROM documents d
|
|
329
|
+
JOIN content_chunks c ON c.mirror_hash = d.mirror_hash WHERE d.active = 1
|
|
330
|
+
`)
|
|
331
|
+
.get()!.count;
|
|
332
|
+
const rate = msPerChunk ?? (await measureEmbedRate(db, port));
|
|
333
|
+
const estimate =
|
|
334
|
+
rate === undefined
|
|
335
|
+
? "no estimate: embedding could not be timed"
|
|
336
|
+
: `estimated ${formatEstimate(rate * chunks)} at the measured ${Math.round(rate)} ms per chunk`;
|
|
337
|
+
return `Embedding would build a separate vector partition (${reason}). It re-embeds all ${chunks} chunks (${estimate}). Confirm with \`gno embed --new-partition\`; --yes alone does not confirm.`;
|
|
338
|
+
}
|
|
339
|
+
|
|
251
340
|
/** Resolve authority before counts, dry runs, forced work, or early returns. */
|
|
252
341
|
export async function prepareEmbeddingBacklog(
|
|
253
342
|
deps: EmbedBacklogDeps
|
|
@@ -259,19 +348,36 @@ export async function prepareEmbeddingBacklog(
|
|
|
259
348
|
const initialized = await deps.embedPort.init();
|
|
260
349
|
if (!initialized.ok) return err("INTERNAL", initialized.error.message);
|
|
261
350
|
const identity = deps.embedPort.getIdentity?.();
|
|
262
|
-
|
|
351
|
+
const primary = embeddingPartitionIdentity(deps.embedPort);
|
|
352
|
+
if (identity && primary) {
|
|
263
353
|
const identitySnapshot = JSON.stringify(identity);
|
|
264
|
-
const dimensions =
|
|
265
|
-
const
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
354
|
+
const dimensions = primary.dimensions;
|
|
355
|
+
const resolved = await resolveRuntimePartition(
|
|
356
|
+
db,
|
|
357
|
+
deps.embedPort,
|
|
358
|
+
primary
|
|
359
|
+
);
|
|
360
|
+
if (resolved.blocked && !deps.allowNewPartition)
|
|
361
|
+
return err(
|
|
362
|
+
"VECTOR_PARTITION_FORK",
|
|
363
|
+
await separatePartitionMessage(
|
|
364
|
+
db,
|
|
365
|
+
deps.embedPort,
|
|
366
|
+
resolved.blocked.reason,
|
|
367
|
+
resolved.msPerChunk
|
|
368
|
+
)
|
|
369
|
+
);
|
|
370
|
+
const variantStore = await createVectorVariantStore(
|
|
371
|
+
db,
|
|
372
|
+
resolved.blocked?.separate ?? resolved.identity,
|
|
373
|
+
identity.runtimeLabel
|
|
374
|
+
);
|
|
375
|
+
if (resolved.blocked || resolved.verdict === "unverified") {
|
|
376
|
+
// Vectors without a current owner cannot be measured; they must not
|
|
377
|
+
// survive to be reused by the runtime that becomes the reference.
|
|
378
|
+
variantStore.collectGarbage();
|
|
379
|
+
recordReferenceRuntime(db, variantStore.partitionId, identity);
|
|
380
|
+
}
|
|
275
381
|
variantStore.selectForEmbedding();
|
|
276
382
|
return ok({
|
|
277
383
|
...deps,
|
package/src/embed/fingerprint.ts
CHANGED
|
@@ -36,16 +36,19 @@ export function getEmbeddingFingerprint(
|
|
|
36
36
|
.digest("hex");
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Partition identity: actual weights plus the formatter policy. Runtime details
|
|
41
|
+
* (Bun, native binding, backend, threads) are provenance, never identity; the
|
|
42
|
+
* measured compatibility check decides whether a runtime may share vectors.
|
|
43
|
+
*/
|
|
40
44
|
export function getVariantModelFingerprint(
|
|
41
45
|
input: EmbeddingFingerprintInput,
|
|
42
|
-
identity: { modelFingerprint: string
|
|
46
|
+
identity: { modelFingerprint: string }
|
|
43
47
|
): string {
|
|
44
48
|
return new Bun.CryptoHasher("sha256")
|
|
45
49
|
.update(
|
|
46
50
|
JSON.stringify([
|
|
47
51
|
identity.modelFingerprint,
|
|
48
|
-
identity.runtimeFingerprint,
|
|
49
52
|
getEmbeddingFingerprint(input),
|
|
50
53
|
])
|
|
51
54
|
)
|
package/src/embed/retry.ts
CHANGED
|
@@ -15,6 +15,28 @@ import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
|
15
15
|
import { embedTextsWithRecovery } from "./batch";
|
|
16
16
|
|
|
17
17
|
export const MAX_EMBED_CHUNK_ATTEMPTS = 2;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Write gate for callers that do not already hold the shared writer lease
|
|
21
|
+
* (the resident scheduler). Returns a release, or null when another writer
|
|
22
|
+
* holds the lease.
|
|
23
|
+
*/
|
|
24
|
+
export type AcquireWriteTurn = () => Promise<(() => Promise<void>) | null>;
|
|
25
|
+
|
|
26
|
+
/** Run `write` inside one write turn, or report that the gate is held elsewhere. */
|
|
27
|
+
export async function inWriteTurn<T>(
|
|
28
|
+
acquire: AcquireWriteTurn | undefined,
|
|
29
|
+
write: () => Promise<T>
|
|
30
|
+
): Promise<{ deferred: true } | { deferred: false; value: T }> {
|
|
31
|
+
if (!acquire) return { deferred: false, value: await write() };
|
|
32
|
+
const release = await acquire();
|
|
33
|
+
if (!release) return { deferred: true };
|
|
34
|
+
try {
|
|
35
|
+
return { deferred: false, value: await write() };
|
|
36
|
+
} finally {
|
|
37
|
+
await release();
|
|
38
|
+
}
|
|
39
|
+
}
|
|
18
40
|
export const MAX_EMBED_FAILURE_SAMPLES = 5;
|
|
19
41
|
|
|
20
42
|
/** Total upsert attempts (initial + retries) when persistence hits SQLITE_BUSY/LOCKED. */
|
|
@@ -44,6 +66,8 @@ export interface EmbedStoreBatchResult {
|
|
|
44
66
|
suggestion?: string;
|
|
45
67
|
batchFailed: boolean;
|
|
46
68
|
batchError?: string;
|
|
69
|
+
/** The write gate was held elsewhere; nothing was persisted. */
|
|
70
|
+
deferred?: boolean;
|
|
47
71
|
}
|
|
48
72
|
|
|
49
73
|
// fn-127 integration: CLI consumers (src/cli/commands/embed.ts,
|
|
@@ -173,6 +197,8 @@ export async function embedAndStoreBatch(params: {
|
|
|
173
197
|
identityStillCurrent?: () => boolean;
|
|
174
198
|
/** Test seam: override contention-retry delays in milliseconds. */
|
|
175
199
|
delays?: number[];
|
|
200
|
+
/** Gate held only around persistence, never around inference. */
|
|
201
|
+
acquireWriteTurn?: AcquireWriteTurn;
|
|
176
202
|
}): Promise<EmbedStoreBatchResult> {
|
|
177
203
|
const { embedPort, vectorIndex, items, modelUri, embedFingerprint } = params;
|
|
178
204
|
const db = params.statsPort && getVectorStatsDatabase(params.statsPort);
|
|
@@ -282,35 +308,48 @@ export async function embedAndStoreBatch(params: {
|
|
|
282
308
|
);
|
|
283
309
|
});
|
|
284
310
|
};
|
|
285
|
-
const
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
311
|
+
const turn = await inWriteTurn(params.acquireWriteTurn, () =>
|
|
312
|
+
upsertVectorsWithContentionRetry(
|
|
313
|
+
{
|
|
314
|
+
upsertVectors: async (rows) => {
|
|
315
|
+
if (vectorIndex.upsertVectorsChecked) {
|
|
316
|
+
const result = await vectorIndex.upsertVectorsChecked(
|
|
317
|
+
rows,
|
|
318
|
+
(candidates) => {
|
|
319
|
+
committedRows = checkpoint(candidates);
|
|
320
|
+
return committedRows;
|
|
321
|
+
}
|
|
322
|
+
);
|
|
323
|
+
if (!result.ok) return result;
|
|
324
|
+
written = result.value;
|
|
325
|
+
return ok(undefined);
|
|
326
|
+
}
|
|
327
|
+
if (db)
|
|
328
|
+
return err("INVALID_INPUT", "Atomic vector checkpoint unavailable");
|
|
329
|
+
const valid = checkpoint(rows);
|
|
330
|
+
const result = await vectorIndex.upsertVectors(valid);
|
|
331
|
+
if (result.ok) {
|
|
332
|
+
written = valid.length;
|
|
333
|
+
committedRows = valid;
|
|
334
|
+
}
|
|
335
|
+
return result;
|
|
336
|
+
},
|
|
309
337
|
},
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
338
|
+
vectors,
|
|
339
|
+
params.delays
|
|
340
|
+
)
|
|
313
341
|
);
|
|
342
|
+
if (turn.deferred)
|
|
343
|
+
return {
|
|
344
|
+
embedded: 0,
|
|
345
|
+
errors: 0,
|
|
346
|
+
contentionErrors: 0,
|
|
347
|
+
retryItems: [],
|
|
348
|
+
errorSamples: [],
|
|
349
|
+
batchFailed: false,
|
|
350
|
+
deferred: true,
|
|
351
|
+
};
|
|
352
|
+
const storeResult = turn.value;
|
|
314
353
|
if (!storeResult.ok) {
|
|
315
354
|
if (isUpsertLockContention(storeResult.error)) {
|
|
316
355
|
return {
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
} from "../llm/inference-scope";
|
|
9
9
|
import { err, ok } from "../store/types";
|
|
10
10
|
import { getVectorStatsDatabase } from "../store/vector/stats";
|
|
11
|
+
import { inWriteTurn } from "./retry";
|
|
11
12
|
import { variantBacklogPage } from "./variant-plan";
|
|
12
13
|
import { embedVariantBatch } from "./variant-retry";
|
|
13
14
|
|
|
@@ -59,7 +60,9 @@ export async function embedVariantBacklog(
|
|
|
59
60
|
owners,
|
|
60
61
|
identityStillCurrent,
|
|
61
62
|
force: deps.force,
|
|
63
|
+
acquireWriteTurn: deps.acquireWriteTurn,
|
|
62
64
|
});
|
|
65
|
+
if (result.deferred) return ok({ ...total, deferred: true });
|
|
63
66
|
total.embedded += result.embedded;
|
|
64
67
|
total.errors += result.errors;
|
|
65
68
|
total.contentionErrors += result.contentionErrors;
|
|
@@ -83,16 +86,18 @@ export async function embedVariantBacklog(
|
|
|
83
86
|
// Capture the epoch before checking completeness; activate rechecks under write lock.
|
|
84
87
|
const epoch = store.epoch();
|
|
85
88
|
if (identityStillCurrent() && !store.pending({ limit: 1 }).length) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}
|
|
89
|
+
const turn = await inWriteTurn(deps.acquireWriteTurn, async () => {
|
|
90
|
+
try {
|
|
91
|
+
if (!store.isActive()) store.syncIndex();
|
|
92
|
+
if (identityStillCurrent()) store.activate(epoch);
|
|
93
|
+
return undefined;
|
|
94
|
+
} catch (cause) {
|
|
95
|
+
return cause instanceof Error ? cause.message : String(cause);
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
if (turn.deferred) return ok({ ...total, deferred: true });
|
|
99
|
+
if (turn.value !== undefined)
|
|
100
|
+
return ok({ ...total, syncError: turn.value });
|
|
96
101
|
}
|
|
97
102
|
assertInferenceActive();
|
|
98
103
|
return ok(total);
|