@gmickel/gno 1.37.0 → 1.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/assets/spa-production.json.gz +0 -0
  2. package/browser-extension/artifacts/{gno-browser-clipper-v1.37.0.zip → gno-browser-clipper-v1.38.0.zip} +0 -0
  3. package/browser-extension/artifacts/gno-browser-clipper-v1.38.0.zip.sha256 +1 -0
  4. package/browser-extension/dist/manifest.json +1 -1
  5. package/package.json +1 -1
  6. package/spec/cli.md +35 -15
  7. package/src/cli/commands/cleanup.ts +8 -2
  8. package/src/cli/commands/collection/clear-embeddings.ts +6 -1
  9. package/src/cli/commands/doctor-activation.ts +5 -1
  10. package/src/cli/commands/doctor.ts +72 -2
  11. package/src/cli/commands/embed.ts +227 -194
  12. package/src/cli/commands/index-cmd.ts +74 -50
  13. package/src/cli/commands/init.ts +5 -1
  14. package/src/cli/commands/profile-apply.ts +5 -1
  15. package/src/cli/commands/setup-activation.ts +2 -1
  16. package/src/cli/commands/setup.ts +2 -1
  17. package/src/cli/commands/shared.ts +5 -1
  18. package/src/cli/commands/status.ts +5 -1
  19. package/src/cli/commands/tags.ts +18 -3
  20. package/src/cli/commands/update.ts +34 -27
  21. package/src/cli/commands/vec.ts +13 -4
  22. package/src/cli/errors.ts +3 -2
  23. package/src/cli/program.ts +345 -194
  24. package/src/config/defaults.ts +2 -0
  25. package/src/config/index.ts +3 -0
  26. package/src/config/types.ts +32 -1
  27. package/src/core/file-lock.ts +16 -4
  28. package/src/core/write-lease.ts +354 -0
  29. package/src/embed/backlog.ts +9 -1
  30. package/src/embed/retry.ts +116 -3
  31. package/src/sdk/client.ts +3 -1
  32. package/src/sdk/embed.ts +8 -3
  33. package/src/sdk/types.ts +2 -0
  34. package/src/serve/embed-scheduler.ts +8 -0
  35. package/src/serve/resident-runtime.ts +5 -1
  36. package/src/serve/spa-production-build.ts +53 -7
  37. package/src/store/sqlite/adapter.ts +28 -4
  38. package/src/store/sqlite/scoped-index.ts +5 -1
  39. package/src/store/vector/sqlite-vec.ts +2 -1
  40. package/browser-extension/artifacts/gno-browser-clipper-v1.37.0.zip.sha256 +0 -1
package/src/sdk/embed.ts CHANGED
@@ -117,9 +117,10 @@ async function forceEmbedAll(
117
117
  vectorIndex: VectorIndexPort,
118
118
  modelUri: string,
119
119
  batchSize: number
120
- ): Promise<{ embedded: number; errors: number }> {
120
+ ): Promise<{ embedded: number; errors: number; contentionErrors: number }> {
121
121
  let embedded = 0;
122
122
  let errors = 0;
123
+ let contentionErrors = 0;
123
124
  let cursor: { mirrorHash: string; seq: number } | undefined;
124
125
  const retryQueue = new Map<string, { item: BacklogItem; attempts: number }>();
125
126
  const embedFingerprint = getEmbeddingFingerprint({
@@ -164,6 +165,7 @@ async function forceEmbedAll(
164
165
  });
165
166
  embedded += retryResult.embedded;
166
167
  errors += retryResult.errors;
168
+ contentionErrors += retryResult.contentionErrors;
167
169
  retryEmbedded += retryResult.embedded;
168
170
 
169
171
  const retryByKey = new Set(
@@ -212,6 +214,7 @@ async function forceEmbedAll(
212
214
  });
213
215
  embedded += embedResult.embedded;
214
216
  errors += embedResult.errors;
217
+ contentionErrors += embedResult.contentionErrors;
215
218
  enqueueRetryItems(embedResult.retryItems, 1);
216
219
 
217
220
  if (embedded > beforeEmbedded) {
@@ -232,7 +235,7 @@ async function forceEmbedAll(
232
235
  }
233
236
  }
234
237
 
235
- return { embedded, errors };
238
+ return { embedded, errors, contentionErrors };
236
239
  }
237
240
 
238
241
  async function checkVecAvailable(db: Database): Promise<boolean> {
@@ -343,7 +346,7 @@ export async function runEmbed(
343
346
  }
344
347
 
345
348
  const startedAt = Date.now();
346
- let result: { embedded: number; errors: number };
349
+ let result: { embedded: number; errors: number; contentionErrors: number };
347
350
  if (force) {
348
351
  result = await forceEmbedAll(
349
352
  db,
@@ -369,12 +372,14 @@ export async function runEmbed(
369
372
  result = {
370
373
  embedded: processed.value.embedded,
371
374
  errors: processed.value.errors,
375
+ contentionErrors: processed.value.contentionErrors ?? 0,
372
376
  };
373
377
  }
374
378
 
375
379
  return {
376
380
  embedded: result.embedded,
377
381
  errors: result.errors,
382
+ contentionErrors: result.contentionErrors,
378
383
  duration: (Date.now() - startedAt) / 1000,
379
384
  model: modelUri,
380
385
  searchAvailable: vectorIndex.searchAvailable,
package/src/sdk/types.ts CHANGED
@@ -204,6 +204,8 @@ export interface GnoEmbedOptions {
204
204
  export interface GnoEmbedResult {
205
205
  embedded: number;
206
206
  errors: number;
207
+ /** Persistence lock-contention failures; distinct from embedding-provider `errors`. */
208
+ contentionErrors?: number;
207
209
  duration: number;
208
210
  model: string;
209
211
  searchAvailable: boolean;
@@ -130,6 +130,14 @@ export function createEmbedScheduler(deps: EmbedSchedulerDeps): EmbedScheduler {
130
130
  console.error("[embed-scheduler] Embed failed:", result.error.message);
131
131
  return { embedded: 0, errors: 0 };
132
132
  }
133
+ if ((result.value.contentionErrors ?? 0) > 0) {
134
+ // Chunks deferred by SQLITE_BUSY stay in the backlog; retry the run
135
+ // instead of silently leaving embeddings a pass behind (fn-127 R6).
136
+ console.error(
137
+ `[embed-scheduler] ${result.value.contentionErrors} chunks deferred by index contention; rescheduling`
138
+ );
139
+ needsRerun = true;
140
+ }
133
141
  if (result.value.embedded > 0) onEmbedded?.(result.value);
134
142
  return result.value;
135
143
  } finally {
@@ -212,7 +212,11 @@ export async function startResidentRuntime(
212
212
  const paths = (deps.getConfigPaths ?? getConfigPaths)();
213
213
  const actualConfigPath = resolve(options.configPath ?? paths.configFile);
214
214
  store.setConfigPath(actualConfigPath);
215
- const openResult = await store.open(dbPath, initialConfig.ftsTokenizer);
215
+ const openResult = await store.open(
216
+ dbPath,
217
+ initialConfig.ftsTokenizer,
218
+ initialConfig.busyTimeoutMs
219
+ );
216
220
  if (!openResult.ok) {
217
221
  await ownerLock.release();
218
222
  return { success: false, error: openResult.error.message };
@@ -1,9 +1,10 @@
1
- // node:fs/promises — no Bun equivalent for mkdir/rm of the temporary SPA outdir.
2
- import { mkdir, rm } from "node:fs/promises";
1
+ // node:fs/promises — no Bun equivalent for mkdir/rm of the temporary SPA
2
+ // outdir, or for recursive directory listing.
3
+ import { mkdir, readdir, rm } from "node:fs/promises";
3
4
  // node:os — no Bun equivalent for the platform temporary directory.
4
5
  import { tmpdir } from "node:os";
5
6
  // node:path — no Bun path utils.
6
- import { basename, join } from "node:path";
7
+ import { basename, join, relative } from "node:path";
7
8
 
8
9
  export const ROOT_MOUNT_MARKER = 'getElementById("root")';
9
10
 
@@ -15,6 +16,50 @@ export type ProductionSpaFile = {
15
16
  export type ProductionSpaAssets = {
16
17
  files: Record<string, ProductionSpaFile>;
17
18
  html: string;
19
+ sourceHash: string;
20
+ };
21
+
22
+ const productionSpaPublicDir = (): string => join(import.meta.dir, "public");
23
+
24
+ export const productionSpaEntryPath = (): string =>
25
+ join(productionSpaPublicDir(), "index.html");
26
+
27
+ /**
28
+ * SHA-256 hex of every file under `src/serve/public/`, in sorted relative-path
29
+ * order. Used to detect a stale `assets/spa-production.json.gz` without
30
+ * comparing Bun.build output (minified symbols and chunk hashes differ
31
+ * across Bun binaries).
32
+ */
33
+ export const computeSpaSourceHash = async (): Promise<string> => {
34
+ const publicDir = productionSpaPublicDir();
35
+ const entries = await readdir(publicDir, {
36
+ recursive: true,
37
+ withFileTypes: true,
38
+ });
39
+ const relativePaths: string[] = [];
40
+ for (const entry of entries) {
41
+ if (!entry.isFile()) {
42
+ continue;
43
+ }
44
+ relativePaths.push(
45
+ relative(publicDir, join(entry.parentPath, entry.name)).replaceAll(
46
+ "\\",
47
+ "/"
48
+ )
49
+ );
50
+ }
51
+ relativePaths.sort();
52
+
53
+ const hasher = new Bun.CryptoHasher("sha256");
54
+ for (const relativePath of relativePaths) {
55
+ const bytes = await Bun.file(join(publicDir, relativePath)).bytes();
56
+ hasher.update(relativePath);
57
+ hasher.update("\0");
58
+ hasher.update(String(bytes.byteLength));
59
+ hasher.update("\0");
60
+ hasher.update(bytes);
61
+ }
62
+ return hasher.digest("hex");
18
63
  };
19
64
 
20
65
  const SCRIPT_TAG_RE = /<script\b[^>]*\bsrc="[^"]+"[^>]*><\/script>/iu;
@@ -41,9 +86,6 @@ export const isStandaloneExecutable = (): boolean =>
41
86
  export const isBunfsPath = (path: string): boolean =>
42
87
  path.includes("/$bunfs/") || path.includes("\\$bunfs\\");
43
88
 
44
- export const productionSpaEntryPath = (): string =>
45
- join(import.meta.dir, "public", "index.html");
46
-
47
89
  const rewriteProductionHtml = (html: string, jsEntryPath: string): string => {
48
90
  const script = `<script type="module" src="/${basename(jsEntryPath)}"></script>`;
49
91
  let next = html.replace(BASE_TAG_RE, "");
@@ -123,7 +165,11 @@ export const buildProductionSpaAssets = async (
123
165
  };
124
166
  }
125
167
 
126
- return { files, html };
168
+ return {
169
+ files,
170
+ html,
171
+ sourceHash: await computeSpaSourceHash(),
172
+ };
127
173
  } finally {
128
174
  await rm(outdir, { recursive: true, force: true });
129
175
  }
@@ -114,9 +114,12 @@ import { buildUri, deriveDocid, stripUriIndex } from "../../app/constants";
114
114
  import {
115
115
  type Collection,
116
116
  type Context,
117
+ DEFAULT_BUSY_TIMEOUT_MS,
117
118
  type EgressPolicy,
118
119
  type EgressPolicySource,
119
120
  type FtsTokenizer,
121
+ MAX_BUSY_TIMEOUT_MS,
122
+ MIN_BUSY_TIMEOUT_MS,
120
123
  resolveConfiguredEgressPolicy,
121
124
  } from "../../config/types";
122
125
  import {
@@ -476,6 +479,19 @@ function isDatabaseLockedError(cause: unknown): boolean {
476
479
  );
477
480
  }
478
481
 
482
+ /** Resolve a caller-supplied busy_timeout, defaulting rather than using 0. */
483
+ function resolveBusyTimeoutMs(busyTimeoutMs?: number): number {
484
+ if (
485
+ busyTimeoutMs === undefined ||
486
+ !Number.isInteger(busyTimeoutMs) ||
487
+ busyTimeoutMs < MIN_BUSY_TIMEOUT_MS ||
488
+ busyTimeoutMs > MAX_BUSY_TIMEOUT_MS
489
+ ) {
490
+ return DEFAULT_BUSY_TIMEOUT_MS;
491
+ }
492
+ return busyTimeoutMs;
493
+ }
494
+
479
495
  export class SqliteAdapter implements StorePort, SqliteDbProvider {
480
496
  private db: Database | null = null;
481
497
  private dbPath = "";
@@ -492,7 +508,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
492
508
 
493
509
  async open(
494
510
  dbPath: string,
495
- ftsTokenizer: FtsTokenizer
511
+ ftsTokenizer: FtsTokenizer,
512
+ busyTimeoutMs: number = DEFAULT_BUSY_TIMEOUT_MS
496
513
  ): Promise<StoreResult<MigrationResult>> {
497
514
  try {
498
515
  this.db = new Database(dbPath, { create: true });
@@ -501,7 +518,9 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
501
518
 
502
519
  // Enable pragmas for performance and safety
503
520
  this.db.exec("PRAGMA foreign_keys = ON");
504
- this.db.exec("PRAGMA busy_timeout = 5000");
521
+ this.db.exec(
522
+ `PRAGMA busy_timeout = ${resolveBusyTimeoutMs(busyTimeoutMs)}`
523
+ );
505
524
 
506
525
  // Keep WAL everywhere so readers can continue while a writer is active.
507
526
  // CI still relaxes fsync/temp-store for speed, but MEMORY journal mode
@@ -558,12 +577,17 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
558
577
  }
559
578
 
560
579
  /** Open an existing index with SQLite enforced query-only semantics. */
561
- openReadOnly(dbPath: string): StoreResult<void> {
580
+ openReadOnly(
581
+ dbPath: string,
582
+ busyTimeoutMs: number = DEFAULT_BUSY_TIMEOUT_MS
583
+ ): StoreResult<void> {
562
584
  try {
563
585
  this.db = new Database(dbPath, { readonly: true, strict: true });
564
586
  this.dbPath = dbPath;
565
587
  this.db.exec("PRAGMA query_only = ON");
566
- this.db.exec("PRAGMA busy_timeout = 5000");
588
+ this.db.exec(
589
+ `PRAGMA busy_timeout = ${resolveBusyTimeoutMs(busyTimeoutMs)}`
590
+ );
567
591
  this.contextGeneration += 1;
568
592
  return ok(undefined);
569
593
  } catch (cause) {
@@ -38,7 +38,11 @@ export async function openScopedIndexStore(options: {
38
38
 
39
39
  const store = new SqliteAdapter();
40
40
  store.setConfigPath(options.configPath ?? "<inline-config>");
41
- const openResult = await store.open(dbPath, options.config.ftsTokenizer);
41
+ const openResult = await store.open(
42
+ dbPath,
43
+ options.config.ftsTokenizer,
44
+ options.config.busyTimeoutMs
45
+ );
42
46
  if (!openResult.ok) {
43
47
  throw new Error(openResult.error.message);
44
48
  }
@@ -193,7 +193,8 @@ export async function createVectorIndexPort(
193
193
  return Promise.resolve(
194
194
  err(
195
195
  "VECTOR_WRITE_FAILED",
196
- `Vector write failed: ${e instanceof Error ? e.message : String(e)}`
196
+ `Vector write failed: ${e instanceof Error ? e.message : String(e)}`,
197
+ e
197
198
  )
198
199
  );
199
200
  }
@@ -1 +0,0 @@
1
- 497d8e90a34b7472eac55da861c3049b7453a33d9fdbd0a27e49f2ce36ccde9b gno-browser-clipper-v1.37.0.zip