@gmickel/gno 2.8.5 → 2.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -139,7 +139,7 @@ See the [guide](docs/COMPILED-CONTEXT.md).
139
139
 
140
140
  <!-- public-truth:current-version -->
141
141
 
142
- > Current source version: **v2.8.5**. See [CHANGELOG.md](./CHANGELOG.md).
142
+ > Current source version: **v2.9.1**. See [CHANGELOG.md](./CHANGELOG.md).
143
143
 
144
144
  <!-- /public-truth -->
145
145
 
@@ -0,0 +1 @@
1
+ f58e51eea9d0b35a61ef55574882c4be8002ed2fdc29966a3cd1bd7c84350e3c gno-browser-clipper-v2.9.1.zip
@@ -21,5 +21,5 @@
21
21
  "content_security_policy": {
22
22
  "extension_pages": "script-src 'self'; object-src 'none'; connect-src http://127.0.0.1:*"
23
23
  },
24
- "version": "2.8.5"
24
+ "version": "2.9.1"
25
25
  }
package/bunfig.toml CHANGED
@@ -2,6 +2,6 @@
2
2
  plugins = ["bun-plugin-tailwind"]
3
3
 
4
4
  [test]
5
- preload = ["./test/preload/happy-dom.ts"]
5
+ preload = ["./test/preload/happy-dom.ts", "./test/preload/env-guard.ts"]
6
6
  # Frozen evidence and local experiments are not the executable regression suite.
7
7
  pathIgnorePatterns = [".flow/artifacts/**", "notes/**"]
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gmickel/gno",
3
- "version": "2.8.5",
3
+ "version": "2.9.1",
4
4
  "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
5
5
  "keywords": [
6
6
  "embeddings",
@@ -20,6 +20,7 @@ import {
20
20
  loadConfig,
21
21
  writeConfigWarningsToStderr,
22
22
  } from "../../config";
23
+ import { tracePhase } from "../../core/phase-trace";
23
24
  import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
24
25
  import { SqliteAdapter } from "../../store/sqlite/adapter";
25
26
  import { assertCliSessionBinding } from "../session-binding";
@@ -63,6 +64,7 @@ export interface InitStoreOptions {
63
64
  export async function initStore(
64
65
  options: InitStoreOptions = {}
65
66
  ): Promise<InitStoreResult> {
67
+ tracePhase("store: init");
66
68
  // Check if initialized
67
69
  const initialized = await isInitialized(options.configPath);
68
70
  if (!initialized) {
@@ -119,11 +121,13 @@ export async function initStore(
119
121
  // Set configPath for status output
120
122
  store.setConfigPath(actualConfigPath);
121
123
 
124
+ tracePhase("store: opening index");
122
125
  const openResult = await store.open(
123
126
  dbPath,
124
127
  config.ftsTokenizer,
125
128
  config.busyTimeoutMs
126
129
  );
130
+ tracePhase("store: index open");
127
131
  if (!openResult.ok) {
128
132
  return { ok: false, error: openResult.error.message };
129
133
  }
@@ -139,6 +143,7 @@ export async function initStore(
139
143
  return { ok: false, error: syncCollResult.error.message };
140
144
  }
141
145
 
146
+ tracePhase("store: collections synced");
142
147
  // Sync contexts from config to DB
143
148
  const syncCtxResult = await store.syncContexts(config.contexts ?? []);
144
149
  if (!syncCtxResult.ok) {
@@ -146,6 +151,7 @@ export async function initStore(
146
151
  return { ok: false, error: syncCtxResult.error.message };
147
152
  }
148
153
 
154
+ tracePhase("store: contexts synced");
149
155
  return { ok: true, store, config, collections, actualConfigPath };
150
156
  }
151
157
 
@@ -20,20 +20,11 @@ import {
20
20
  } from "../../errors";
21
21
  import { ADAPTER_VERSIONS } from "../../versions";
22
22
  import { hasPrefix, isPasswordProtectedXlsx } from "../shared/ooxml-protection";
23
+ import { MARKITDOWN_CONVERTER_ID, markitdownCanHandle } from "./match";
23
24
 
24
- const CONVERTER_ID = "adapter/markitdown-ts" as const;
25
+ const CONVERTER_ID = MARKITDOWN_CONVERTER_ID;
25
26
  const CONVERTER_VERSION = ADAPTER_VERSIONS["markitdown-ts"];
26
27
 
27
- /** Supported extensions for this adapter */
28
- const SUPPORTED_EXTENSIONS = [".pdf", ".docx", ".xlsx"];
29
-
30
- /** Supported MIME types */
31
- const SUPPORTED_MIMES = [
32
- "application/pdf",
33
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
34
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
35
- ];
36
-
37
28
  const PDF_SIGNATURE = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]);
38
29
  const PDF_TRAILER_SCAN_BYTES = 2048;
39
30
  const MAX_MESSAGE_LENGTH = 200;
@@ -114,9 +105,7 @@ export const markitdownAdapter: Converter = {
114
105
  id: CONVERTER_ID,
115
106
  version: CONVERTER_VERSION,
116
107
 
117
- canHandle(mime: string, ext: string): boolean {
118
- return SUPPORTED_EXTENSIONS.includes(ext) || SUPPORTED_MIMES.includes(mime);
119
- },
108
+ canHandle: markitdownCanHandle,
120
109
 
121
110
  async convert(input: ConvertInput): Promise<ConvertResult> {
122
111
  // 1. Check size limit (defense in depth; EPIC 5 does stat-based pre-check)
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Routing for the markitdown-ts adapter, kept apart from the adapter so the
3
+ * registry can route files without loading pdf.js and the Office parsers.
4
+ */
5
+
6
+ export const MARKITDOWN_CONVERTER_ID = "adapter/markitdown-ts" as const;
7
+
8
+ /** Supported extensions for this adapter */
9
+ const SUPPORTED_EXTENSIONS = [".pdf", ".docx", ".xlsx"];
10
+
11
+ /** Supported MIME types */
12
+ const SUPPORTED_MIMES = [
13
+ "application/pdf",
14
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
15
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
16
+ ];
17
+
18
+ export function markitdownCanHandle(mime: string, ext: string): boolean {
19
+ return SUPPORTED_EXTENSIONS.includes(ext) || SUPPORTED_MIMES.includes(mime);
20
+ }
@@ -14,14 +14,11 @@ import { parseOffice } from "../../../../vendor/converters/officeparser";
14
14
  import { adapterError, corruptError, tooLargeError } from "../../errors";
15
15
  import { basenameWithoutExt } from "../../path";
16
16
  import { ADAPTER_VERSIONS } from "../../versions";
17
+ import { OFFICEPARSER_CONVERTER_ID, officeparserCanHandle } from "./match";
17
18
 
18
- const CONVERTER_ID = "adapter/officeparser" as const;
19
+ const CONVERTER_ID = OFFICEPARSER_CONVERTER_ID;
19
20
  const CONVERTER_VERSION = ADAPTER_VERSIONS.officeparser;
20
21
 
21
- /** Supported MIME type */
22
- const PPTX_MIME =
23
- "application/vnd.openxmlformats-officedocument.presentationml.presentation";
24
-
25
22
  /**
26
23
  * Control character pattern built dynamically to avoid lint issues.
27
24
  * Matches U+0000-U+001F and U+007F (all ASCII control chars).
@@ -62,9 +59,7 @@ export const officeparserAdapter: Converter = {
62
59
  id: CONVERTER_ID,
63
60
  version: CONVERTER_VERSION,
64
61
 
65
- canHandle(mime: string, ext: string): boolean {
66
- return ext === ".pptx" || mime === PPTX_MIME;
67
- },
62
+ canHandle: officeparserCanHandle,
68
63
 
69
64
  async convert(input: ConvertInput): Promise<ConvertResult> {
70
65
  // Size check (defense in depth; EPIC 5 does stat-based pre-check)
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Routing for the officeparser adapter, kept apart from the adapter so the
3
+ * registry can route files without loading the parser.
4
+ */
5
+
6
+ export const OFFICEPARSER_CONVERTER_ID = "adapter/officeparser" as const;
7
+
8
+ const PPTX_MIME =
9
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation";
10
+
11
+ export function officeparserCanHandle(mime: string, ext: string): boolean {
12
+ return ext === ".pptx" || mime === PPTX_MIME;
13
+ }
@@ -5,12 +5,22 @@
5
5
 
6
6
  import type {
7
7
  Converter,
8
+ ConverterId,
8
9
  ConvertInput,
9
10
  ConvertResult,
10
11
  RecordAdapter,
11
12
  } from "./types";
12
13
 
14
+ import {
15
+ MARKITDOWN_CONVERTER_ID,
16
+ markitdownCanHandle,
17
+ } from "./adapters/markitdownTs/match";
18
+ import {
19
+ OFFICEPARSER_CONVERTER_ID,
20
+ officeparserCanHandle,
21
+ } from "./adapters/officeparser/match";
13
22
  import { adapterError, unsupportedError } from "./errors";
23
+ import { ADAPTER_VERSIONS } from "./versions";
14
24
 
15
25
  export class ConverterRegistry {
16
26
  private readonly converters: Converter[] = [];
@@ -71,38 +81,56 @@ export class ConverterRegistry {
71
81
  }
72
82
  }
73
83
 
84
+ /** Identity and routing of an adapter whose module loads on first use. */
85
+ export interface LazyAdapterSpec {
86
+ id: ConverterId;
87
+ version: string;
88
+ canHandle: Converter["canHandle"];
89
+ }
90
+
74
91
  /**
75
- * Load an adapter module; if it cannot load in this runtime, stand in a
76
- * converter that fails its file types with ADAPTER_FAILURE and the reason.
92
+ * An adapter that loads its module on its first conversion. Routing and
93
+ * identity come from the spec, so selecting a converter or a record adapter
94
+ * never loads the parser: the PDF/Office parsers took 9-11 s to load on a
95
+ * cold Windows runner, inside every `gno update` (fn-201). If the module
96
+ * cannot load in this runtime (pdf.js without its native canvas binding in a
97
+ * standalone executable), its file types fail with ADAPTER_FAILURE and the
98
+ * reason, and other file types are unaffected.
77
99
  */
78
- export async function loadAdapter(
79
- load: () => Promise<Converter>,
80
- id: string,
81
- extensions: readonly string[],
82
- mimes: readonly string[]
83
- ): Promise<Converter> {
84
- try {
85
- return await load();
86
- } catch (cause) {
87
- const reason = cause instanceof Error ? cause.message : String(cause);
88
- return {
89
- id,
90
- version: "unavailable",
91
- // Same matching as the real adapter, so a sniffed MIME without an
92
- // extension still reports ADAPTER_FAILURE, not UNSUPPORTED.
93
- canHandle: (mime, ext) =>
94
- extensions.includes(ext) || mimes.includes(mime),
95
- convert: (input) =>
96
- Promise.resolve({
97
- ok: false,
98
- error: adapterError(
99
- input,
100
- id,
101
- `${id} is unavailable in this build: ${reason}`
102
- ),
103
- }),
104
- };
105
- }
100
+ export function lazyAdapter(
101
+ spec: LazyAdapterSpec,
102
+ load: () => Promise<Converter>
103
+ ): Converter {
104
+ let loaded: Promise<Converter> | undefined;
105
+ return {
106
+ id: spec.id,
107
+ version: spec.version,
108
+ canHandle: spec.canHandle,
109
+ async convert(input) {
110
+ loaded ??= load().catch((cause: unknown) =>
111
+ unavailableAdapter(
112
+ spec,
113
+ cause instanceof Error ? cause.message : String(cause)
114
+ )
115
+ );
116
+ return (await loaded).convert(input);
117
+ },
118
+ };
119
+ }
120
+
121
+ function unavailableAdapter(spec: LazyAdapterSpec, reason: string): Converter {
122
+ return {
123
+ ...spec,
124
+ convert: (input) =>
125
+ Promise.resolve({
126
+ ok: false,
127
+ error: adapterError(
128
+ input,
129
+ spec.id,
130
+ `${spec.id} is unavailable in this build: ${reason}`
131
+ ),
132
+ }),
133
+ };
106
134
  }
107
135
 
108
136
  /**
@@ -121,27 +149,23 @@ export async function createDefaultRegistry(): Promise<ConverterRegistry> {
121
149
  const { markdownConverter } = await import("./native/markdown");
122
150
  const { plaintextConverter } = await import("./native/plaintext");
123
151
  const { xlsxAdapter } = await import("./adapters/xlsx/adapter");
124
- // The PDF/Office adapters load pdf.js, which cannot initialize where its
125
- // native canvas binding is missing (a standalone compiled executable). A
126
- // failed adapter only fails its own file types.
127
- const markitdownAdapter = await loadAdapter(
152
+ const markitdownAdapter = lazyAdapter(
153
+ {
154
+ id: MARKITDOWN_CONVERTER_ID,
155
+ version: ADAPTER_VERSIONS["markitdown-ts"],
156
+ canHandle: markitdownCanHandle,
157
+ },
128
158
  async () =>
129
- (await import("./adapters/markitdownTs/adapter")).markitdownAdapter,
130
- "adapter/markitdown-ts",
131
- [".pdf", ".docx"],
132
- [
133
- "application/pdf",
134
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
135
- ]
159
+ (await import("./adapters/markitdownTs/adapter")).markitdownAdapter
136
160
  );
137
- const officeparserAdapter = await loadAdapter(
161
+ const officeparserAdapter = lazyAdapter(
162
+ {
163
+ id: OFFICEPARSER_CONVERTER_ID,
164
+ version: ADAPTER_VERSIONS.officeparser,
165
+ canHandle: officeparserCanHandle,
166
+ },
138
167
  async () =>
139
- (await import("./adapters/officeparser/adapter")).officeparserAdapter,
140
- "adapter/officeparser",
141
- [".pptx"],
142
- [
143
- "application/vnd.openxmlformats-officedocument.presentationml.presentation",
144
- ]
168
+ (await import("./adapters/officeparser/adapter")).officeparserAdapter
145
169
  );
146
170
  const { jsonlAdapter } = await import("./adapters/jsonl/adapter");
147
171
  const { transcriptAdapter } = await import("./adapters/transcript/adapter");
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Opt-in startup phase trace for diagnosing stalled CLI runs.
3
+ *
4
+ * With `GNO_PHASE_TRACE=1`, each call writes one line to stderr:
5
+ * `gno-phase <ms since process start> <pid> <label>`. The write is
6
+ * synchronous so a line reaches the pipe before a later blocking call (a
7
+ * SQLite busy wait, a native load) can stall the process; the last line a
8
+ * killed process printed names the phase it stalled in.
9
+ *
10
+ * @module src/core/phase-trace
11
+ */
12
+
13
+ // node:fs writeSync: Bun has no synchronous stderr write; process.stderr.write
14
+ // to a pipe can still be queued when the event loop blocks.
15
+ import { writeSync } from "node:fs";
16
+
17
+ export const PHASE_TRACE_ENV = "GNO_PHASE_TRACE";
18
+
19
+ const STDERR_FD = 2;
20
+
21
+ const enabled = process.env[PHASE_TRACE_ENV] === "1";
22
+
23
+ /** Record that the process reached `label` (no-op unless tracing is on). */
24
+ export function tracePhase(label: string): void {
25
+ if (!enabled) return;
26
+ try {
27
+ writeSync(
28
+ STDERR_FD,
29
+ `gno-phase ${Math.round(performance.now())} ${process.pid} ${label}\n`
30
+ );
31
+ } catch {
32
+ // A closed stderr must never fail the traced command.
33
+ }
34
+ }
@@ -11,6 +11,7 @@ import { dirname, join } from "node:path";
11
11
 
12
12
  import { getIndexDbPath } from "../app/constants";
13
13
  import { acquireWriteLock } from "./file-lock";
14
+ import { tracePhase } from "./phase-trace";
14
15
 
15
16
  export const DEFAULT_LOCK_WAIT_MS = 120_000;
16
17
  export const WRITE_LEASE_BUSY_MESSAGE =
@@ -149,6 +150,7 @@ export async function acquireCliWriteLease(
149
150
  options: AcquireCliWriteLeaseOptions
150
151
  ): Promise<WriteLeaseResult> {
151
152
  const lockPath = writeLeasePath(options.dbPath);
153
+ tracePhase("lease: acquiring");
152
154
  // A non-finite or out-of-range wait must never loop unbounded (fn-127 review).
153
155
  const requestedWaitMs = Number.isFinite(options.waitMs)
154
156
  ? Math.min(Math.max(0, options.waitMs), MAX_LOCK_WAIT_MS)
@@ -163,7 +165,9 @@ export async function acquireCliWriteLease(
163
165
  const sliceMs = options.noWait ? 0 : Math.min(LOCK_SLICE_MS, remaining);
164
166
  const handle = await acquireWriteLock(lockPath, sliceMs);
165
167
  if (handle) {
168
+ tracePhase("lease: acquired");
166
169
  await writeHolderSidecar(lockPath, options.command);
170
+ tracePhase("lease: holder recorded");
167
171
  return {
168
172
  ok: true,
169
173
  release: async () => {
@@ -213,6 +217,7 @@ export async function withCliWriteLease<T>(
213
217
  } finally {
214
218
  try {
215
219
  await result.release();
220
+ tracePhase("lease: released");
216
221
  } catch (cause) {
217
222
  const message = cause instanceof Error ? cause.message : String(cause);
218
223
  process.stderr.write(`gno: failed to release write lease: ${message}\n`);
package/src/index.ts CHANGED
@@ -7,6 +7,7 @@
7
7
  */
8
8
 
9
9
  import { runCli } from "./cli/run";
10
+ import { tracePhase } from "./core/phase-trace";
10
11
  import { FILE_PROCESSOR_CHILD_ENV } from "./ingestion/file-child-env";
11
12
  import { resetModelManager } from "./llm/nodeLlamaCpp/lifecycle";
12
13
  import { IMPORT_CHILD_ENV } from "./sessions/import-child-env";
@@ -50,6 +51,7 @@ async function cleanupAndExit(code: number): Promise<never> {
50
51
  await resetModelManager().catch(() => {
51
52
  // Ignore cleanup errors on exit
52
53
  });
54
+ tracePhase("exit: models reset");
53
55
  await Promise.all([flushStream(process.stdout), flushStream(process.stderr)]);
54
56
  process.exit(code);
55
57
  }
@@ -89,8 +91,12 @@ if (process.env[FILE_PROCESSOR_CHILD_ENV] === "1") {
89
91
  }
90
92
 
91
93
  // Await module completion so pending piped stdin keeps Windows Bun alive.
94
+ tracePhase("cli: start");
92
95
  await runCli(process.argv)
93
- .then((code) => cleanupAndExit(interruptExitCode || code))
96
+ .then((code) => {
97
+ tracePhase(`cli: done (${code})`);
98
+ return cleanupAndExit(interruptExitCode || code);
99
+ })
94
100
  .catch((err) => {
95
101
  process.stderr.write(
96
102
  `Fatal error: ${err instanceof Error ? err.message : String(err)}\n`
@@ -37,6 +37,7 @@ import type {
37
37
  PrepareOutcome,
38
38
  } from "./prepare-file";
39
39
 
40
+ import { tracePhase } from "../core/phase-trace";
40
41
  import { isBunfsPath } from "../serve/spa-production-build";
41
42
  import { FILE_PROCESSOR_CHILD_ENV } from "./file-child-env";
42
43
  import { PREPARE_PHASES } from "./prepare-file";
@@ -452,7 +453,9 @@ const runOne = async (
452
453
  }
453
454
  // Loading the processor is not the file's time: its clock starts once the
454
455
  // processor can take it.
456
+ tracePhase(`processor: awaiting ${active.kind}`);
455
457
  const notReady = await awaitReady(active);
458
+ tracePhase(`processor: ${notReady ?? "ready"}`);
456
459
  if (notReady) {
457
460
  if (processor === active) stopProcessor();
458
461
  return { ok: false, error: unavailableFailure(notReady) };
@@ -43,6 +43,7 @@ import {
43
43
  type DocumentStructureSnapshot,
44
44
  } from "../core/change-diff";
45
45
  import { enforceCollectionEgress } from "../core/egress-enforcement";
46
+ import { tracePhase } from "../core/phase-trace";
46
47
  import { defaultChunker } from "./chunker";
47
48
  import { persistChunkLayout, prepareChunking } from "./chunking";
48
49
  import {
@@ -433,6 +434,7 @@ export class SyncService {
433
434
  "Compiled context is derived material; index original sources instead",
434
435
  };
435
436
  if (isCompiledContextPath(entry.relPath)) return generatedResult;
437
+ tracePhase(`sync: file ${entry.relPath}`);
436
438
  const limits = {
437
439
  maxBytes: options.limits?.maxBytes ?? DEFAULT_LIMITS.maxBytes,
438
440
  timeoutMs: options.limits?.timeoutMs ?? DEFAULT_LIMITS.timeoutMs,
@@ -499,6 +501,7 @@ export class SyncService {
499
501
  extensionMime.mime,
500
502
  extensionMime.ext
501
503
  );
504
+ tracePhase("sync: record adapter selected");
502
505
  const contentTypeRules = options.contentTypeRules ?? [];
503
506
  const contentTypeRulesFingerprint =
504
507
  options.contentTypeRulesFingerprint ??
@@ -673,6 +676,7 @@ export class SyncService {
673
676
  let previousStructure: DocumentStructureSnapshot | null | undefined =
674
677
  existing ? undefined : null;
675
678
  const previous = await this.readPreviousRevision(store, existing);
679
+ tracePhase("sync: preparing file");
676
680
  const prepared = await this.prepare(
677
681
  {
678
682
  input: {
@@ -1626,7 +1630,9 @@ export class SyncService {
1626
1630
  sourceAvailability: availabilityMode,
1627
1631
  directoryAvailability,
1628
1632
  };
1633
+ tracePhase(`sync: walking ${collection.name}`);
1629
1634
  const { entries, skipped } = await this.walker.walk(walkConfig);
1635
+ tracePhase(`sync: walked ${entries.length} entries`);
1630
1636
 
1631
1637
  // Track seen paths for marking inactive
1632
1638
  // Only include TOO_LARGE files (they exist but are unprocessable)
@@ -1945,7 +1951,9 @@ export class SyncService {
1945
1951
  options: SyncOptions = {}
1946
1952
  ): Promise<SyncResult> {
1947
1953
  const startTime = Date.now();
1954
+ tracePhase("sync: preparing chunking");
1948
1955
  const prepared = await prepareChunking(store, this.chunker, options);
1956
+ tracePhase("sync: chunking prepared");
1949
1957
  const results: CollectionSyncResult[] = [];
1950
1958
  const deferredProjectionOptions: SyncOptions = {
1951
1959
  ...prepared.options,
@@ -1961,6 +1969,7 @@ export class SyncService {
1961
1969
  results.push(result);
1962
1970
  }
1963
1971
 
1972
+ tracePhase("sync: collections done");
1964
1973
  let graphRebuild: SyncResult["graphRebuild"];
1965
1974
  if (results.length > 0) {
1966
1975
  const projectionErrors = await this.projectTypedEdges(store, {
@@ -1973,6 +1982,7 @@ export class SyncService {
1973
1982
  results.at(-1)?.errors.push(...projectionErrors);
1974
1983
  }
1975
1984
 
1985
+ tracePhase("sync: typed edges projected");
1976
1986
  // Aggregate totals
1977
1987
  const totals = results.reduce(
1978
1988
  (acc, r) => ({
@@ -42,9 +42,20 @@ export const WATCHER_MAX_DIRTY_HINTS = 4_096;
42
42
  /** Cap on application-write suppression history entries. */
43
43
  export const WATCHER_MAX_SUPPRESSION_ENTRIES = 4_096;
44
44
 
45
- /** Bounded retry delay after failed classification/sync. */
45
+ /** Retry delay after the first failed classification/sync of a collection. */
46
46
  export const WATCHER_RETRY_BACKOFF_MS = 500;
47
47
 
48
+ /** Ceiling for the retry delay while a collection keeps failing. */
49
+ export const WATCHER_MAX_RETRY_BACKOFF_MS = 5 * 60_000;
50
+
51
+ /** Delay before the retry that follows `failures` consecutive failed flushes. */
52
+ export function watcherRetryDelayMs(failures: number): number {
53
+ return Math.min(
54
+ WATCHER_RETRY_BACKOFF_MS * 2 ** Math.max(0, failures - 1),
55
+ WATCHER_MAX_RETRY_BACKOFF_MS
56
+ );
57
+ }
58
+
48
59
  /** Retry delay while another writer holds the shared writer lease. */
49
60
  export const WATCHER_LEASE_RETRY_MS = 5_000;
50
61
 
@@ -28,6 +28,7 @@ export {
28
28
  WATCHER_MAX_DIRTY_HINTS,
29
29
  WATCHER_MAX_EXACT_PATHS,
30
30
  WATCHER_MAX_FLUSH_DELAY_MS,
31
+ WATCHER_MAX_RETRY_BACKOFF_MS,
31
32
  WATCHER_MAX_SUPPRESSION_ENTRIES,
32
33
  WATCHER_RETRY_BACKOFF_MS,
33
34
  addToCappedSet,
@@ -39,6 +40,7 @@ export {
39
40
  mergeSyncPathBatch,
40
41
  pruneSuppressionMap,
41
42
  successfulChangedPaths,
43
+ watcherRetryDelayMs,
42
44
  widenVanishedExactPaths,
43
45
  type ClassificationResult,
44
46
  type ExactPathKind,
@@ -12,8 +12,8 @@ import type { Collection } from "../config/types";
12
12
  import {
13
13
  WATCHER_FLUSH_DEBOUNCE_MS,
14
14
  WATCHER_MAX_FLUSH_DELAY_MS,
15
- WATCHER_RETRY_BACKOFF_MS,
16
15
  classifyWatcherFilename,
16
+ watcherRetryDelayMs,
17
17
  } from "./watch-reconciliation";
18
18
  import {
19
19
  applyPendingForceFlags,
@@ -25,6 +25,12 @@ import {
25
25
  type PendingForceFlags,
26
26
  } from "./watch-service-state";
27
27
 
28
+ /**
29
+ * Why an explicit retry timer is armed: "backoff" after a failed flush, "fixed"
30
+ * for an explicit delay such as writer-lease contention (R3: stays at 5 s).
31
+ */
32
+ export type WatcherRetryKind = "backoff" | "fixed";
33
+
28
34
  export interface WatchEventHost {
29
35
  disposed: () => boolean;
30
36
  findCollection: (collectionName: string) => Collection | undefined;
@@ -45,8 +51,12 @@ export interface WatchQueueHost {
45
51
  pendingByCollection: Map<string, CollectionPending>;
46
52
  flushDeadlineAt: Map<string, number>;
47
53
  timers: Map<string, ReturnType<typeof setTimeout>>;
48
- /** Collections with an explicit retry timer; finally must not bypass. */
49
- retryScheduled: Set<string>;
54
+ /** Collections with an explicit retry timer, by kind; finally must not bypass. */
55
+ retryScheduled: Map<string, WatcherRetryKind>;
56
+ /** Consecutive failed flushes per collection; cleared by a successful flush. */
57
+ retryFailures: Map<string, number>;
58
+ /** Collections with watcher events queued since their last flush took its work. */
59
+ freshEvents: Set<string>;
50
60
  snapshotReady: Map<string, boolean>;
51
61
  inFlightSyncs: Set<Promise<void>>;
52
62
  runFlush: (collectionName: string) => Promise<void>;
@@ -105,6 +115,7 @@ export function enqueueExactPath(
105
115
  collectionName,
106
116
  queueExactPath(pending, relPath, host.maxExactPaths)
107
117
  );
118
+ host.freshEvents.add(collectionName);
108
119
  scheduleFlush(host, collectionName);
109
120
  }
110
121
 
@@ -123,6 +134,7 @@ export function enqueueDirtyHint(
123
134
  collectionName,
124
135
  queueDirtyHint(pending, hint, host.maxDirtyHints)
125
136
  );
137
+ host.freshEvents.add(collectionName);
126
138
  scheduleFlush(host, collectionName);
127
139
  }
128
140
 
@@ -152,9 +164,17 @@ export function scheduleFlush(
152
164
  if (host.disposed()) {
153
165
  return;
154
166
  }
155
- // Explicit retry owns the single timer; do not replace it with debounce.
156
- if (host.retryScheduled.has(collectionName)) {
157
- return;
167
+ const retryKind = host.retryScheduled.get(collectionName);
168
+ if (retryKind !== undefined) {
169
+ // Explicit retry owns the single timer; do not replace it with debounce.
170
+ if (retryKind === "fixed" || !host.freshEvents.has(collectionName)) {
171
+ return;
172
+ }
173
+ // A fresh event never waits out a failure backoff: drop the retry timer and
174
+ // flush on the normal debounce. The failure count stays until a flush succeeds.
175
+ // A fixed (lease) retry keeps its timer so a held lease is not probed per event.
176
+ host.retryScheduled.delete(collectionName);
177
+ host.flushDeadlineAt.delete(collectionName);
158
178
  }
159
179
  const schedule = computeFlushDelay({
160
180
  nowMs: host.clock(),
@@ -213,6 +233,8 @@ export function startFlush(host: WatchQueueHost, collectionName: string): void {
213
233
  * Re-arm pending work after a file-level failure with retry backoff.
214
234
  * At most one retry timer per collection; finally must not startFlush while set.
215
235
  * forceFallback/overflow survive until forced classification + sync succeed.
236
+ * Without an explicit delay, each consecutive failure doubles the wait (capped),
237
+ * so work that fails the same way every time cannot spin the resident.
216
238
  */
217
239
  export function requeueAfterFailure(
218
240
  host: WatchQueueHost,
@@ -220,7 +242,7 @@ export function requeueAfterFailure(
220
242
  exact: string[],
221
243
  dirty: string[],
222
244
  forceFlags?: PendingForceFlags,
223
- delayMs = WATCHER_RETRY_BACKOFF_MS
245
+ delayMs?: number
224
246
  ): void {
225
247
  queueWithoutSchedule(host, collectionName, exact, dirty, forceFlags);
226
248
  if (host.disposed()) {
@@ -230,7 +252,19 @@ export function requeueAfterFailure(
230
252
  // Pending already merged; existing retry timer remains the sole attempt.
231
253
  return;
232
254
  }
233
- host.retryScheduled.add(collectionName);
255
+ host.retryScheduled.set(
256
+ collectionName,
257
+ delayMs === undefined ? "backoff" : "fixed"
258
+ );
259
+ let retryDelayMs = delayMs;
260
+ if (retryDelayMs === undefined) {
261
+ const failures = (host.retryFailures.get(collectionName) ?? 0) + 1;
262
+ host.retryFailures.set(collectionName, failures);
263
+ // Events that arrived while the failed flush ran flush on the debounce.
264
+ retryDelayMs = host.freshEvents.has(collectionName)
265
+ ? host.flushDebounceMs
266
+ : watcherRetryDelayMs(failures);
267
+ }
234
268
  const existingTimer = host.timers.get(collectionName);
235
269
  if (existingTimer) {
236
270
  clearTimeout(existingTimer);
@@ -242,7 +276,7 @@ export function requeueAfterFailure(
242
276
  host.timers.delete(collectionName);
243
277
  host.retryScheduled.delete(collectionName);
244
278
  startFlush(host, collectionName);
245
- }, delayMs)
279
+ }, retryDelayMs)
246
280
  );
247
281
  }
248
282
 
@@ -8,7 +8,11 @@ import type { FSWatcher } from "node:fs";
8
8
 
9
9
  import type { Collection } from "../config/types";
10
10
  import type { SyncOptions } from "../ingestion";
11
- import type { WatchEventHost, WatchQueueHost } from "./watch-service-events";
11
+ import type {
12
+ WatchEventHost,
13
+ WatchQueueHost,
14
+ WatcherRetryKind,
15
+ } from "./watch-service-events";
12
16
  import type { WatchLifecycleHost } from "./watch-service-lifecycle";
13
17
  import type { CollectionPending } from "./watch-service-state";
14
18
  import type { WatcherSnapshot } from "./watch-snapshot";
@@ -46,7 +50,9 @@ export interface WatchServiceHostState {
46
50
  maxDirtyHints: number;
47
51
  flushDeadlineAt: Map<string, number>;
48
52
  timers: Map<string, ReturnType<typeof setTimeout>>;
49
- retryScheduled: Set<string>;
53
+ retryScheduled: Map<string, WatcherRetryKind>;
54
+ retryFailures: Map<string, number>;
55
+ freshEvents: Set<string>;
50
56
  inFlightSyncs: Set<Promise<void>>;
51
57
  runFlush: (collectionName: string) => Promise<void>;
52
58
  }
@@ -71,6 +77,8 @@ export function buildLifecycleHost(
71
77
  snapshotInit: state.snapshotInit,
72
78
  syncing: state.syncing,
73
79
  pendingByCollection: state.pendingByCollection,
80
+ retryFailures: state.retryFailures,
81
+ freshEvents: state.freshEvents,
74
82
  clearCollectionRuntimeState: state.clearCollectionRuntimeState,
75
83
  beginSnapshotInit: state.beginSnapshotInit,
76
84
  watchFactory: state.watchFactory,
@@ -102,6 +110,8 @@ export function buildQueueHost(state: WatchServiceHostState): WatchQueueHost {
102
110
  flushDeadlineAt: state.flushDeadlineAt,
103
111
  timers: state.timers,
104
112
  retryScheduled: state.retryScheduled,
113
+ retryFailures: state.retryFailures,
114
+ freshEvents: state.freshEvents,
105
115
  snapshotReady: state.snapshotReady,
106
116
  inFlightSyncs: state.inFlightSyncs,
107
117
  runFlush: state.runFlush,
@@ -38,6 +38,10 @@ export interface WatchLifecycleHost {
38
38
  * forceFallback + durable generation reconcile so a new baseline cannot absorb it.
39
39
  */
40
40
  pendingByCollection: Map<string, CollectionPending>;
41
+ /** Consecutive failed flushes per collection (failure backoff input). */
42
+ retryFailures: Map<string, number>;
43
+ /** Collections whose queued work is new user action, not a failure retry. */
44
+ freshEvents: Set<string>;
41
45
  clearCollectionRuntimeState: (collectionName: string) => void;
42
46
  beginSnapshotInit: (collection: Collection) => void;
43
47
  watchFactory: (
@@ -186,6 +190,10 @@ export function applyCollectionUpdate(
186
190
  pending.forceFallback = true;
187
191
  }
188
192
  host.pendingByCollection.set(collection.name, pending);
193
+ // A config edit is user action that may fix the failure cause: it must
194
+ // not wait out an armed failure backoff, and the count starts over.
195
+ host.freshEvents.add(collection.name);
196
+ host.retryFailures.delete(collection.name);
189
197
  }
190
198
  if (host.watchers.has(collection.name)) {
191
199
  host.beginSnapshotInit(collection);
@@ -10,7 +10,7 @@ import { normalize } from "node:path";
10
10
  import type { Collection } from "../config/types";
11
11
  import type { CollectionSyncResult, SyncOptions } from "../ingestion";
12
12
  import type { SqliteAdapter } from "../store/sqlite/adapter";
13
- import type { WatchQueueHost } from "./watch-service-events";
13
+ import type { WatchQueueHost, WatcherRetryKind } from "./watch-service-events";
14
14
  import type { CollectionPending } from "./watch-service-state";
15
15
  import type { WatcherSnapshot, WatcherSnapshotFs } from "./watch-snapshot";
16
16
 
@@ -36,7 +36,7 @@ export interface RunFlushContext {
36
36
  pendingByCollection: Map<string, CollectionPending>;
37
37
  flushDeadlineAt: Map<string, number>;
38
38
  syncing: Set<string>;
39
- retryScheduled: Set<string>;
39
+ retryScheduled: Map<string, WatcherRetryKind>;
40
40
  collectionGenerations: Map<string, number>;
41
41
  snapshots: Map<string, WatcherSnapshot>;
42
42
  snapshotReady: Map<string, boolean>;
@@ -125,6 +125,7 @@ export async function runOwnedCollectionFlush(
125
125
  }
126
126
 
127
127
  // Events that arrived while the lease was being taken join this flush.
128
+ ctx.queueHost.freshEvents.delete(collectionName);
128
129
  const taken = takePending(
129
130
  ctx.pendingByCollection.get(collectionName) ?? pending
130
131
  );
@@ -148,6 +149,9 @@ export async function runOwnedCollectionFlush(
148
149
  );
149
150
  };
150
151
 
152
+ // A flush can requeue failed work (e.g. a dirty classification error) and
153
+ // still settle as synced/idle; that must not reset the failure backoff.
154
+ let requeuedFailure = false;
151
155
  try {
152
156
  const outcome = await flushCollectionOnce({
153
157
  collection,
@@ -218,6 +222,7 @@ export async function runOwnedCollectionFlush(
218
222
  if (!stillOwner()) {
219
223
  return;
220
224
  }
225
+ requeuedFailure = true;
221
226
  requeueAfterFailure(
222
227
  ctx.queueHost,
223
228
  collectionName,
@@ -236,6 +241,12 @@ export async function runOwnedCollectionFlush(
236
241
  requeueGenerationReconcile(ctx.queueHost, collectionName);
237
242
  },
238
243
  });
244
+ if (
245
+ (outcome.status === "synced" || outcome.status === "idle") &&
246
+ !requeuedFailure
247
+ ) {
248
+ ctx.queueHost.retryFailures.delete(collectionName);
249
+ }
239
250
  if (
240
251
  outcome.status === "failed" &&
241
252
  outcome.error &&
@@ -15,6 +15,7 @@ import type { CollectionSyncResult, SyncOptions } from "../ingestion";
15
15
  import type { SqliteAdapter } from "../store/sqlite/adapter";
16
16
  import type { DocumentEvent, DocumentEventBus } from "./doc-events";
17
17
  import type { EmbedScheduler } from "./embed-scheduler";
18
+ import type { WatcherRetryKind } from "./watch-service-events";
18
19
  import type {
19
20
  WatcherSnapshot,
20
21
  WatcherSnapshotBuildResult,
@@ -126,7 +127,9 @@ export class CollectionWatchService {
126
127
  readonly #pendingByCollection = new Map<string, CollectionPending>();
127
128
  readonly #timers = new Map<string, ReturnType<typeof setTimeout>>();
128
129
  readonly #flushDeadlineAt = new Map<string, number>();
129
- readonly #retryScheduled = new Set<string>();
130
+ readonly #retryScheduled = new Map<string, WatcherRetryKind>();
131
+ readonly #retryFailures = new Map<string, number>();
132
+ readonly #freshEvents = new Set<string>();
130
133
  readonly #syncing = new Set<string>();
131
134
  readonly #inFlightSyncs = new Set<Promise<void>>();
132
135
  readonly #suppressedPaths = new Map<string, number>();
@@ -216,6 +219,8 @@ export class CollectionWatchService {
216
219
  this.#timers.clear();
217
220
  this.#flushDeadlineAt.clear();
218
221
  this.#retryScheduled.clear();
222
+ this.#retryFailures.clear();
223
+ this.#freshEvents.clear();
219
224
  this.#watchers.clear();
220
225
  this.#watchRoots.clear();
221
226
  this.#collectionGenerations.clear();
@@ -303,6 +308,8 @@ export class CollectionWatchService {
303
308
  flushDeadlineAt: this.#flushDeadlineAt,
304
309
  timers: this.#timers,
305
310
  retryScheduled: this.#retryScheduled,
311
+ retryFailures: this.#retryFailures,
312
+ freshEvents: this.#freshEvents,
306
313
  inFlightSyncs: this.#inFlightSyncs,
307
314
  runFlush: (name) => this.#flushCollection(name),
308
315
  };
@@ -316,6 +323,8 @@ export class CollectionWatchService {
316
323
  this.#snapshotInit.delete(collectionName);
317
324
  this.#flushDeadlineAt.delete(collectionName);
318
325
  this.#retryScheduled.delete(collectionName);
326
+ this.#retryFailures.delete(collectionName);
327
+ this.#freshEvents.delete(collectionName);
319
328
  const timer = this.#timers.get(collectionName);
320
329
  if (timer) {
321
330
  clearTimeout(timer);
@@ -145,6 +145,7 @@ import {
145
145
  type LinkWorkspaceSource,
146
146
  } from "../../core/link-workspace";
147
147
  import { normalizeWikiName, stripWikiMdExt } from "../../core/links";
148
+ import { tracePhase } from "../../core/phase-trace";
148
149
  import {
149
150
  TYPED_METADATA_INGEST_VERSION,
150
151
  typedMetadataSchema,
@@ -615,6 +616,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
615
616
  ): Promise<StoreResult<MigrationResult>> {
616
617
  try {
617
618
  this.db = new Database(dbPath, { create: true });
619
+ tracePhase("db: connection open");
618
620
  this.shutdownFenced = false;
619
621
  this.shutdownDeadline = undefined;
620
622
  this.dbPath = dbPath;
@@ -633,8 +635,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
633
635
  const journalMode = this.db
634
636
  .query<{ journal_mode: string }, []>("PRAGMA journal_mode")
635
637
  .get()?.journal_mode;
638
+ tracePhase(`db: journal_mode ${journalMode}`);
636
639
  if (journalMode?.toLowerCase() !== "wal") {
637
640
  this.db.exec("PRAGMA journal_mode = WAL");
641
+ tracePhase("db: journal_mode set to wal");
638
642
  }
639
643
  } catch (cause) {
640
644
  if (!isDatabaseLockedError(cause)) {
@@ -664,7 +668,9 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
664
668
  }
665
669
 
666
670
  // Run migrations
671
+ tracePhase("db: migrations");
667
672
  const result = runMigrations(this.db, migrations, ftsTokenizer);
673
+ tracePhase("db: migrations done");
668
674
  if (!result.ok) {
669
675
  this.db.close();
670
676
  this.db = null;
@@ -1 +0,0 @@
1
- 347695588c724660218a91ae70cd7bd8420445d2f78dd293cf661626380db7f2 gno-browser-clipper-v2.8.5.zip