@gmickel/gno 2.8.3 → 2.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.8.3.zip → gno-browser-clipper-v2.8.4.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.4.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +11 -2
- package/spec/output-schemas/doctor.schema.json +10 -4
- package/spec/output-schemas/status.schema.json +4 -2
- package/src/cli/commands/doctor-activation.ts +38 -11
- package/src/cli/commands/doctor.ts +4 -1
- package/src/cli/commands/status.ts +16 -4
- package/src/core/activation-empty.ts +16 -0
- package/src/core/activation-status.ts +23 -5
- package/src/core/links.ts +53 -15
- package/src/ingestion/strip.ts +69 -27
- package/src/serve/activation-health.ts +25 -7
- package/src/serve/public/components/BootstrapStatus.tsx +9 -4
- package/src/serve/status.ts +27 -8
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.3.zip.sha256 +0 -1
package/README.md
CHANGED
|
@@ -139,7 +139,7 @@ See the [guide](docs/COMPILED-CONTEXT.md).
|
|
|
139
139
|
|
|
140
140
|
<!-- public-truth:current-version -->
|
|
141
141
|
|
|
142
|
-
> Current source version: **v2.8.
|
|
142
|
+
> Current source version: **v2.8.4**. See [CHANGELOG.md](./CHANGELOG.md).
|
|
143
143
|
|
|
144
144
|
<!-- /public-truth -->
|
|
145
145
|
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
956ba0ce42d3b72635b61ca710d7adc8599fbdf08f3e32000d979158cac432c6 gno-browser-clipper-v2.8.4.zip
|
package/package.json
CHANGED
package/spec/cli.md
CHANGED
|
@@ -351,7 +351,12 @@ gno status [--json|--md]
|
|
|
351
351
|
```
|
|
352
352
|
|
|
353
353
|
`activation.usable` means at least one configured collection passed its local
|
|
354
|
-
lexical proof. `activation.healthy` means
|
|
354
|
+
lexical proof. `activation.healthy` means at least one collection is configured
|
|
355
|
+
and none failed its proof. A collection with no documents yet (index stage
|
|
356
|
+
`no_documents`, remediation `gno update`) is informational, not a failure: it
|
|
357
|
+
leaves `healthy` true, projects no connector proofs, and terminal output
|
|
358
|
+
lists it as `no documents yet` (`Lexical activation: NO DOCUMENTS YET` when no
|
|
359
|
+
collection has documents).
|
|
355
360
|
Semantic and connector stages remain independent; passive status never starts a
|
|
356
361
|
model runtime or connector process. `gno status` still exits 0 when activation
|
|
357
362
|
is unhealthy so scripts can inspect the structured state.
|
|
@@ -3022,7 +3027,11 @@ The additive `activation` object uses the same contract as `gno status` and
|
|
|
3022
3027
|
`GET /api/status`. Doctor performs only the local lexical proof. It never starts
|
|
3023
3028
|
connector children or initializes/downloads models. A failed lexical proof adds
|
|
3024
3029
|
the `retrieval-activation` error check and exits 2 after writing the complete
|
|
3025
|
-
result; no duplicate error is written to stderr.
|
|
3030
|
+
result; no duplicate error is written to stderr. A collection with no
|
|
3031
|
+
documents yet is not a failed proof: when it is the only activation problem,
|
|
3032
|
+
`retrieval-activation` has status `info` (terminal icon `i`) and lists the
|
|
3033
|
+
collection with its remediation (add files, then `gno update`). Check status
|
|
3034
|
+
is one of `ok`, `info`, `warn` or `error`; only `error` fails doctor. Connector failure or projection
|
|
3026
3035
|
truncation adds a warning and makes the structured doctor result non-healthy,
|
|
3027
3036
|
but preserves exit 0 when lexical proof and all other required checks pass. An
|
|
3028
3037
|
omitted target/collection pair has no inferred result.
|
|
@@ -24,8 +24,14 @@
|
|
|
24
24
|
],
|
|
25
25
|
"properties": {
|
|
26
26
|
"schemaVersion": { "const": "1.0" },
|
|
27
|
-
"usable": {
|
|
28
|
-
|
|
27
|
+
"usable": {
|
|
28
|
+
"type": "boolean",
|
|
29
|
+
"description": "At least one collection passed the lexical retrieval proof"
|
|
30
|
+
},
|
|
31
|
+
"healthy": {
|
|
32
|
+
"type": "boolean",
|
|
33
|
+
"description": "At least one collection is configured and none failed its lexical retrieval proof. A collection with no documents yet (remediation code no_documents, command gno update) is informational, not a failure."
|
|
34
|
+
},
|
|
29
35
|
"collections": {
|
|
30
36
|
"type": "array",
|
|
31
37
|
"items": {
|
|
@@ -229,8 +235,8 @@
|
|
|
229
235
|
},
|
|
230
236
|
"status": {
|
|
231
237
|
"type": "string",
|
|
232
|
-
"enum": ["ok", "warn", "error"],
|
|
233
|
-
"description": "Check result status"
|
|
238
|
+
"enum": ["ok", "info", "warn", "error"],
|
|
239
|
+
"description": "Check result status. info reports a state that needs no repair, such as a collection with no documents yet"
|
|
234
240
|
},
|
|
235
241
|
"message": {
|
|
236
242
|
"type": "string",
|
|
@@ -347,10 +347,12 @@
|
|
|
347
347
|
"const": "1.0"
|
|
348
348
|
},
|
|
349
349
|
"usable": {
|
|
350
|
-
"type": "boolean"
|
|
350
|
+
"type": "boolean",
|
|
351
|
+
"description": "At least one collection passed the lexical retrieval proof"
|
|
351
352
|
},
|
|
352
353
|
"healthy": {
|
|
353
|
-
"type": "boolean"
|
|
354
|
+
"type": "boolean",
|
|
355
|
+
"description": "At least one collection is configured and none failed its lexical retrieval proof. A collection with no documents yet (remediation code no_documents, command gno update) is informational, not a failure."
|
|
354
356
|
},
|
|
355
357
|
"collections": {
|
|
356
358
|
"type": "array",
|
|
@@ -6,6 +6,7 @@ import type { StorePort } from "../../store/types";
|
|
|
6
6
|
import type { DoctorCheck } from "./doctor";
|
|
7
7
|
|
|
8
8
|
import { getIndexDbPath, getModelsCachePath } from "../../app/constants";
|
|
9
|
+
import { isEmptyActivationCollection } from "../../core/activation-empty";
|
|
9
10
|
import { buildActivationStatus } from "../../core/activation-status";
|
|
10
11
|
import { ModelCache } from "../../llm/cache";
|
|
11
12
|
import { getActivePreset } from "../../llm/registry";
|
|
@@ -75,28 +76,54 @@ export async function buildDoctorActivation(
|
|
|
75
76
|
}
|
|
76
77
|
}
|
|
77
78
|
|
|
79
|
+
const plural = (count: number, noun: string): string =>
|
|
80
|
+
`${count} ${noun}${count === 1 ? "" : "s"}`;
|
|
81
|
+
|
|
78
82
|
export function checkRetrievalActivation(
|
|
79
83
|
activation: ActivationStatus
|
|
80
84
|
): DoctorCheck {
|
|
85
|
+
const ready = activation.collections.filter((collection) => collection.ready);
|
|
86
|
+
const empty = activation.collections.filter(isEmptyActivationCollection);
|
|
87
|
+
const emptyDetails = empty.flatMap(({ collection, remediation }) => [
|
|
88
|
+
`${collection}: no documents indexed yet (informational)`,
|
|
89
|
+
`Add files, then run: ${remediation?.command ?? "gno update"}`,
|
|
90
|
+
]);
|
|
91
|
+
|
|
81
92
|
if (activation.healthy) {
|
|
82
93
|
const semanticStates = [
|
|
83
94
|
...new Set(
|
|
84
|
-
|
|
85
|
-
({ semanticAvailability }) => semanticAvailability.code
|
|
86
|
-
)
|
|
95
|
+
ready.map(({ semanticAvailability }) => semanticAvailability.code)
|
|
87
96
|
),
|
|
88
97
|
];
|
|
98
|
+
const semanticDetail =
|
|
99
|
+
ready.length > 0
|
|
100
|
+
? [
|
|
101
|
+
`Semantic retrieval remains separate (${semanticStates.join(", ")}).`,
|
|
102
|
+
]
|
|
103
|
+
: [];
|
|
104
|
+
if (empty.length === 0) {
|
|
105
|
+
return {
|
|
106
|
+
name: "retrieval-activation",
|
|
107
|
+
status: "ok",
|
|
108
|
+
message: `${plural(ready.length, "collection")} passed lexical retrieval proof`,
|
|
109
|
+
details: semanticDetail,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
89
112
|
return {
|
|
90
113
|
name: "retrieval-activation",
|
|
91
|
-
status: "
|
|
92
|
-
message:
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
114
|
+
status: "info",
|
|
115
|
+
message:
|
|
116
|
+
ready.length > 0
|
|
117
|
+
? `${plural(ready.length, "collection")} passed lexical retrieval proof; ${plural(empty.length, "collection")} with no documents yet`
|
|
118
|
+
: `No documents indexed yet in ${plural(empty.length, "collection")}`,
|
|
119
|
+
details: [...emptyDetails, ...semanticDetail],
|
|
96
120
|
};
|
|
97
121
|
}
|
|
98
122
|
|
|
99
|
-
const failed = activation.collections.filter(
|
|
123
|
+
const failed = activation.collections.filter(
|
|
124
|
+
(collection) =>
|
|
125
|
+
!collection.ready && !isEmptyActivationCollection(collection)
|
|
126
|
+
);
|
|
100
127
|
const details = failed.flatMap(({ collection, remediation }) =>
|
|
101
128
|
remediation
|
|
102
129
|
? [
|
|
@@ -112,9 +139,9 @@ export function checkRetrievalActivation(
|
|
|
112
139
|
activation.collections.length === 0
|
|
113
140
|
? "No collections configured. Run: gno collection add"
|
|
114
141
|
: activation.usable
|
|
115
|
-
? `${failed.length
|
|
142
|
+
? `${plural(failed.length, "collection")} failed lexical retrieval proof`
|
|
116
143
|
: "No configured collection passed lexical retrieval proof",
|
|
117
|
-
details,
|
|
144
|
+
details: [...details, ...emptyDetails],
|
|
118
145
|
};
|
|
119
146
|
}
|
|
120
147
|
|
|
@@ -55,7 +55,8 @@ import {
|
|
|
55
55
|
// Types
|
|
56
56
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
57
57
|
|
|
58
|
-
|
|
58
|
+
/** `info` reports a state that needs no repair, such as an empty collection. */
|
|
59
|
+
export type DoctorCheckStatus = "ok" | "info" | "warn" | "error";
|
|
59
60
|
|
|
60
61
|
export interface DoctorCheck {
|
|
61
62
|
name: string;
|
|
@@ -754,6 +755,8 @@ function statusIcon(status: DoctorCheckStatus): string {
|
|
|
754
755
|
switch (status) {
|
|
755
756
|
case "ok":
|
|
756
757
|
return "✓";
|
|
758
|
+
case "info":
|
|
759
|
+
return "i";
|
|
757
760
|
case "warn":
|
|
758
761
|
return "!";
|
|
759
762
|
case "error":
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
loadConfig,
|
|
20
20
|
} from "../../config";
|
|
21
21
|
import { isConnectorActivationComplete } from "../../core/activation-connector-health";
|
|
22
|
+
import { isEmptyActivationCollection } from "../../core/activation-empty";
|
|
22
23
|
import { buildActivationStatus } from "../../core/activation-status";
|
|
23
24
|
import { formatChunkingStatus } from "../../core/chunking-status";
|
|
24
25
|
import { formatLinkWorkspace } from "../../core/link-workspace";
|
|
@@ -229,10 +230,21 @@ function formatTerminal(
|
|
|
229
230
|
`Health: ${isStatusHealthy(indexStatus, activation) ? "OK" : "DEGRADED"}`
|
|
230
231
|
);
|
|
231
232
|
lines.push("");
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
233
|
+
const activationState = activation.healthy
|
|
234
|
+
? activation.usable
|
|
235
|
+
? "READY"
|
|
236
|
+
: "NO DOCUMENTS YET"
|
|
237
|
+
: activation.usable
|
|
238
|
+
? "DEGRADED"
|
|
239
|
+
: "BLOCKED";
|
|
240
|
+
lines.push(`Lexical activation: ${activationState}`);
|
|
235
241
|
for (const collection of activation.collections) {
|
|
242
|
+
if (isEmptyActivationCollection(collection)) {
|
|
243
|
+
lines.push(
|
|
244
|
+
` ${collection.collection}: no documents yet (add files, then run ${collection.remediation?.command})`
|
|
245
|
+
);
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
236
248
|
const failedStage = collection.remediation?.stage;
|
|
237
249
|
const suffix = failedStage
|
|
238
250
|
? ` (${failedStage}: ${collection.remediation?.code}; ${collection.remediation?.command})`
|
|
@@ -325,7 +337,7 @@ function formatMarkdown(
|
|
|
325
337
|
lines.push(`- **Lexically healthy**: ${activation.healthy}`);
|
|
326
338
|
for (const collection of activation.collections) {
|
|
327
339
|
lines.push(
|
|
328
|
-
`- **${collection.collection}**: ${collection.ready ? "lexical ready" : `${collection.remediation?.stage ?? "index"} ${collection.remediation?.code ?? "index_query_failed"}`}`
|
|
340
|
+
`- **${collection.collection}**: ${collection.ready ? "lexical ready" : isEmptyActivationCollection(collection) ? `no documents yet (${collection.remediation?.command})` : `${collection.remediation?.stage ?? "index"} ${collection.remediation?.code ?? "index_query_failed"}`}`
|
|
329
341
|
);
|
|
330
342
|
}
|
|
331
343
|
for (const connector of activation.connectors) {
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Empty-collection rule shared by server surfaces and the Web UI bundle, so
|
|
3
|
+
* it carries type imports only.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { ActivationCollectionStatus } from "./activation-status";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A collection with nothing indexed yet: its only activation problem is
|
|
10
|
+
* `no_documents`. Surfaces show it as informational, never as a failure.
|
|
11
|
+
*/
|
|
12
|
+
export function isEmptyActivationCollection(
|
|
13
|
+
collection: Pick<ActivationCollectionStatus, "ready" | "remediation">
|
|
14
|
+
): boolean {
|
|
15
|
+
return !collection.ready && collection.remediation?.code === "no_documents";
|
|
16
|
+
}
|
|
@@ -14,6 +14,7 @@ import type {
|
|
|
14
14
|
ConnectorVerificationTarget,
|
|
15
15
|
} from "./connector-verifier";
|
|
16
16
|
|
|
17
|
+
import { isEmptyActivationCollection } from "./activation-empty";
|
|
17
18
|
import { createEphemeralActivationProbePlan } from "./activation-probe-plan";
|
|
18
19
|
import { verifyLexicalActivation } from "./activation-verifier";
|
|
19
20
|
import {
|
|
@@ -73,7 +74,13 @@ export interface ActivationConnectorStatus {
|
|
|
73
74
|
|
|
74
75
|
export interface ActivationStatus {
|
|
75
76
|
schemaVersion: "1.0";
|
|
77
|
+
/** At least one collection passed the lexical proof. */
|
|
76
78
|
usable: boolean;
|
|
79
|
+
/**
|
|
80
|
+
* At least one collection is configured and none failed its proof. A
|
|
81
|
+
* collection with no documents yet is not a failure (see
|
|
82
|
+
* `isEmptyActivationCollection`), so it does not clear this flag.
|
|
83
|
+
*/
|
|
77
84
|
healthy: boolean;
|
|
78
85
|
collections: ActivationCollectionStatus[];
|
|
79
86
|
/** Only fingerprint-current persisted connector receipts may appear here. */
|
|
@@ -178,10 +185,13 @@ function remediationFor(
|
|
|
178
185
|
stage: ActivationStageName,
|
|
179
186
|
code: ActivationVerificationCode
|
|
180
187
|
): ActivationRemediation {
|
|
181
|
-
const command =
|
|
188
|
+
const command =
|
|
189
|
+
code === "no_documents"
|
|
190
|
+
? "gno update"
|
|
191
|
+
: `gno index ${collection} --no-embed`;
|
|
182
192
|
const messages: Partial<Record<ActivationVerificationCode, string>> = {
|
|
183
193
|
no_documents:
|
|
184
|
-
"
|
|
194
|
+
"No documents are indexed yet. Add supported files to this collection's folder, then run gno update.",
|
|
185
195
|
no_probe_term:
|
|
186
196
|
"Add searchable text or adjust the collection filters, then reindex.",
|
|
187
197
|
index_query_failed:
|
|
@@ -322,7 +332,11 @@ async function buildConnectorStatuses(
|
|
|
322
332
|
): Promise<{ items: ActivationConnectorStatus[]; total: number }> {
|
|
323
333
|
const sortedTargets = [...targets].sort((a, b) => a.id.localeCompare(b.id));
|
|
324
334
|
const boundedTargets = sortedTargets.slice(0, MAX_CONNECTOR_TARGETS);
|
|
325
|
-
|
|
335
|
+
// A collection with no documents gives a connector nothing to prove.
|
|
336
|
+
const provable = collections.filter(
|
|
337
|
+
({ projected }) => !isEmptyActivationCollection(projected)
|
|
338
|
+
);
|
|
339
|
+
const allPairs = provable.flatMap((collection) =>
|
|
326
340
|
boundedTargets.map((target) => ({ collection, target }))
|
|
327
341
|
);
|
|
328
342
|
const items = await mapBounded(
|
|
@@ -363,7 +377,7 @@ async function buildConnectorStatuses(
|
|
|
363
377
|
};
|
|
364
378
|
}
|
|
365
379
|
);
|
|
366
|
-
return { items, total:
|
|
380
|
+
return { items, total: provable.length * sortedTargets.length };
|
|
367
381
|
}
|
|
368
382
|
|
|
369
383
|
async function verifyCoalesced(
|
|
@@ -501,7 +515,11 @@ export async function buildActivationStatus(
|
|
|
501
515
|
schemaVersion: "1.0",
|
|
502
516
|
usable: projected.some((collection) => collection.ready),
|
|
503
517
|
healthy:
|
|
504
|
-
projected.length > 0 &&
|
|
518
|
+
projected.length > 0 &&
|
|
519
|
+
projected.every(
|
|
520
|
+
(collection) =>
|
|
521
|
+
collection.ready || isEmptyActivationCollection(collection)
|
|
522
|
+
),
|
|
505
523
|
collections: projected,
|
|
506
524
|
connectors: connectorStatuses.items,
|
|
507
525
|
connectorProjection: {
|
package/src/core/links.ts
CHANGED
|
@@ -80,11 +80,53 @@ const UNSAFE_PERCENT_CODES = new Set(["%2F", "%5C", "%00", "%2f", "%5c"]);
|
|
|
80
80
|
// Regex Patterns
|
|
81
81
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
82
82
|
|
|
83
|
+
interface WikiLinkMatch {
|
|
84
|
+
index: number;
|
|
85
|
+
raw: string;
|
|
86
|
+
content: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
83
89
|
/**
|
|
84
|
-
* Wiki
|
|
85
|
-
*
|
|
90
|
+
* Wiki links: [[target]], [[target|alias]], [[target#anchor]] or
|
|
91
|
+
* [[collection:target]]. Matches exactly what
|
|
92
|
+
* `/\[\[([^\]|]+(?:\|[^\]]+)?)\]\]/g` matches, in one pass: the content runs
|
|
93
|
+
* to the first `]`, which must be doubled; it must not start with `|`, and a
|
|
94
|
+
* `|` in it needs text after it. The regex rescanned to that `]` from every
|
|
95
|
+
* unclosed `[[`, so a note with thousands of them took minutes.
|
|
86
96
|
*/
|
|
87
|
-
|
|
97
|
+
export function* findWikiLinks(
|
|
98
|
+
markdown: string
|
|
99
|
+
): Generator<WikiLinkMatch, void, undefined> {
|
|
100
|
+
let nextClose = -2;
|
|
101
|
+
let nextPipe = -2;
|
|
102
|
+
let start = markdown.indexOf("[[");
|
|
103
|
+
while (start !== -1) {
|
|
104
|
+
const contentStart = start + 2;
|
|
105
|
+
if (nextClose < contentStart) {
|
|
106
|
+
nextClose = markdown.indexOf("]", contentStart);
|
|
107
|
+
if (nextClose === -1) return;
|
|
108
|
+
}
|
|
109
|
+
if (nextPipe !== -1 && nextPipe < contentStart) {
|
|
110
|
+
nextPipe = markdown.indexOf("|", contentStart);
|
|
111
|
+
}
|
|
112
|
+
const pipeInContent = nextPipe !== -1 && nextPipe < nextClose;
|
|
113
|
+
const matched =
|
|
114
|
+
markdown.charCodeAt(nextClose + 1) === 93 /* ] */ &&
|
|
115
|
+
nextClose > contentStart &&
|
|
116
|
+
nextPipe !== contentStart &&
|
|
117
|
+
(!pipeInContent || nextClose > nextPipe + 1);
|
|
118
|
+
if (matched) {
|
|
119
|
+
yield {
|
|
120
|
+
index: start,
|
|
121
|
+
raw: markdown.slice(start, nextClose + 2),
|
|
122
|
+
content: markdown.slice(contentStart, nextClose),
|
|
123
|
+
};
|
|
124
|
+
start = markdown.indexOf("[[", nextClose + 2);
|
|
125
|
+
} else {
|
|
126
|
+
start = markdown.indexOf("[[", start + 1);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
88
130
|
|
|
89
131
|
/** Logseq embed: {{embed [[Page]]}} or {{embed ((block-id))}} */
|
|
90
132
|
const LOGSEQ_EMBED_REGEX =
|
|
@@ -385,12 +427,9 @@ export function parseLinks(
|
|
|
385
427
|
};
|
|
386
428
|
|
|
387
429
|
// Parse wiki links
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
while ((match = WIKI_LINK_REGEX.exec(markdown)) !== null) {
|
|
392
|
-
const startOffset = match.index;
|
|
393
|
-
const endOffset = startOffset + match[0].length;
|
|
430
|
+
for (const wiki of findWikiLinks(markdown)) {
|
|
431
|
+
const startOffset = wiki.index;
|
|
432
|
+
const endOffset = startOffset + wiki.raw.length;
|
|
394
433
|
|
|
395
434
|
// Skip [[target]] nested in Logseq alias syntax: [Display]([[target]])
|
|
396
435
|
if (
|
|
@@ -405,12 +444,10 @@ export function parseLinks(
|
|
|
405
444
|
continue;
|
|
406
445
|
}
|
|
407
446
|
|
|
408
|
-
const content = match[1];
|
|
409
|
-
if (!content) continue;
|
|
410
|
-
|
|
411
447
|
// Parse [[target|alias]] (and table-escaped [[target\|alias]]) format
|
|
412
|
-
const { target: targetPart, alias: aliasText } =
|
|
413
|
-
|
|
448
|
+
const { target: targetPart, alias: aliasText } = splitWikiLinkContent(
|
|
449
|
+
wiki.content
|
|
450
|
+
);
|
|
414
451
|
// Only set displayText if different from target
|
|
415
452
|
const displayText =
|
|
416
453
|
aliasText !== undefined && aliasText !== targetPart
|
|
@@ -421,11 +458,12 @@ export function parseLinks(
|
|
|
421
458
|
if (!trimmedTarget) {
|
|
422
459
|
continue;
|
|
423
460
|
}
|
|
424
|
-
pushWikiLink(
|
|
461
|
+
pushWikiLink(wiki.raw, trimmedTarget, startOffset, endOffset, displayText);
|
|
425
462
|
}
|
|
426
463
|
|
|
427
464
|
// Parse Logseq embeds as links
|
|
428
465
|
LOGSEQ_EMBED_REGEX.lastIndex = 0;
|
|
466
|
+
let match: RegExpExecArray | null;
|
|
429
467
|
|
|
430
468
|
while ((match = LOGSEQ_EMBED_REGEX.exec(markdown)) !== null) {
|
|
431
469
|
const startOffset = match.index;
|
package/src/ingestion/strip.ts
CHANGED
|
@@ -14,8 +14,10 @@
|
|
|
14
14
|
* Code regions come from the CommonMark + GFM parser. GFM table parsing is
|
|
15
15
|
* quadratic in the cells of one table (a 20,000-row table took minutes), so
|
|
16
16
|
* text with a table over the cell budget is parsed without the table
|
|
17
|
-
* extension
|
|
18
|
-
*
|
|
17
|
+
* extension and without inline links and inline HTML, which rescan a
|
|
18
|
+
* table-sized paragraph once per row: code spans, fences, indented code,
|
|
19
|
+
* blockquotes and lists keep their CommonMark meaning; table-cell
|
|
20
|
+
* boundaries, link destinations and HTML tags no longer hide a backtick.
|
|
19
21
|
*
|
|
20
22
|
* @module src/ingestion/strip
|
|
21
23
|
*/
|
|
@@ -59,9 +61,6 @@ export interface ExcludedRange {
|
|
|
59
61
|
/** Frontmatter at start of file (YAML between --- delimiters) */
|
|
60
62
|
const FRONTMATTER_REGEX = /^---\r?\n[\s\S]*?(?:\r?\n)?---(?:\r?\n|$)/;
|
|
61
63
|
|
|
62
|
-
/** HTML comments */
|
|
63
|
-
const HTML_COMMENT_REGEX = /<!--[\s\S]*?-->/g;
|
|
64
|
-
|
|
65
64
|
/**
|
|
66
65
|
* Most `|` characters one blank-line-delimited block may hold before the
|
|
67
66
|
* GFM table extension is left out of the parse. A GFM table only ends at a
|
|
@@ -132,13 +131,33 @@ const hasOversizedTable = (markdown: string): boolean => {
|
|
|
132
131
|
return false;
|
|
133
132
|
};
|
|
134
133
|
|
|
135
|
-
/**
|
|
134
|
+
/**
|
|
135
|
+
* Inline constructs left out with the table extension. Without table rules a
|
|
136
|
+
* large table is one paragraph, and these rescan it once per row: on every
|
|
137
|
+
* `]` link and image ends walk back through all the paragraph's earlier
|
|
138
|
+
* events looking for an opening `[` (an unmatched `]` per row made 8,000
|
|
139
|
+
* rows take 66 s), and inline HTML reads forward from every unclosed `<!--`
|
|
140
|
+
* to the paragraph's end. They only matter to code detection through a
|
|
141
|
+
* backtick inside a link destination or title or an inline HTML tag, which
|
|
142
|
+
* then reads as a code-span delimiter.
|
|
143
|
+
*/
|
|
144
|
+
const PARAGRAPH_RESCANNING_CONSTRUCTS = [
|
|
145
|
+
"labelEnd",
|
|
146
|
+
"gfmPotentialFootnoteCall",
|
|
147
|
+
"htmlText",
|
|
148
|
+
];
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* GFM without tables: every GFM construct except table rows and cells, and
|
|
152
|
+
* without the constructs in `PARAGRAPH_RESCANNING_CONSTRUCTS`.
|
|
153
|
+
*/
|
|
136
154
|
const GFM_WITHOUT_TABLES = {
|
|
137
155
|
extensions: [
|
|
138
156
|
gfmAutolinkLiteral(),
|
|
139
157
|
gfmFootnote(),
|
|
140
158
|
gfmStrikethrough(),
|
|
141
159
|
gfmTaskListItem(),
|
|
160
|
+
{ disable: { null: PARAGRAPH_RESCANNING_CONSTRUCTS } },
|
|
142
161
|
],
|
|
143
162
|
mdastExtensions: [
|
|
144
163
|
gfmAutolinkLiteralFromMarkdown(),
|
|
@@ -226,15 +245,15 @@ export function getExcludedRanges(markdown: string): ExcludedRange[] {
|
|
|
226
245
|
});
|
|
227
246
|
}
|
|
228
247
|
|
|
229
|
-
// 2. HTML comments
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
248
|
+
// 2. HTML comments: each `<!--` to the first `-->` after it. Once no
|
|
249
|
+
// `-->` follows, no later comment can close either, so the scan stops;
|
|
250
|
+
// the regex /<!--[\s\S]*?-->/g rescanned to the end from every opener.
|
|
251
|
+
let open = markdown.indexOf("<!--");
|
|
252
|
+
while (open !== -1) {
|
|
253
|
+
const close = markdown.indexOf("-->", open + 4);
|
|
254
|
+
if (close === -1) break;
|
|
255
|
+
ranges.push({ start: open, end: close + 3, kind: "html_comment" });
|
|
256
|
+
open = markdown.indexOf("<!--", close + 3);
|
|
238
257
|
}
|
|
239
258
|
|
|
240
259
|
// 3. Code spans and fenced or indented code blocks, from the parser.
|
|
@@ -281,24 +300,47 @@ export function isExcluded(
|
|
|
281
300
|
return false;
|
|
282
301
|
}
|
|
283
302
|
|
|
303
|
+
/**
|
|
304
|
+
* Largest `end` among each prefix of a start-sorted range list, built once
|
|
305
|
+
* per list. Ranges can overlap (a comment around a code span), so a range
|
|
306
|
+
* list's ends are not sorted and the prefix maximum stands in for them.
|
|
307
|
+
*/
|
|
308
|
+
const prefixMaxEnds = new WeakMap<readonly ExcludedRange[], Int32Array>();
|
|
309
|
+
|
|
310
|
+
const getPrefixMaxEnds = (ranges: readonly ExcludedRange[]): Int32Array => {
|
|
311
|
+
let maxEnds = prefixMaxEnds.get(ranges);
|
|
312
|
+
if (maxEnds?.length !== ranges.length) {
|
|
313
|
+
maxEnds = new Int32Array(ranges.length);
|
|
314
|
+
let maxEnd = -1;
|
|
315
|
+
for (const [index, range] of ranges.entries()) {
|
|
316
|
+
maxEnd = Math.max(maxEnd, range.end);
|
|
317
|
+
maxEnds[index] = maxEnd;
|
|
318
|
+
}
|
|
319
|
+
prefixMaxEnds.set(ranges, maxEnds);
|
|
320
|
+
}
|
|
321
|
+
return maxEnds;
|
|
322
|
+
};
|
|
323
|
+
|
|
284
324
|
/**
|
|
285
325
|
* Check if a range [start, end) intersects any excluded range.
|
|
286
|
-
* More precise than isExcluded for multi-character matches.
|
|
326
|
+
* More precise than isExcluded for multi-character matches. Ranges must be
|
|
327
|
+
* sorted by start, as getExcludedRanges returns them; one lookup is a binary
|
|
328
|
+
* search, so checking every link of a large note stays linear.
|
|
287
329
|
*/
|
|
288
330
|
export function rangeIntersectsExcluded(
|
|
289
331
|
start: number,
|
|
290
332
|
end: number,
|
|
291
|
-
excludedRanges: ExcludedRange[]
|
|
333
|
+
excludedRanges: readonly ExcludedRange[]
|
|
292
334
|
): boolean {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
if (
|
|
300
|
-
|
|
301
|
-
}
|
|
335
|
+
// Ranges starting before `end` form a prefix; one of them intersects
|
|
336
|
+
// [start, end) exactly when the largest end among them passes `start`.
|
|
337
|
+
let low = 0;
|
|
338
|
+
let high = excludedRanges.length;
|
|
339
|
+
while (low < high) {
|
|
340
|
+
const mid = (low + high) >>> 1;
|
|
341
|
+
if ((excludedRanges[mid]?.start ?? end) < end) low = mid + 1;
|
|
342
|
+
else high = mid;
|
|
302
343
|
}
|
|
303
|
-
return false;
|
|
344
|
+
if (low === 0) return false;
|
|
345
|
+
return (getPrefixMaxEnds(excludedRanges)[low - 1] ?? -1) > start;
|
|
304
346
|
}
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
import type { ActivationStatus } from "../core/activation-status";
|
|
4
4
|
import type { HealthCheck } from "./status-model";
|
|
5
5
|
|
|
6
|
+
import { isEmptyActivationCollection } from "../core/activation-empty";
|
|
7
|
+
|
|
6
8
|
function countLabel(count: number, singular: string): string {
|
|
7
9
|
return `${count} ${count === 1 ? singular : `${singular}s`}`;
|
|
8
10
|
}
|
|
@@ -10,26 +12,42 @@ function countLabel(count: number, singular: string): string {
|
|
|
10
12
|
export function buildActivationCheck(
|
|
11
13
|
activation: ActivationStatus
|
|
12
14
|
): HealthCheck {
|
|
15
|
+
const ready = activation.collections.filter((collection) => collection.ready);
|
|
16
|
+
const empty = activation.collections.filter(isEmptyActivationCollection);
|
|
17
|
+
const emptyDetail =
|
|
18
|
+
empty.length > 0
|
|
19
|
+
? `${countLabel(empty.length, "folder")} ${empty.length === 1 ? "has" : "have"} no documents yet (${empty.map(({ collection }) => collection).join(", ")}): add files, then run update.`
|
|
20
|
+
: null;
|
|
13
21
|
if (activation.healthy) {
|
|
14
22
|
const semanticReasons = [
|
|
15
23
|
...new Set(
|
|
16
|
-
|
|
17
|
-
({ semanticAvailability }) => semanticAvailability.code
|
|
18
|
-
)
|
|
24
|
+
ready.map(({ semanticAvailability }) => semanticAvailability.code)
|
|
19
25
|
),
|
|
20
26
|
];
|
|
27
|
+
const semanticDetail =
|
|
28
|
+
ready.length > 0
|
|
29
|
+
? `Lexical search is proven. Semantic availability is separate (${semanticReasons.join(", ")}).`
|
|
30
|
+
: null;
|
|
21
31
|
return {
|
|
22
32
|
id: "retrieval-activation",
|
|
23
33
|
title: "Retrieval proof",
|
|
24
34
|
status: "ok",
|
|
25
|
-
summary:
|
|
26
|
-
|
|
35
|
+
summary:
|
|
36
|
+
ready.length === 0
|
|
37
|
+
? `No documents indexed yet in ${countLabel(empty.length, "folder")}`
|
|
38
|
+
: empty.length > 0
|
|
39
|
+
? `${countLabel(ready.length, "folder")} passed lexical retrieval; ${countLabel(empty.length, "folder")} with no documents yet`
|
|
40
|
+
: `${countLabel(ready.length, "folder")} passed lexical retrieval`,
|
|
41
|
+
detail: [semanticDetail, emptyDetail].filter(Boolean).join(" "),
|
|
27
42
|
actionLabel: "Run update",
|
|
28
43
|
actionKind: "sync",
|
|
29
44
|
};
|
|
30
45
|
}
|
|
31
46
|
|
|
32
|
-
const failed = activation.collections.filter(
|
|
47
|
+
const failed = activation.collections.filter(
|
|
48
|
+
(collection) =>
|
|
49
|
+
!collection.ready && !isEmptyActivationCollection(collection)
|
|
50
|
+
);
|
|
33
51
|
const first = failed[0];
|
|
34
52
|
const detail = first?.remediation
|
|
35
53
|
? `${first.collection}: ${first.remediation.stage}/${first.remediation.code}. Run: ${first.remediation.command}`
|
|
@@ -41,7 +59,7 @@ export function buildActivationCheck(
|
|
|
41
59
|
summary: activation.usable
|
|
42
60
|
? `${countLabel(failed.length, "folder")} failed lexical retrieval`
|
|
43
61
|
: "No folder passed lexical retrieval",
|
|
44
|
-
detail,
|
|
62
|
+
detail: [detail, emptyDetail].filter(Boolean).join(" "),
|
|
45
63
|
actionLabel: "Run update",
|
|
46
64
|
actionKind: "sync",
|
|
47
65
|
};
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
|
|
10
10
|
import type { AppStatusResponse } from "../../status-model";
|
|
11
11
|
|
|
12
|
+
import { isEmptyActivationCollection } from "../../../core/activation-empty";
|
|
12
13
|
import { buildConnectorActivationCheck } from "../../activation-health";
|
|
13
14
|
import { Button } from "./ui/button";
|
|
14
15
|
import {
|
|
@@ -147,9 +148,9 @@ export function BootstrapStatus({
|
|
|
147
148
|
<Card className="border-border/60 bg-card/70">
|
|
148
149
|
<CardHeader className="pb-3">
|
|
149
150
|
<div className="flex items-center gap-2">
|
|
150
|
-
{activation.healthy && connectorsHealthy ? (
|
|
151
|
+
{activation.healthy && activation.usable && connectorsHealthy ? (
|
|
151
152
|
<CheckCircle2Icon className="size-4 text-emerald-500" />
|
|
152
|
-
) : activation.usable ? (
|
|
153
|
+
) : activation.usable || activation.healthy ? (
|
|
153
154
|
<ServerCogIcon className="size-4 text-amber-500" />
|
|
154
155
|
) : (
|
|
155
156
|
<AlertCircleIcon className="size-4 text-destructive" />
|
|
@@ -158,7 +159,9 @@ export function BootstrapStatus({
|
|
|
158
159
|
</div>
|
|
159
160
|
<CardDescription>
|
|
160
161
|
{activation.healthy
|
|
161
|
-
?
|
|
162
|
+
? activation.usable
|
|
163
|
+
? "Lexical retrieval proven"
|
|
164
|
+
: "No documents indexed yet"
|
|
162
165
|
: activation.usable
|
|
163
166
|
? `Search usable in ${activation.collections.filter(({ ready }) => ready).length}/${activation.collections.length} folders`
|
|
164
167
|
: "Retrieval proof failed"}
|
|
@@ -179,7 +182,9 @@ export function BootstrapStatus({
|
|
|
179
182
|
<p className="text-muted-foreground text-xs">
|
|
180
183
|
{collection.ready
|
|
181
184
|
? `Lexical passed; semantic ${collection.semanticAvailability.code}`
|
|
182
|
-
:
|
|
185
|
+
: isEmptyActivationCollection(collection)
|
|
186
|
+
? "No documents yet. Add files, then run update."
|
|
187
|
+
: `${collection.remediation?.stage ?? "index"}/${collection.remediation?.code ?? "index_query_failed"}`}
|
|
183
188
|
</p>
|
|
184
189
|
{collection.remediation && (
|
|
185
190
|
<p className="mt-1 font-mono text-muted-foreground text-xs">
|
package/src/serve/status.ts
CHANGED
|
@@ -14,6 +14,7 @@ import type {
|
|
|
14
14
|
|
|
15
15
|
import { getModelsCachePath } from "../app/constants";
|
|
16
16
|
import { buildContentTypeBoostStatus } from "../config/content-types";
|
|
17
|
+
import { isEmptyActivationCollection } from "../core/activation-empty";
|
|
17
18
|
import {
|
|
18
19
|
type ActivationStatus,
|
|
19
20
|
buildActivationStatus,
|
|
@@ -555,7 +556,13 @@ function buildOnboarding(
|
|
|
555
556
|
): AppStatusResponse["onboarding"] {
|
|
556
557
|
const foldersReady = activation.collections.length > 0;
|
|
557
558
|
const modelsReady = modelCheck.status === "ok";
|
|
558
|
-
|
|
559
|
+
// An empty folder is not a failure, but a workspace with no documents yet
|
|
560
|
+
// has not finished its first proof.
|
|
561
|
+
const indexedReady = activation.healthy && activation.usable;
|
|
562
|
+
const readyCount = activation.collections.filter(({ ready }) => ready).length;
|
|
563
|
+
const emptyCount = activation.collections.filter(
|
|
564
|
+
isEmptyActivationCollection
|
|
565
|
+
).length;
|
|
559
566
|
const semanticStates = [
|
|
560
567
|
...new Set(
|
|
561
568
|
activation.collections.map(
|
|
@@ -563,10 +570,22 @@ function buildOnboarding(
|
|
|
563
570
|
)
|
|
564
571
|
),
|
|
565
572
|
];
|
|
566
|
-
const failedActivation =
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
573
|
+
const failedActivation =
|
|
574
|
+
activation.collections.find(
|
|
575
|
+
(collection) =>
|
|
576
|
+
!collection.ready && !isEmptyActivationCollection(collection)
|
|
577
|
+
) ?? activation.collections.find(({ ready }) => !ready);
|
|
578
|
+
const activationDetail = !failedActivation?.remediation
|
|
579
|
+
? "Run the first sync to populate a searchable lexical index."
|
|
580
|
+
: isEmptyActivationCollection(failedActivation)
|
|
581
|
+
? `${failedActivation.collection} has no documents yet. Add files, then run: ${failedActivation.remediation.command}`
|
|
582
|
+
: `${failedActivation.collection}: ${failedActivation.remediation.stage}/${failedActivation.remediation.code}. Run: ${failedActivation.remediation.command}`;
|
|
583
|
+
const emptyNote =
|
|
584
|
+
emptyCount > 0
|
|
585
|
+
? ` ${summarizeCount(emptyCount, "folder")} with no documents yet.`
|
|
586
|
+
: "";
|
|
587
|
+
const passedFolders =
|
|
588
|
+
emptyCount > 0 ? "Every folder with documents" : "Every folder";
|
|
570
589
|
|
|
571
590
|
const steps = [
|
|
572
591
|
{
|
|
@@ -600,7 +619,7 @@ function buildOnboarding(
|
|
|
600
619
|
title: "Prove lexical retrieval",
|
|
601
620
|
status: indexedReady ? "complete" : foldersReady ? "current" : "upcoming",
|
|
602
621
|
detail: indexedReady
|
|
603
|
-
? `${summarizeCount(
|
|
622
|
+
? `${summarizeCount(readyCount, "folder")} passed a corpus-derived lexical proof.${emptyNote}`
|
|
604
623
|
: activationDetail,
|
|
605
624
|
action: "sync",
|
|
606
625
|
},
|
|
@@ -636,8 +655,8 @@ function buildOnboarding(
|
|
|
636
655
|
stage: "ready",
|
|
637
656
|
headline: "Workspace ready",
|
|
638
657
|
detail: modelsReady
|
|
639
|
-
?
|
|
640
|
-
:
|
|
658
|
+
? `${passedFolders} passed lexical retrieval.${emptyNote} Local model files are cached; semantic retrieval remains separate (${semanticStates.join(", ")}).`
|
|
659
|
+
: `${passedFolders} passed lexical retrieval.${emptyNote} Semantic retrieval remains an optional next step (${semanticStates.join(", ")}).`,
|
|
641
660
|
suggestedCollections: suggestions,
|
|
642
661
|
steps,
|
|
643
662
|
};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
7fe15ce08a24f4dae4ac6694f62bb26d94bbc47d6abc09826f29267ace10558d gno-browser-clipper-v2.8.3.zip
|