@gmickel/gno 0.41.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -0
- package/assets/screenshots/publish-reader.jpg +0 -0
- package/assets/skill/SKILL.md +2 -0
- package/assets/skill/cli-reference.md +28 -0
- package/package.json +1 -1
- package/src/cli/commands/embed.ts +216 -8
- package/src/cli/commands/index.ts +6 -0
- package/src/cli/commands/publish.ts +140 -0
- package/src/cli/options.ts +2 -0
- package/src/cli/program.ts +72 -0
- package/src/embed/batch.ts +154 -3
- package/src/publish/artifact.ts +252 -0
- package/src/publish/export-service.ts +238 -0
- package/src/serve/AGENTS.md +17 -16
- package/src/serve/CLAUDE.md +17 -16
- package/src/serve/public/lib/publish-export.ts +21 -0
- package/src/serve/public/pages/Collections.tsx +63 -0
- package/src/serve/public/pages/DocView.tsx +71 -0
- package/src/serve/routes/api.ts +82 -0
- package/src/serve/server.ts +12 -0
- package/src/store/vector/sqlite-vec.ts +11 -6
package/README.md
CHANGED
|
@@ -8,6 +8,9 @@
|
|
|
8
8
|
[](https://twitter.com/gmickel)
|
|
9
9
|
[](https://discord.gg/nHEmyJB5tg)
|
|
10
10
|
|
|
11
|
+
> [!TIP]
|
|
12
|
+
> **[gno.sh/publish](https://gno.sh/publish) is live.** Turn any GNO note or collection into a polished, reader-first URL — editorial typography, scoped search, and four visibility modes from public to end-to-end encrypted. **[See the reader →](#publish-to-gnosh)**
|
|
13
|
+
|
|
11
14
|
> **ClawdHub**: GNO skills bundled for Clawdbot — [clawdhub.com/gmickel/gno](https://clawdhub.com/gmickel/gno)
|
|
12
15
|
|
|
13
16
|

|
|
@@ -27,6 +30,7 @@ Use it when:
|
|
|
27
30
|
- **Real retrieval surfaces**: CLI, Web UI, REST API, MCP, SDK
|
|
28
31
|
- **Local-first answers**: grounded synthesis with citations when you want answers, raw retrieval when you do not
|
|
29
32
|
- **Connected knowledge**: backlinks, related notes, graph view, cross-collection navigation
|
|
33
|
+
- **Shareable, not synced**: export a note or collection to [gno.sh](https://gno.sh/publish) as a polished reader page — public, secret, invite-only, or end-to-end encrypted
|
|
30
34
|
- **Operational fit**: daemon mode, model presets, remote GPU backends, safe config/state on disk
|
|
31
35
|
|
|
32
36
|
### One-Minute Tour
|
|
@@ -74,6 +78,7 @@ gno daemon
|
|
|
74
78
|
- [Search Modes](#search-modes)
|
|
75
79
|
- [Agent Integration](#agent-integration)
|
|
76
80
|
- [Web UI](#web-ui)
|
|
81
|
+
- [Publish to gno.sh](#publish-to-gnosh)
|
|
77
82
|
- [REST API](#rest-api)
|
|
78
83
|
- [SDK](#sdk)
|
|
79
84
|
- [How It Works](#how-it-works)
|
|
@@ -90,6 +95,7 @@ gno daemon
|
|
|
90
95
|
> Latest release: [v0.40.2](./CHANGELOG.md#0402---2026-04-06)
|
|
91
96
|
> Full release history: [CHANGELOG.md](./CHANGELOG.md)
|
|
92
97
|
|
|
98
|
+
- **Publish to [gno.sh](https://gno.sh/publish)**: new `gno publish export` CLI and Web UI action produce a self-contained artifact you upload to the hosted reader — public, secret, invite-only, or end-to-end encrypted
|
|
93
99
|
- **Retrieval Quality Upgrade**: stronger BM25 lexical handling, code-aware chunking, terminal result hyperlinks, and per-collection model overrides
|
|
94
100
|
- **Code Embedding Benchmarks**: new benchmark workflow across canonical, real-GNO, and pinned OSS slices for comparing alternate embedding models
|
|
95
101
|
- **Default Embed Model**: built-in presets now use `Qwen3-Embedding-0.6B-GGUF` after it beat `bge-m3` on both code and multilingual prose benchmark lanes
|
|
@@ -122,6 +128,15 @@ gno collection clear-embeddings my-collection --all
|
|
|
122
128
|
gno embed my-collection
|
|
123
129
|
```
|
|
124
130
|
|
|
131
|
+
If a re-embed run still reports failures, rerun with:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
gno --verbose embed --force
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Recent releases now print sample embedding errors and a concrete retry hint when
|
|
138
|
+
batch recovery cannot fully recover on its own.
|
|
139
|
+
|
|
125
140
|
Model guides:
|
|
126
141
|
|
|
127
142
|
- [Code Embeddings](./docs/guides/code-embeddings.md)
|
|
@@ -401,8 +416,22 @@ gno query "refresh token rotation" --explain
|
|
|
401
416
|
# Work with filters
|
|
402
417
|
gno query "meeting notes" --since "last month" --category "meeting,notes"
|
|
403
418
|
gno search "incident review" --tags-all "status/active,team/platform"
|
|
419
|
+
|
|
420
|
+
# Export a publish artifact for gno.sh
|
|
421
|
+
gno publish export work-docs --out ~/Downloads/work-docs.json
|
|
422
|
+
gno publish export "gno://work-docs/runbooks/deploy.md" --out ~/Downloads/deploy.json
|
|
423
|
+
# Or let GNO choose ~/Downloads/<slug>-<YYYYMMDD>.json automatically
|
|
424
|
+
gno publish export work-docs
|
|
404
425
|
```
|
|
405
426
|
|
|
427
|
+
The local web UI exposes the same export flow:
|
|
428
|
+
|
|
429
|
+
- Collections page → collection menu → `Export for gno.sh`
|
|
430
|
+
- Document view → `Export for gno.sh`
|
|
431
|
+
|
|
432
|
+
Both actions download the same JSON artifact the CLI writes, ready for upload at
|
|
433
|
+
`https://gno.sh/studio`.
|
|
434
|
+
|
|
406
435
|
### Retrieval V2 Controls
|
|
407
436
|
|
|
408
437
|
Existing query calls still work. Retrieval v2 adds optional structured intent control and deeper explain output.
|
|
@@ -581,6 +610,47 @@ Everything runs locally. No cloud, no accounts, no data leaving your machine.
|
|
|
581
610
|
|
|
582
611
|
---
|
|
583
612
|
|
|
613
|
+
## Publish to gno.sh
|
|
614
|
+
|
|
615
|
+
GNO is local-first, but sometimes you want a URL to send someone. [**gno.sh**](https://gno.sh/publish) is the hosted reader on top of GNO — a polished, reading-first page for a single note or a whole collection, without mounting your vault or syncing anything.
|
|
616
|
+
|
|
617
|
+

|
|
618
|
+
|
|
619
|
+
The workflow is deliberately explicit: **export locally → upload artifact → share URL**. Your private notes and metadata stay on your machine. Only what you export leaves.
|
|
620
|
+
|
|
621
|
+
```bash
|
|
622
|
+
# Export a single note
|
|
623
|
+
gno publish export "gno://work-docs/runbooks/deploy.md" --out ~/Downloads/deploy.json
|
|
624
|
+
|
|
625
|
+
# Export a whole collection
|
|
626
|
+
gno publish export work-docs --out ~/Downloads/work-docs.json
|
|
627
|
+
|
|
628
|
+
# Let GNO pick the path (~/Downloads/<slug>-<YYYYMMDD>.json)
|
|
629
|
+
gno publish export work-docs
|
|
630
|
+
```
|
|
631
|
+
|
|
632
|
+
Or use the Web UI:
|
|
633
|
+
|
|
634
|
+
- **Collections page** → collection menu → **Export for gno.sh**
|
|
635
|
+
- **Document view** → **Export for gno.sh**
|
|
636
|
+
|
|
637
|
+
Upload the artifact at [gno.sh/studio](https://gno.sh/studio) and pick a visibility mode:
|
|
638
|
+
|
|
639
|
+
| Mode | Use When |
|
|
640
|
+
| :----------------------- | :-------------------------------------------------- |
|
|
641
|
+
| **Public** | Open URL, indexable — talks, blog posts, portfolios |
|
|
642
|
+
| **Secret link** | Unguessable token, rotate / revoke / expire |
|
|
643
|
+
| **Invite-only** | Private space for specific people |
|
|
644
|
+
| **End-to-end encrypted** | WebCrypto bundle, passphrase decrypts in-browser |
|
|
645
|
+
|
|
646
|
+
**Reader experience**: editorial serif typography, drop caps, hanging punctuation, table of contents, keyboard shortcuts (`j/k`, `/`), scoped Pagefind-style search, and backlinks restricted to the published subset. Nothing leaks that you didn't publish.
|
|
647
|
+
|
|
648
|
+
Republishing an artifact updates the same URL — no churn for your readers.
|
|
649
|
+
|
|
650
|
+
> **Full story**: [gno.sh/publish](https://gno.sh/publish) · **Try it**: [gno.sh/studio](https://gno.sh/studio)
|
|
651
|
+
|
|
652
|
+
---
|
|
653
|
+
|
|
584
654
|
## REST API
|
|
585
655
|
|
|
586
656
|
Programmatic access to all GNO features via HTTP.
|
|
Binary file
|
package/assets/skill/SKILL.md
CHANGED
|
@@ -22,6 +22,7 @@ Fast local semantic search. Index once, search instantly. No cloud, no API keys.
|
|
|
22
22
|
- User wants to **tag, categorize, or filter** documents
|
|
23
23
|
- User asks about **backlinks, wiki links, or related notes**
|
|
24
24
|
- User wants to **visualize document connections** or see a **knowledge graph**
|
|
25
|
+
- User wants to **export a note or collection for gno.sh publishing**
|
|
25
26
|
|
|
26
27
|
## Quick Start
|
|
27
28
|
|
|
@@ -44,6 +45,7 @@ gno search "your query" # BM25 keyword search
|
|
|
44
45
|
| **Context** | `context add/list/rm/check` | Add hints to improve search relevance |
|
|
45
46
|
| **Models** | `models list/use/pull/clear/path` | Manage local AI models |
|
|
46
47
|
| **Serve** | `serve` | Web UI for browsing and searching |
|
|
48
|
+
| **Publish** | `publish export` | Export gno.sh publish artifacts |
|
|
47
49
|
| **MCP** | `mcp`, `mcp install/uninstall/status` | AI assistant integration |
|
|
48
50
|
| **Skill** | `skill install/uninstall/show/paths` | Install skill for AI agents |
|
|
49
51
|
| **Admin** | `status`, `doctor`, `cleanup`, `reset`, `vec`, `completion` | Maintenance and diagnostics |
|
|
@@ -494,6 +494,34 @@ gno serve [options]
|
|
|
494
494
|
|
|
495
495
|
Features: Dashboard, search, browse collections, document viewer, AI Q&A with citations.
|
|
496
496
|
|
|
497
|
+
## Publish
|
|
498
|
+
|
|
499
|
+
### gno publish export
|
|
500
|
+
|
|
501
|
+
Export a note or collection as a gno.sh publish artifact JSON.
|
|
502
|
+
|
|
503
|
+
```bash
|
|
504
|
+
gno publish export <target> [--out <path.json>] [options]
|
|
505
|
+
```
|
|
506
|
+
|
|
507
|
+
| Option | Default | Description |
|
|
508
|
+
| -------------- | ------- | ------------------------------------------------------------- |
|
|
509
|
+
| `--out` | auto | Output path, defaults to `~/Downloads/<slug>-<YYYYMMDD>.json` |
|
|
510
|
+
| `--visibility` | public | One of `public`, `secret-link`, `invite-only`, `encrypted` |
|
|
511
|
+
| `--slug` | auto | Override the published route slug |
|
|
512
|
+
| `--title` | auto | Override the exported title |
|
|
513
|
+
| `--summary` | auto | Override the exported summary |
|
|
514
|
+
| `--json` | false | Structured result output |
|
|
515
|
+
|
|
516
|
+
Examples:
|
|
517
|
+
|
|
518
|
+
```bash
|
|
519
|
+
gno publish export work-docs --out ~/Downloads/work-docs.json
|
|
520
|
+
gno publish export "gno://work-docs/runbooks/deploy.md" --out ~/Downloads/deploy.json
|
|
521
|
+
```
|
|
522
|
+
|
|
523
|
+
On success, upload the JSON file at `https://gno.sh/studio`.
|
|
524
|
+
|
|
497
525
|
## Skill Management
|
|
498
526
|
|
|
499
527
|
### gno skill install
|
package/package.json
CHANGED
|
@@ -71,6 +71,9 @@ export type EmbedResult =
|
|
|
71
71
|
duration: number;
|
|
72
72
|
model: string;
|
|
73
73
|
searchAvailable: boolean;
|
|
74
|
+
errorSamples?: string[];
|
|
75
|
+
suggestion?: string;
|
|
76
|
+
syncError?: string;
|
|
74
77
|
}
|
|
75
78
|
| { success: false; error: string };
|
|
76
79
|
|
|
@@ -87,6 +90,30 @@ function formatDuration(seconds: number): string {
|
|
|
87
90
|
return `${mins}m ${secs.toFixed(0)}s`;
|
|
88
91
|
}
|
|
89
92
|
|
|
93
|
+
function formatLlmFailure(
|
|
94
|
+
error: { message: string; cause?: unknown } | undefined
|
|
95
|
+
): string {
|
|
96
|
+
if (!error) {
|
|
97
|
+
return "Unknown embedding failure";
|
|
98
|
+
}
|
|
99
|
+
const cause =
|
|
100
|
+
error.cause &&
|
|
101
|
+
typeof error.cause === "object" &&
|
|
102
|
+
"message" in error.cause &&
|
|
103
|
+
typeof error.cause.message === "string"
|
|
104
|
+
? error.cause.message
|
|
105
|
+
: typeof error.cause === "string"
|
|
106
|
+
? error.cause
|
|
107
|
+
: "";
|
|
108
|
+
return cause && cause !== error.message
|
|
109
|
+
? `${error.message} - ${cause}`
|
|
110
|
+
: error.message;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function isDisposedBatchError(message: string): boolean {
|
|
114
|
+
return message.toLowerCase().includes("object is disposed");
|
|
115
|
+
}
|
|
116
|
+
|
|
90
117
|
async function checkVecAvailable(
|
|
91
118
|
db: import("bun:sqlite").Database
|
|
92
119
|
): Promise<boolean> {
|
|
@@ -111,10 +138,20 @@ interface BatchContext {
|
|
|
111
138
|
showProgress: boolean;
|
|
112
139
|
totalToEmbed: number;
|
|
113
140
|
verbose: boolean;
|
|
141
|
+
recreateEmbedPort?: () => Promise<
|
|
142
|
+
{ ok: true; value: EmbeddingPort } | { ok: false; error: string }
|
|
143
|
+
>;
|
|
114
144
|
}
|
|
115
145
|
|
|
116
146
|
type BatchResult =
|
|
117
|
-
| {
|
|
147
|
+
| {
|
|
148
|
+
ok: true;
|
|
149
|
+
embedded: number;
|
|
150
|
+
errors: number;
|
|
151
|
+
duration: number;
|
|
152
|
+
errorSamples: string[];
|
|
153
|
+
suggestion?: string;
|
|
154
|
+
}
|
|
118
155
|
| { ok: false; error: string };
|
|
119
156
|
|
|
120
157
|
interface Cursor {
|
|
@@ -126,8 +163,21 @@ async function processBatches(ctx: BatchContext): Promise<BatchResult> {
|
|
|
126
163
|
const startTime = Date.now();
|
|
127
164
|
let embedded = 0;
|
|
128
165
|
let errors = 0;
|
|
166
|
+
const errorSamples: string[] = [];
|
|
167
|
+
let suggestion: string | undefined;
|
|
129
168
|
let cursor: Cursor | undefined;
|
|
130
169
|
|
|
170
|
+
const pushErrorSamples = (samples: string[]): void => {
|
|
171
|
+
for (const sample of samples) {
|
|
172
|
+
if (errorSamples.length >= 5) {
|
|
173
|
+
break;
|
|
174
|
+
}
|
|
175
|
+
if (!errorSamples.includes(sample)) {
|
|
176
|
+
errorSamples.push(sample);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
|
|
131
181
|
while (embedded + errors < ctx.totalToEmbed) {
|
|
132
182
|
// Get next batch using seek pagination (cursor-based)
|
|
133
183
|
const batchResult = ctx.force
|
|
@@ -161,6 +211,89 @@ async function processBatches(ctx: BatchContext): Promise<BatchResult> {
|
|
|
161
211
|
)
|
|
162
212
|
);
|
|
163
213
|
if (!batchEmbedResult.ok) {
|
|
214
|
+
const formattedError = formatLlmFailure(batchEmbedResult.error);
|
|
215
|
+
if (ctx.recreateEmbedPort && isDisposedBatchError(formattedError)) {
|
|
216
|
+
if (ctx.verbose) {
|
|
217
|
+
process.stderr.write(
|
|
218
|
+
"\n[embed] Embedding port disposed; recreating model/contexts and retrying batch once\n"
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
const recreated = await ctx.recreateEmbedPort();
|
|
222
|
+
if (recreated.ok) {
|
|
223
|
+
ctx.embedPort = recreated.value;
|
|
224
|
+
const retryResult = await embedTextsWithRecovery(
|
|
225
|
+
ctx.embedPort,
|
|
226
|
+
batch.map((b) =>
|
|
227
|
+
formatDocForEmbedding(b.text, b.title ?? undefined, ctx.modelUri)
|
|
228
|
+
)
|
|
229
|
+
);
|
|
230
|
+
if (retryResult.ok) {
|
|
231
|
+
if (ctx.verbose) {
|
|
232
|
+
process.stderr.write(
|
|
233
|
+
"\n[embed] Retry after port reset succeeded\n"
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
pushErrorSamples(retryResult.value.failureSamples);
|
|
237
|
+
suggestion ||= retryResult.value.retrySuggestion;
|
|
238
|
+
|
|
239
|
+
const retryVectors: VectorRow[] = [];
|
|
240
|
+
for (const [idx, item] of batch.entries()) {
|
|
241
|
+
const embedding = retryResult.value.vectors[idx];
|
|
242
|
+
if (!embedding) {
|
|
243
|
+
errors += 1;
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
retryVectors.push({
|
|
247
|
+
mirrorHash: item.mirrorHash,
|
|
248
|
+
seq: item.seq,
|
|
249
|
+
model: ctx.modelUri,
|
|
250
|
+
embedding: new Float32Array(embedding),
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
if (retryVectors.length === 0) {
|
|
255
|
+
if (ctx.verbose) {
|
|
256
|
+
process.stderr.write(
|
|
257
|
+
"\n[embed] No recoverable embeddings in retry batch\n"
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const retryStoreResult =
|
|
264
|
+
await ctx.vectorIndex.upsertVectors(retryVectors);
|
|
265
|
+
if (!retryStoreResult.ok) {
|
|
266
|
+
if (ctx.verbose) {
|
|
267
|
+
process.stderr.write(
|
|
268
|
+
`\n[embed] Store failed: ${retryStoreResult.error.message}\n`
|
|
269
|
+
);
|
|
270
|
+
}
|
|
271
|
+
pushErrorSamples([retryStoreResult.error.message]);
|
|
272
|
+
suggestion ??=
|
|
273
|
+
"Store write failed. Rerun `gno embed` once more; if it repeats, run `gno doctor` and `gno vec sync`.";
|
|
274
|
+
errors += retryVectors.length;
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
embedded += retryVectors.length;
|
|
279
|
+
if (ctx.showProgress) {
|
|
280
|
+
const embeddedDisplay = Math.min(embedded, ctx.totalToEmbed);
|
|
281
|
+
const completed = Math.min(embedded + errors, ctx.totalToEmbed);
|
|
282
|
+
const pct = (completed / ctx.totalToEmbed) * 100;
|
|
283
|
+
const elapsed = (Date.now() - startTime) / 1000;
|
|
284
|
+
const rate = embedded / Math.max(elapsed, 0.001);
|
|
285
|
+
const eta =
|
|
286
|
+
Math.max(0, ctx.totalToEmbed - completed) /
|
|
287
|
+
Math.max(rate, 0.001);
|
|
288
|
+
process.stdout.write(
|
|
289
|
+
`\rEmbedding: ${embeddedDisplay.toLocaleString()}/${ctx.totalToEmbed.toLocaleString()} (${pct.toFixed(1)}%) | ${rate.toFixed(1)} chunks/s | ETA ${formatDuration(eta)}`
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
164
297
|
if (ctx.verbose) {
|
|
165
298
|
const err = batchEmbedResult.error;
|
|
166
299
|
const cause = err.cause;
|
|
@@ -178,6 +311,9 @@ async function processBatches(ctx: BatchContext): Promise<BatchResult> {
|
|
|
178
311
|
`\n[embed] Batch failed (${batch.length} chunks: ${titles}${batch.length > 3 ? "..." : ""}): ${err.message}${causeMsg ? ` - ${causeMsg}` : ""}\n`
|
|
179
312
|
);
|
|
180
313
|
}
|
|
314
|
+
pushErrorSamples([formattedError]);
|
|
315
|
+
suggestion =
|
|
316
|
+
"Try rerunning the same command. If failures persist, rerun with `gno --verbose embed --batch-size 1` to isolate failing chunks.";
|
|
181
317
|
errors += batch.length;
|
|
182
318
|
continue;
|
|
183
319
|
}
|
|
@@ -191,6 +327,13 @@ async function processBatches(ctx: BatchContext): Promise<BatchResult> {
|
|
|
191
327
|
`\n[embed] Batch fallback (${batch.length} chunks: ${titles}${batch.length > 3 ? "..." : ""}): ${batchEmbedResult.value.batchError ?? "unknown batch error"}\n`
|
|
192
328
|
);
|
|
193
329
|
}
|
|
330
|
+
pushErrorSamples(batchEmbedResult.value.failureSamples);
|
|
331
|
+
suggestion ||= batchEmbedResult.value.retrySuggestion;
|
|
332
|
+
if (ctx.verbose && batchEmbedResult.value.failureSamples.length > 0) {
|
|
333
|
+
for (const sample of batchEmbedResult.value.failureSamples) {
|
|
334
|
+
process.stderr.write(`\n[embed] Sample failure: ${sample}\n`);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
194
337
|
|
|
195
338
|
const vectors: VectorRow[] = [];
|
|
196
339
|
for (const [idx, item] of batch.entries()) {
|
|
@@ -221,6 +364,9 @@ async function processBatches(ctx: BatchContext): Promise<BatchResult> {
|
|
|
221
364
|
`\n[embed] Store failed: ${storeResult.error.message}\n`
|
|
222
365
|
);
|
|
223
366
|
}
|
|
367
|
+
pushErrorSamples([storeResult.error.message]);
|
|
368
|
+
suggestion ??=
|
|
369
|
+
"Store write failed. Rerun `gno embed` once more; if it repeats, run `gno doctor` and `gno vec sync`.";
|
|
224
370
|
errors += vectors.length;
|
|
225
371
|
continue;
|
|
226
372
|
}
|
|
@@ -229,13 +375,15 @@ async function processBatches(ctx: BatchContext): Promise<BatchResult> {
|
|
|
229
375
|
|
|
230
376
|
// Progress output
|
|
231
377
|
if (ctx.showProgress) {
|
|
232
|
-
const
|
|
378
|
+
const embeddedDisplay = Math.min(embedded, ctx.totalToEmbed);
|
|
379
|
+
const completed = Math.min(embedded + errors, ctx.totalToEmbed);
|
|
380
|
+
const pct = (completed / ctx.totalToEmbed) * 100;
|
|
233
381
|
const elapsed = (Date.now() - startTime) / 1000;
|
|
234
382
|
const rate = embedded / Math.max(elapsed, 0.001);
|
|
235
383
|
const eta =
|
|
236
|
-
(ctx.totalToEmbed -
|
|
384
|
+
Math.max(0, ctx.totalToEmbed - completed) / Math.max(rate, 0.001);
|
|
237
385
|
process.stdout.write(
|
|
238
|
-
`\rEmbedding: ${
|
|
386
|
+
`\rEmbedding: ${embeddedDisplay.toLocaleString()}/${ctx.totalToEmbed.toLocaleString()} (${pct.toFixed(1)}%) | ${rate.toFixed(1)} chunks/s | ETA ${formatDuration(eta)}`
|
|
239
387
|
);
|
|
240
388
|
}
|
|
241
389
|
}
|
|
@@ -249,6 +397,8 @@ async function processBatches(ctx: BatchContext): Promise<BatchResult> {
|
|
|
249
397
|
embedded,
|
|
250
398
|
errors,
|
|
251
399
|
duration: (Date.now() - startTime) / 1000,
|
|
400
|
+
errorSamples,
|
|
401
|
+
suggestion,
|
|
252
402
|
};
|
|
253
403
|
}
|
|
254
404
|
|
|
@@ -354,6 +504,7 @@ export async function embed(options: EmbedOptions = {}): Promise<EmbedResult> {
|
|
|
354
504
|
duration: 0,
|
|
355
505
|
model: modelUri,
|
|
356
506
|
searchAvailable: vecAvailable,
|
|
507
|
+
errorSamples: [],
|
|
357
508
|
};
|
|
358
509
|
}
|
|
359
510
|
|
|
@@ -366,6 +517,7 @@ export async function embed(options: EmbedOptions = {}): Promise<EmbedResult> {
|
|
|
366
517
|
duration: 0,
|
|
367
518
|
model: modelUri,
|
|
368
519
|
searchAvailable: vecAvailable,
|
|
520
|
+
errorSamples: [],
|
|
369
521
|
};
|
|
370
522
|
}
|
|
371
523
|
|
|
@@ -382,6 +534,27 @@ export async function embed(options: EmbedOptions = {}): Promise<EmbedResult> {
|
|
|
382
534
|
: undefined;
|
|
383
535
|
|
|
384
536
|
const llm = new LlmAdapter(config);
|
|
537
|
+
const recreateEmbedPort = async () => {
|
|
538
|
+
if (embedPort) {
|
|
539
|
+
await embedPort.dispose();
|
|
540
|
+
}
|
|
541
|
+
await llm.getManager().dispose(modelUri);
|
|
542
|
+
const recreated = await llm.createEmbeddingPort(modelUri, {
|
|
543
|
+
policy,
|
|
544
|
+
onProgress: downloadProgress
|
|
545
|
+
? (progress) => downloadProgress("embed", progress)
|
|
546
|
+
: undefined,
|
|
547
|
+
});
|
|
548
|
+
if (!recreated.ok) {
|
|
549
|
+
return { ok: false as const, error: recreated.error.message };
|
|
550
|
+
}
|
|
551
|
+
const initResult = await recreated.value.init();
|
|
552
|
+
if (!initResult.ok) {
|
|
553
|
+
await recreated.value.dispose();
|
|
554
|
+
return { ok: false as const, error: initResult.error.message };
|
|
555
|
+
}
|
|
556
|
+
return { ok: true as const, value: recreated.value };
|
|
557
|
+
};
|
|
385
558
|
const embedResult = await llm.createEmbeddingPort(modelUri, {
|
|
386
559
|
policy,
|
|
387
560
|
onProgress: downloadProgress
|
|
@@ -428,6 +601,7 @@ export async function embed(options: EmbedOptions = {}): Promise<EmbedResult> {
|
|
|
428
601
|
showProgress: !options.json,
|
|
429
602
|
totalToEmbed,
|
|
430
603
|
verbose: options.verbose ?? false,
|
|
604
|
+
recreateEmbedPort,
|
|
431
605
|
});
|
|
432
606
|
|
|
433
607
|
if (!result.ok) {
|
|
@@ -447,10 +621,27 @@ export async function embed(options: EmbedOptions = {}): Promise<EmbedResult> {
|
|
|
447
621
|
}
|
|
448
622
|
}
|
|
449
623
|
vectorIndex.vecDirty = false;
|
|
450
|
-
} else
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
624
|
+
} else {
|
|
625
|
+
if (!options.json) {
|
|
626
|
+
process.stdout.write(
|
|
627
|
+
`\n[vec] Sync failed: ${syncResult.error.message}\n`
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
return {
|
|
631
|
+
success: true,
|
|
632
|
+
embedded: result.embedded,
|
|
633
|
+
errors: result.errors,
|
|
634
|
+
duration: result.duration,
|
|
635
|
+
model: modelUri,
|
|
636
|
+
searchAvailable: vectorIndex.searchAvailable,
|
|
637
|
+
errorSamples: [
|
|
638
|
+
...result.errorSamples,
|
|
639
|
+
syncResult.error.message,
|
|
640
|
+
].slice(0, 5),
|
|
641
|
+
suggestion:
|
|
642
|
+
"Vector index sync failed after embedding. Rerun `gno embed` once more. If it repeats, run `gno vec sync`.",
|
|
643
|
+
syncError: syncResult.error.message,
|
|
644
|
+
};
|
|
454
645
|
}
|
|
455
646
|
}
|
|
456
647
|
|
|
@@ -461,6 +652,8 @@ export async function embed(options: EmbedOptions = {}): Promise<EmbedResult> {
|
|
|
461
652
|
duration: result.duration,
|
|
462
653
|
model: modelUri,
|
|
463
654
|
searchAvailable: vectorIndex.searchAvailable,
|
|
655
|
+
errorSamples: result.errorSamples,
|
|
656
|
+
suggestion: result.suggestion,
|
|
464
657
|
};
|
|
465
658
|
} finally {
|
|
466
659
|
if (embedPort) {
|
|
@@ -585,6 +778,9 @@ export function formatEmbed(
|
|
|
585
778
|
duration: result.duration,
|
|
586
779
|
model: result.model,
|
|
587
780
|
searchAvailable: result.searchAvailable,
|
|
781
|
+
errorSamples: result.errorSamples ?? [],
|
|
782
|
+
suggestion: result.suggestion,
|
|
783
|
+
syncError: result.syncError,
|
|
588
784
|
},
|
|
589
785
|
null,
|
|
590
786
|
2
|
|
@@ -606,6 +802,14 @@ export function formatEmbed(
|
|
|
606
802
|
|
|
607
803
|
if (result.errors > 0) {
|
|
608
804
|
lines.push(`${result.errors} chunks failed to embed.`);
|
|
805
|
+
if ((result.errorSamples?.length ?? 0) > 0) {
|
|
806
|
+
for (const sample of result.errorSamples ?? []) {
|
|
807
|
+
lines.push(`Sample error: ${sample}`);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
if (result.suggestion) {
|
|
811
|
+
lines.push(`Hint: ${result.suggestion}`);
|
|
812
|
+
}
|
|
609
813
|
}
|
|
610
814
|
|
|
611
815
|
if (!result.searchAvailable) {
|
|
@@ -614,5 +818,9 @@ export function formatEmbed(
|
|
|
614
818
|
);
|
|
615
819
|
}
|
|
616
820
|
|
|
821
|
+
if (result.syncError) {
|
|
822
|
+
lines.push(`Vec sync error: ${result.syncError}`);
|
|
823
|
+
}
|
|
824
|
+
|
|
617
825
|
return lines.join("\n");
|
|
618
826
|
}
|
|
@@ -88,6 +88,12 @@ export {
|
|
|
88
88
|
type StatusResult,
|
|
89
89
|
status,
|
|
90
90
|
} from "./status";
|
|
91
|
+
export {
|
|
92
|
+
formatPublishExport,
|
|
93
|
+
publishExport,
|
|
94
|
+
type PublishExportOptions,
|
|
95
|
+
type PublishExportResult,
|
|
96
|
+
} from "./publish";
|
|
91
97
|
export {
|
|
92
98
|
formatUpdate,
|
|
93
99
|
type UpdateOptions,
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* gno publish export command.
|
|
3
|
+
*
|
|
4
|
+
* @module src/cli/commands/publish
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
8
|
+
import { homedir } from "node:os";
|
|
9
|
+
import { dirname } from "node:path";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
|
|
12
|
+
import type {
|
|
13
|
+
PublishArtifact,
|
|
14
|
+
PublishVisibility,
|
|
15
|
+
} from "../../publish/artifact";
|
|
16
|
+
|
|
17
|
+
import { derivePublishArtifactFilename, slugify } from "../../publish/artifact";
|
|
18
|
+
import { exportPublishArtifact } from "../../publish/export-service";
|
|
19
|
+
import { initStore } from "./shared";
|
|
20
|
+
|
|
21
|
+
export interface PublishExportOptions {
|
|
22
|
+
configPath?: string;
|
|
23
|
+
json?: boolean;
|
|
24
|
+
out?: string;
|
|
25
|
+
slug?: string;
|
|
26
|
+
summary?: string;
|
|
27
|
+
title?: string;
|
|
28
|
+
visibility?: PublishVisibility;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export type PublishExportResult =
|
|
32
|
+
| {
|
|
33
|
+
success: true;
|
|
34
|
+
data: {
|
|
35
|
+
artifact: PublishArtifact;
|
|
36
|
+
outPath: string;
|
|
37
|
+
uploadUrl: string;
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
| { success: false; error: string; isValidation?: boolean };
|
|
41
|
+
|
|
42
|
+
function formatExportDateStamp(isoTimestamp: string): string {
|
|
43
|
+
return isoTimestamp.slice(0, 10).replaceAll("-", "");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function buildDefaultPublishExportPath(
|
|
47
|
+
artifact: PublishArtifact
|
|
48
|
+
): string {
|
|
49
|
+
const fileName = derivePublishArtifactFilename(artifact).replace(
|
|
50
|
+
/\.json$/u,
|
|
51
|
+
""
|
|
52
|
+
);
|
|
53
|
+
return join(
|
|
54
|
+
homedir(),
|
|
55
|
+
"Downloads",
|
|
56
|
+
`${fileName}-${formatExportDateStamp(artifact.exportedAt)}.json`
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export async function publishExport(
|
|
61
|
+
target: string,
|
|
62
|
+
options: PublishExportOptions
|
|
63
|
+
): Promise<PublishExportResult> {
|
|
64
|
+
const initResult = await initStore({
|
|
65
|
+
configPath: options.configPath,
|
|
66
|
+
syncConfig: false,
|
|
67
|
+
});
|
|
68
|
+
if (!initResult.ok) {
|
|
69
|
+
return { success: false, error: initResult.error };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const { collections, store } = initResult;
|
|
73
|
+
|
|
74
|
+
try {
|
|
75
|
+
const artifact = await exportPublishArtifact({
|
|
76
|
+
collections,
|
|
77
|
+
options: {
|
|
78
|
+
routeSlug: options.slug,
|
|
79
|
+
summary: options.summary,
|
|
80
|
+
title: options.title,
|
|
81
|
+
visibility: options.visibility,
|
|
82
|
+
},
|
|
83
|
+
store,
|
|
84
|
+
target,
|
|
85
|
+
});
|
|
86
|
+
const outPath =
|
|
87
|
+
options.out?.trim() || buildDefaultPublishExportPath(artifact);
|
|
88
|
+
|
|
89
|
+
await mkdir(dirname(outPath), { recursive: true });
|
|
90
|
+
await writeFile(outPath, JSON.stringify(artifact, null, 2));
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
success: true,
|
|
94
|
+
data: {
|
|
95
|
+
artifact,
|
|
96
|
+
outPath,
|
|
97
|
+
uploadUrl: "https://gno.sh/studio",
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
} catch (error) {
|
|
101
|
+
return {
|
|
102
|
+
success: false,
|
|
103
|
+
error: error instanceof Error ? error.message : String(error),
|
|
104
|
+
};
|
|
105
|
+
} finally {
|
|
106
|
+
await store.close();
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function formatPublishExport(
|
|
111
|
+
result: PublishExportResult,
|
|
112
|
+
options: Pick<PublishExportOptions, "json">
|
|
113
|
+
): string {
|
|
114
|
+
if (!result.success) {
|
|
115
|
+
if (options.json) {
|
|
116
|
+
return JSON.stringify({
|
|
117
|
+
error: {
|
|
118
|
+
code: result.isValidation ? "VALIDATION" : "RUNTIME",
|
|
119
|
+
message: result.error,
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
return `Error: ${result.error}`;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (options.json) {
|
|
127
|
+
return JSON.stringify(result.data, null, 2);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const { artifact, outPath, uploadUrl } = result.data;
|
|
131
|
+
const space = artifact.spaces[0];
|
|
132
|
+
|
|
133
|
+
return [
|
|
134
|
+
`Exported ${space?.sourceType ?? "artifact"} to ${outPath}`,
|
|
135
|
+
`Route slug: ${space?.routeSlug ?? slugify(artifact.source)}`,
|
|
136
|
+
`Visibility: ${space?.visibility ?? "public"}`,
|
|
137
|
+
`Filename: ${derivePublishArtifactFilename(artifact)}`,
|
|
138
|
+
`Next: open ${uploadUrl} and drop ${outPath} into the upload zone.`,
|
|
139
|
+
].join("\n");
|
|
140
|
+
}
|