@gmickel/gno 0.41.0 → 0.42.0

package/src/cli/options.ts

@@ -27,6 +27,7 @@ export const CMD = {
   multiGet: "multi-get",
   ls: "ls",
   status: "status",
+  publishExport: "publish.export",
   collectionList: "collection.list",
   contextList: "context.list",
   contextCheck: "context.check",
@@ -49,6 +50,7 @@ const FORMAT_SUPPORT: Record<CommandId, OutputFormat[]> = {
   [CMD.multiGet]: ["terminal", "json", "files", "md"],
   [CMD.ls]: ["terminal", "json", "files", "md"],
   [CMD.status]: ["terminal", "json"],
+  [CMD.publishExport]: ["terminal", "json"],
   [CMD.collectionList]: ["terminal", "json", "md"],
   [CMD.contextList]: ["terminal", "json", "md"],
   [CMD.contextCheck]: ["terminal", "json", "md"],

package/src/cli/program.ts

@@ -216,6 +216,7 @@ export function createProgram(): Command {
   wireSearchCommands(program);
   wireOnboardingCommands(program);
   wireManagementCommands(program);
+  wirePublishCommand(program);
   wireVecCommands(program);
   wireRetrievalCommands(program);
   wireTagsCommands(program);
@@ -1623,6 +1624,77 @@ function wireVecCommands(program: Command): void {
     });
 }
 
+function wirePublishCommand(program: Command): void {
+  const visibilityValues = [
+    "public",
+    "secret-link",
+    "invite-only",
+    "encrypted",
+  ] as const;
+  const publishCmd = program
+    .command("publish")
+    .description("Export publish artifacts for gno.sh");
+
+  publishCmd
+    .command("export <target>")
+    .description("Export a collection or document ref as a gno.sh artifact")
+    .option("--out <path>", "output artifact path")
+    .option(
+      "--visibility <mode>",
+      "visibility hint (public, secret-link, invite-only, encrypted)",
+      "public"
+    )
+    .option("--slug <slug>", "route slug override")
+    .option("--title <title>", "space title override")
+    .option("--summary <summary>", "space summary override")
+    .option("--json", "JSON output")
+    .action(async (target: string, cmdOpts: Record<string, unknown>) => {
+      const format = getFormat(cmdOpts);
+      assertFormatSupported(CMD.publishExport, format);
+      const globals = getGlobals();
+
+      const visibility = cmdOpts.visibility as string;
+      if (
+        !visibilityValues.includes(
+          visibility as (typeof visibilityValues)[number]
+        )
+      ) {
+        throw new CliError(
+          "VALIDATION",
+          `Invalid visibility: ${visibility}. Must be public, secret-link, invite-only, or encrypted.`
+        );
+      }
+
+      const { formatPublishExport, publishExport } =
+        await import("./commands/publish");
+      const result = await publishExport(target, {
+        configPath: globals.config,
+        json: format === "json",
+        out: typeof cmdOpts.out === "string" ? cmdOpts.out : undefined,
+        slug: cmdOpts.slug as string | undefined,
+        summary: cmdOpts.summary as string | undefined,
+        title: cmdOpts.title as string | undefined,
+        visibility: visibility as
+          | "encrypted"
+          | "invite-only"
+          | "public"
+          | "secret-link",
+      });
+
+      if (!result.success) {
+        throw new CliError(
+          result.isValidation ? "VALIDATION" : "RUNTIME",
+          result.error
+        );
+      }
+
+      await writeOutput(
+        formatPublishExport(result, { json: format === "json" }),
+        format
+      );
+    });
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Skill Commands (install, uninstall, show, paths)
 // ─────────────────────────────────────────────────────────────────────────────

package/src/embed/batch.ts

@@ -14,8 +14,12 @@ export interface EmbedBatchRecoveryResult {
   batchFailed: boolean;
   batchError?: string;
   fallbackErrors: number;
+  failureSamples: string[];
+  retrySuggestion?: string;
 }
 
+const MAX_FAILURE_SAMPLES = 5;
+
 function errorMessage(error: unknown): string {
   if (
     error &&
@@ -28,6 +32,27 @@ function errorMessage(error: unknown): string {
   return String(error);
 }
 
+function formatFailureMessage(error: {
+  message: string;
+  cause?: unknown;
+}): string {
+  const cause = error.cause ? errorMessage(error.cause) : "";
+  return cause && cause !== error.message
+    ? `${error.message} - ${cause}`
+    : error.message;
+}
+
+function isDisposedFailure(message: string): boolean {
+  return message.toLowerCase().includes("object is disposed");
+}
+
+async function resetEmbeddingPort(
+  embedPort: EmbeddingPort
+): Promise<LlmResult<void>> {
+  await embedPort.dispose();
+  return embedPort.init();
+}
+
 export async function embedTextsWithRecovery(
   embedPort: EmbeddingPort,
   texts: string[]
@@ -39,13 +64,24 @@ export async function embedTextsWithRecovery(
         vectors: [],
         batchFailed: false,
         fallbackErrors: 0,
+        failureSamples: [],
       },
     };
   }
 
   const profile = getEmbeddingCompatibilityProfile(embedPort.modelUri);
   if (profile.batchEmbeddingTrusted) {
-    const batchResult = await embedPort.embedBatch(texts);
+    let batchResult = await embedPort.embedBatch(texts);
+    if (!batchResult.ok) {
+      const formattedBatchError = formatFailureMessage(batchResult.error);
+      if (isDisposedFailure(formattedBatchError)) {
+        const reset = await resetEmbeddingPort(embedPort);
+        if (!reset.ok) {
+          return reset;
+        }
+        batchResult = await embedPort.embedBatch(texts);
+      }
+    }
     if (batchResult.ok && batchResult.value.length === texts.length) {
       return {
         ok: true,
@@ -53,11 +89,14 @@ export async function embedTextsWithRecovery(
           vectors: batchResult.value,
           batchFailed: false,
           fallbackErrors: 0,
+          failureSamples: [],
         },
       };
     }
 
-    const recovered = await recoverIndividually(embedPort, texts);
+    const recovered = await recoverWithAdaptiveBatches(embedPort, texts, {
+      rootBatchAlreadyFailed: true,
+    });
     if (!recovered.ok) {
       return recovered;
     }
@@ -68,7 +107,11 @@ export async function embedTextsWithRecovery(
         batchFailed: true,
         batchError: batchResult.ok
           ? `Embedding count mismatch: got ${batchResult.value.length}, expected ${texts.length}`
-          : batchResult.error.message,
+          : formatFailureMessage(batchResult.error),
+        retrySuggestion:
+          recovered.value.fallbackErrors > 0
+            ? "Try rerunning the same command. If failures persist, rerun with `gno --verbose embed --batch-size 1` to isolate failing chunks."
+            : undefined,
       },
     };
   }
@@ -83,10 +126,113 @@ export async function embedTextsWithRecovery(
       ...recovered.value,
       batchFailed: true,
       batchError: "Batch embedding disabled for this compatibility profile",
+      retrySuggestion:
+        recovered.value.fallbackErrors > 0
+          ? "Some chunks still failed individually. Rerun with `gno --verbose embed --batch-size 1` for exact chunk errors."
+          : undefined,
     },
   };
 }
 
+async function recoverWithAdaptiveBatches(
+  embedPort: EmbeddingPort,
+  texts: string[],
+  options: { rootBatchAlreadyFailed?: boolean } = {}
+): Promise<
+  LlmResult<Omit<EmbedBatchRecoveryResult, "batchFailed" | "batchError">>
+> {
+  try {
+    const vectors: Array<number[] | null> = Array.from(
+      { length: texts.length },
+      () => null
+    );
+    const failureSamples: string[] = [];
+    let fallbackErrors = 0;
+
+    const recordFailure = (message: string): void => {
+      if (failureSamples.length < MAX_FAILURE_SAMPLES) {
+        failureSamples.push(message);
+      }
+    };
+
+    const processRange = async (
+      rangeTexts: string[],
+      offset: number,
+      batchAlreadyFailed = false
+    ): Promise<void> => {
+      if (rangeTexts.length === 0) {
+        return;
+      }
+
+      if (rangeTexts.length === 1) {
+        const result = await embedPort.embed(rangeTexts[0] ?? "");
+        if (result.ok) {
+          vectors[offset] = result.value;
+          return;
+        }
+        fallbackErrors += 1;
+        recordFailure(formatFailureMessage(result.error));
+        return;
+      }
+
+      let batchResult: Awaited<ReturnType<typeof embedPort.embedBatch>> | null =
+        null;
+      if (!batchAlreadyFailed) {
+        batchResult = await embedPort.embedBatch(rangeTexts);
+      }
+      if (
+        batchResult &&
+        batchResult.ok &&
+        batchResult.value.length === rangeTexts.length
+      ) {
+        for (const [index, vector] of batchResult.value.entries()) {
+          vectors[offset + index] = vector;
+        }
+        return;
+      }
+
+      const mid = Math.ceil(rangeTexts.length / 2);
+      await processRange(rangeTexts.slice(0, mid), offset);
+      await processRange(rangeTexts.slice(mid), offset + mid);
+    };
+
+    await processRange(texts, 0, options.rootBatchAlreadyFailed ?? false);
+
+    if (fallbackErrors === texts.length) {
+      const reinit = await resetEmbeddingPort(embedPort);
+      if (!reinit.ok) {
+        return reinit;
+      }
+
+      const retry = await recoverIndividually(embedPort, texts);
+      if (!retry.ok) {
+        return retry;
+      }
+      return {
+        ok: true,
+        value: retry.value,
+      };
+    }
+
+    return {
+      ok: true,
+      value: {
+        vectors,
+        fallbackErrors,
+        failureSamples,
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      error: inferenceFailedError(
+        embedPort.modelUri,
+        new Error(errorMessage(error))
+      ),
+    };
+  }
+}
+
 async function recoverIndividually(
   embedPort: EmbeddingPort,
   texts: string[]
@@ -95,6 +241,7 @@ async function recoverIndividually(
 > {
   try {
     const vectors: Array<number[] | null> = [];
+    const failureSamples: string[] = [];
     let fallbackErrors = 0;
 
     for (const text of texts) {
@@ -104,6 +251,9 @@ async function recoverIndividually(
       } else {
         vectors.push(null);
         fallbackErrors += 1;
+        if (failureSamples.length < MAX_FAILURE_SAMPLES) {
+          failureSamples.push(formatFailureMessage(result.error));
+        }
       }
     }
 
@@ -112,6 +262,7 @@ async function recoverIndividually(
       value: {
         vectors,
         fallbackErrors,
+        failureSamples,
       },
     };
   } catch (error) {

package/src/publish/artifact.ts

@@ -0,0 +1,252 @@
+/**
+ * Publish artifact types and builders for gno.sh export.
+ *
+ * @module src/publish/artifact
+ */
+
+import type { DocumentRow, TagRow } from "../store/types";
+
+import { stripFrontmatter } from "../ingestion/frontmatter";
+
+export type PublishVisibility =
+  | "encrypted"
+  | "invite-only"
+  | "public"
+  | "secret-link";
+
+export interface PublishArtifactNote {
+  markdown: string;
+  metadata?: Record<string, string | string[]>;
+  slug: string;
+  summary: string;
+  title: string;
+}
+
+export interface PublishArtifactSpace {
+  homeNoteSlug?: string;
+  notes: PublishArtifactNote[];
+  routeSlug: string;
+  sourceType: "note" | "collection";
+  summary: string;
+  title: string;
+  visibility: PublishVisibility;
+}
+
+export interface PublishArtifact {
+  exportedAt: string;
+  source: string;
+  spaces: PublishArtifactSpace[];
+  version: 1;
+}
+
+export const PUBLISH_VISIBILITY_VALUES = [
+  "public",
+  "secret-link",
+  "invite-only",
+  "encrypted",
+] as const;
+
+export const MAX_PUBLISH_SLUG_LENGTH = 80;
+
+const ALLOWED_FRONTMATTER_METADATA_KEYS = new Set([
+  "audience",
+  "canonical",
+  "canonicalUrl",
+  "canonicalURL",
+  "coverAlt",
+  "coverImage",
+  "icon",
+  "image",
+  "layout",
+  "publishedAt",
+  "readingTime",
+  "series",
+  "seriesOrder",
+  "status",
+  "subtitle",
+  "theme",
+  "topic",
+  "topics",
+]);
+
+export const isPublishVisibility = (
+  value: unknown
+): value is PublishVisibility =>
+  typeof value === "string" &&
+  PUBLISH_VISIBILITY_VALUES.includes(value as PublishVisibility);
+
+export const slugify = (value: string) =>
+  value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/(^-|-$)/g, "");
+
+function toPublishSlugCandidate(value: string): string {
+  return slugify(value).slice(0, MAX_PUBLISH_SLUG_LENGTH).replace(/-+$/g, "");
+}
+
+export function derivePublishSlug(
+  candidates: Array<string>,
+  fallback = "untitled"
+): string {
+  for (const candidate of candidates) {
+    const slug = toPublishSlugCandidate(candidate);
+    if (slug.length > 0) {
+      return slug;
+    }
+  }
+
+  return fallback;
+}
+
+export const normalizePublishSlug = (value: string, fallback?: string) =>
+  derivePublishSlug([value], fallback);
+
+const basenameWithoutExt = (value: string) =>
+  value
+    .split("/")
+    .pop()
+    ?.replace(/\.[^.]+$/, "") ?? value;
+
+export const deriveExportedTitle = (
+  doc: Pick<DocumentRow, "relPath" | "title">
+) => doc.title?.trim() || basenameWithoutExt(doc.relPath);
+
+export const deriveExportedSlug = (
+  doc: Pick<DocumentRow, "relPath" | "title">
+) =>
+  derivePublishSlug([
+    deriveExportedTitle(doc),
+    doc.relPath.replace(/\.[^.]+$/, "").replaceAll("/", "-"),
+  ]);
+
+export const deriveExportedSummary = (
+  markdown: string,
+  metadata: Record<string, unknown>
+) => {
+  const metadataSummary =
+    typeof metadata.description === "string"
+      ? metadata.description
+      : typeof metadata.summary === "string"
+        ? metadata.summary
+        : null;
+  if (metadataSummary?.trim()) {
+    return metadataSummary.trim();
+  }
+
+  const plain = stripFrontmatter(markdown)
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(
+      (line) =>
+        line.length > 0 &&
+        !line.startsWith("#") &&
+        !line.startsWith("!") &&
+        !line.startsWith("```")
+    )
+    .join(" ")
+    .replace(/\s+/g, " ")
+    .trim();
+
+  return plain.slice(0, 200).trim();
+};
+
+export const buildExportedMetadata = (
+  doc: Pick<
+    DocumentRow,
+    | "author"
+    | "categories"
+    | "collection"
+    | "contentType"
+    | "frontmatterDate"
+    | "languageHint"
+    | "relPath"
+  >,
+  parsedFrontmatter: Record<string, unknown>,
+  tags: TagRow[]
+) => {
+  const metadata: Record<string, string | string[]> = {};
+
+  if (doc.author) {
+    metadata.author = doc.author;
+  }
+  if (doc.contentType) {
+    metadata.contentType = doc.contentType;
+  }
+  if (doc.languageHint) {
+    metadata.language = doc.languageHint;
+  }
+  if (doc.frontmatterDate) {
+    metadata.date = doc.frontmatterDate;
+  }
+  if (doc.categories?.length) {
+    metadata.categories = doc.categories;
+  }
+
+  const tagValues = tags.map((tag) => tag.tag);
+  if (tagValues.length) {
+    metadata.tags = tagValues;
+  }
+
+  metadata.collection = doc.collection;
+  metadata.sourceRelPath = doc.relPath;
+
+  for (const [key, value] of Object.entries(parsedFrontmatter)) {
+    if (
+      key === "tags" ||
+      key === "title" ||
+      key === "summary" ||
+      !ALLOWED_FRONTMATTER_METADATA_KEYS.has(key)
+    ) {
+      continue;
+    }
+    if (typeof value === "string" && value.trim()) {
+      metadata[key] = value.trim();
+      continue;
+    }
+    if (Array.isArray(value)) {
+      const cleaned = value
+        .filter((entry): entry is string => typeof entry === "string")
+        .map((entry) => entry.trim())
+        .filter(Boolean);
+      if (cleaned.length > 0) {
+        metadata[key] = cleaned;
+      }
+    }
+  }
+
+  return metadata;
+};
+
+export const buildPublishArtifact = (input: {
+  homeNoteSlug?: string;
+  notes: PublishArtifactNote[];
+  routeSlug: string;
+  source: string;
+  sourceType: "note" | "collection";
+  summary: string;
+  title: string;
+  visibility: PublishVisibility;
+}) => ({
+  exportedAt: new Date().toISOString(),
+  source: input.source,
+  spaces: [
+    {
+      homeNoteSlug: input.homeNoteSlug,
+      notes: input.notes,
+      routeSlug: input.routeSlug,
+      sourceType: input.sourceType,
+      summary: input.summary,
+      title: input.title,
+      visibility: input.visibility,
+    },
+  ],
+  version: 1 as const,
+});
+
+export const derivePublishArtifactFilename = (artifact: PublishArtifact) => {
+  const routeSlug =
+    artifact.spaces[0]?.routeSlug.trim() ||
+    normalizePublishSlug(artifact.source, "publish-artifact");
+  return `${routeSlug || "publish-artifact"}.json`;
+};