@gmickel/gno 2.8.2 → 2.8.4

package/src/cli/commands/status.ts

@@ -19,6 +19,7 @@ import {
   loadConfig,
 } from "../../config";
 import { isConnectorActivationComplete } from "../../core/activation-connector-health";
+import { isEmptyActivationCollection } from "../../core/activation-empty";
 import { buildActivationStatus } from "../../core/activation-status";
 import { formatChunkingStatus } from "../../core/chunking-status";
 import { formatLinkWorkspace } from "../../core/link-workspace";
@@ -27,6 +28,7 @@ import {
   formatMemoryStatusLines,
 } from "../../core/memory-diagnostics";
 import { formatVectorPartitionLines } from "../../core/vector-partition-status";
+import { BUDGET_ERROR_CODES } from "../../ingestion/file-processor";
 import { ModelCache } from "../../llm/cache";
 import { getActivePreset, resolveModelUri } from "../../llm/registry";
 import { getConnectorVerificationTargets } from "../../serve/connectors";
@@ -64,9 +66,56 @@ export type StatusResult =
       contentTypeBoost: ContentTypeBoostStatus;
       memory: MemoryStatus;
       backgroundIssues: ResidentBackgroundIssue[];
+      /** Files whose latest conversion stopped at the per-file budget. */
+      budgetStops: BudgetStop[];
     }
   | { success: false; error: string };
 
+/** A file left unindexed because its conversion overran its budget. */
+export interface BudgetStop {
+  collection: string;
+  relPath: string;
+  code: string;
+  message: string;
+}
+
+/** Most recent ingest errors inspected for budget stops. */
+const BUDGET_STOP_SCAN_LIMIT = 100;
+/** Most budget stops listed in status output. */
+const MAX_BUDGET_STOPS = 10;
+
+/**
+ * Files whose latest indexing attempt stopped at the conversion budget, from
+ * the recent error log, kept only while the document still carries that code.
+ */
+async function collectBudgetStops(store: SqliteAdapter): Promise<BudgetStop[]> {
+  const recent = await store.getRecentErrors(BUDGET_STOP_SCAN_LIMIT);
+  if (!recent.ok) return [];
+  const stops: BudgetStop[] = [];
+  const seen = new Set<string>();
+  for (const row of recent.value) {
+    const key = `${row.collection}\u0000${row.relPath}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    if (!BUDGET_ERROR_CODES.has(row.code)) continue;
+    const document = await store.getDocument(row.collection, row.relPath);
+    if (
+      document.ok &&
+      document.value?.active &&
+      document.value.lastErrorCode === row.code
+    ) {
+      stops.push({
+        collection: row.collection,
+        relPath: row.relPath,
+        code: row.code,
+        message: row.message,
+      });
+      if (stops.length >= MAX_BUDGET_STOPS) break;
+    }
+  }
+  return stops;
+}
+
 /** A background issue reported by (or about) a running detached resident. */
 export type ResidentBackgroundIssue = BackgroundIssue & {
   process: "serve" | "daemon";
@@ -181,10 +230,21 @@ function formatTerminal(
     `Health: ${isStatusHealthy(indexStatus, activation) ? "OK" : "DEGRADED"}`
   );
   lines.push("");
-  lines.push(
-    `Lexical activation: ${activation.healthy ? "READY" : activation.usable ? "DEGRADED" : "BLOCKED"}`
-  );
+  const activationState = activation.healthy
+    ? activation.usable
+      ? "READY"
+      : "NO DOCUMENTS YET"
+    : activation.usable
+      ? "DEGRADED"
+      : "BLOCKED";
+  lines.push(`Lexical activation: ${activationState}`);
   for (const collection of activation.collections) {
+    if (isEmptyActivationCollection(collection)) {
+      lines.push(
+        `  ${collection.collection}: no documents yet (add files, then run ${collection.remediation?.command})`
+      );
+      continue;
+    }
     const failedStage = collection.remediation?.stage;
     const suffix = failedStage
       ? ` (${failedStage}: ${collection.remediation?.code}; ${collection.remediation?.command})`
@@ -277,7 +337,7 @@ function formatMarkdown(
   lines.push(`- **Lexically healthy**: ${activation.healthy}`);
   for (const collection of activation.collections) {
     lines.push(
-      `- **${collection.collection}**: ${collection.ready ? "lexical ready" : `${collection.remediation?.stage ?? "index"} ${collection.remediation?.code ?? "index_query_failed"}`}`
+      `- **${collection.collection}**: ${collection.ready ? "lexical ready" : isEmptyActivationCollection(collection) ? `no documents yet (${collection.remediation?.command})` : `${collection.remediation?.stage ?? "index"} ${collection.remediation?.code ?? "index_query_failed"}`}`
     );
   }
   for (const connector of activation.connectors) {
@@ -368,6 +428,7 @@ export async function status(
       contentTypeBoost: buildContentTypeBoostStatus(config.contentTypes ?? []),
       memory: await buildMemoryStatus(store, config.collections),
       backgroundIssues: await collectResidentIssues(),
+      budgetStops: await collectBudgetStops(store),
     };
   } finally {
     await store.close();
@@ -439,6 +500,10 @@ export function formatStatus(
     (issue) =>
       `  ${issue.process} (pid ${issue.pid}): ${formatBackgroundIssue(issue)}`
   );
+  const stopLines = result.budgetStops.map(
+    (stop) =>
+      `  ${stop.collection}/${stop.relPath} [${stop.code}] ${stop.message}`
+  );
   return [
     formatTerminal(
       result.status,
@@ -446,6 +511,13 @@ export function formatStatus(
       result.contentTypeBoost,
       result.memory
     ),
+    ...(stopLines.length
+      ? [
+          "",
+          "Stopped at conversion budget (retried by the next update):",
+          ...stopLines,
+        ]
+      : []),
     ...(issueLines.length ? ["", "Background issues:", ...issueLines] : []),
   ].join("\n");
 }

package/src/cli/commands/update.ts

@@ -15,7 +15,11 @@ import {
   type SyncResult,
   withContentTypeRules,
 } from "../../ingestion";
-import { formatSyncResultLines, initStore } from "./shared";
+import {
+  formatSyncResultLines,
+  initStore,
+  reportSlowConversion,
+} from "./shared";
 
 /**
  * Options for update command.
@@ -66,6 +70,7 @@ export async function update(
           {
             gitPull: options.gitPull,
             runUpdateCmd: true,
+            onSlowConversion: reportSlowConversion,
           },
           config
         )

package/src/cli/commands/vec.ts

@@ -13,10 +13,16 @@ import {
   createVectorIndexPort,
   createVectorStatsPort,
 } from "../../store/vector";
+import {
+  rebuildPartitionIndex,
+  syncPartitionIndex,
+} from "../../store/vector/partition-index";
 import {
   dropVectorPartition,
+  storedVectorPartition,
   type VectorPartitionStatus,
 } from "../../store/vector/status";
+import { loadSqliteVec } from "../../store/vector/variants";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -76,7 +82,9 @@ function inferDimensions(
 // ─────────────────────────────────────────────────────────────────────────────
 
 /**
- * Sync vec0 index with content_vectors (add missing, remove orphans).
+ * Sync the vec0 index with stored vectors (add missing, remove orphans): the
+ * active vector partition's variants, or legacy `content_vectors` on an
+ * index that never activated a partition.
  */
 export async function vecSync(
   options: VecOptions = {}
@@ -112,6 +120,21 @@ export async function vecSync(
   try {
     const db = store.getRawDb();
 
+    const partition = storedVectorPartition(db, modelUri);
+    if (partition) {
+      if (!(await loadSqliteVec(db))) {
+        return {
+          success: false,
+          error: "sqlite-vec not available. Cannot sync index.",
+        };
+      }
+      return {
+        success: true,
+        ...syncPartitionIndex(db, partition),
+        model: modelUri,
+      };
+    }
+
     // Infer dimensions from stored vectors for this model
     const dimensions = inferDimensions(db, modelUri);
     if (dimensions === null) {
@@ -154,7 +177,9 @@ export async function vecSync(
 }
 
 /**
- * Rebuild vec0 index from content_vectors (drop + recreate + repopulate).
+ * Rebuild the vec0 index (drop + recreate + repopulate) from the active
+ * vector partition's variants, or from legacy `content_vectors` on an index
+ * that never activated a partition.
  */
 export async function vecRebuild(
   options: VecOptions = {}
@@ -190,6 +215,21 @@ export async function vecRebuild(
   try {
     const db = store.getRawDb();
 
+    const partition = storedVectorPartition(db, modelUri);
+    if (partition) {
+      if (!(await loadSqliteVec(db))) {
+        return {
+          success: false,
+          error: "sqlite-vec not available. Cannot rebuild index.",
+        };
+      }
+      return {
+        success: true,
+        count: rebuildPartitionIndex(db, partition),
+        model: modelUri,
+      };
+    }
+
     // Infer dimensions from stored vectors for this model
     const dimensions = inferDimensions(db, modelUri);
     if (dimensions === null) {

package/src/cli/program.ts

@@ -3655,7 +3655,7 @@ function wireVecCommands(program: Command): void {
   addWriteLeaseFlags(
     vecCmd
       .command("sync")
-      .description("Sync vec0 index with content_vectors")
+      .description("Sync the vec0 index with the active vector partition")
       .option("--json", "JSON output")
   ).action(async (cmdOpts: Record<string, unknown>) => {
     const format = getFormat(cmdOpts);
@@ -3728,7 +3728,7 @@ function wireVecCommands(program: Command): void {
   addWriteLeaseFlags(
     vecCmd
       .command("rebuild")
-      .description("Rebuild vec0 index from content_vectors")
+      .description("Rebuild the vec0 index from the active vector partition")
       .option("--json", "JSON output")
   ).action(async (cmdOpts: Record<string, unknown>) => {
     const format = getFormat(cmdOpts);

package/src/config/types.ts

@@ -429,6 +429,22 @@ export const DEFAULT_MODEL_PRESETS: ModelPreset[] = [
   },
 ];
 
+/**
+ * Per-file conversion budget. A file that overruns either limit is recorded
+ * with TIMEOUT or MEMORY_LIMIT and retried by the next update.
+ */
+export const ConversionConfigSchema = z
+  .object({
+    /** Wall-clock budget for converting one file, in ms (default 60000). */
+    timeoutMs: z.number().int().min(1000).max(86_400_000).optional(),
+    /**
+     * Resident-memory ceiling while a file converts, in MB (default: half
+     * of physical memory, at least 2048).
+     */
+    maxMemoryMb: z.number().int().min(256).max(1_048_576).optional(),
+  })
+  .strict();
+
 export const ModelConfigSchema = z.object({
   /** Active preset ID */
   activePreset: z.string().default("slim-tuned"),
@@ -608,6 +624,9 @@ export const ConfigSchema = z.object({
   /** Optional index-wide chunk size/overlap; omitted preserves legacy defaults. */
   chunking: ChunkingConfigSchema.optional(),
 
+  /** Per-file conversion budget; omitted fields use the runtime defaults. */
+  conversion: ConversionConfigSchema.optional(),
+
   /** Model configuration */
   models: ModelConfigSchema.optional(),
 

package/src/converters/adapters/markitdownTs/adapter.ts

@@ -19,6 +19,7 @@ import {
   tooLargeError,
 } from "../../errors";
 import { ADAPTER_VERSIONS } from "../../versions";
+import { hasPrefix, isPasswordProtectedXlsx } from "../shared/ooxml-protection";
 
 const CONVERTER_ID = "adapter/markitdown-ts" as const;
 const CONVERTER_VERSION = ADAPTER_VERSIONS["markitdown-ts"];
@@ -34,11 +35,6 @@ const SUPPORTED_MIMES = [
 ];
 
 const PDF_SIGNATURE = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]);
-const CFB_SIGNATURE = new Uint8Array([
-  0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1,
-]);
-const ENCRYPTION_INFO = utf16le("EncryptionInfo");
-const ENCRYPTED_PACKAGE = utf16le("EncryptedPackage");
 const PDF_TRAILER_SCAN_BYTES = 2048;
 const MAX_MESSAGE_LENGTH = 200;
 const PASSWORD_ERROR_REGEX = /password(?:-protected)?|no password given/i;
@@ -51,45 +47,6 @@ function toBuffer(bytes: Uint8Array): Buffer {
   return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
 }
 
-function utf16le(value: string): Uint8Array {
-  return new Uint8Array(Buffer.from(value, "utf16le"));
-}
-
-function hasPrefix(bytes: Uint8Array, prefix: Uint8Array): boolean {
-  if (bytes.length < prefix.length) {
-    return false;
-  }
-
-  for (let index = 0; index < prefix.length; index += 1) {
-    if (bytes[index] !== prefix[index]) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-function includesBytes(bytes: Uint8Array, needle: Uint8Array): boolean {
-  if (needle.length === 0 || bytes.length < needle.length) {
-    return false;
-  }
-
-  outer: for (
-    let index = 0;
-    index <= bytes.length - needle.length;
-    index += 1
-  ) {
-    for (let offset = 0; offset < needle.length; offset += 1) {
-      if (bytes[index + offset] !== needle[offset]) {
-        continue outer;
-      }
-    }
-    return true;
-  }
-
-  return false;
-}
-
 function isPasswordProtectedPdf(bytes: Uint8Array): boolean {
   if (!hasPrefix(bytes, PDF_SIGNATURE)) {
     return false;
@@ -113,14 +70,6 @@ function hasCompletePdfTrailer(bytes: Uint8Array): boolean {
   return startXrefIndex >= 0 && eofIndex > startXrefIndex;
 }
 
-function isPasswordProtectedXlsx(bytes: Uint8Array): boolean {
-  return (
-    hasPrefix(bytes, CFB_SIGNATURE) &&
-    includesBytes(bytes, ENCRYPTION_INFO) &&
-    includesBytes(bytes, ENCRYPTED_PACKAGE)
-  );
-}
-
 function isPasswordProtected(input: ConvertInput): boolean {
   if (input.ext === ".pdf") {
     return isPasswordProtectedPdf(input.bytes);

package/src/converters/adapters/shared/ooxml-protection.ts

@@ -0,0 +1,58 @@
+/**
+ * Byte checks shared by document adapters: signature prefixes and the
+ * encrypted-OOXML (CFB container) test. Dependency-free so an adapter can
+ * use them without loading another adapter's parser stack.
+ *
+ * @module src/converters/adapters/shared/ooxml-protection
+ */
+
+const CFB_SIGNATURE = new Uint8Array([
+  0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1,
+]);
+
+export function utf16le(value: string): Uint8Array {
+  return new Uint8Array(Buffer.from(value, "utf16le"));
+}
+
+const ENCRYPTION_INFO = utf16le("EncryptionInfo");
+const ENCRYPTED_PACKAGE = utf16le("EncryptedPackage");
+
+export function hasPrefix(bytes: Uint8Array, prefix: Uint8Array): boolean {
+  if (bytes.length < prefix.length) {
+    return false;
+  }
+  for (let index = 0; index < prefix.length; index += 1) {
+    if (bytes[index] !== prefix[index]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+export function includesBytes(bytes: Uint8Array, needle: Uint8Array): boolean {
+  if (needle.length === 0 || bytes.length < needle.length) {
+    return false;
+  }
+  outer: for (
+    let index = 0;
+    index <= bytes.length - needle.length;
+    index += 1
+  ) {
+    for (let offset = 0; offset < needle.length; offset += 1) {
+      if (bytes[index + offset] !== needle[offset]) {
+        continue outer;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+/** An encrypted (password-protected) OOXML workbook: a CFB container. */
+export function isPasswordProtectedXlsx(bytes: Uint8Array): boolean {
+  return (
+    hasPrefix(bytes, CFB_SIGNATURE) &&
+    includesBytes(bytes, ENCRYPTION_INFO) &&
+    includesBytes(bytes, ENCRYPTED_PACKAGE)
+  );
+}

package/src/converters/adapters/xlsx/adapter.ts

@@ -0,0 +1,132 @@
+/**
+ * Excel (.xlsx) converter: SheetJS workbook -> Markdown tables, linear in
+ * cells (fn-198).
+ *
+ * Produces the Markdown markitdown-ts produced for workbooks (one `## sheet`
+ * section and GFM table per sheet with a range) but renders each table
+ * directly from the sheet data instead of through HTML, jsdom and turndown,
+ * whose cost grew with columns x rows^2 and about 20 KB per cell. Pivot
+ * cache parts are not user content and SheetJS does not read them, as
+ * before.
+ *
+ * @module src/converters/adapters/xlsx/adapter
+ */
+
+import * as XLSX from "xlsx";
+
+import type { Converter, ConvertInput, ConvertResult } from "../../types";
+
+import { corruptError, permissionError, tooLargeError } from "../../errors";
+import { ADAPTER_VERSIONS } from "../../versions";
+import { isPasswordProtectedXlsx } from "../shared/ooxml-protection";
+import { sheetToMarkdown } from "./sheet-markdown";
+
+const CONVERTER_ID = "adapter/xlsx" as const;
+const CONVERTER_VERSION = ADAPTER_VERSIONS.xlsx;
+const XLSX_MIME =
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+const PASSWORD_ERROR_REGEX = /password(?:-protected)?|no password given/i;
+/** Output under this length from a larger file is flagged as lossy. */
+const SHORT_OUTPUT_CHARS = 10;
+const SHORT_OUTPUT_MIN_BYTES = 1000;
+
+/**
+ * Markdown for a parsed workbook: one section per sheet with a range,
+ * normalized as markitdown-ts normalizes every result (line endings, outer
+ * whitespace, at most one blank line).
+ */
+export function workbookToMarkdown(workbook: XLSX.WorkBook): string {
+  let markdown = "";
+  for (const name of workbook.SheetNames) {
+    const sheet = workbook.Sheets[name];
+    if (sheet?.["!ref"]) {
+      markdown += `## ${name}\n${sheetToMarkdown(sheet)}\n\n`;
+    }
+  }
+  return markdown
+    .replace(/\r\n|\r|\n/g, "\n")
+    .trim()
+    .replace(/\n{3,}/g, "\n\n");
+}
+
+export const xlsxAdapter: Converter = {
+  id: CONVERTER_ID,
+  version: CONVERTER_VERSION,
+
+  canHandle(mime: string, ext: string): boolean {
+    return ext === ".xlsx" || mime === XLSX_MIME;
+  },
+
+  convert(input: ConvertInput): Promise<ConvertResult> {
+    return Promise.resolve(convertWorkbook(input));
+  },
+};
+
+function convertWorkbook(input: ConvertInput): ConvertResult {
+  if (input.bytes.length > input.limits.maxBytes) {
+    return { ok: false, error: tooLargeError(input, CONVERTER_ID) };
+  }
+  if (isPasswordProtectedXlsx(input.bytes)) {
+    return {
+      ok: false,
+      error: permissionError(
+        input,
+        CONVERTER_ID,
+        "File is password-protected",
+        undefined,
+        { protection: "xlsx" }
+      ),
+    };
+  }
+
+  let workbook: XLSX.WorkBook;
+  try {
+    // Dense sheets store rows as arrays: smaller than one object per cell.
+    workbook = XLSX.read(input.bytes, { type: "buffer", dense: true });
+  } catch (cause) {
+    const message = cause instanceof Error ? cause.message : String(cause);
+    return {
+      ok: false,
+      error: PASSWORD_ERROR_REGEX.test(message)
+        ? permissionError(
+            input,
+            CONVERTER_ID,
+            "File is password-protected",
+            cause,
+            { protection: "xlsx" }
+          )
+        : corruptError(
+            input,
+            CONVERTER_ID,
+            "Could not read the .xlsx workbook",
+            cause
+          ),
+    };
+  }
+
+  const markdown = workbookToMarkdown(workbook);
+  if (!markdown) {
+    return {
+      ok: false,
+      error: corruptError(input, CONVERTER_ID, "Empty conversion result"),
+    };
+  }
+  const lossy =
+    markdown.length < SHORT_OUTPUT_CHARS &&
+    input.bytes.length > SHORT_OUTPUT_MIN_BYTES;
+  return {
+    ok: true,
+    value: {
+      markdown,
+      title: workbook.Props?.Title || "Untitled",
+      meta: {
+        converterId: CONVERTER_ID,
+        converterVersion: CONVERTER_VERSION,
+        sourceMime: input.mime,
+        warnings: lossy
+          ? [{ code: "LOSSY", message: "Suspiciously short output" }]
+          : undefined,
+      },
+    },
+  };
+}