@gmickel/gno 2.8.1 → 2.8.3

package/src/cli/commands/vec.ts

@@ -13,10 +13,16 @@ import {
   createVectorIndexPort,
   createVectorStatsPort,
 } from "../../store/vector";
+import {
+  rebuildPartitionIndex,
+  syncPartitionIndex,
+} from "../../store/vector/partition-index";
 import {
   dropVectorPartition,
+  storedVectorPartition,
   type VectorPartitionStatus,
 } from "../../store/vector/status";
+import { loadSqliteVec } from "../../store/vector/variants";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -76,7 +82,9 @@ function inferDimensions(
 // ─────────────────────────────────────────────────────────────────────────────
 
 /**
- * Sync vec0 index with content_vectors (add missing, remove orphans).
+ * Sync the vec0 index with stored vectors (add missing, remove orphans): the
+ * active vector partition's variants, or legacy `content_vectors` on an
+ * index that never activated a partition.
  */
 export async function vecSync(
   options: VecOptions = {}
@@ -112,6 +120,21 @@ export async function vecSync(
   try {
     const db = store.getRawDb();
 
+    const partition = storedVectorPartition(db, modelUri);
+    if (partition) {
+      if (!(await loadSqliteVec(db))) {
+        return {
+          success: false,
+          error: "sqlite-vec not available. Cannot sync index.",
+        };
+      }
+      return {
+        success: true,
+        ...syncPartitionIndex(db, partition),
+        model: modelUri,
+      };
+    }
+
     // Infer dimensions from stored vectors for this model
     const dimensions = inferDimensions(db, modelUri);
     if (dimensions === null) {
@@ -154,7 +177,9 @@ export async function vecSync(
 }
 
 /**
- * Rebuild vec0 index from content_vectors (drop + recreate + repopulate).
+ * Rebuild the vec0 index (drop + recreate + repopulate) from the active
+ * vector partition's variants, or from legacy `content_vectors` on an index
+ * that never activated a partition.
  */
 export async function vecRebuild(
   options: VecOptions = {}
@@ -190,6 +215,21 @@ export async function vecRebuild(
   try {
     const db = store.getRawDb();
 
+    const partition = storedVectorPartition(db, modelUri);
+    if (partition) {
+      if (!(await loadSqliteVec(db))) {
+        return {
+          success: false,
+          error: "sqlite-vec not available. Cannot rebuild index.",
+        };
+      }
+      return {
+        success: true,
+        count: rebuildPartitionIndex(db, partition),
+        model: modelUri,
+      };
+    }
+
     // Infer dimensions from stored vectors for this model
     const dimensions = inferDimensions(db, modelUri);
     if (dimensions === null) {

package/src/cli/program.ts

@@ -3655,7 +3655,7 @@ function wireVecCommands(program: Command): void {
   addWriteLeaseFlags(
     vecCmd
       .command("sync")
-      .description("Sync vec0 index with content_vectors")
+      .description("Sync the vec0 index with the active vector partition")
       .option("--json", "JSON output")
   ).action(async (cmdOpts: Record<string, unknown>) => {
     const format = getFormat(cmdOpts);
@@ -3728,7 +3728,7 @@ function wireVecCommands(program: Command): void {
   addWriteLeaseFlags(
     vecCmd
       .command("rebuild")
-      .description("Rebuild vec0 index from content_vectors")
+      .description("Rebuild the vec0 index from the active vector partition")
       .option("--json", "JSON output")
   ).action(async (cmdOpts: Record<string, unknown>) => {
     const format = getFormat(cmdOpts);

package/src/config/types.ts

@@ -429,6 +429,22 @@ export const DEFAULT_MODEL_PRESETS: ModelPreset[] = [
   },
 ];
 
+/**
+ * Per-file conversion budget. A file that overruns either limit is recorded
+ * with TIMEOUT or MEMORY_LIMIT and retried by the next update.
+ */
+export const ConversionConfigSchema = z
+  .object({
+    /** Wall-clock budget for converting one file, in ms (default 60000). */
+    timeoutMs: z.number().int().min(1000).max(86_400_000).optional(),
+    /**
+     * Resident-memory ceiling while a file converts, in MB (default: half
+     * of physical memory, at least 2048).
+     */
+    maxMemoryMb: z.number().int().min(256).max(1_048_576).optional(),
+  })
+  .strict();
+
 export const ModelConfigSchema = z.object({
   /** Active preset ID */
   activePreset: z.string().default("slim-tuned"),
@@ -608,6 +624,9 @@ export const ConfigSchema = z.object({
   /** Optional index-wide chunk size/overlap; omitted preserves legacy defaults. */
   chunking: ChunkingConfigSchema.optional(),
 
+  /** Per-file conversion budget; omitted fields use the runtime defaults. */
+  conversion: ConversionConfigSchema.optional(),
+
   /** Model configuration */
   models: ModelConfigSchema.optional(),
 

package/src/converters/adapters/markitdownTs/adapter.ts

@@ -19,6 +19,7 @@ import {
   tooLargeError,
 } from "../../errors";
 import { ADAPTER_VERSIONS } from "../../versions";
+import { hasPrefix, isPasswordProtectedXlsx } from "../shared/ooxml-protection";
 
 const CONVERTER_ID = "adapter/markitdown-ts" as const;
 const CONVERTER_VERSION = ADAPTER_VERSIONS["markitdown-ts"];
@@ -34,11 +35,6 @@ const SUPPORTED_MIMES = [
 ];
 
 const PDF_SIGNATURE = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]);
-const CFB_SIGNATURE = new Uint8Array([
-  0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1,
-]);
-const ENCRYPTION_INFO = utf16le("EncryptionInfo");
-const ENCRYPTED_PACKAGE = utf16le("EncryptedPackage");
 const PDF_TRAILER_SCAN_BYTES = 2048;
 const MAX_MESSAGE_LENGTH = 200;
 const PASSWORD_ERROR_REGEX = /password(?:-protected)?|no password given/i;
@@ -51,45 +47,6 @@ function toBuffer(bytes: Uint8Array): Buffer {
   return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
 }
 
-function utf16le(value: string): Uint8Array {
-  return new Uint8Array(Buffer.from(value, "utf16le"));
-}
-
-function hasPrefix(bytes: Uint8Array, prefix: Uint8Array): boolean {
-  if (bytes.length < prefix.length) {
-    return false;
-  }
-
-  for (let index = 0; index < prefix.length; index += 1) {
-    if (bytes[index] !== prefix[index]) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-function includesBytes(bytes: Uint8Array, needle: Uint8Array): boolean {
-  if (needle.length === 0 || bytes.length < needle.length) {
-    return false;
-  }
-
-  outer: for (
-    let index = 0;
-    index <= bytes.length - needle.length;
-    index += 1
-  ) {
-    for (let offset = 0; offset < needle.length; offset += 1) {
-      if (bytes[index + offset] !== needle[offset]) {
-        continue outer;
-      }
-    }
-    return true;
-  }
-
-  return false;
-}
-
 function isPasswordProtectedPdf(bytes: Uint8Array): boolean {
   if (!hasPrefix(bytes, PDF_SIGNATURE)) {
     return false;
@@ -113,14 +70,6 @@ function hasCompletePdfTrailer(bytes: Uint8Array): boolean {
   return startXrefIndex >= 0 && eofIndex > startXrefIndex;
 }
 
-function isPasswordProtectedXlsx(bytes: Uint8Array): boolean {
-  return (
-    hasPrefix(bytes, CFB_SIGNATURE) &&
-    includesBytes(bytes, ENCRYPTION_INFO) &&
-    includesBytes(bytes, ENCRYPTED_PACKAGE)
-  );
-}
-
 function isPasswordProtected(input: ConvertInput): boolean {
   if (input.ext === ".pdf") {
     return isPasswordProtectedPdf(input.bytes);

package/src/converters/adapters/shared/ooxml-protection.ts

@@ -0,0 +1,58 @@
+/**
+ * Byte checks shared by document adapters: signature prefixes and the
+ * encrypted-OOXML (CFB container) test. Dependency-free so an adapter can
+ * use them without loading another adapter's parser stack.
+ *
+ * @module src/converters/adapters/shared/ooxml-protection
+ */
+
+const CFB_SIGNATURE = new Uint8Array([
+  0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1,
+]);
+
+export function utf16le(value: string): Uint8Array {
+  return new Uint8Array(Buffer.from(value, "utf16le"));
+}
+
+const ENCRYPTION_INFO = utf16le("EncryptionInfo");
+const ENCRYPTED_PACKAGE = utf16le("EncryptedPackage");
+
+export function hasPrefix(bytes: Uint8Array, prefix: Uint8Array): boolean {
+  if (bytes.length < prefix.length) {
+    return false;
+  }
+  for (let index = 0; index < prefix.length; index += 1) {
+    if (bytes[index] !== prefix[index]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+export function includesBytes(bytes: Uint8Array, needle: Uint8Array): boolean {
+  if (needle.length === 0 || bytes.length < needle.length) {
+    return false;
+  }
+  outer: for (
+    let index = 0;
+    index <= bytes.length - needle.length;
+    index += 1
+  ) {
+    for (let offset = 0; offset < needle.length; offset += 1) {
+      if (bytes[index + offset] !== needle[offset]) {
+        continue outer;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+/** An encrypted (password-protected) OOXML workbook: a CFB container. */
+export function isPasswordProtectedXlsx(bytes: Uint8Array): boolean {
+  return (
+    hasPrefix(bytes, CFB_SIGNATURE) &&
+    includesBytes(bytes, ENCRYPTION_INFO) &&
+    includesBytes(bytes, ENCRYPTED_PACKAGE)
+  );
+}

package/src/converters/adapters/xlsx/adapter.ts

@@ -0,0 +1,132 @@
+/**
+ * Excel (.xlsx) converter: SheetJS workbook -> Markdown tables, linear in
+ * cells (fn-198).
+ *
+ * Produces the Markdown markitdown-ts produced for workbooks (one `## sheet`
+ * section and GFM table per sheet with a range) but renders each table
+ * directly from the sheet data instead of through HTML, jsdom and turndown,
+ * whose cost grew with columns x rows^2 and about 20 KB per cell. Pivot
+ * cache parts are not user content and SheetJS does not read them, as
+ * before.
+ *
+ * @module src/converters/adapters/xlsx/adapter
+ */
+
+import * as XLSX from "xlsx";
+
+import type { Converter, ConvertInput, ConvertResult } from "../../types";
+
+import { corruptError, permissionError, tooLargeError } from "../../errors";
+import { ADAPTER_VERSIONS } from "../../versions";
+import { isPasswordProtectedXlsx } from "../shared/ooxml-protection";
+import { sheetToMarkdown } from "./sheet-markdown";
+
+const CONVERTER_ID = "adapter/xlsx" as const;
+const CONVERTER_VERSION = ADAPTER_VERSIONS.xlsx;
+const XLSX_MIME =
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+const PASSWORD_ERROR_REGEX = /password(?:-protected)?|no password given/i;
+/** Output under this length from a larger file is flagged as lossy. */
+const SHORT_OUTPUT_CHARS = 10;
+const SHORT_OUTPUT_MIN_BYTES = 1000;
+
+/**
+ * Markdown for a parsed workbook: one section per sheet with a range,
+ * normalized as markitdown-ts normalizes every result (line endings, outer
+ * whitespace, at most one blank line).
+ */
+export function workbookToMarkdown(workbook: XLSX.WorkBook): string {
+  let markdown = "";
+  for (const name of workbook.SheetNames) {
+    const sheet = workbook.Sheets[name];
+    if (sheet?.["!ref"]) {
+      markdown += `## ${name}\n${sheetToMarkdown(sheet)}\n\n`;
+    }
+  }
+  return markdown
+    .replace(/\r\n|\r|\n/g, "\n")
+    .trim()
+    .replace(/\n{3,}/g, "\n\n");
+}
+
+export const xlsxAdapter: Converter = {
+  id: CONVERTER_ID,
+  version: CONVERTER_VERSION,
+
+  canHandle(mime: string, ext: string): boolean {
+    return ext === ".xlsx" || mime === XLSX_MIME;
+  },
+
+  convert(input: ConvertInput): Promise<ConvertResult> {
+    return Promise.resolve(convertWorkbook(input));
+  },
+};
+
+function convertWorkbook(input: ConvertInput): ConvertResult {
+  if (input.bytes.length > input.limits.maxBytes) {
+    return { ok: false, error: tooLargeError(input, CONVERTER_ID) };
+  }
+  if (isPasswordProtectedXlsx(input.bytes)) {
+    return {
+      ok: false,
+      error: permissionError(
+        input,
+        CONVERTER_ID,
+        "File is password-protected",
+        undefined,
+        { protection: "xlsx" }
+      ),
+    };
+  }
+
+  let workbook: XLSX.WorkBook;
+  try {
+    // Dense sheets store rows as arrays: smaller than one object per cell.
+    workbook = XLSX.read(input.bytes, { type: "buffer", dense: true });
+  } catch (cause) {
+    const message = cause instanceof Error ? cause.message : String(cause);
+    return {
+      ok: false,
+      error: PASSWORD_ERROR_REGEX.test(message)
+        ? permissionError(
+            input,
+            CONVERTER_ID,
+            "File is password-protected",
+            cause,
+            { protection: "xlsx" }
+          )
+        : corruptError(
+            input,
+            CONVERTER_ID,
+            "Could not read the .xlsx workbook",
+            cause
+          ),
+    };
+  }
+
+  const markdown = workbookToMarkdown(workbook);
+  if (!markdown) {
+    return {
+      ok: false,
+      error: corruptError(input, CONVERTER_ID, "Empty conversion result"),
+    };
+  }
+  const lossy =
+    markdown.length < SHORT_OUTPUT_CHARS &&
+    input.bytes.length > SHORT_OUTPUT_MIN_BYTES;
+  return {
+    ok: true,
+    value: {
+      markdown,
+      title: workbook.Props?.Title || "Untitled",
+      meta: {
+        converterId: CONVERTER_ID,
+        converterVersion: CONVERTER_VERSION,
+        sourceMime: input.mime,
+        warnings: lossy
+          ? [{ code: "LOSSY", message: "Suspiciously short output" }]
+          : undefined,
+      },
+    },
+  };
+}