@aigne/doc-smith 0.9.8 → 0.9.9-beta.1

package/utils/diagram-version-utils.mjs

@@ -0,0 +1,14 @@
+import fs from "fs-extra";
+
+/**
+ * Calculate timestamp for diagram image
+ * Uses file modification time (mtime) as the version identifier
+ * @param {string} imagePath - Absolute path to the image file
+ * @returns {Promise<string>} Unix timestamp in seconds (as string)
+ */
+export async function calculateImageTimestamp(imagePath) {
+  const stats = await fs.stat(imagePath);
+  // Use modification time, convert to Unix timestamp (seconds)
+  const timestamp = Math.floor(stats.mtimeMs / 1000);
+  return timestamp.toString();
+}

package/utils/file-utils.mjs

@@ -530,9 +530,38 @@ export async function readFileContents(files, baseDir = process.cwd(), options =
   return results.filter((result) => result !== null);
 }
 
+/**
+ * Sanitize text by removing or escaping disallowed LLM special tokens
+ * This prevents errors when encoding text that contains special tokens like <|endoftext|>
+ * @param {string} text - Text to sanitize
+ * @returns {string} Sanitized text safe for tokenization
+ */
+function sanitizeForTokenization(text) {
+  if (typeof text !== "string") return text;
+  // Replace <|endoftext|> with a safe alternative that won't trigger special token parsing
+  // We replace it with a space-separated version to prevent tokenizer from recognizing it as a special token
+  return text.replace(/<\|endoftext\|>/g, "<| endoftext |>");
+}
+
 export function calculateTokens(text) {
-  const tokens = encode(text);
-  return tokens.length;
+  try {
+    // Sanitize text before encoding to avoid errors with special tokens
+    const sanitizedText = sanitizeForTokenization(text);
+    const tokens = encode(sanitizedText);
+    return tokens.length;
+  } catch (error) {
+    // If encoding still fails, try with more aggressive sanitization
+    console.warn(`Token calculation warning: ${error.message}`);
+    const fallbackText = sanitizeForTokenization(text).replace(/<\|[^|]+\|>/g, "");
+    try {
+      const tokens = encode(fallbackText);
+      return tokens.length;
+    } catch {
+      // Last resort: estimate tokens based on character count (rough approximation)
+      console.warn(`Token calculation fallback: using character-based estimation`);
+      return Math.ceil(fallbackText.length / 4); // Rough estimate: ~4 chars per token
+    }
+  }
 }
 
 /**
@@ -547,9 +576,15 @@ export function calculateFileStats(sourceFiles) {
   for (const source of sourceFiles) {
     const { content } = source;
     if (content) {
-      // Count tokens using gpt-tokenizer
-      const tokens = encode(content);
-      totalTokens += tokens.length;
+      // Count tokens using gpt-tokenizer with sanitization
+      try {
+        const sanitizedContent = sanitizeForTokenization(content);
+        const tokens = encode(sanitizedContent);
+        totalTokens += tokens.length;
+      } catch {
+        // Fallback: use calculateTokens which has its own error handling
+        totalTokens += calculateTokens(content);
+      }
 
       // Count lines (excluding empty lines)
       totalLines += content.split("\n").filter((line) => line.trim() !== "").length;

package/utils/image-compress.mjs

@@ -68,7 +68,7 @@ export async function compressImage(inputPath, options = {}) {
 
     return finalOutputPath;
   } catch (error) {
-    debug(`⚠️  Failed to compress image ${inputPath}: ${error.message}`);
+    debug(`❌ Failed to compress image ${inputPath}`, error);
     // Return original path if compression fails
     return inputPath;
   }

package/utils/sync-diagram-to-translations.mjs

@@ -61,12 +61,12 @@ async function findTranslationFiles(docPath, docsDir, locale) {
  */
 function extractDiagramImagePaths(content) {
   const images = [];
-  const matches = Array.from(content.matchAll(diagramImageWithPathRegex));
+  const matches = Array.from((content || "").matchAll(diagramImageWithPathRegex));
 
   for (const match of matches) {
     images.push({
-      path: match[1],
-      fullMatch: match[0],
+      path: match[4] || "",
+      fullMatch: match[0] || "",
       index: match.index,
     });
   }