@umituz/web-ai 1.0.8 → 1.0.9

package/dist/index.cjs

@@ -1141,6 +1141,83 @@ ${paragraph}`;
   return chunks.map((chunk) => chunk.length > maxChars ? chunk.slice(0, maxChars) : chunk).filter((chunk) => chunk.length > 0);
 }
 
+// src/core/parsers/text-extract.ts
+var TEXT_MIME_PREFIX = "text/";
+var TEXT_MIME_TYPES = /* @__PURE__ */ new Set([
+  "application/json",
+  "application/xml",
+  "application/yaml",
+  "application/x-yaml",
+  "application/javascript",
+  "application/typescript",
+  "application/x-sh",
+  "application/sql",
+  "text/csv"
+]);
+var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
+  "txt",
+  "md",
+  "markdown",
+  "csv",
+  "tsv",
+  "json",
+  "xml",
+  "yaml",
+  "yml",
+  "toml",
+  "ini",
+  "js",
+  "mjs",
+  "cjs",
+  "ts",
+  "tsx",
+  "jsx",
+  "py",
+  "rb",
+  "go",
+  "rs",
+  "java",
+  "kt",
+  "swift",
+  "c",
+  "h",
+  "cpp",
+  "hpp",
+  "cs",
+  "php",
+  "sh",
+  "bash",
+  "zsh",
+  "sql",
+  "html",
+  "css",
+  "scss",
+  "env"
+]);
+var TEXT_EXTRACT_PER_FILE_LIMIT = 24e3;
+var TEXT_EXTRACT_TOTAL_LIMIT = 48e3;
+function isTextLike(mimeType, name) {
+  if (mimeType.startsWith(TEXT_MIME_PREFIX) || TEXT_MIME_TYPES.has(mimeType)) return true;
+  const extension = name.split(".").pop()?.toLowerCase() ?? "";
+  return TEXT_EXTENSIONS.has(extension);
+}
+function extractText(bytes) {
+  let out = "";
+  const decoder = new TextDecoder("utf-8", { fatal: false });
+  const chunkSize = 65536;
+  for (let offset = 0; offset < bytes.length; offset += chunkSize) {
+    out += decoder.decode(bytes.subarray(offset, Math.min(offset + chunkSize, bytes.length)), {
+      stream: offset + chunkSize < bytes.length
+    });
+  }
+  return out.split("\0").join("");
+}
+function capText(text, limit) {
+  if (text.length <= limit) return text;
+  return `${text.slice(0, limit)}
+[truncated]`;
+}
+
 // src/core/calculations/random-base36-suffix.ts
 var BASE_36_RADIX = 36;
 var DECIMAL_OFFSET = 2;
@@ -1588,11 +1665,7 @@ function resolveTimestamp(value) {
   const parsed = Date.parse(value);
   return Number.isNaN(parsed) ? 0 : parsed;
 }
-function truncateString(str, maxLength) {
-  if (str.length <= maxLength) return str;
-  return str.slice(0, Math.max(maxLength, 0));
-}
-function selectMemories(rows, maxItems = MEMORY_MAX_ITEMS, totalLimit = MEMORY_TOTAL_LIMIT) {
+function selectMemories(rows, maxItems = MEMORY_MAX_ITEMS, totalLimit = MEMORY_TOTAL_LIMIT, capper = capText) {
   const ordered = [...rows].sort((a, b) => {
     if (a.pinned !== b.pinned) return a.pinned ? -1 : 1;
     return resolveTimestamp(b.updatedAt) - resolveTimestamp(a.updatedAt);
@@ -1603,7 +1676,7 @@ function selectMemories(rows, maxItems = MEMORY_MAX_ITEMS, totalLimit = MEMORY_T
     if (out.length >= maxItems || used >= totalLimit) break;
     const content = row.content.trim();
     if (content.length === 0) continue;
-    const capped = truncateString(content, totalLimit - used);
+    const capped = capper(content, totalLimit - used);
     out.push(capped);
     used += capped.length;
   }
@@ -2602,7 +2675,7 @@ var AnthropicProvider = class {
     }
   }
   toGenerationResult(response, fallbackModel) {
-    const text = extractText(response.content);
+    const text = extractText2(response.content);
     const finishReason = mapAnthropicStopReason(response.stop_reason);
     if (text.length === 0) {
       if (finishReason === "refusal") {
@@ -2630,7 +2703,7 @@ var AnthropicProvider = class {
     };
   }
 };
-function extractText(blocks) {
+function extractText2(blocks) {
   return blocks.filter((block) => block.type === "text").map((block) => block.text).join("");
 }
 function refusalTextFor(finishReason) {
@@ -7142,6 +7215,8 @@ exports.SeoService = SeoService;
 exports.SocialService = SocialService;
 exports.StreamingResponseParser = StreamingResponseParser;
 exports.StructuredPromptBuilder = StructuredPromptBuilder;
+exports.TEXT_EXTRACT_PER_FILE_LIMIT = TEXT_EXTRACT_PER_FILE_LIMIT;
+exports.TEXT_EXTRACT_TOTAL_LIMIT = TEXT_EXTRACT_TOTAL_LIMIT;
 exports.TEXT_TIERS = TEXT_TIERS;
 exports.TIMEOUTS = TIMEOUTS;
 exports.TRANSIENT_ERROR_CODES = TRANSIENT_ERROR_CODES;
@@ -7181,6 +7256,7 @@ exports.calculateExponentialBackoff = calculateExponentialBackoff;
 exports.calculateTrailingDelay = calculateTrailingDelay;
 exports.canFailover = canFailover;
 exports.canStartTask = canStartTask;
+exports.capText = capText;
 exports.chatIdGenerator = chatIdGenerator;
 exports.chunkText = chunkText;
 exports.clampScore = clampScore;
@@ -7227,6 +7303,7 @@ exports.extractJsonCandidate = extractJsonCandidate;
 exports.extractJsonFromCodeFence = extractJsonFromCodeFence;
 exports.extractJsonObject = extractJsonObject;
 exports.extractSseDataPayload = extractSseDataPayload;
+exports.extractText = extractText;
 exports.extractTextFromMessages = extractTextFromMessages;
 exports.extractTokenUsage = extractTokenUsage;
 exports.failoverCandidate = failoverCandidate;
@@ -7247,6 +7324,7 @@ exports.isMediaProvider = isMediaProvider;
 exports.isRetryableErrorType = isRetryableErrorType;
 exports.isSseDataLine = isSseDataLine;
 exports.isSseDoneMarker = isSseDoneMarker;
+exports.isTextLike = isTextLike;
 exports.isTextProvider = isTextProvider;
 exports.isTextTier = isTextTier;
 exports.isTransientError = isTransientError;