@umituz/web-ai 1.0.8 → 1.0.9

package/dist/core/index.cjs

@@ -1134,6 +1134,83 @@ ${paragraph}`;
   return chunks.map((chunk) => chunk.length > maxChars ? chunk.slice(0, maxChars) : chunk).filter((chunk) => chunk.length > 0);
 }
 
+// src/core/parsers/text-extract.ts
+var TEXT_MIME_PREFIX = "text/";
+var TEXT_MIME_TYPES = /* @__PURE__ */ new Set([
+  "application/json",
+  "application/xml",
+  "application/yaml",
+  "application/x-yaml",
+  "application/javascript",
+  "application/typescript",
+  "application/x-sh",
+  "application/sql",
+  "text/csv"
+]);
+var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
+  "txt",
+  "md",
+  "markdown",
+  "csv",
+  "tsv",
+  "json",
+  "xml",
+  "yaml",
+  "yml",
+  "toml",
+  "ini",
+  "js",
+  "mjs",
+  "cjs",
+  "ts",
+  "tsx",
+  "jsx",
+  "py",
+  "rb",
+  "go",
+  "rs",
+  "java",
+  "kt",
+  "swift",
+  "c",
+  "h",
+  "cpp",
+  "hpp",
+  "cs",
+  "php",
+  "sh",
+  "bash",
+  "zsh",
+  "sql",
+  "html",
+  "css",
+  "scss",
+  "env"
+]);
+var TEXT_EXTRACT_PER_FILE_LIMIT = 24e3;
+var TEXT_EXTRACT_TOTAL_LIMIT = 48e3;
+function isTextLike(mimeType, name) {
+  if (mimeType.startsWith(TEXT_MIME_PREFIX) || TEXT_MIME_TYPES.has(mimeType)) return true;
+  const extension = name.split(".").pop()?.toLowerCase() ?? "";
+  return TEXT_EXTENSIONS.has(extension);
+}
+function extractText(bytes) {
+  let out = "";
+  const decoder = new TextDecoder("utf-8", { fatal: false });
+  const chunkSize = 65536;
+  for (let offset = 0; offset < bytes.length; offset += chunkSize) {
+    out += decoder.decode(bytes.subarray(offset, Math.min(offset + chunkSize, bytes.length)), {
+      stream: offset + chunkSize < bytes.length
+    });
+  }
+  return out.split("\0").join("");
+}
+function capText(text, limit) {
+  if (text.length <= limit) return text;
+  return `${text.slice(0, limit)}
+[truncated]`;
+}
+
 // src/core/calculations/random-base36-suffix.ts
 var BASE_36_RADIX = 36;
 var DECIMAL_OFFSET = 2;
@@ -1581,11 +1658,7 @@ function resolveTimestamp(value) {
   const parsed = Date.parse(value);
   return Number.isNaN(parsed) ? 0 : parsed;
 }
-function truncateString(str, maxLength) {
-  if (str.length <= maxLength) return str;
-  return str.slice(0, Math.max(maxLength, 0));
-}
-function selectMemories(rows, maxItems = MEMORY_MAX_ITEMS, totalLimit = MEMORY_TOTAL_LIMIT) {
+function selectMemories(rows, maxItems = MEMORY_MAX_ITEMS, totalLimit = MEMORY_TOTAL_LIMIT, capper = capText) {
   const ordered = [...rows].sort((a, b) => {
     if (a.pinned !== b.pinned) return a.pinned ? -1 : 1;
     return resolveTimestamp(b.updatedAt) - resolveTimestamp(a.updatedAt);
@@ -1596,7 +1669,7 @@ function selectMemories(rows, maxItems = MEMORY_MAX_ITEMS, totalLimit = MEMORY_T
     if (out.length >= maxItems || used >= totalLimit) break;
     const content = row.content.trim();
     if (content.length === 0) continue;
-    const capped = truncateString(content, totalLimit - used);
+    const capped = capper(content, totalLimit - used);
     out.push(capped);
     used += capped.length;
   }
@@ -1879,6 +1952,8 @@ exports.STATUS_TO_CODE = STATUS_TO_CODE;
 exports.SYSTEM_PROMPT_VERSION = SYSTEM_PROMPT_VERSION;
 exports.StreamingResponseParser = StreamingResponseParser;
 exports.StructuredPromptBuilder = StructuredPromptBuilder;
+exports.TEXT_EXTRACT_PER_FILE_LIMIT = TEXT_EXTRACT_PER_FILE_LIMIT;
+exports.TEXT_EXTRACT_TOTAL_LIMIT = TEXT_EXTRACT_TOTAL_LIMIT;
 exports.TEXT_TIERS = TEXT_TIERS;
 exports.TRANSIENT_ERROR_CODES = TRANSIENT_ERROR_CODES;
 exports.ThrottleManager = ThrottleManager;
@@ -1903,6 +1978,7 @@ exports.calculateExponentialBackoff = calculateExponentialBackoff;
 exports.calculateTrailingDelay = calculateTrailingDelay;
 exports.canFailover = canFailover;
 exports.canStartTask = canStartTask;
+exports.capText = capText;
 exports.chunkText = chunkText;
 exports.clampScore = clampScore;
 exports.classifyHttpStatus = classifyHttpStatus;
@@ -1928,6 +2004,7 @@ exports.extractJsonCandidate = extractJsonCandidate;
 exports.extractJsonFromCodeFence = extractJsonFromCodeFence;
 exports.extractJsonObject = extractJsonObject;
 exports.extractSseDataPayload = extractSseDataPayload;
+exports.extractText = extractText;
 exports.failoverCandidate = failoverCandidate;
 exports.findTopKSimilar = findTopKSimilar;
 exports.formatCost = formatCost;
@@ -1938,6 +2015,7 @@ exports.isAuthErrorType = isAuthErrorType;
 exports.isRetryableErrorType = isRetryableErrorType;
 exports.isSseDataLine = isSseDataLine;
 exports.isSseDoneMarker = isSseDoneMarker;
+exports.isTextLike = isTextLike;
 exports.isTextTier = isTextTier;
 exports.isTransientError = isTransientError;
 exports.isValidApiKey = isValidApiKey;