@xmem-ai/memcode 4.0.2

package/dist/utils/bash/heredoc.js

@@ -0,0 +1,283 @@
+import { randomBytes } from "crypto";
+const HEREDOC_PLACEHOLDER_PREFIX = "__HEREDOC_";
+const HEREDOC_PLACEHOLDER_SUFFIX = "__";
+function generatePlaceholderSalt() {
+  return randomBytes(8).toString("hex");
+}
+const HEREDOC_START_PATTERN = (
+  // eslint-disable-next-line custom-rules/no-lookbehind-regex -- gated by command.includes('<<') at extractHeredocs() entry
+  /(?<!<)<<(?!<)(-)?[ \t]*(?:(['"])(\\?\w+)\2|\\?(\w+))/
+);
+function extractHeredocs(command, options) {
+  const heredocs = /* @__PURE__ */ new Map();
+  if (!command.includes("<<")) {
+    return { processedCommand: command, heredocs };
+  }
+  if (/\$['"]/.test(command)) {
+    return { processedCommand: command, heredocs };
+  }
+  const firstHeredocPos = command.indexOf("<<");
+  if (firstHeredocPos > 0 && command.slice(0, firstHeredocPos).includes("`")) {
+    return { processedCommand: command, heredocs };
+  }
+  if (firstHeredocPos > 0) {
+    const beforeHeredoc = command.slice(0, firstHeredocPos);
+    const openArith = (beforeHeredoc.match(/\(\(/g) || []).length;
+    const closeArith = (beforeHeredoc.match(/\)\)/g) || []).length;
+    if (openArith > closeArith) {
+      return { processedCommand: command, heredocs };
+    }
+  }
+  const heredocStartPattern = new RegExp(HEREDOC_START_PATTERN.source, "g");
+  const heredocMatches = [];
+  const skippedHeredocRanges = [];
+  let match;
+  let scanPos = 0;
+  let scanInSingleQuote = false;
+  let scanInDoubleQuote = false;
+  let scanInComment = false;
+  let scanDqEscapeNext = false;
+  let scanPendingBackslashes = 0;
+  const advanceScan = (target) => {
+    for (let i = scanPos; i < target; i++) {
+      const ch = command[i];
+      if (ch === "\n") scanInComment = false;
+      if (scanInSingleQuote) {
+        if (ch === "'") scanInSingleQuote = false;
+        continue;
+      }
+      if (scanInDoubleQuote) {
+        if (scanDqEscapeNext) {
+          scanDqEscapeNext = false;
+          continue;
+        }
+        if (ch === "\\") {
+          scanDqEscapeNext = true;
+          continue;
+        }
+        if (ch === '"') scanInDoubleQuote = false;
+        continue;
+      }
+      if (ch === "\\") {
+        scanPendingBackslashes++;
+        continue;
+      }
+      const escaped = scanPendingBackslashes % 2 === 1;
+      scanPendingBackslashes = 0;
+      if (escaped) continue;
+      if (ch === "'") scanInSingleQuote = true;
+      else if (ch === '"') scanInDoubleQuote = true;
+      else if (!scanInComment && ch === "#") scanInComment = true;
+    }
+    scanPos = target;
+  };
+  while ((match = heredocStartPattern.exec(command)) !== null) {
+    const startIndex = match.index;
+    advanceScan(startIndex);
+    if (scanInSingleQuote || scanInDoubleQuote) {
+      continue;
+    }
+    if (scanInComment) {
+      continue;
+    }
+    if (scanPendingBackslashes % 2 === 1) {
+      continue;
+    }
+    let insideSkipped = false;
+    for (const skipped of skippedHeredocRanges) {
+      if (startIndex > skipped.contentStartIndex && startIndex < skipped.contentEndIndex) {
+        insideSkipped = true;
+        break;
+      }
+    }
+    if (insideSkipped) {
+      continue;
+    }
+    const fullMatch = match[0];
+    const isDash = match[1] === "-";
+    const delimiter = match[3] || match[4];
+    const operatorEndIndex = startIndex + fullMatch.length;
+    const quoteChar = match[2];
+    if (quoteChar && command[operatorEndIndex - 1] !== quoteChar) {
+      continue;
+    }
+    const isEscapedDelimiter = fullMatch.includes("\\");
+    const isQuotedOrEscaped = !!quoteChar || isEscapedDelimiter;
+    if (operatorEndIndex < command.length) {
+      const nextChar = command[operatorEndIndex];
+      if (!/^[ \t\n|&;()<>]$/.test(nextChar)) {
+        continue;
+      }
+    }
+    let firstNewlineOffset = -1;
+    {
+      let inSingleQuote = false;
+      let inDoubleQuote = false;
+      for (let k = operatorEndIndex; k < command.length; k++) {
+        const ch = command[k];
+        if (inSingleQuote) {
+          if (ch === "'") inSingleQuote = false;
+          continue;
+        }
+        if (inDoubleQuote) {
+          if (ch === "\\") {
+            k++;
+            continue;
+          }
+          if (ch === '"') inDoubleQuote = false;
+          continue;
+        }
+        if (ch === "\n") {
+          firstNewlineOffset = k - operatorEndIndex;
+          break;
+        }
+        let backslashCount = 0;
+        for (let j = k - 1; j >= operatorEndIndex && command[j] === "\\"; j--) {
+          backslashCount++;
+        }
+        if (backslashCount % 2 === 1) continue;
+        if (ch === "'") inSingleQuote = true;
+        else if (ch === '"') inDoubleQuote = true;
+      }
+    }
+    if (firstNewlineOffset === -1) {
+      continue;
+    }
+    const sameLineContent = command.slice(
+      operatorEndIndex,
+      operatorEndIndex + firstNewlineOffset
+    );
+    let trailingBackslashes = 0;
+    for (let j = sameLineContent.length - 1; j >= 0; j--) {
+      if (sameLineContent[j] === "\\") {
+        trailingBackslashes++;
+      } else {
+        break;
+      }
+    }
+    if (trailingBackslashes % 2 === 1) {
+      continue;
+    }
+    const contentStartIndex = operatorEndIndex + firstNewlineOffset;
+    const afterNewline = command.slice(contentStartIndex + 1);
+    const contentLines = afterNewline.split("\n");
+    let closingLineIndex = -1;
+    for (let i = 0; i < contentLines.length; i++) {
+      const line = contentLines[i];
+      if (isDash) {
+        const stripped = line.replace(/^\t*/, "");
+        if (stripped === delimiter) {
+          closingLineIndex = i;
+          break;
+        }
+      } else {
+        if (line === delimiter) {
+          closingLineIndex = i;
+          break;
+        }
+      }
+      const eofCheckLine = isDash ? line.replace(/^\t*/, "") : line;
+      if (eofCheckLine.length > delimiter.length && eofCheckLine.startsWith(delimiter)) {
+        const charAfterDelimiter = eofCheckLine[delimiter.length];
+        if (/^[)}`|&;(<>]$/.test(charAfterDelimiter)) {
+          closingLineIndex = -1;
+          break;
+        }
+      }
+    }
+    if (options?.quotedOnly && !isQuotedOrEscaped) {
+      let skipContentEndIndex;
+      if (closingLineIndex === -1) {
+        skipContentEndIndex = command.length;
+      } else {
+        const skipLinesUpToClosing = contentLines.slice(0, closingLineIndex + 1);
+        const skipContentLength = skipLinesUpToClosing.join("\n").length;
+        skipContentEndIndex = contentStartIndex + 1 + skipContentLength;
+      }
+      skippedHeredocRanges.push({
+        contentStartIndex,
+        contentEndIndex: skipContentEndIndex
+      });
+      continue;
+    }
+    if (closingLineIndex === -1) {
+      continue;
+    }
+    const linesUpToClosing = contentLines.slice(0, closingLineIndex + 1);
+    const contentLength = linesUpToClosing.join("\n").length;
+    const contentEndIndex = contentStartIndex + 1 + contentLength;
+    let overlapsSkipped = false;
+    for (const skipped of skippedHeredocRanges) {
+      if (contentStartIndex < skipped.contentEndIndex && skipped.contentStartIndex < contentEndIndex) {
+        overlapsSkipped = true;
+        break;
+      }
+    }
+    if (overlapsSkipped) {
+      continue;
+    }
+    const operatorText = command.slice(startIndex, operatorEndIndex);
+    const contentText = command.slice(contentStartIndex, contentEndIndex);
+    const fullText = operatorText + contentText;
+    heredocMatches.push({
+      fullText,
+      delimiter,
+      operatorStartIndex: startIndex,
+      operatorEndIndex,
+      contentStartIndex,
+      contentEndIndex
+    });
+  }
+  if (heredocMatches.length === 0) {
+    return { processedCommand: command, heredocs };
+  }
+  const topLevelHeredocs = heredocMatches.filter((candidate, _i, all) => {
+    for (const other of all) {
+      if (candidate === other) continue;
+      if (candidate.operatorStartIndex > other.contentStartIndex && candidate.operatorStartIndex < other.contentEndIndex) {
+        return false;
+      }
+    }
+    return true;
+  });
+  if (topLevelHeredocs.length === 0) {
+    return { processedCommand: command, heredocs };
+  }
+  const contentStartPositions = new Set(
+    topLevelHeredocs.map((h) => h.contentStartIndex)
+  );
+  if (contentStartPositions.size < topLevelHeredocs.length) {
+    return { processedCommand: command, heredocs };
+  }
+  topLevelHeredocs.sort((a, b) => b.contentEndIndex - a.contentEndIndex);
+  const salt = generatePlaceholderSalt();
+  let processedCommand = command;
+  topLevelHeredocs.forEach((info, index) => {
+    const placeholderIndex = topLevelHeredocs.length - 1 - index;
+    const placeholder = `${HEREDOC_PLACEHOLDER_PREFIX}${placeholderIndex}_${salt}${HEREDOC_PLACEHOLDER_SUFFIX}`;
+    heredocs.set(placeholder, info);
+    processedCommand = processedCommand.slice(0, info.operatorStartIndex) + placeholder + processedCommand.slice(info.operatorEndIndex, info.contentStartIndex) + processedCommand.slice(info.contentEndIndex);
+  });
+  return { processedCommand, heredocs };
+}
+function restoreHeredocsInString(text, heredocs) {
+  let result = text;
+  for (const [placeholder, info] of heredocs) {
+    result = result.replaceAll(placeholder, info.fullText);
+  }
+  return result;
+}
+function restoreHeredocs(parts, heredocs) {
+  if (heredocs.size === 0) {
+    return parts;
+  }
+  return parts.map((part) => restoreHeredocsInString(part, heredocs));
+}
+function containsHeredoc(command) {
+  return HEREDOC_START_PATTERN.test(command);
+}
+export {
+  containsHeredoc,
+  extractHeredocs,
+  restoreHeredocs
+};

package/dist/utils/bash/parser.js

@@ -0,0 +1,152 @@
+import { feature } from "bun:bundle";
+import { logEvent } from "../../services/analytics/index.js";
+import { logForDebugging } from "../debug.js";
+import {
+  ensureParserInitialized,
+  getParserModule
+} from "./bashParser.js";
+const MAX_COMMAND_LENGTH = 1e4;
+const DECLARATION_COMMANDS = /* @__PURE__ */ new Set([
+  "export",
+  "declare",
+  "typeset",
+  "readonly",
+  "local",
+  "unset",
+  "unsetenv"
+]);
+const ARGUMENT_TYPES = /* @__PURE__ */ new Set(["word", "string", "raw_string", "number"]);
+const SUBSTITUTION_TYPES = /* @__PURE__ */ new Set([
+  "command_substitution",
+  "process_substitution"
+]);
+const COMMAND_TYPES = /* @__PURE__ */ new Set(["command", "declaration_command"]);
+let logged = false;
+function logLoadOnce(success) {
+  if (logged) return;
+  logged = true;
+  logForDebugging(
+    success ? "tree-sitter: native module loaded" : "tree-sitter: unavailable"
+  );
+  logEvent("tengu_tree_sitter_load", { success });
+}
+async function ensureInitialized() {
+  if (feature("TREE_SITTER_BASH") || feature("TREE_SITTER_BASH_SHADOW")) {
+    await ensureParserInitialized();
+  }
+}
+async function parseCommand(command) {
+  if (!command || command.length > MAX_COMMAND_LENGTH) return null;
+  if (feature("TREE_SITTER_BASH")) {
+    await ensureParserInitialized();
+    const mod = getParserModule();
+    logLoadOnce(mod !== null);
+    if (!mod) return null;
+    try {
+      const rootNode = mod.parse(command);
+      if (!rootNode) return null;
+      const commandNode = findCommandNode(rootNode, null);
+      const envVars = extractEnvVars(commandNode);
+      return { rootNode, envVars, commandNode, originalCommand: command };
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+const PARSE_ABORTED = /* @__PURE__ */ Symbol("parse-aborted");
+async function parseCommandRaw(command) {
+  if (!command || command.length > MAX_COMMAND_LENGTH) return null;
+  if (feature("TREE_SITTER_BASH") || feature("TREE_SITTER_BASH_SHADOW")) {
+    await ensureParserInitialized();
+    const mod = getParserModule();
+    logLoadOnce(mod !== null);
+    if (!mod) return null;
+    try {
+      const result = mod.parse(command);
+      if (result === null) {
+        logEvent("tengu_tree_sitter_parse_abort", {
+          cmdLength: command.length,
+          panic: false
+        });
+        return PARSE_ABORTED;
+      }
+      return result;
+    } catch {
+      logEvent("tengu_tree_sitter_parse_abort", {
+        cmdLength: command.length,
+        panic: true
+      });
+      return PARSE_ABORTED;
+    }
+  }
+  return null;
+}
+function findCommandNode(node, parent) {
+  const { type, children } = node;
+  if (COMMAND_TYPES.has(type)) return node;
+  if (type === "variable_assignment" && parent) {
+    return parent.children.find(
+      (c) => COMMAND_TYPES.has(c.type) && c.startIndex > node.startIndex
+    ) ?? null;
+  }
+  if (type === "pipeline") {
+    for (const child of children) {
+      const result = findCommandNode(child, node);
+      if (result) return result;
+    }
+    return null;
+  }
+  if (type === "redirected_statement") {
+    return children.find((c) => COMMAND_TYPES.has(c.type)) ?? null;
+  }
+  for (const child of children) {
+    const result = findCommandNode(child, node);
+    if (result) return result;
+  }
+  return null;
+}
+function extractEnvVars(commandNode) {
+  if (!commandNode || commandNode.type !== "command") return [];
+  const envVars = [];
+  for (const child of commandNode.children) {
+    if (child.type === "variable_assignment") {
+      envVars.push(child.text);
+    } else if (child.type === "command_name" || child.type === "word") {
+      break;
+    }
+  }
+  return envVars;
+}
+function extractCommandArguments(commandNode) {
+  if (commandNode.type === "declaration_command") {
+    const firstChild = commandNode.children[0];
+    return firstChild && DECLARATION_COMMANDS.has(firstChild.text) ? [firstChild.text] : [];
+  }
+  const args = [];
+  let foundCommandName = false;
+  for (const child of commandNode.children) {
+    if (child.type === "variable_assignment") continue;
+    if (child.type === "command_name" || !foundCommandName && child.type === "word") {
+      foundCommandName = true;
+      args.push(child.text);
+      continue;
+    }
+    if (ARGUMENT_TYPES.has(child.type)) {
+      args.push(stripQuotes(child.text));
+    } else if (SUBSTITUTION_TYPES.has(child.type)) {
+      break;
+    }
+  }
+  return args;
+}
+function stripQuotes(text) {
+  return text.length >= 2 && (text[0] === '"' && text.at(-1) === '"' || text[0] === "'" && text.at(-1) === "'") ? text.slice(1, -1) : text;
+}
+export {
+  PARSE_ABORTED,
+  ensureInitialized,
+  extractCommandArguments,
+  parseCommand,
+  parseCommandRaw
+};

package/dist/utils/bash/prefix.js

@@ -0,0 +1,127 @@
+import { buildPrefix } from "../shell/specPrefix.js";
+import { splitCommand_DEPRECATED } from "./commands.js";
+import { extractCommandArguments, parseCommand } from "./parser.js";
+import { getCommandSpec } from "./registry.js";
+const NUMERIC = /^\d+$/;
+const ENV_VAR = /^[A-Za-z_][A-Za-z0-9_]*=/;
+const WRAPPER_COMMANDS = /* @__PURE__ */ new Set([
+  "nice"
+  // command position varies based on options
+]);
+const toArray = (val) => Array.isArray(val) ? val : [val];
+function isKnownSubcommand(arg, spec) {
+  if (!spec?.subcommands?.length) return false;
+  return spec.subcommands.some(
+    (sub) => Array.isArray(sub.name) ? sub.name.includes(arg) : sub.name === arg
+  );
+}
+async function getCommandPrefixStatic(command, recursionDepth = 0, wrapperCount = 0) {
+  if (wrapperCount > 2 || recursionDepth > 10) return null;
+  const parsed = await parseCommand(command);
+  if (!parsed) return null;
+  if (!parsed.commandNode) {
+    return { commandPrefix: null };
+  }
+  const { envVars, commandNode } = parsed;
+  const cmdArgs = extractCommandArguments(commandNode);
+  const [cmd, ...args] = cmdArgs;
+  if (!cmd) return { commandPrefix: null };
+  const spec = await getCommandSpec(cmd);
+  let isWrapper = WRAPPER_COMMANDS.has(cmd) || spec?.args && toArray(spec.args).some((arg) => arg?.isCommand);
+  if (isWrapper && args[0] && isKnownSubcommand(args[0], spec)) {
+    isWrapper = false;
+  }
+  const prefix = isWrapper ? await handleWrapper(cmd, args, recursionDepth, wrapperCount) : await buildPrefix(cmd, args, spec);
+  if (prefix === null && recursionDepth === 0 && isWrapper) {
+    return null;
+  }
+  const envPrefix = envVars.length ? `${envVars.join(" ")} ` : "";
+  return { commandPrefix: prefix ? envPrefix + prefix : null };
+}
+async function handleWrapper(command, args, recursionDepth, wrapperCount) {
+  const spec = await getCommandSpec(command);
+  if (spec?.args) {
+    const commandArgIndex = toArray(spec.args).findIndex((arg) => arg?.isCommand);
+    if (commandArgIndex !== -1) {
+      const parts = [command];
+      for (let i = 0; i < args.length && i <= commandArgIndex; i++) {
+        if (i === commandArgIndex) {
+          const result2 = await getCommandPrefixStatic(
+            args.slice(i).join(" "),
+            recursionDepth + 1,
+            wrapperCount + 1
+          );
+          if (result2?.commandPrefix) {
+            parts.push(...result2.commandPrefix.split(" "));
+            return parts.join(" ");
+          }
+          break;
+        } else if (args[i] && !args[i].startsWith("-") && !ENV_VAR.test(args[i])) {
+          parts.push(args[i]);
+        }
+      }
+    }
+  }
+  const wrapped = args.find(
+    (arg) => !arg.startsWith("-") && !NUMERIC.test(arg) && !ENV_VAR.test(arg)
+  );
+  if (!wrapped) return command;
+  const result = await getCommandPrefixStatic(
+    args.slice(args.indexOf(wrapped)).join(" "),
+    recursionDepth + 1,
+    wrapperCount + 1
+  );
+  return !result?.commandPrefix ? null : `${command} ${result.commandPrefix}`;
+}
+async function getCompoundCommandPrefixesStatic(command, excludeSubcommand) {
+  const subcommands = splitCommand_DEPRECATED(command);
+  if (subcommands.length <= 1) {
+    const result = await getCommandPrefixStatic(command);
+    return result?.commandPrefix ? [result.commandPrefix] : [];
+  }
+  const prefixes = [];
+  for (const subcmd of subcommands) {
+    const trimmed = subcmd.trim();
+    if (excludeSubcommand?.(trimmed)) continue;
+    const result = await getCommandPrefixStatic(trimmed);
+    if (result?.commandPrefix) {
+      prefixes.push(result.commandPrefix);
+    }
+  }
+  if (prefixes.length === 0) return [];
+  const groups = /* @__PURE__ */ new Map();
+  for (const prefix of prefixes) {
+    const root = prefix.split(" ")[0];
+    const group = groups.get(root);
+    if (group) {
+      group.push(prefix);
+    } else {
+      groups.set(root, [prefix]);
+    }
+  }
+  const collapsed = [];
+  for (const [, group] of groups) {
+    collapsed.push(longestCommonPrefix(group));
+  }
+  return collapsed;
+}
+function longestCommonPrefix(strings) {
+  if (strings.length === 0) return "";
+  if (strings.length === 1) return strings[0];
+  const first = strings[0];
+  const words = first.split(" ");
+  let commonWords = words.length;
+  for (let i = 1; i < strings.length; i++) {
+    const otherWords = strings[i].split(" ");
+    let shared = 0;
+    while (shared < commonWords && shared < otherWords.length && words[shared] === otherWords[shared]) {
+      shared++;
+    }
+    commonWords = shared;
+  }
+  return words.slice(0, Math.max(1, commonWords)).join(" ");
+}
+export {
+  getCommandPrefixStatic,
+  getCompoundCommandPrefixesStatic
+};

package/dist/utils/bash/registry.js

@@ -0,0 +1,24 @@
+import { memoizeWithLRU } from "../memoize.js";
+import specs from "./specs/index.js";
+async function loadFigSpec(command) {
+  if (!command || command.includes("/") || command.includes("\\")) return null;
+  if (command.includes("..")) return null;
+  if (command.startsWith("-") && command !== "-") return null;
+  try {
+    const module = await import(`@withfig/autocomplete/build/${command}.js`);
+    return module.default || module;
+  } catch {
+    return null;
+  }
+}
+const getCommandSpec = memoizeWithLRU(
+  async (command) => {
+    const spec = specs.find((s) => s.name === command) || await loadFigSpec(command) || null;
+    return spec;
+  },
+  (command) => command
+);
+export {
+  getCommandSpec,
+  loadFigSpec
+};