@contentrain/mcp 0.3.1 → 1.0.3

package/dist/scanner-BdTiv8sw.mjs

@@ -0,0 +1,619 @@
+import { o as readText } from "./fs-BbuGnVcG.mjs";
+import { i as autoDetectSourceDirs, o as discoverFiles } from "./scan-config-AJGxCdNd.mjs";
+import { t as parseTsx } from "./tsx-parser-CYVyBWjn.mjs";
+import { extname, join } from "node:path";
+//#region src/core/ast-scanner/pre-filter.ts
+const PURE_NUMBER_RE = /^\d+(\.\d+)?$/;
+const HEX_COLOR_RE = /^#[0-9a-f]{3,8}$/i;
+const FILE_EXT_RE$1 = /\.(png|jpg|jpeg|gif|svg|webp|ico|css|scss|less|js|ts|tsx|jsx|json|md|html|xml|yaml|yml|woff|woff2|ttf|eot|mp4|webm|mp3|wav|pdf)$/i;
+const HTML_PROP_VALUES = new Set([
+	"class",
+	"variant",
+	"secondary",
+	"outline",
+	"ghost",
+	"destructive",
+	"default",
+	"primary",
+	"_blank",
+	"noopener",
+	"noreferrer",
+	"button",
+	"submit",
+	"reset",
+	"text",
+	"numeric",
+	"password",
+	"email",
+	"checkbox",
+	"radio",
+	"hidden",
+	"none",
+	"auto",
+	"inherit",
+	"initial",
+	"sm",
+	"md",
+	"lg",
+	"xl",
+	"xs",
+	"2xl",
+	"3xl",
+	"icon",
+	"icon-sm",
+	"icon-lg"
+]);
+const SLOT_EVENT_NAMES = new Set([
+	"header",
+	"footer",
+	"default",
+	"trigger",
+	"content",
+	"sidebar",
+	"overlay",
+	"body",
+	"actions",
+	"click",
+	"change",
+	"input",
+	"focus",
+	"blur",
+	"submit",
+	"mounted",
+	"unmounted",
+	"updated"
+]);
+const TAILWIND_SEGMENT_RE = /^(?:bg-|text-|border-|flex|grid|p-|px-|py-|pt-|pb-|pl-|pr-|m-|mx-|my-|mt-|mb-|ml-|mr-|rounded|shadow|w-|h-|min-|max-|gap-|space-|items-|justify-|self-|overflow-|z-|opacity-|transition|duration-|ease-|animate-|font-|leading-|tracking-|decoration-|underline|line-through|uppercase|lowercase|capitalize|truncate|whitespace-|break-|sr-only|not-sr-only|hover:|focus:|active:|disabled:|dark:|sm:|md:|lg:|xl:|2xl:|group-|peer-|ring-|outline-|divide-|table-|col-|row-|aspect-|object-|inset-|top-|right-|bottom-|left-|translate-|rotate-|scale-|skew-|origin-|cursor-|select-|resize-|fill-|stroke-|block|inline|absolute|relative|fixed|sticky|static|float-|clear-|isolate|visible|invisible|grow|shrink|basis-|order-|place-)/;
+/**
+* Returns true if the string looks like a CSS class list (Tailwind or similar).
+* Requires 2+ space-separated segments with majority matching utility patterns.
+*/
+function isCssClassList(value) {
+	const segments = value.trim().split(/\s+/);
+	if (segments.length < 2) return false;
+	let matched = 0;
+	for (const seg of segments) if (TAILWIND_SEGMENT_RE.test(seg)) matched++;
+	return matched / segments.length >= .5;
+}
+/**
+* Returns true if the string is a single CSS utility token (e.g. "bg-blue-500").
+*/
+function isSingleCssUtility(value) {
+	const trimmed = value.trim();
+	if (trimmed.includes(" ")) return false;
+	return TAILWIND_SEGMENT_RE.test(trimmed);
+}
+const PRE_FILTER_RULES = [
+	{
+		context: "import_path",
+		reason: "import_path"
+	},
+	{
+		context: "type_annotation",
+		reason: "type_annotation"
+	},
+	{
+		context: "css_class",
+		reason: "css_class"
+	},
+	{
+		context: "css_utility_call",
+		reason: "css_utility_call"
+	},
+	{
+		context: "console_call",
+		reason: "console_call"
+	},
+	{
+		context: "test_assertion",
+		reason: "test_assertion"
+	},
+	{
+		context: "switch_case",
+		reason: "switch_case"
+	},
+	{
+		condition: (v) => v.length <= 1,
+		reason: "single_char"
+	},
+	{
+		condition: (v) => PURE_NUMBER_RE.test(v),
+		reason: "pure_number"
+	},
+	{
+		condition: (v) => v.startsWith("--"),
+		reason: "cli_flag"
+	},
+	{
+		condition: (v) => HEX_COLOR_RE.test(v),
+		reason: "hex_color"
+	},
+	{
+		condition: (v) => FILE_EXT_RE$1.test(v),
+		reason: "file_extension"
+	},
+	{
+		condition: (v) => HTML_PROP_VALUES.has(v.toLowerCase()),
+		reason: "html_prop_value"
+	},
+	{
+		condition: (v) => isCssClassList(v),
+		reason: "css_class_list"
+	},
+	{
+		condition: (v) => isSingleCssUtility(v),
+		reason: "css_utility_token"
+	},
+	{
+		condition: (v) => {
+			const lower = v.toLowerCase();
+			return v === lower && !v.includes(" ") && SLOT_EVENT_NAMES.has(lower);
+		},
+		reason: "slot_event_name"
+	}
+];
+/**
+* Structural pre-filter: removes strings that are 100% NOT content.
+*
+* Conservative — when in doubt, INCLUDE the string.
+* Returns candidates (passed), count of filtered, and reason breakdown.
+*/
+function applyPreFilter(strings) {
+	const candidates = [];
+	const filterReasons = {};
+	let filtered = 0;
+	for (const str of strings) {
+		const matchedRule = findMatchingRule(str);
+		if (matchedRule) {
+			filtered++;
+			filterReasons[matchedRule.reason] = (filterReasons[matchedRule.reason] ?? 0) + 1;
+		} else candidates.push(str);
+	}
+	return {
+		candidates,
+		filtered,
+		filterReasons
+	};
+}
+/**
+* Find the first pre-filter rule that matches this string.
+* Returns the rule if matched, undefined if the string should pass through.
+*/
+function findMatchingRule(str) {
+	for (const rule of PRE_FILTER_RULES) {
+		if (rule.context !== void 0 && str.context === rule.context) return rule;
+		if (rule.condition !== void 0 && rule.condition(str.value)) return rule;
+	}
+}
+//#endregion
+//#region src/core/ast-scanner/index.ts
+const TSX_EXTENSIONS = new Set([
+	".tsx",
+	".jsx",
+	".ts",
+	".js",
+	".mjs"
+]);
+const VUE_EXTENSIONS = new Set([".vue"]);
+const SVELTE_EXTENSIONS = new Set([".svelte"]);
+const ASTRO_EXTENSIONS = new Set([".astro"]);
+/**
+* Lazily import vue-parser. Returns undefined if not available
+* (@vue/compiler-sfc is an optional dependency).
+*/
+async function loadVueParser() {
+	try {
+		return (await import("./vue-parser-CjS4YAoQ.mjs")).parseVue;
+	} catch {
+		return;
+	}
+}
+/**
+* Lazily import svelte-parser. Returns undefined if not available
+* (svelte is an optional dependency).
+*/
+async function loadSvelteParser() {
+	try {
+		return (await import("./svelte-parser-COTNG92w.mjs")).parseSvelte;
+	} catch {
+		return;
+	}
+}
+/**
+* Lazily import astro-parser. Returns undefined if not available
+* (@astrojs/compiler is an optional dependency).
+*/
+async function loadAstroParser() {
+	try {
+		return (await import("./astro-parser-RofKmKQi.mjs")).parseAstro;
+	} catch {
+		return;
+	}
+}
+const SURROUNDING_MAX = 120;
+/**
+* Minimal regex-based extractor for file types without AST parsers.
+* Extracts quoted strings and tag text — conservative, low accuracy.
+* Used as fallback when dedicated parsers are unavailable.
+*/
+function extractWithRegex(content, _filePath) {
+	const lines = content.split("\n");
+	const results = [];
+	const stringRe = /(['"`])(?:(?!\1|\\).|\\.)*?\1/g;
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+		const trimmed = line.trim();
+		if (trimmed.startsWith("//") || trimmed.startsWith("/*") || trimmed.startsWith("*")) continue;
+		if (/^\s*(import|export)\s/.test(line)) continue;
+		let match;
+		stringRe.lastIndex = 0;
+		while ((match = stringRe.exec(line)) !== null) {
+			const value = match[0].slice(1, -1);
+			if (value.length === 0) continue;
+			const surrounding = buildSurrounding(lines, i);
+			results.push({
+				value,
+				line: i + 1,
+				column: match.index + 1,
+				context: "other",
+				scope: "script",
+				parent: "",
+				surrounding
+			});
+		}
+		const tagTextRe = />([^<>{]+)</g;
+		let tagMatch;
+		tagTextRe.lastIndex = 0;
+		while ((tagMatch = tagTextRe.exec(line)) !== null) {
+			const text = tagMatch[1].trim();
+			if (text.length === 0) continue;
+			if (/^[\s\W]*$/.test(text) && !/[a-zA-Z]/.test(text)) continue;
+			results.push({
+				value: text,
+				line: i + 1,
+				column: tagMatch.index + 1,
+				context: "template_text",
+				scope: "template",
+				parent: "",
+				surrounding: buildSurrounding(lines, i)
+			});
+		}
+	}
+	return results;
+}
+function buildSurrounding(lines, lineIdx) {
+	const start = Math.max(0, lineIdx - 1);
+	const end = Math.min(lines.length - 1, lineIdx + 1);
+	const parts = [];
+	for (let i = start; i <= end; i++) {
+		const line = lines[i];
+		if (line !== void 0) parts.push(line);
+	}
+	const joined = parts.join("\n");
+	if (joined.length > SURROUNDING_MAX) return joined.slice(0, SURROUNDING_MAX);
+	return joined;
+}
+/**
+* Extract strings from a source file with structural context.
+*
+* Routes to the correct parser based on file extension:
+* - .tsx/.jsx/.ts/.js/.mjs -> tsx-parser (AST-based)
+* - .vue -> vue-parser (lazy-loaded, AST-based)
+* - .svelte -> svelte-parser (lazy-loaded, AST-based)
+* - .astro -> astro-parser (lazy-loaded, AST-based)
+* - unknown -> empty array
+*
+* Falls back to regex extraction when the dedicated parser's
+* optional dependency is not installed.
+*
+* Applies structural pre-filter to remove 100% non-content strings.
+* Returns only candidates that should be sent to the agent.
+*/
+async function extractStrings(filePath, content, ext) {
+	const normalizedExt = ext.startsWith(".") ? ext : `.${ext}`;
+	let rawStrings;
+	if (TSX_EXTENSIONS.has(normalizedExt)) rawStrings = parseTsx(content, filePath);
+	else if (VUE_EXTENSIONS.has(normalizedExt)) {
+		const parseVue = await loadVueParser();
+		if (parseVue) rawStrings = await parseVue(content, filePath);
+		else rawStrings = extractWithRegex(content, filePath);
+	} else if (SVELTE_EXTENSIONS.has(normalizedExt)) {
+		const parseSvelte = await loadSvelteParser();
+		if (parseSvelte) try {
+			rawStrings = await parseSvelte(content, filePath);
+		} catch {
+			rawStrings = extractWithRegex(content, filePath);
+		}
+		else rawStrings = extractWithRegex(content, filePath);
+	} else if (ASTRO_EXTENSIONS.has(normalizedExt)) {
+		const parseAstro = await loadAstroParser();
+		if (parseAstro) try {
+			rawStrings = await parseAstro(content, filePath);
+		} catch {
+			rawStrings = extractWithRegex(content, filePath);
+		}
+		else rawStrings = extractWithRegex(content, filePath);
+	} else return [];
+	const { candidates } = applyPreFilter(rawStrings);
+	return candidates;
+}
+//#endregion
+//#region src/core/scanner.ts
+const DEFAULT_LIMIT = 50;
+const DEFAULT_OFFSET = 0;
+const DEFAULT_MIN_LENGTH = 2;
+const DEFAULT_MAX_LENGTH = 500;
+const SUMMARY_SAMPLE_SIZE = 10;
+const CONTEXT_MAP = {
+	"template_text": "template_text",
+	"template_attribute": "template_attribute",
+	"jsx_text": "jsx_text",
+	"jsx_attribute": "jsx_attribute",
+	"variable_assignment": "variable_assignment",
+	"object_property": "object_value",
+	"function_argument": "function_argument",
+	"array_element": "other",
+	"enum_value": "other",
+	"template_literal": "other",
+	"switch_case": "other",
+	"other": "other",
+	"import_path": "other",
+	"type_annotation": "other",
+	"css_class": "other",
+	"css_utility_call": "other",
+	"console_call": "other",
+	"test_assertion": "other"
+};
+const IMPORT_PATH_RE = /^(\.\/|\.\.\/|@\/|@[a-z]|[a-z][a-z0-9-]*\/)/;
+const CSS_CLASS_PREFIX_RE = /^(flex|grid|block|inline|hidden|absolute|relative|fixed|sticky|p-|m-|w-|h-|bg-|text-|border-|rounded|shadow|gap-|space-|font-|leading-|tracking-|opacity-|z-|overflow-|cursor-|transition|transform|animate-|hover:|focus:|active:|dark:|sm:|md:|lg:|xl:)/;
+const URL_ROUTE_RE = /^(https?:\/\/|\/api\/|\/v[0-9]|#\/|mailto:|tel:)/;
+const ROUTE_LIKE_RE = /^\/[a-z0-9/:_-]*$/i;
+const COLOR_RE = /^(#[0-9a-f]{3,8}|rgba?\(|hsla?\(|transparent|inherit|currentColor)$/i;
+const CAMEL_CASE_RE = /^[a-z][a-zA-Z0-9]*$/;
+const SCREAMING_SNAKE_RE = /^[A-Z][A-Z0-9_]+$/;
+const KEBAB_TECH_RE = /^[a-z][a-z0-9]*(-[a-z0-9]+)+$/;
+const FILE_EXT_RE = /\.(png|jpg|jpeg|gif|svg|webp|ico|css|scss|less|js|ts|tsx|jsx|json|md|html|xml|yaml|yml|woff|woff2|ttf|eot|mp4|webm|mp3|wav|pdf)$/i;
+const REGEX_LIKE_RE = /^\^|^\(.*\)$|\$$|\\d|\\w|\\s|\.\*|\.\+/;
+const CONSOLE_RE = /console\.(log|warn|error|debug|info|trace)\s*\(/;
+const CONTENT_WORDS = new Set([
+	"sign",
+	"up",
+	"log",
+	"in",
+	"out",
+	"get",
+	"started",
+	"learn",
+	"more",
+	"read",
+	"click",
+	"here",
+	"go",
+	"back",
+	"next",
+	"prev",
+	"previous",
+	"see",
+	"all",
+	"view",
+	"show",
+	"hide",
+	"open",
+	"close",
+	"add",
+	"new",
+	"save",
+	"edit",
+	"delete",
+	"remove",
+	"submit",
+	"cancel",
+	"confirm",
+	"welcome",
+	"hello",
+	"thank",
+	"you",
+	"good",
+	"bye",
+	"hi",
+	"hey",
+	"no",
+	"yes",
+	"ok",
+	"sorry",
+	"please",
+	"help",
+	"about",
+	"contact",
+	"home",
+	"page",
+	"not",
+	"found",
+	"error",
+	"success",
+	"loading"
+]);
+/**
+* Determines if a string is non-content (technical) and should be filtered out.
+* Pure function — no side effects, no semantic decisions.
+* Conservative: if in doubt, returns false (keeps the string).
+*
+* @deprecated Kept for backward compatibility. Scanner v2 uses structural pre-filter instead.
+*/
+function isNonContent(value, lineContext, minLength = DEFAULT_MIN_LENGTH, maxLength = DEFAULT_MAX_LENGTH) {
+	if (value.length < minLength) return true;
+	if (value.length > maxLength) return true;
+	if (IMPORT_PATH_RE.test(value) && !value.includes(" ")) return true;
+	if (CSS_CLASS_PREFIX_RE.test(value) && !value.includes(" ")) return true;
+	if (value.includes(" ") && looksLikeCssClasses(value)) return true;
+	if (URL_ROUTE_RE.test(value)) return true;
+	if (value.startsWith("/") && ROUTE_LIKE_RE.test(value)) return true;
+	if (COLOR_RE.test(value)) return true;
+	if (isTechnicalIdentifier(value)) return true;
+	if (FILE_EXT_RE.test(value)) return true;
+	if (REGEX_LIKE_RE.test(value)) return true;
+	if (CONSOLE_RE.test(lineContext)) return true;
+	return false;
+}
+function isTechnicalIdentifier(value) {
+	if (value.includes(" ")) return false;
+	if (CAMEL_CASE_RE.test(value) && /[A-Z]/.test(value)) return true;
+	if (SCREAMING_SNAKE_RE.test(value) && value.includes("_")) return true;
+	if (KEBAB_TECH_RE.test(value)) {
+		if (!value.split("-").some((p) => CONTENT_WORDS.has(p))) return true;
+	}
+	if (/^[a-z][a-z0-9-]*$/.test(value) && value.length < 40 && !value.includes(" ")) {
+		if (value.includes("/")) return true;
+	}
+	return false;
+}
+function looksLikeCssClasses(value) {
+	const tokens = value.split(/\s+/);
+	if (tokens.length < 2) return false;
+	return tokens.filter((t) => isCssClassToken(t)).length === tokens.length;
+}
+function isCssClassToken(token) {
+	if (CSS_CLASS_PREFIX_RE.test(token)) return true;
+	if (/^(flex|grid|block|inline|hidden|container|relative|absolute|fixed|sticky|static|isolate|overflow-hidden|overflow-auto|overflow-scroll|underline|italic|uppercase|lowercase|capitalize|truncate|break-words|break-all|whitespace-nowrap|sr-only|not-sr-only)$/.test(token)) return true;
+	if (/^[a-z]+-\[.+\]$/.test(token)) return true;
+	if (/^(sm|md|lg|xl|2xl|hover|focus|active|disabled|first|last|odd|even|dark|group-hover|peer-checked):/.test(token)) return true;
+	return false;
+}
+/**
+* Apply legacy isNonContent filter as a secondary pass on v2 candidates.
+* This catches heuristic patterns (URLs, regex, camelCase identifiers, etc.)
+* that the structural pre-filter does not cover.
+*/
+function applySecondaryFilter(value, _surrounding, minLength, maxLength) {
+	return isNonContent(value, "", minLength, maxLength);
+}
+async function scanCandidates(projectRoot, options) {
+	const limit = options?.limit ?? DEFAULT_LIMIT;
+	const offset = options?.offset ?? DEFAULT_OFFSET;
+	const minLength = options?.min_length ?? DEFAULT_MIN_LENGTH;
+	const maxLength = options?.max_length ?? DEFAULT_MAX_LENGTH;
+	const files = await discoverFiles(projectRoot, {
+		paths: options?.paths ?? await autoDetectSourceDirs(projectRoot),
+		include: options?.include,
+		exclude: options?.exclude
+	});
+	const allCandidates = [];
+	const dupeMap = /* @__PURE__ */ new Map();
+	let rawStringsFound = 0;
+	const filePromises = files.map(async (relPath) => {
+		const filePath = join(projectRoot, relPath);
+		const content = await readText(filePath);
+		if (!content) return {
+			relPath,
+			extractions: []
+		};
+		return {
+			relPath,
+			extractions: await extractStrings(filePath, content, extname(filePath))
+		};
+	});
+	const fileResults = await Promise.all(filePromises);
+	for (const { relPath, extractions } of fileResults) {
+		rawStringsFound += extractions.length;
+		for (const extraction of extractions) {
+			if (applySecondaryFilter(extraction.value, extraction.surrounding, minLength, maxLength)) continue;
+			const mappedContext = CONTEXT_MAP[extraction.context];
+			const candidate = {
+				file: relPath,
+				line: extraction.line,
+				column: extraction.column,
+				value: extraction.value,
+				context: mappedContext,
+				surrounding: extraction.surrounding
+			};
+			allCandidates.push(candidate);
+			const key = extraction.value;
+			if (!dupeMap.has(key)) dupeMap.set(key, []);
+			dupeMap.get(key).push({
+				file: relPath,
+				line: extraction.line
+			});
+		}
+	}
+	const duplicates = [...dupeMap.entries()].filter(([, occurrences]) => occurrences.length >= 2).map(([value, occurrences]) => ({
+		value,
+		count: occurrences.length,
+		occurrences
+	})).toSorted((a, b) => b.count - a.count);
+	const afterFiltering = allCandidates.length;
+	const paginated = allCandidates.slice(offset, offset + limit);
+	const hasMore = afterFiltering > offset + limit;
+	return {
+		candidates: paginated,
+		duplicates,
+		stats: {
+			files_scanned: files.length,
+			raw_strings_found: rawStringsFound,
+			after_filtering: afterFiltering,
+			candidates_returned: paginated.length,
+			has_more: hasMore
+		}
+	};
+}
+async function scanSummary(projectRoot, options) {
+	const minLength = options?.min_length ?? DEFAULT_MIN_LENGTH;
+	const maxLength = options?.max_length ?? DEFAULT_MAX_LENGTH;
+	const files = await discoverFiles(projectRoot, {
+		paths: options?.paths ?? await autoDetectSourceDirs(projectRoot),
+		include: options?.include,
+		exclude: options?.exclude
+	});
+	const dirFiles = /* @__PURE__ */ new Map();
+	const fileTypes = {};
+	for (const relPath of files) {
+		const dir = relPath.split("/").slice(0, -1).join("/") || ".";
+		const ext = extname(relPath);
+		if (!dirFiles.has(dir)) dirFiles.set(dir, []);
+		dirFiles.get(dir).push(relPath);
+		fileTypes[ext] = (fileTypes[ext] ?? 0) + 1;
+	}
+	const byDirectory = {};
+	const freqMap = /* @__PURE__ */ new Map();
+	let totalCandidatesEstimate = 0;
+	for (const [dir, dirFileList] of dirFiles) {
+		const totalInDir = dirFileList.length;
+		const sampleFiles = dirFileList.slice(0, SUMMARY_SAMPLE_SIZE);
+		let sampleCandidates = 0;
+		const samplePromises = sampleFiles.map(async (relPath) => {
+			const filePath = join(projectRoot, relPath);
+			const content = await readText(filePath);
+			if (!content) return [];
+			return extractStrings(filePath, content, extname(filePath));
+		});
+		const sampleResults = await Promise.all(samplePromises);
+		for (const extractions of sampleResults) for (const extraction of extractions) {
+			if (applySecondaryFilter(extraction.value, extraction.surrounding, minLength, maxLength)) continue;
+			sampleCandidates++;
+			const prev = freqMap.get(extraction.value) ?? 0;
+			freqMap.set(extraction.value, prev + 1);
+		}
+		const avgPerFile = sampleFiles.length > 0 ? sampleCandidates / sampleFiles.length : 0;
+		const estimatedCandidates = Math.round(avgPerFile * totalInDir);
+		byDirectory[dir] = {
+			files: totalInDir,
+			candidates: estimatedCandidates
+		};
+		totalCandidatesEstimate += estimatedCandidates;
+	}
+	const topRepeated = [...freqMap.entries()].filter(([, count]) => count >= 2).toSorted((a, b) => b[1] - a[1]).slice(0, 20).map(([value, count]) => ({
+		value,
+		count
+	}));
+	return {
+		total_files: files.length,
+		total_candidates_estimate: totalCandidatesEstimate,
+		by_directory: byDirectory,
+		top_repeated: topRepeated,
+		sampling_note: `Based on first ${SUMMARY_SAMPLE_SIZE} files per directory. Counts are from sampled subset, not project-wide.`,
+		file_types: fileTypes
+	};
+}
+//#endregion
+export { scanCandidates as n, scanSummary as r, isNonContent as t };
+
+//# sourceMappingURL=scanner-BdTiv8sw.mjs.map