@omfalos/mokosh 0.3.0 → 0.3.2

package/dist/index.d.mts

@@ -442,6 +442,102 @@ declare function saveChangeImpactCache(cache: ChangeImpactCache, cachePath: stri
  */
 declare function loadChangeImpactCache(cachePath: string): ChangeImpactCache | null;
 
+/**
+ * Language-family partitioning for duplicate detection. `findDuplicates` never compares files
+ * across a family boundary — see docs/adr-013-duplicate-detection-noise-reduction.md. The
+ * immediate driver is CSS-family declarations: they share a small, finite vocabulary (property
+ * names, common value keywords) that the shared `KEYWORDS` denylist in `tokenizer.ts` doesn't
+ * cover, so unrelated style rules with the same declaration *shape* but different selectors/
+ * properties/values were hashing identically to unrelated TS/JS/Python shapes too. Splitting by
+ * family also lets later phases tune `windowSize`/`minLines`/vocabulary per family without one
+ * family's tuning affecting another's.
+ */
+
+/** One partition of `findDuplicates` matching — files in different families are never compared. */
+type DuplicateFamily = "style" | "code";
+
+/**
+ * Sliding-window (shingle) hashing over a normalized token stream, plus chain-merging of
+ * consecutive matching windows into contiguous duplicate blocks, a structural-punctuation-density
+ * gate that drops blocks that are mostly object/array-literal shape (e.g. MCP tool `inputSchema`
+ * boilerplate) rather than substantive shared logic, and exact-occurrence clustering of pair-
+ * matches into one N-occurrence group instead of reporting C(N,2) near-identical pairs for a block
+ * repeated N times. Shared by every language `tokenize()` supports — the shingling step itself has
+ * no language awareness at all. See docs/adr-013-duplicate-detection-noise-reduction.md for the
+ * noise this addresses.
+ */
+
+interface DuplicateOccurrence {
+    file: string;
+    startLine: number;
+    endLine: number;
+}
+interface DuplicateGroup {
+    /** Every location sharing this duplicated block (two or more) — locations that pairwise
+     *  chain-match are clustered into one group instead of being reported once per pair, so a
+     *  block repeated N times produces one N-occurrence group, not C(N,2) near-identical ones. */
+    occurrences: DuplicateOccurrence[];
+    /** Line span of the largest single pairwise match clustered into this group (each pair's own
+     *  span is the shorter of its two occurrences) — the best-verified size for this block, not an
+     *  average or a value shrunk by a more weakly-matching cluster member. */
+    lines: number;
+    /** Token-window length backing this block, after chain-merging adjacent windows. */
+    tokens: number;
+    /** Which language family both occurrences belong to (set by `findDuplicates`, which never
+     *  matches across families — see docs/adr-013-duplicate-detection-noise-reduction.md).
+     *  Absent when called directly with a token stream that isn't family-scoped, e.g. in tests. */
+    family?: DuplicateFamily | undefined;
+}
+
+interface FindDuplicatesOptions {
+    /** Minimum duplicated block size, in source lines, to report (default 6). */
+    minLines?: number | undefined;
+    /** Shingle window size, in tokens — the smallest duplicate the scan can detect (default 15). */
+    windowSize?: number | undefined;
+    /** When true (default), string/number literals are normalized too, so only structural shape
+     *  — not the specific values used — drives a match. Set false for stricter, Type-1-only matching. */
+    ignoreLiterals?: boolean | undefined;
+    /** Maximum fraction of a token-shingled block's window that may be object/array-literal
+     *  structural punctuation (`{ } : , [ ]`) (default 0.5) — gates out blocks that are mostly
+     *  schema/object-literal shape (e.g. MCP tool `inputSchema` boilerplate repeated across
+     *  unrelated tool definitions) rather than substantive shared logic. Does not apply to the
+     *  CSS/Less/SCSS structural comparator, which already matches on literal declaration content.
+     *  Set to 1 to disable. See docs/adr-013-duplicate-detection-noise-reduction.md. */
+    maxPunctuationRatio?: number | undefined;
+    /** Caps the number of duplicate blocks returned, largest-first (default 50). */
+    limit?: number | undefined;
+    /** Directory names to exclude, matched against any path segment (default `DEFAULT_IGNORE_DIRS`
+     *  — `node_modules`, `dist`, `.git`, `mokosh-cache`, `coverage`, etc.). Pass `[]` to disable. */
+    ignoreDirs?: readonly string[] | undefined;
+}
+/**
+ * @description Scans every file already present in `graph` for cross-file (and within-file)
+ *   duplicated code. CSS/Less/SCSS files are compared structurally — by their rule bodies'
+ *   literal, ordered `property: value` declarations, independent of selector name — via
+ *   {@link findStyleBlockDuplicates}. Every other language (TS/JS, Python, Go, CoffeeScript,
+ *   LiveScript, Lua, Gherkin, Markdown, and Stylus, which has no shared PostCSS AST here) runs
+ *   the generic token-shingling pipeline instead: comments are stripped per `FileType`,
+ *   identifiers (and, by default, literals) are normalized to placeholders so renamed-variable
+ *   copies still match, then a sliding token window is hashed and chain-merged into contiguous
+ *   blocks. Token-shingled files are additionally partitioned into language families
+ *   ({@link getDuplicateFamily} — `"style"` for Stylus, `"code"` for everything else) so
+ *   matching never crosses that boundary — see docs/adr-013-duplicate-detection-noise-reduction.md.
+ * @param graph - The graph to scan; its node paths (already ignore-rule-filtered) are the file
+ *   list, re-read from disk since duplication data isn't cached on `FileNode`.
+ * @param rootDir - Absolute project root that graph paths are relative to.
+ * @param options - `minLines`/`windowSize` tune token-shingle sensitivity (`minLines` also caps
+ *   CSS/Less/SCSS block size); `ignoreLiterals` toggles Type-2 vs Type-1 matching for the
+ *   token-shingle path only (CSS/Less/SCSS always match on literal declaration content);
+ *   `maxPunctuationRatio` gates out token-shingle blocks that are mostly object/array-literal
+ *   structural punctuation (e.g. schema/object-literal boilerplate) rather than substantive
+ *   shared logic; `ignoreDirs` excludes files under matching directory names; `limit` caps
+ *   results. Lock files are always excluded, independent of `ignoreDirs`.
+ * @returns Duplicate blocks (each tagged with its `family`), two or more occurrences per block —
+ *   every block that pairwise chain-matches another is clustered into one group instead of one
+ *   per pair — sorted largest-first across all families.
+ */
+declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<DuplicateGroup[]>;
+
 /** Controls how aggressively `detectFeatures` promotes files to features. */
 interface FeatureDetectionOptions {
     /**
@@ -1303,4 +1399,4 @@ declare function createWorkspaceGraph(rootDir: string, options?: {
  */
 declare function getAllProjectFiles(rootDir: string, options?: ScanOptions): string[];
 
-export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };
+export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };

package/dist/index.d.ts

@@ -442,6 +442,102 @@ declare function saveChangeImpactCache(cache: ChangeImpactCache, cachePath: stri
  */
 declare function loadChangeImpactCache(cachePath: string): ChangeImpactCache | null;
 
+/**
+ * Language-family partitioning for duplicate detection. `findDuplicates` never compares files
+ * across a family boundary — see docs/adr-013-duplicate-detection-noise-reduction.md. The
+ * immediate driver is CSS-family declarations: they share a small, finite vocabulary (property
+ * names, common value keywords) that the shared `KEYWORDS` denylist in `tokenizer.ts` doesn't
+ * cover, so unrelated style rules with the same declaration *shape* but different selectors/
+ * properties/values were hashing identically to unrelated TS/JS/Python shapes too. Splitting by
+ * family also lets later phases tune `windowSize`/`minLines`/vocabulary per family without one
+ * family's tuning affecting another's.
+ */
+
+/** One partition of `findDuplicates` matching — files in different families are never compared. */
+type DuplicateFamily = "style" | "code";
+
+/**
+ * Sliding-window (shingle) hashing over a normalized token stream, plus chain-merging of
+ * consecutive matching windows into contiguous duplicate blocks, a structural-punctuation-density
+ * gate that drops blocks that are mostly object/array-literal shape (e.g. MCP tool `inputSchema`
+ * boilerplate) rather than substantive shared logic, and exact-occurrence clustering of pair-
+ * matches into one N-occurrence group instead of reporting C(N,2) near-identical pairs for a block
+ * repeated N times. Shared by every language `tokenize()` supports — the shingling step itself has
+ * no language awareness at all. See docs/adr-013-duplicate-detection-noise-reduction.md for the
+ * noise this addresses.
+ */
+
+interface DuplicateOccurrence {
+    file: string;
+    startLine: number;
+    endLine: number;
+}
+interface DuplicateGroup {
+    /** Every location sharing this duplicated block (two or more) — locations that pairwise
+     *  chain-match are clustered into one group instead of being reported once per pair, so a
+     *  block repeated N times produces one N-occurrence group, not C(N,2) near-identical ones. */
+    occurrences: DuplicateOccurrence[];
+    /** Line span of the largest single pairwise match clustered into this group (each pair's own
+     *  span is the shorter of its two occurrences) — the best-verified size for this block, not an
+     *  average or a value shrunk by a more weakly-matching cluster member. */
+    lines: number;
+    /** Token-window length backing this block, after chain-merging adjacent windows. */
+    tokens: number;
+    /** Which language family both occurrences belong to (set by `findDuplicates`, which never
+     *  matches across families — see docs/adr-013-duplicate-detection-noise-reduction.md).
+     *  Absent when called directly with a token stream that isn't family-scoped, e.g. in tests. */
+    family?: DuplicateFamily | undefined;
+}
+
+interface FindDuplicatesOptions {
+    /** Minimum duplicated block size, in source lines, to report (default 6). */
+    minLines?: number | undefined;
+    /** Shingle window size, in tokens — the smallest duplicate the scan can detect (default 15). */
+    windowSize?: number | undefined;
+    /** When true (default), string/number literals are normalized too, so only structural shape
+     *  — not the specific values used — drives a match. Set false for stricter, Type-1-only matching. */
+    ignoreLiterals?: boolean | undefined;
+    /** Maximum fraction of a token-shingled block's window that may be object/array-literal
+     *  structural punctuation (`{ } : , [ ]`) (default 0.5) — gates out blocks that are mostly
+     *  schema/object-literal shape (e.g. MCP tool `inputSchema` boilerplate repeated across
+     *  unrelated tool definitions) rather than substantive shared logic. Does not apply to the
+     *  CSS/Less/SCSS structural comparator, which already matches on literal declaration content.
+     *  Set to 1 to disable. See docs/adr-013-duplicate-detection-noise-reduction.md. */
+    maxPunctuationRatio?: number | undefined;
+    /** Caps the number of duplicate blocks returned, largest-first (default 50). */
+    limit?: number | undefined;
+    /** Directory names to exclude, matched against any path segment (default `DEFAULT_IGNORE_DIRS`
+     *  — `node_modules`, `dist`, `.git`, `mokosh-cache`, `coverage`, etc.). Pass `[]` to disable. */
+    ignoreDirs?: readonly string[] | undefined;
+}
+/**
+ * @description Scans every file already present in `graph` for cross-file (and within-file)
+ *   duplicated code. CSS/Less/SCSS files are compared structurally — by their rule bodies'
+ *   literal, ordered `property: value` declarations, independent of selector name — via
+ *   {@link findStyleBlockDuplicates}. Every other language (TS/JS, Python, Go, CoffeeScript,
+ *   LiveScript, Lua, Gherkin, Markdown, and Stylus, which has no shared PostCSS AST here) runs
+ *   the generic token-shingling pipeline instead: comments are stripped per `FileType`,
+ *   identifiers (and, by default, literals) are normalized to placeholders so renamed-variable
+ *   copies still match, then a sliding token window is hashed and chain-merged into contiguous
+ *   blocks. Token-shingled files are additionally partitioned into language families
+ *   ({@link getDuplicateFamily} — `"style"` for Stylus, `"code"` for everything else) so
+ *   matching never crosses that boundary — see docs/adr-013-duplicate-detection-noise-reduction.md.
+ * @param graph - The graph to scan; its node paths (already ignore-rule-filtered) are the file
+ *   list, re-read from disk since duplication data isn't cached on `FileNode`.
+ * @param rootDir - Absolute project root that graph paths are relative to.
+ * @param options - `minLines`/`windowSize` tune token-shingle sensitivity (`minLines` also caps
+ *   CSS/Less/SCSS block size); `ignoreLiterals` toggles Type-2 vs Type-1 matching for the
+ *   token-shingle path only (CSS/Less/SCSS always match on literal declaration content);
+ *   `maxPunctuationRatio` gates out token-shingle blocks that are mostly object/array-literal
+ *   structural punctuation (e.g. schema/object-literal boilerplate) rather than substantive
+ *   shared logic; `ignoreDirs` excludes files under matching directory names; `limit` caps
+ *   results. Lock files are always excluded, independent of `ignoreDirs`.
+ * @returns Duplicate blocks (each tagged with its `family`), two or more occurrences per block —
+ *   every block that pairwise chain-matches another is clustered into one group instead of one
+ *   per pair — sorted largest-first across all families.
+ */
+declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<DuplicateGroup[]>;
+
 /** Controls how aggressively `detectFeatures` promotes files to features. */
 interface FeatureDetectionOptions {
     /**
@@ -1303,4 +1399,4 @@ declare function createWorkspaceGraph(rootDir: string, options?: {
  */
 declare function getAllProjectFiles(rootDir: string, options?: ScanOptions): string[];
 
-export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };
+export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };