@omfalos/mokosh 0.3.3 → 0.4.0

package/dist/duplication-worker.d.mts

@@ -1,19 +1,6 @@
+import { N as NormalizedToken } from './tokenizer-BaF1eUBQ.mjs';
 import { F as FileType } from './parse-CAKctgb6.mjs';
 
-/**
- * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language
- * comment syntax, then splits what remains into a normalized token stream shared by every
- * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language
- * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, CoffeeScript,
- * LiveScript, Lua, Gherkin, style files, and Markdown.
- */
-
-/** One normalized token plus the 1-based source line it came from. */
-interface NormalizedToken {
-    text: string;
-    line: number;
-}
-
 /** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */
 
 declare function tokenizeInWorker(payload: {

package/dist/duplication-worker.d.ts

@@ -1,19 +1,6 @@
+import { N as NormalizedToken } from './tokenizer-BaF1eUBQ.js';
 import { F as FileType } from './parse-CAKctgb6.js';
 
-/**
- * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language
- * comment syntax, then splits what remains into a normalized token stream shared by every
- * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language
- * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, CoffeeScript,
- * LiveScript, Lua, Gherkin, style files, and Markdown.
- */
-
-/** One normalized token plus the 1-based source line it came from. */
-interface NormalizedToken {
-    text: string;
-    line: number;
-}
-
 /** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */
 
 declare function tokenizeInWorker(payload: {

package/dist/index.d.mts

@@ -1,5 +1,6 @@
 import { F as FileNode, C as CallEdge, I as ImportEdge, P as ParseResult, S as StructuredTag } from './types-BtSqoqbZ.mjs';
 export { E as ExportedSymbol } from './types-BtSqoqbZ.mjs';
+import { N as NormalizedToken } from './tokenizer-BaF1eUBQ.mjs';
 import { F as FileType, N as NodeCategory } from './parse-CAKctgb6.mjs';
 export { I as ImportType, T as TagKind } from './parse-CAKctgb6.mjs';
 
@@ -499,6 +500,22 @@ type ParallelTokenizingOption = boolean | {
     maxThreads?: number;
 };
 
+/** One file's cached tokenize result, fingerprinted by `mtime`/`size`/`ignoreLiterals` — any
+ *  mismatch against the current `FileNode` (or against the `ignoreLiterals` this scan is running
+ *  with) means the entry is stale and must be recomputed, exactly like `GraphBuilder`'s
+ *  mtime+size node reuse for incremental graph builds. */
+interface CachedFileTokens {
+    mtime: number;
+    size: number;
+    ignoreLiterals: boolean;
+    tokens: NormalizedToken[];
+}
+/** Caller-owned cache, keyed by project-relative path, reused across repeated `findDuplicates`
+ *  calls against the same root (e.g. successive MCP tool calls in one session) so unchanged files
+ *  never pay tokenizing cost twice. `findDuplicates` itself is stateless — callers that want this
+ *  benefit own the `Map` and pass it in; the CLI's one-shot process has nothing to gain and omits
+ *  it. See docs/adr-014-duplicate-detection-scale.md. */
+type DuplicationTokenCache = Map<string, CachedFileTokens>;
 interface FindDuplicatesOptions {
     /** Minimum duplicated block size, in source lines, to report (default 6). */
     minLines?: number | undefined;
@@ -514,11 +531,6 @@ interface FindDuplicatesOptions {
      *  CSS/Less/SCSS structural comparator, which already matches on literal declaration content.
      *  Set to 1 to disable. See docs/adr-013-duplicate-detection-noise-reduction.md. */
     maxPunctuationRatio?: number | undefined;
-    /** Skip a hash bucket's O(k²) pairwise comparison once it holds more than this many locations
-     *  (default 400) — bounds worst-case scan time on large repos where a single ubiquitous token
-     *  window (a common import line, a boilerplate header) would otherwise blow past what a single
-     *  scan can finish in. Set `Infinity` to disable. See docs/adr-014-duplicate-detection-scale.md. */
-    maxBucketSize?: number | undefined;
     /** Caps the number of duplicate blocks returned, largest-first (default 50). */
     limit?: number | undefined;
     /** Directory names to exclude, matched against any path segment (default `DEFAULT_IGNORE_DIRS`
@@ -529,13 +541,15 @@ interface FindDuplicatesOptions {
      *  `false` always tokenizes in-process; an object overrides `minFiles`/`maxThreads`. See
      *  docs/adr-014-duplicate-detection-scale.md. */
     parallelTokenizing?: ParallelTokenizingOption | undefined;
+    /** Optional caller-owned cache reused across calls against the same root — files whose
+     *  `mtime`/`size` are unchanged since the cached entry (and whose `ignoreLiterals` matches this
+     *  call's) skip tokenizing entirely. Mutated in place; omit for one-shot callers (e.g. the CLI).
+     *  See {@link DuplicationTokenCache} and docs/adr-014-duplicate-detection-scale.md. */
+    tokenCache?: DuplicationTokenCache | undefined;
 }
 interface FindDuplicatesResult {
     /** Duplicate blocks, largest-first, capped at `limit`. */
     groups: DuplicateGroup[];
-    /** How many hash buckets were skipped for exceeding `maxBucketSize` — a non-zero count means
-     *  results may under-report duplication that's unusually widespread (see `maxBucketSize`). */
-    skippedBuckets: number;
 }
 /**
  * @description Scans every file already present in `graph` for cross-file (and within-file)
@@ -557,15 +571,13 @@ interface FindDuplicatesResult {
  *   token-shingle path only (CSS/Less/SCSS always match on literal declaration content);
  *   `maxPunctuationRatio` gates out token-shingle blocks that are mostly object/array-literal
  *   structural punctuation (e.g. schema/object-literal boilerplate) rather than substantive
- *   shared logic; `maxBucketSize` bounds worst-case scan time on large repos by skipping
- *   pathologically common hash buckets; `ignoreDirs` excludes files under matching directory
- *   names; `limit` caps results; `parallelTokenizing` offloads per-file tokenizing to a worker
- *   pool once the candidate file count is large enough to be worth it. Lock files are always
- *   excluded, independent of `ignoreDirs`.
+ *   shared logic; `ignoreDirs` excludes files under matching directory names; `limit` caps
+ *   results; `parallelTokenizing` offloads per-file tokenizing to a worker pool once the
+ *   candidate file count is large enough to be worth it. Lock files are always excluded,
+ *   independent of `ignoreDirs`.
  * @returns `groups` — duplicate blocks (each tagged with its `family`), two or more occurrences
  *   per block, every block that pairwise chain-matches another clustered into one group instead
- *   of one per pair, sorted largest-first across all families — plus `skippedBuckets` (see
- *   {@link FindDuplicatesResult}).
+ *   of one per pair, sorted largest-first across all families.
  */
 declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<FindDuplicatesResult>;
 
@@ -1430,4 +1442,4 @@ declare function createWorkspaceGraph(rootDir: string, options?: {
  */
 declare function getAllProjectFiles(rootDir: string, options?: ScanOptions): string[];
 
-export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };
+export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, type CachedFileTokens, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, type DuplicationTokenCache, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FindDuplicatesResult, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 import { F as FileNode, C as CallEdge, I as ImportEdge, P as ParseResult, S as StructuredTag } from './types-BlN-U5AM.js';
 export { E as ExportedSymbol } from './types-BlN-U5AM.js';
+import { N as NormalizedToken } from './tokenizer-BaF1eUBQ.js';
 import { F as FileType, N as NodeCategory } from './parse-CAKctgb6.js';
 export { I as ImportType, T as TagKind } from './parse-CAKctgb6.js';
 
@@ -499,6 +500,22 @@ type ParallelTokenizingOption = boolean | {
     maxThreads?: number;
 };
 
+/** One file's cached tokenize result, fingerprinted by `mtime`/`size`/`ignoreLiterals` — any
+ *  mismatch against the current `FileNode` (or against the `ignoreLiterals` this scan is running
+ *  with) means the entry is stale and must be recomputed, exactly like `GraphBuilder`'s
+ *  mtime+size node reuse for incremental graph builds. */
+interface CachedFileTokens {
+    mtime: number;
+    size: number;
+    ignoreLiterals: boolean;
+    tokens: NormalizedToken[];
+}
+/** Caller-owned cache, keyed by project-relative path, reused across repeated `findDuplicates`
+ *  calls against the same root (e.g. successive MCP tool calls in one session) so unchanged files
+ *  never pay tokenizing cost twice. `findDuplicates` itself is stateless — callers that want this
+ *  benefit own the `Map` and pass it in; the CLI's one-shot process has nothing to gain and omits
+ *  it. See docs/adr-014-duplicate-detection-scale.md. */
+type DuplicationTokenCache = Map<string, CachedFileTokens>;
 interface FindDuplicatesOptions {
     /** Minimum duplicated block size, in source lines, to report (default 6). */
     minLines?: number | undefined;
@@ -514,11 +531,6 @@ interface FindDuplicatesOptions {
      *  CSS/Less/SCSS structural comparator, which already matches on literal declaration content.
      *  Set to 1 to disable. See docs/adr-013-duplicate-detection-noise-reduction.md. */
     maxPunctuationRatio?: number | undefined;
-    /** Skip a hash bucket's O(k²) pairwise comparison once it holds more than this many locations
-     *  (default 400) — bounds worst-case scan time on large repos where a single ubiquitous token
-     *  window (a common import line, a boilerplate header) would otherwise blow past what a single
-     *  scan can finish in. Set `Infinity` to disable. See docs/adr-014-duplicate-detection-scale.md. */
-    maxBucketSize?: number | undefined;
     /** Caps the number of duplicate blocks returned, largest-first (default 50). */
     limit?: number | undefined;
     /** Directory names to exclude, matched against any path segment (default `DEFAULT_IGNORE_DIRS`
@@ -529,13 +541,15 @@ interface FindDuplicatesOptions {
      *  `false` always tokenizes in-process; an object overrides `minFiles`/`maxThreads`. See
      *  docs/adr-014-duplicate-detection-scale.md. */
     parallelTokenizing?: ParallelTokenizingOption | undefined;
+    /** Optional caller-owned cache reused across calls against the same root — files whose
+     *  `mtime`/`size` are unchanged since the cached entry (and whose `ignoreLiterals` matches this
+     *  call's) skip tokenizing entirely. Mutated in place; omit for one-shot callers (e.g. the CLI).
+     *  See {@link DuplicationTokenCache} and docs/adr-014-duplicate-detection-scale.md. */
+    tokenCache?: DuplicationTokenCache | undefined;
 }
 interface FindDuplicatesResult {
     /** Duplicate blocks, largest-first, capped at `limit`. */
     groups: DuplicateGroup[];
-    /** How many hash buckets were skipped for exceeding `maxBucketSize` — a non-zero count means
-     *  results may under-report duplication that's unusually widespread (see `maxBucketSize`). */
-    skippedBuckets: number;
 }
 /**
  * @description Scans every file already present in `graph` for cross-file (and within-file)
@@ -557,15 +571,13 @@ interface FindDuplicatesResult {
  *   token-shingle path only (CSS/Less/SCSS always match on literal declaration content);
  *   `maxPunctuationRatio` gates out token-shingle blocks that are mostly object/array-literal
  *   structural punctuation (e.g. schema/object-literal boilerplate) rather than substantive
- *   shared logic; `maxBucketSize` bounds worst-case scan time on large repos by skipping
- *   pathologically common hash buckets; `ignoreDirs` excludes files under matching directory
- *   names; `limit` caps results; `parallelTokenizing` offloads per-file tokenizing to a worker
- *   pool once the candidate file count is large enough to be worth it. Lock files are always
- *   excluded, independent of `ignoreDirs`.
+ *   shared logic; `ignoreDirs` excludes files under matching directory names; `limit` caps
+ *   results; `parallelTokenizing` offloads per-file tokenizing to a worker pool once the
+ *   candidate file count is large enough to be worth it. Lock files are always excluded,
+ *   independent of `ignoreDirs`.
  * @returns `groups` — duplicate blocks (each tagged with its `family`), two or more occurrences
  *   per block, every block that pairwise chain-matches another clustered into one group instead
- *   of one per pair, sorted largest-first across all families — plus `skippedBuckets` (see
- *   {@link FindDuplicatesResult}).
+ *   of one per pair, sorted largest-first across all families.
  */
 declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<FindDuplicatesResult>;
 
@@ -1430,4 +1442,4 @@ declare function createWorkspaceGraph(rootDir: string, options?: {
  */
 declare function getAllProjectFiles(rootDir: string, options?: ScanOptions): string[];
 
-export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };
+export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, type CachedFileTokens, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, type DuplicationTokenCache, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FindDuplicatesResult, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };