@skill-map/cli 0.45.1 → 0.47.0

package/dist/conformance/index.js

@@ -1,4 +1,6 @@
 // conformance/index.ts
+
+!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="e3097683-993f-599d-92ef-d1e724637586")}catch(e){}}();
 import { spawnSync } from "child_process";
 import { cpSync, existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "fs";
 import { tmpdir } from "os";
@@ -422,4 +424,5 @@ export {
   assertSpecRoot,
   runConformanceCase
 };
-//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map
+//# debugId=e3097683-993f-599d-92ef-d1e724637586

package/dist/index.js

@@ -1,4 +1,6 @@
 // kernel/i18n/registry.texts.ts
+
+!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="e5a349ec-19a4-5568-bcab-36d7ea370839")}catch(e){}}();
 var REGISTRY_TEXTS = {
   duplicateExtension: "Extension already registered: {{kind}}:{{qualifiedId}}",
   unknownKind: "Unknown extension kind: {{kind}}",
@@ -96,12 +98,11 @@ var Registry = class {
 import { existsSync as existsSync11, statSync as statSync4 } from "fs";
 import { isAbsolute as isAbsolute4, resolve as resolve11 } from "path";
 import { Tiktoken as Tiktoken2 } from "js-tiktoken/lite";
-import cl100k_base from "js-tiktoken/ranks/cl100k_base";
 
 // package.json
 var package_default = {
   name: "@skill-map/cli",
-  version: "0.45.1",
+  version: "0.47.0",
   description: "skill-map reference implementation \u2014 kernel + CLI + adapters.",
   license: "MIT",
   type: "module",
@@ -765,16 +766,13 @@ function strip(value) {
 // config/defaults.json
 var defaults_default = {
   schemaVersion: 1,
-  autoMigrate: true,
   allowEditSmFiles: false,
   tokenizer: "cl100k_base",
-  providers: [],
   roots: [],
   ignore: [],
   scan: {
     tokenize: true,
     strict: false,
-    followSymlinks: false,
     maxFileSizeBytes: 1048576,
     maxNodes: 256,
     watch: {
@@ -783,9 +781,6 @@ var defaults_default = {
     referencePaths: []
   },
   plugins: {},
-  history: {
-    share: false
-  },
   jobs: {
     ttlSeconds: 3600,
     graceMultiplier: 3,
@@ -796,9 +791,6 @@ var defaults_default = {
       completed: 2592e3,
       failed: null
     }
-  },
-  i18n: {
-    locale: "en"
   }
 };
 
@@ -2363,8 +2355,9 @@ async function* walkContent(roots, options) {
   if (!parser) throw new UnknownParserError(options.parser);
   const filter = options.ignoreFilter ?? buildIgnoreFilter();
   const extensions = options.extensions;
+  const sizeLimit = buildSizeLimit(options);
   for (const root of roots) {
-    for await (const file of walkRoot(root, root, filter, extensions)) {
+    for await (const file of walkRoot(root, root, filter, extensions, sizeLimit)) {
       const relPath = relative2(root, file).split(sep2).join("/");
       let raw;
       try {
@@ -2387,7 +2380,15 @@ async function* walkContent(roots, options) {
     }
   }
 }
-async function* walkRoot(root, current, filter, extensions) {
+function buildSizeLimit(options) {
+  const sizeLimit = {};
+  if (options.maxFileSizeBytes !== void 0) {
+    sizeLimit.maxFileSizeBytes = options.maxFileSizeBytes;
+  }
+  if (options.onOversizedFile) sizeLimit.onOversizedFile = options.onOversizedFile;
+  return sizeLimit;
+}
+async function* walkRoot(root, current, filter, extensions, sizeLimit) {
   let entries;
   try {
     entries = await readdir(current, { withFileTypes: true, encoding: "utf8" });
@@ -2401,11 +2402,16 @@ async function* walkRoot(root, current, filter, extensions) {
     if (filter.ignores(rel)) continue;
     if (entry.isSymbolicLink()) continue;
     if (entry.isDirectory()) {
-      yield* walkRoot(root, full, filter, extensions);
+      yield* walkRoot(root, full, filter, extensions, sizeLimit);
     } else if (entry.isFile() && hasMatchingExtension(name, extensions)) {
       try {
         const s = await lstat(full);
-        if (s.isFile()) yield full;
+        if (!s.isFile()) continue;
+        if (sizeLimit.maxFileSizeBytes !== void 0 && s.size > sizeLimit.maxFileSizeBytes) {
+          sizeLimit.onOversizedFile?.({ path: rel, bytes: s.size });
+          continue;
+        }
+        yield full;
       } catch {
       }
     }
@@ -2435,6 +2441,10 @@ function resolveProviderWalk(provider) {
       parser: read.parser
     };
     if (options?.ignoreFilter) walkOptions.ignoreFilter = options.ignoreFilter;
+    if (options?.maxFileSizeBytes !== void 0) {
+      walkOptions.maxFileSizeBytes = options.maxFileSizeBytes;
+    }
+    if (options?.onOversizedFile) walkOptions.onOversizedFile = options.onOversizedFile;
     return walkContent(roots, walkOptions);
   };
 }
@@ -2843,7 +2853,18 @@ async function walkAndExtract(opts) {
   const accum = createWalkAccumulators();
   const wctx = buildWalkContext(opts);
   const claimedPaths = /* @__PURE__ */ new Set();
-  const walkOptions = opts.ignoreFilter ? { ignoreFilter: opts.ignoreFilter } : {};
+  const oversizedFiles = [];
+  const oversizedSeen = /* @__PURE__ */ new Set();
+  const onOversizedFile = (info) => {
+    if (oversizedSeen.has(info.path)) return;
+    oversizedSeen.add(info.path);
+    oversizedFiles.push(info);
+  };
+  const walkOptions = {
+    ...opts.ignoreFilter ? { ignoreFilter: opts.ignoreFilter } : {},
+    onOversizedFile,
+    ...opts.maxFileSizeBytes !== void 0 ? { maxFileSizeBytes: opts.maxFileSizeBytes } : {}
+  };
   let filesWalked = 0;
   let index = 0;
   const effectiveMaxNodes = opts.overrideMaxNodes ?? opts.recommendedNodeLimit;
@@ -2876,6 +2897,7 @@ async function walkAndExtract(opts) {
     cachedPaths: accum.cachedPaths,
     frontmatterIssues: accum.frontmatterIssues,
     filesWalked,
+    oversizedFiles,
     recommendedNodeLimit: opts.recommendedNodeLimit,
     overrideMaxNodes: opts.overrideMaxNodes,
     capReached,
@@ -2926,7 +2948,11 @@ async function processRawNode(raw, provider, wctx, accum, claimedPaths, nextInde
   }
   claimedPaths.add(raw.path);
   const priorNode = wctx.priorNodesByPath.get(raw.path);
-  const nodeHashCacheEligible = wctx.opts.enableCache && wctx.opts.prior !== null && priorNode !== void 0 && priorNode.bodyHash === bodyHash && priorNode.frontmatterHash === frontmatterHash;
+  const nodeHashCacheEligible = wctx.opts.enableCache && // Tokenizer-change invalidation: when the resolved encoder differs
+  // from the one that produced the prior snapshot's counts, no node is
+  // cache-eligible, every node rebuilds so `buildNode` re-tokenizes
+  // with the current encoder. See `tokenizerChanged` on the options.
+  !wctx.opts.tokenizerChanged && wctx.opts.prior !== null && priorNode !== void 0 && priorNode.bodyHash === bodyHash && priorNode.frontmatterHash === frontmatterHash;
   const sidecarResolution = resolveSidecarOverlay(
     raw.path,
     raw.path,
@@ -3128,6 +3154,16 @@ function resolveSpecVersionSafe() {
     return "unknown";
   }
 }
+var DEFAULT_TOKENIZER = "cl100k_base";
+function resolveTokenizerName(name) {
+  return name === "o200k_base" ? "o200k_base" : DEFAULT_TOKENIZER;
+}
+async function loadTokenizerRanks(name) {
+  if (name === "o200k_base") {
+    return (await import("js-tiktoken/ranks/o200k_base")).default;
+  }
+  return (await import("js-tiktoken/ranks/cl100k_base")).default;
+}
 async function runScanWithRenames(_kernel, options) {
   return runScanInternal(_kernel, options);
 }
@@ -3137,7 +3173,7 @@ async function runScan(_kernel, options) {
 }
 async function runScanInternal(_kernel, options) {
   validateRoots(options.roots);
-  const setup = buildScanSetup(options);
+  const setup = await buildScanSetup(options);
   const { emitter, exts, hookDispatcher, encoder, prior, start } = setup;
   const scanStartedEvent = makeEvent("scan.started", { roots: options.roots });
   emitter.emit(scanStartedEvent);
@@ -3147,6 +3183,7 @@ async function runScanInternal(_kernel, options) {
     options.roots,
     exts.providers
   );
+  const tokenizerChanged = encoder !== null && prior !== null && prior.tokenizer !== setup.tokenizer;
   const walked = await walkAndExtract({
     providers: exts.providers,
     extractors: exts.extractors,
@@ -3156,6 +3193,7 @@ async function runScanInternal(_kernel, options) {
     encoder,
     strict: setup.strict,
     enableCache: setup.enableCache,
+    tokenizerChanged,
     prior,
     priorIndex: setup.priorIndex,
     priorExtractorRuns: setup.priorExtractorRuns,
@@ -3163,7 +3201,8 @@ async function runScanInternal(_kernel, options) {
     pluginStores: options.pluginStores,
     activeProvider: activeProviderId,
     recommendedNodeLimit: options.recommendedNodeLimit ?? 256,
-    overrideMaxNodes: options.overrideMaxNodes ?? null
+    overrideMaxNodes: options.overrideMaxNodes ?? null,
+    ...options.maxFileSizeBytes !== void 0 ? { maxFileSizeBytes: options.maxFileSizeBytes } : {}
   });
   const activeProvider = activeProviderId ? exts.providers.find((p) => p.id === activeProviderId) ?? null : null;
   const resolved = resolveSignals({
@@ -3270,13 +3309,14 @@ function buildReservedNodePaths(nodes, kindRegistry, reservedNamesByProviderKind
 function hasEntries(set) {
   return set !== void 0 && set.size > 0;
 }
-function buildScanSetup(options) {
+async function buildScanSetup(options) {
   const start = Date.now();
   const emitter = options.emitter ?? new InMemoryProgressEmitter();
   const exts = options.extensions ?? { providers: [], extractors: [], analyzers: [] };
   const hookDispatcher = makeHookDispatcher(exts.hooks ?? [], emitter);
   const tokenize = options.tokenize !== false;
-  const encoder = tokenize ? new Tiktoken2(cl100k_base) : null;
+  const tokenizer = resolveTokenizerName(options.tokenizer);
+  const encoder = tokenize ? new Tiktoken2(await loadTokenizerRanks(tokenizer)) : null;
   const prior = options.priorSnapshot ?? null;
   const priorIndex = indexPriorSnapshot(prior);
   const providerFrontmatter = buildProviderFrontmatterValidator(exts.providers);
@@ -3287,6 +3327,7 @@ function buildScanSetup(options) {
     exts,
     hookDispatcher,
     encoder,
+    tokenizer,
     prior,
     priorIndex,
     priorExtractorRuns: options.priorExtractorRuns,
@@ -3322,6 +3363,7 @@ function buildScanStats(walked, issues, start) {
     // Providers compete.
     filesWalked: walked.filesWalked,
     filesSkipped: 0,
+    filesOversized: walked.oversizedFiles.length,
     nodesCount: walked.nodes.length,
     linksCount: walked.internalLinks.length,
     issuesCount: issues.length,
@@ -3336,8 +3378,10 @@ function buildScanReturn(walked, issues, renameOps, stats, options, setup) {
       roots: options.roots,
       providers: setup.exts.providers.map((a) => a.id),
       scannedBy: SCANNED_BY,
+      tokenizer: setup.tokenizer,
       recommendedNodeLimit: walked.recommendedNodeLimit,
       overrideMaxNodes: walked.overrideMaxNodes,
+      oversizedFiles: walked.oversizedFiles,
       nodes: walked.nodes,
       links: walked.internalLinks,
       issues,
@@ -3775,4 +3819,5 @@ export {
   runScan,
   runScanWithRenames
 };
-//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map
+//# debugId=e5a349ec-19a4-5568-bcab-36d7ea370839

package/dist/kernel/index.d.ts

@@ -442,7 +442,7 @@ interface IExtensionBase {
  *   4. **Internal interfaces**, option bags, result records, config
  *      slices, anything declared as `interface` and passed across
  *      function boundaries inside the kernel / CLI but not part of the
- *      spec: `IPluginRuntimeBundle`, `IPruneResult`, `IMigrationFile`,
+ *      spec: `IPluginRuntime`, `IPruneResult`, `IMigrationFile`,
  *      `IDbLocationOptions`. **`I` prefix.** The prefix matches
  *      category 3 because both are "shapes that live in TypeScript
  *      only, never in JSON".
@@ -912,6 +912,13 @@ interface ScanStats {
      * multiple Providers can claim the same file.
      */
     filesSkipped: number;
+    /**
+     * Files skipped by the walker BEFORE reading because their on-disk
+     * size exceeded `scan.maxFileSizeBytes`. Equals
+     * `ScanResult.oversizedFiles.length`. Absent on synthetic fixtures /
+     * loaders that predate the field; defaults to 0 when omitted.
+     */
+    filesOversized?: number;
     nodesCount: number;
     linksCount: number;
     issuesCount: number;
@@ -1021,6 +1028,17 @@ interface ScanResult {
     providers: string[];
     /** Implementation metadata. Populated by `runScan` for self-describing output. */
     scannedBy?: ScanScannedBy;
+    /**
+     * Resolved offline tokenizer (encoder) that produced the per-node token
+     * counts in this scan. One of the closed allow-list in
+     * `project-config.schema.json#/properties/tokenizer` (`cl100k_base`
+     * default, `o200k_base`). Mirrors `scan_meta.tokenizer`. Populated by
+     * `runScan` from the resolved `RunScanOptions.tokenizer`; the
+     * incremental path compares the persisted value against the resolved
+     * one and force-recomputes counts when they differ. Absent on synthetic
+     * fixtures / loaders that predate the field.
+     */
+    tokenizer?: string;
     /**
      * Effective recommended cap on the number of files the walker accepted
      * during this scan (`scan.maxNodes` from settings, default 256). The UI
@@ -1035,11 +1053,30 @@ interface ScanResult {
      * setting). Bidirectional: can raise OR lower the recommended limit.
      */
     overrideMaxNodes?: number | null;
+    /**
+     * Files the walker skipped because their on-disk size exceeded
+     * `scan.maxFileSizeBytes` (default 1 MiB). Each entry is the
+     * root-relative, forward-slash path (same form as `node.path`) plus
+     * the byte size. Drives the CLI / serve terminal WARN and the UI
+     * banner. Defaults to `[]`; absent on synthetic fixtures that bypass
+     * the walker.
+     */
+    oversizedFiles?: OversizedFile[];
     nodes: Node[];
     links: Link[];
     issues: Issue[];
     stats: ScanStats;
 }
+/**
+ * One file the walker skipped for exceeding `scan.maxFileSizeBytes`.
+ * Mirrors `scan-result.schema.json#/properties/oversizedFiles/items`.
+ */
+interface OversizedFile {
+    /** Root-relative, forward-slash path (same form as `node.path`). */
+    path: string;
+    /** On-disk size of the skipped file, in bytes. */
+    bytes: number;
+}
 
 /**
  * Plugin-surface types, hand-written to mirror
@@ -1396,7 +1433,7 @@ interface IPersistOptions {
      * `<pluginId>/<extensionId>/<contributionId>`. Passed to the
      * `scan_contributions` upsert so the catalog sweep can drop rows
      * belonging to plugins / extensions that are no longer in the
-     * catalog (uninstalled plugins, disabled bundles, removed
+     * catalog (uninstalled plugins, disabled plugins, removed
      * contributions). Empty / absent set = no catalog sweep (legacy
      * behaviour, leaves disabled-plugin rows stale per design F24
      * pre-fix).
@@ -1712,9 +1749,10 @@ interface IContributionRecord {
  * new / modified nodes from a fresh extractor pass.
  *
  * Meta envelope: the `scan_meta` table persists `roots` /
- * `scannedAt` / `scannedBy` / `providers` / `stats.filesWalked` /
- * `stats.filesSkipped` / `stats.durationMs`. When the row exists,
- * those fields come back authoritatively. When it does not (DB
+ * `scannedAt` / `scannedBy` / `tokenizer` / `providers` /
+ * `stats.filesWalked` / `stats.filesSkipped` / `stats.filesOversized` /
+ * `stats.durationMs` / `oversizedFiles`. When the row exists, those fields come back
+ * authoritatively. When it does not (DB
  * freshly migrated but never scanned, or a legacy DB never
  * re-persisted), the loader degrades to a synthetic envelope:
  *
@@ -2163,9 +2201,7 @@ interface IProvider extends IExtensionBase {
      * directly, it goes through `resolveProviderWalk(provider)` which
      * picks `walk` over `read`.
      */
-    walk?(roots: string[], options?: {
-        ignoreFilter?: IIgnoreFilter;
-    }): AsyncIterable<IRawNode>;
+    walk?(roots: string[], options?: IProviderWalkOptions): AsyncIterable<IRawNode>;
     /**
      * Given a path and its parsed frontmatter, decide the node kind, or
      * `null` to disclaim the file. The classifier is called after walk()
@@ -2186,7 +2222,7 @@ interface IProvider extends IExtensionBase {
     /**
      * Strict resolution matrix consumed by the post-walk confidence-lift
      * transform: maps a `link.kind` (emitted by an Extractor in this
-     * Provider's bundle, e.g. `'mentions'`, `'invokes'`) to the set of
+     * Provider's plugin, e.g. `'mentions'`, `'invokes'`) to the set of
      * target `node.kind` values that count as a valid resolution.
      *
      * Used to decide whether to bump a link's confidence to 1.0 when its
@@ -2317,6 +2353,28 @@ interface IResolverRules {
      */
     kindPriority?: readonly LinkKind[];
 }
+/**
+ * Per-invocation options the orchestrator threads into a Provider walk
+ * (and through `resolveProviderWalk` into the kernel walker). All
+ * optional, so a bare `provider.walk(roots)` keeps working.
+ *
+ *   - `ignoreFilter`, the composed `.skillmapignore` + config.ignore +
+ *     bundled-defaults filter.
+ *   - `maxFileSizeBytes` / `onOversizedFile`, mirror of
+ *     `scan.maxFileSizeBytes` and the collector that records skipped
+ *     files into `ScanResult.oversizedFiles`. A Provider that ships its
+ *     own `walk()` SHOULD forward both into `walkContent` (or apply the
+ *     same size guard) so oversized files stay skipped + reported
+ *     regardless of which discovery path runs.
+ */
+interface IProviderWalkOptions {
+    ignoreFilter?: IIgnoreFilter;
+    maxFileSizeBytes?: number;
+    onOversizedFile?: (info: {
+        path: string;
+        bytes: number;
+    }) => void;
+}
 /**
  * Declarative read config a Provider declares via `IProvider.read`.
  * Mirrors `extensions/provider.schema.json#/properties/read` at the
@@ -3313,11 +3371,25 @@ interface RunScanOptions {
     viewContributions?: readonly IRegisteredViewContribution[];
     /**
      * Compute per-node token counts (frontmatter / body / total) using the
-     * cl100k_base BPE (the modern OpenAI tokenizer used by GPT-4 / GPT-3.5).
-     * Defaults to true. Set false to skip tokenization; `node.tokens` is
-     * left undefined (spec-valid: the field is optional).
+     * encoder named by `tokenizer` (default `cl100k_base`). Defaults to
+     * true. Set false to skip tokenization; `node.tokens` is left undefined
+     * (spec-valid: the field is optional).
      */
     tokenize?: boolean;
+    /**
+     * Offline tokenizer (encoder) used to build the per-node token counts.
+     * Closed allow-list mirroring `project-config.schema.json#/properties/tokenizer`:
+     * `cl100k_base` (default) or `o200k_base`. Threaded from `cfg.tokenizer`
+     * by the driving adapters (scan-runner, watcher). Absent → `cl100k_base`.
+     * The orchestrator guards the override layer: any value that is neither
+     * allow-list member falls back to `cl100k_base` (the AJV enum on the
+     * config schema already guarantees this for the config layers, the
+     * guard covers out-of-band callers and the `override` layer). The
+     * resolved value is carried onto `ScanResult.tokenizer` so the
+     * persistence layer can record which encoder produced the counts and
+     * the incremental path can detect an encoder switch.
+     */
+    tokenizer?: string;
     /**
      * Prior snapshot for two purposes (decoupled by design):
      *
@@ -3468,6 +3540,14 @@ interface RunScanOptions {
      * replaces the recommended limit for the duration of this scan.
      */
     overrideMaxNodes?: number | null;
+    /**
+     * Mirror of `scan.maxFileSizeBytes` (default 1 MiB). Threaded into
+     * `walkAndExtract` so the walker skips any file larger than this
+     * BEFORE reading it; skipped files surface in
+     * `ScanResult.oversizedFiles` and `stats.filesOversized`. Absent → no
+     * size limit (out-of-band callers and synthetic fixtures stay safe).
+     */
+    maxFileSizeBytes?: number;
 }
 /**
  * Same as `runScan` but also returns the rename heuristic's `RenameOp[]`
@@ -4015,7 +4095,7 @@ interface StoragePort {
         lookup(pluginId: string, contributionId: string, nodePath: string, extensionId?: string): Promise<IPersistedContribution[]>;
         /**
          * Drop rows for a plugin (optionally narrowed to a single
-         * extension within the bundle). Returns the number of deleted
+         * extension within the plugin). Returns the number of deleted
          * rows. Called by `sm plugins disable` so the UI stops rendering
          * the disabled plugin's chips before the next scan.
          */