@vertekum/schema-builder 0.3.8 → 0.3.11

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @vertekum/schema-builder
 
+## 0.3.11
+
+### Patch Changes
+
+- [`6265f42`](https://github.com/moros-oxus/vertekum/commit/6265f4285ed7b2e9a5e1dd2f6801911ddf9247d0) Thanks [@tschemmer](https://github.com/tschemmer)! - `vertekum schema mock` renders the matrix a vocabulary grants: a grouped name listing (`--style names`), a sample DTCG token file with per-type default values (`--style tokens`; types via the `mock.types` glob map, then `--type`), at `least` (every name adjacency once) or `full` (the whole matrix) coverage — and with `--break <p>`, a separate deliberately-broken sibling file that `check` must refuse. Deterministic at a fixed `--seed`; outputs land in the configured `mock.out`.
+- Updated dependencies []:
+  - @vertekum/core@0.3.11
+
+## 0.3.10
+
+### Patch Changes
+
+- [`351366b`](https://github.com/moros-oxus/vertekum/commit/351366b8a92c60e9186cb0c74d774000c0d5366c) Thanks [@tschemmer](https://github.com/tschemmer)! - Emitted schemas share repeated subtrees: identical tails (optional-slot syntagms expand them once per branch) hoist into `$defs` as content-named `shared-*` entries. A consumer module that emitted 12 MB with 22k ref-sites — enough to overflow ajv's call stack at validation — now emits ~30 KB with identical validation behaviour.
+- Updated dependencies []:
+  - @vertekum/core@0.3.10
+
+## 0.3.9
+
+### Patch Changes
+
+- Updated dependencies [[`a35485a`](https://github.com/moros-oxus/vertekum/commit/a35485a227a9e94271b041bc020f8edf28e0137b)]:
+  - @vertekum/core@0.3.9
+
 ## 0.3.8
 
 ### Patch Changes

package/README.md

@@ -62,6 +62,7 @@ root = color.<property>.<role>?.<emphasis>?.<state>?
 - [`schema build`](./docs/build.md) — the command, the sweep, the ownership stamp, the
   `--check` CI gate, programmatic use.
 - [`schema lint`](./docs/lint.md) — validate the `.dfn` sources themselves: fragments
+- [`schema mock`](./docs/mock.md) — render the granted matrix: name listings, sample token files, deliberate breakage.
   included, every production, findings collected with positions; `--fix` repairs the
   mechanical ones.
 - [`schema fmt`](./docs/format.md) — canonical formatting: JS-literal blocks, indent

package/docs/mock.md

@@ -0,0 +1,63 @@
+# `schema mock` — the matrix, made tangible
+
+A vocabulary grants a space of names. `schema mock` renders that space as files you
+can read, validate, and break on purpose:
+
+```bash
+# vertekum schema mock [module] [--style names|tokens] [--coverage least|full] [--break <p>] [--type <t>] [--seed <n>]
+vertekum schema mock                          # every module in the configured source
+vertekum schema mock schemas/color.dfn       # one module
+vertekum schema mock --style tokens --break 0.3
+```
+
+| Flag | Values | Default | What it does |
+| --- | --- | --- | --- |
+| `--style` | `names` \| `tokens` | both | `names` writes the markdown listing; `tokens` writes a sample DTCG file. |
+| `--coverage` | `least` \| `full` | `least` | How much of the matrix each output holds (below). |
+| `--break` | probability `0..1` | `0` | Each token independently breaks with this probability — emitted as a SEPARATE `*.broken.tokens.json`; the clean mock stays valid. |
+| `--type` | a DTCG `$type` | `color` | Fallback type for mock tokens (the settings `mock.types` map wins). |
+| `--seed` | integer | `1` | The breakage RNG seed — identical invocations produce identical files; vary it to vary the breaks. |
+
+Outputs land in the configured `mock.out` (default `./mocks`), one trio per module:
+`<module>.names.md` (grouped by top-level segment, with counts),
+`<module>.mock.tokens.json`, and — with `--break` — `<module>.broken.tokens.json`.
+
+## Coverage
+
+- **`least`** — every parent→child name adjacency appears at least once: each member
+  of each position is exercised somewhere, without the cross-product. The smoke-test
+  size.
+- **`full`** — every complete token name the vocabulary grants. On a generative
+  module this is the whole matrix; expect thousands.
+
+Token names are the tree's **leaves** — an interior name whose optional tail was
+skipped is a group, never a token (that is what `$root` exists for).
+
+## Types and values
+
+A mock token's `$type` resolves through: the settings map → `--type` → `color`.
+
+```ts
+schemaBuilderExtension({
+  mock: {
+    out: './mocks',
+    types: { 'color.*': 'color', 'space.*': 'dimension' },  // name-glob → $type
+  },
+})
+```
+
+Values come from a fixed per-type table: color → white, dimension → `0px`,
+duration → `0ms`, number → `0`, fontWeight → `400`, fontFamily → `["sans-serif"]`,
+cubicBezier → `[0,0,1,1]`, strokeStyle → `"solid"`, and minimal spec objects for the
+composites (border, shadow, transition, gradient, typography). An unknown type gets
+the string `"mock"`.
+
+## Breakage
+
+With `--break 0.5`, each token has a 50% chance of one deliberate violation — a coin
+picks **name** (an ungranted `-broken` sibling, caught by the vocabulary seal) or
+**value** (a type-mismatched literal, caught by the format binding; types without a
+published value schema break by name instead, so every break stays catchable). Each
+broken token carries `$description: "deliberately broken (name|value)"`. Copy the
+broken file into a collection and watch `vertekum check` refuse it — a self-test of
+the whole validation stack.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vertekum/schema-builder",
-  "version": "0.3.8",
+  "version": "0.3.11",
   "description": "Declare token vocabularies as grammar; build them into JSON Schema files",
   "license": "Apache-2.0",
   "keywords": [
@@ -37,13 +37,13 @@
     "./cli": "./src/cli.ts"
   },
   "peerDependencies": {
-    "@vertekum/core": "0.3.8"
+    "@vertekum/core": "0.3.11"
   },
   "dependencies": {
     "zod": "^3.25.76"
   },
   "devDependencies": {
-    "@vertekum/core": "0.3.8",
+    "@vertekum/core": "0.3.11",
     "@vertekum/schema-vocabulary": "0.3.0"
   }
 }

package/src/api.ts

@@ -1,5 +1,10 @@
 import type { ActivateContext } from '@vertekum/core';
-import { schemaBuildCommand, schemaFmtCommand, schemaLintCommand } from './cli';
+import {
+  schemaBuildCommand,
+  schemaFmtCommand,
+  schemaLintCommand,
+  schemaMockCommand,
+} from './cli';
 import type { schemaBuilderManifest } from './index';
 
 export {
@@ -33,4 +38,5 @@ export function activate(
   ctx.commands.register(schemaBuildCommand);
   ctx.commands.register(schemaLintCommand);
   ctx.commands.register(schemaFmtCommand);
+  ctx.commands.register(schemaMockCommand);
 }

package/src/cli.ts

@@ -12,6 +12,16 @@ import { emit, isStamped } from './emit';
 import { DfnError } from './error';
 import { fixSource, formatSource, resolveIndent } from './format';
 import { lintModule } from './lint';
+import {
+  breakTokens,
+  fullPaths,
+  leastPaths,
+  mockTokens,
+  renderNames,
+  renderTokens,
+  rng,
+  typeResolver,
+} from './mock';
 import { type ResolvedModule, resolveModule } from './resolve';
 
 /**
@@ -37,6 +47,7 @@ function settingsOf(ctx: { project: unknown }): {
   out?: string;
   link: boolean;
   schemaId?: string;
+  mock: { out: string; types?: Record<string, string> };
 } {
   const kernel = (ctx.project as ProjectDir).kernel;
   const slice = kernel?.config.get<{
@@ -44,12 +55,17 @@ function settingsOf(ctx: { project: unknown }): {
     out?: string;
     link?: boolean;
     schemaId?: string;
+    mock?: { out?: string; types?: Record<string, string> };
   }>('vtk.schema-builder');
   return {
     source: slice?.source ?? './schemas',
     out: slice?.out,
     link: slice?.link === true,
     schemaId: slice?.schemaId,
+    mock: {
+      out: slice?.mock?.out ?? './mocks',
+      ...(slice?.mock?.types ? { types: slice.mock.types } : {}),
+    },
   };
 }
 
@@ -495,3 +511,135 @@ export const schemaBuildCommand: CommandDescriptor = {
     return { summary: notes.join('\n'), files, data: { skipped, fragments } };
   },
 };
+
+/**
+ * `vertekum schema mock` — make the granted matrix tangible: a names listing, a usable sample
+ * token file, and (with `--break`) a deliberately broken sibling that `check` must catch.
+ * Coverage `least` exercises every parent→child name adjacency once; `full` is the whole
+ * matrix. Deterministic at a fixed `--seed`, so mocks are diff-stable.
+ */
+export const schemaMockCommand: CommandDescriptor = {
+  name: 'schema mock',
+  description:
+    'generate the name matrix and sample token files a vocabulary grants',
+  args: [
+    {
+      name: 'module',
+      required: false,
+      description:
+        "a .dfn file or directory; default: the configured source ('./schemas')",
+    },
+  ],
+  options: [
+    {
+      flag: '--style <style>',
+      description: "'names' (markdown) or 'tokens' (DTCG sample); default both",
+    },
+    {
+      flag: '--coverage <coverage>',
+      description:
+        "'least' (every name adjacency once; default) or 'full' (the whole matrix)",
+    },
+    {
+      flag: '--break <p>',
+      description:
+        'probability 0..1 that each token breaks (name or value) — emitted as a separate *.broken.tokens.json',
+    },
+    {
+      flag: '--type <type>',
+      description: "fallback DTCG $type for mock tokens (default 'color')",
+    },
+    {
+      flag: '--seed <n>',
+      description: 'RNG seed for the breakage pass (default 1)',
+    },
+  ],
+  run(ctx): CommandResult {
+    const { projectDir } = ctx.project as ProjectDir;
+    const settings = settingsOf(ctx);
+    const style = (ctx.options.style as string | undefined) ?? 'both';
+    if (!['names', 'tokens', 'both'].includes(style)) {
+      throw new Error(`--style must be 'names' or 'tokens', got '${style}'`);
+    }
+    const coverage = (ctx.options.coverage as string | undefined) ?? 'least';
+    if (!['least', 'full'].includes(coverage)) {
+      throw new Error(
+        `--coverage must be 'least' or 'full', got '${coverage}'`,
+      );
+    }
+    const breakP = Number((ctx.options.break as string | undefined) ?? '0');
+    if (!Number.isFinite(breakP) || breakP < 0 || breakP > 1) {
+      throw new Error(`--break must be a probability 0..1`);
+    }
+    const seed = Number((ctx.options.seed as string | undefined) ?? '1');
+    const typeOf = typeResolver(
+      settings.mock.types,
+      ctx.options.type as string | undefined,
+    );
+
+    const { modules, explicitFile, root } = resolveModules(
+      projectDir,
+      ctx.args.module,
+      settings.source,
+    );
+    if (modules.length === 0) {
+      return { summary: 'no .dfn modules found', files: [] };
+    }
+
+    const files: Array<{ path: string; content: string }> = [];
+    const skipped: string[] = [];
+    let names = 0;
+    let tokensOut = 0;
+
+    for (const modulePath of modules) {
+      const resolved = resolveModule(modulePath);
+      if (natureOf(resolved) === 'inline' || !resolved.module.root) {
+        // Inline modules never emit; a rootless fragment has no matrix of its own. In a sweep
+        // both are notices; an explicit fragment is worth an error.
+        if (explicitFile) {
+          throw new Error(
+            `${relative(projectDir, modulePath)} declares no root — nothing to mock`,
+          );
+        }
+        skipped.push(relative(projectDir, modulePath));
+        continue;
+      }
+      const tree = build(resolved);
+      const paths = coverage === 'full' ? fullPaths(tree) : leastPaths(tree);
+      const rel = relative(root, modulePath).replace(/\.dfn$/, '');
+      const base = join(settings.mock.out, rel);
+      const moduleName = rel.split('/').pop() as string;
+
+      if (style !== 'tokens') {
+        files.push({
+          path: `${base}.names.md`,
+          content: renderNames(moduleName, coverage, paths),
+        });
+        names += paths.length;
+      }
+      if (style !== 'names') {
+        const tokens = mockTokens(paths, typeOf);
+        files.push({
+          path: `${base}.mock.tokens.json`,
+          content: renderTokens(tokens),
+        });
+        tokensOut += tokens.length;
+        if (breakP > 0) {
+          const broken = breakTokens(tokens, breakP, rng(seed));
+          files.push({
+            path: `${base}.broken.tokens.json`,
+            content: renderTokens(broken),
+          });
+        }
+      }
+    }
+
+    const parts = [
+      `mocked ${files.length} file(s) (${coverage})`,
+      style !== 'tokens' ? `${names} name(s)` : null,
+      style !== 'names' ? `${tokensOut} token(s)` : null,
+      skipped.length > 0 ? `skipped ${skipped.join(', ')}` : null,
+    ].filter(Boolean);
+    return { summary: parts.join(' — '), files };
+  },
+};

package/src/dedupe.ts

@@ -0,0 +1,181 @@
+/**
+ * Structural sharing for emitted schemas.
+ *
+ * A generative syntagm with optional slots (`color.<pattern>?.<property>.<role>?.<state>?`)
+ * repeats the TAIL of the expression under every enumerated sibling name — byte-identical
+ * subtrees, hundreds of kilobytes each, multiplied into megabytes (a consumer module reached
+ * 12 MB / 22k ref-sites, and ajv's compiled validator overflowed the call stack). This pass
+ * hoists every repeated schema subtree into `$defs` once and references it — validation
+ * behaviour is unchanged because 2020-12 `unevaluatedProperties` sees through `$ref`.
+ *
+ * Only nodes in SCHEMA POSITION are candidates: hoisting the object under `properties` itself
+ * would turn a keyword map into `{ "$ref": … }`, which is a property named `$ref`, not a
+ * reference. Names are content-derived (`shared-<hash>`), so rebuilds stay diff-stable.
+ */
+
+type Json = Record<string, unknown>;
+
+/** Keywords whose value is a map of name → schema. */
+const SCHEMA_MAPS = new Set(['properties', 'patternProperties', '$defs']);
+/** Keywords whose value is an array of schemas. */
+const SCHEMA_LISTS = new Set(['allOf', 'anyOf', 'oneOf']);
+/** Keywords whose value is a single schema (when it is an object). */
+const SCHEMA_VALUES = new Set([
+  'additionalProperties',
+  'unevaluatedProperties',
+  'items',
+  'not',
+  'if',
+  'then',
+  'else',
+]);
+
+/**
+ * Only subtrees at least this large hoist. Small repetition is deliberate emission style (a
+ * shared `allOf` pair reads better inline, and tests pin those shapes); the pathology this pass
+ * exists for repeats subtrees of hundreds of kilobytes.
+ */
+const MIN_BYTES = 512;
+
+/** FNV-1a over a string, as fixed-width hex — composition-friendly and dependency-free. */
+function fnv(input: string): string {
+  let hash = 0xcbf29ce484222325n;
+  const prime = 0x100000001b3n;
+  for (let i = 0; i < input.length; i++) {
+    hash ^= BigInt(input.charCodeAt(i));
+    hash = (hash * prime) & 0xffffffffffffffffn;
+  }
+  return hash.toString(16).padStart(16, '0');
+}
+
+/** Visit every schema-position object under `schema` (excluding `schema` itself). */
+function visitChildren(
+  schema: Json,
+  visit: (child: Json, replace: (next: Json) => void) => void,
+): void {
+  for (const [key, value] of Object.entries(schema)) {
+    if (SCHEMA_MAPS.has(key) && value && typeof value === 'object') {
+      const map = value as Json;
+      for (const [name, child] of Object.entries(map)) {
+        if (child && typeof child === 'object' && !Array.isArray(child)) {
+          visit(child as Json, (next) => {
+            map[name] = next;
+          });
+        }
+      }
+    } else if (SCHEMA_LISTS.has(key) && Array.isArray(value)) {
+      value.forEach((child, index) => {
+        if (child && typeof child === 'object' && !Array.isArray(child)) {
+          visit(child as Json, (next) => {
+            (value as unknown[])[index] = next;
+          });
+        }
+      });
+    } else if (
+      SCHEMA_VALUES.has(key) &&
+      value &&
+      typeof value === 'object' &&
+      !Array.isArray(value)
+    ) {
+      visit(value as Json, (next) => {
+        schema[key] = next;
+      });
+    }
+  }
+}
+
+/**
+ * Hoist repeated subtrees of `document` into its `$defs`, in place. Iterates to a fixpoint:
+ * hoisting inner duplicates makes outer subtrees identical, which the next round catches.
+ */
+export function dedupeSubtrees(document: Json): void {
+  const defs = (): Json => {
+    if (!document.$defs || typeof document.$defs !== 'object') {
+      document.$defs = {};
+    }
+    return document.$defs as Json;
+  };
+  const reserved = new Set(Object.keys((document.$defs as Json) ?? {}));
+
+  for (let round = 0; round < 32; round++) {
+    // Pass 1: post-order signatures (composed from child hashes — compact and linear).
+    const signature = new Map<Json, string>();
+    const size = new Map<Json, number>();
+    const buckets = new Map<string, Json[]>();
+    const sign = (node: Json): string => {
+      const held = signature.get(node);
+      if (held) return held;
+      visitChildren(node, (child) => {
+        sign(child);
+      });
+      const parts: string[] = [];
+      let bytes = 2;
+      for (const key of Object.keys(node).sort()) {
+        const value = node[key];
+        parts.push(`${key}:${serialize(value)}`);
+        bytes += key.length + byteLength(value);
+      }
+      const hash = fnv(parts.join('|'));
+      signature.set(node, hash);
+      size.set(node, bytes);
+      const bucket = buckets.get(hash) ?? [];
+      bucket.push(node);
+      buckets.set(hash, bucket);
+      return hash;
+    };
+    const serialize = (value: unknown): string => {
+      if (value && typeof value === 'object' && !Array.isArray(value)) {
+        return `#${sign(value as Json)}`;
+      }
+      return JSON.stringify(value) ?? 'null';
+    };
+    const byteLength = (value: unknown): number => {
+      if (value && typeof value === 'object' && !Array.isArray(value)) {
+        return size.get(value as Json) ?? 16;
+      }
+      return (JSON.stringify(value) ?? 'null').length;
+    };
+
+    // The whole forest: the document body plus every existing def.
+    const roots: Json[] = [document];
+    signature.set(document, 'root'); // the document itself is never a candidate
+    visitChildren(document, (child) => {
+      sign(child);
+    });
+
+    // Pass 2: pick duplicated, big-enough subtrees; verify true equality (hash paranoia).
+    const hoisted = new Map<string, { name: string; canonical: string }>();
+    for (const [hash, bucket] of buckets) {
+      if (bucket.length < 2) continue;
+      const first = bucket[0] as Json;
+      if ((size.get(first) ?? 0) < MIN_BYTES) continue;
+      if (Object.keys(first).length === 1 && '$ref' in first) continue;
+      const canonical = JSON.stringify(first);
+      if (bucket.some((node) => JSON.stringify(node) !== canonical)) continue;
+      let name = `shared-${hash.slice(0, 8)}`;
+      while (reserved.has(name)) name = `shared-${fnv(name).slice(0, 8)}`;
+      hoisted.set(hash, { name, canonical });
+    }
+    if (hoisted.size === 0) return;
+
+    // Pass 3: replace every occurrence top-down (an outer hoist swallows its inner ones this
+    // round; the next round dedupes inside the new defs).
+    const defsMap = defs();
+    const replaceIn = (node: Json): void => {
+      visitChildren(node, (child, replace) => {
+        const hash = signature.get(child);
+        const hit = hash === undefined ? undefined : hoisted.get(hash);
+        if (hit) {
+          if (!(hit.name in defsMap)) {
+            defsMap[hit.name] = JSON.parse(hit.canonical);
+            reserved.add(hit.name);
+          }
+          replace({ $ref: `#/$defs/${hit.name}` });
+          return;
+        }
+        replaceIn(child);
+      });
+    };
+    for (const root of roots) replaceIn(root);
+  }
+}

package/src/emit.ts

@@ -1,4 +1,5 @@
 import type { PatternRef, TreeNode } from './build';
+import { dedupeSubtrees } from './dedupe';
 import type { ResolvedModule } from './resolve';
 
 /**
@@ -230,6 +231,9 @@ export function emit(tree: TreeNode | undefined, options: EmitOptions): string {
     }
   }
 
+  // Structural sharing: repeated tails (optional-slot syntagms) hoist into $defs once.
+  dedupeSubtrees(document as Record<string, unknown>);
+
   return `${JSON.stringify(document, null, 2)}\n`;
 }
 

package/src/index.ts

@@ -24,6 +24,13 @@ export const SchemaBuilderSettings = z.object({
    * A file's `id` pragma wins. Unset: no `$id` unless the pragma supplies one.
    */
   schemaId: z.string().optional(),
+  /** `schema mock` output: where mocks land, and a name-glob → DTCG `$type` map. */
+  mock: z
+    .object({
+      out: z.string().default('./mocks'),
+      types: z.record(z.string()).optional(),
+    })
+    .default({}),
 });
 
 export const schemaBuilderManifest = {