@terrazzo/parser 0.0.0

package/lint/index.js

@@ -0,0 +1,59 @@
+import { pluralize } from '@terrazzo/token-tools';
+
+const listFormat = new Intl.ListFormat('en-us');
+
+export default async function lintRunner({ tokens, ast, config = {}, logger }) {
+  const { plugins = [], lint = { rules: {} } } = config;
+  const unusedLintRules = Object.keys(lint?.rules ?? {});
+
+  for (const plugin of plugins) {
+    if (typeof plugin.lint === 'function') {
+      const s = performance.now();
+      logger.debug({ group: 'plugin', task: plugin.name, message: 'Start linting' });
+
+      const rules = plugin.lint();
+      const errorEntries = [];
+      const warnEntries = [];
+      await Promise.all(
+        Object.entries(rules).map(async ([id, linter]) => {
+          const { severity = 'warn' } = lint.rules[id] || {};
+          const results = await linter({ tokens, ast, rule: { id, severity } });
+          for (const result of results ?? []) {
+            const noticeList = severity === 'error' ? errorEntries : warnEntries;
+            noticeList.push({
+              message: result.message,
+              ast,
+              node: result.node,
+              continueOnError: true,
+            });
+          }
+
+          // tick off used rule
+          const unusedLintRuleI = unusedLintRules.indexOf(id);
+          if (unusedLintRuleI !== -1) {
+            unusedLintRules.splice(unusedLintRuleI, 1);
+          }
+        }),
+      );
+      for (const entry of errorEntries) {
+        logger.error(entry);
+      }
+      for (const entry of warnEntries) {
+        logger.warn(entry);
+      }
+      logger.debug({ group: 'plugin', task: plugin.name, message: 'Finish linting', timing: performance.now() - s });
+      if (errorEntries.length) {
+        const counts = [pluralize(errorEntries.length, 'error', 'errors')];
+        if (warnEntries.length) {
+          counts.push(pluralize(warnEntries.length, 'warning', 'warnings'));
+        }
+        logger.error({ message: `Lint failed with ${listFormat.format(counts)}`, label: plugin.name });
+      }
+    }
+  }
+
+  // warn user if they have unused lint rules (they might have meant to configure something!)
+  for (const unusedRule of unusedLintRules) {
+    logger.warn({ group: 'parser', task: 'lint', message: `Unknown lint rule "${unusedRule}"` });
+  }
+}

package/lint/plugin-core/index.d.ts

@@ -0,0 +1,3 @@
+import type { Plugin } from '../../config.js';
+
+export default function coreLintPlugin(): Plugin;

package/lint/plugin-core/index.js

@@ -0,0 +1,12 @@
+import ruleDuplicateValues from './rules/duplicate-values.js';
+
+export default function coreLintPlugin() {
+  return {
+    name: '@terrazzo/plugin-lint-core',
+    lint() {
+      return {
+        duplicateValues: ruleDuplicateValues,
+      };
+    },
+  };
+}

package/lint/plugin-core/rules/duplicate-values.d.ts

@@ -0,0 +1,10 @@
+import type { LintNotice, LinterOptions } from '../../index.js';
+
+export interface RuleDuplicateValueOptions {
+  /** (optional) Token IDs to ignore. Supports globs (`*`). */
+  ignore?: string[];
+}
+
+export default function ruleDuplicateValues(
+  options: LinterOptions<RuleDuplicateValueOptions>,
+): Promise<LintNotice[] | undefined>;

package/lint/plugin-core/rules/duplicate-values.js

@@ -0,0 +1,69 @@
+import { isAlias, isTokenMatch } from '@terrazzo/token-tools';
+import deepEqual from 'deep-equal';
+
+export default function ruleDuplicateValues({ tokens, rule: { severity }, options }) {
+  if (severity === 'off') {
+    return;
+  }
+
+  const notices = [];
+  const values = {};
+
+  for (const id in tokens) {
+    if (!Object.hasOwn(tokens, id)) {
+      continue;
+    }
+
+    const t = tokens[id];
+
+    // skip ignored tokens
+    if (options?.ignore && isTokenMatch(id, options.ignore)) {
+      return;
+    }
+
+    if (!values[t.$type]) {
+      values[t.$type] = new Set();
+    }
+
+    // primitives: direct comparison is easy
+    if (
+      t.$type === 'color' ||
+      t.$type === 'dimension' ||
+      t.$type === 'duration' ||
+      t.$type === 'link' ||
+      t.$type === 'number' ||
+      t.$type === 'fontWeight'
+    ) {
+      // skip aliases (note: $value will be resolved)
+      if (isAlias(t._original.$value)) {
+        return;
+      }
+
+      if (values[t.$type]?.has(t.$value)) {
+        notices.push({ message: `Duplicated value: "${t.$value}" (${t.id})`, node: t.sourceNode });
+        return;
+      }
+
+      values[t.$type]?.add(t.$value);
+      return;
+    }
+
+    // everything else: use deepEqual
+    let isDuplicate = false;
+    for (const v of values[t.$type]?.values() ?? []) {
+      if (deepEqual(t.$value, v)) {
+        notices.push({ message: `Duplicated value (${t.id})`, node: t.sourceNode });
+        isDuplicate = true;
+        break;
+      }
+    }
+
+    if (isDuplicate) {
+      continue;
+    }
+
+    values[t.$type]?.add(t.$value);
+  }
+
+  return notices;
+}

package/logger.d.ts

@@ -0,0 +1,66 @@
+import type { SourceLocation } from '@babel/code-frame';
+import type { AnyNode, DocumentNode } from '@humanwhocodes/momoa';
+
+declare const LOG_ORDER: readonly ['error', 'warn', 'info', 'debug'];
+
+export type LogSeverity = 'error' | 'warn' | 'info' | 'debug';
+
+export type LogLevel = LogSeverity | 'silent';
+
+export type LogGroup = 'core' | 'plugin';
+
+export interface LogEntry {
+  /** Error message to be logged */
+  message: string;
+  /** (optional) Prefix message with label */
+  label?: string;
+  /** Continue on error? (default: false) */
+  continueOnError?: boolean;
+  /** (optional) Show a code frame for the erring node */
+  node?: AnyNode;
+  /** (optional) To show code frame, provide entire AST to show which line erred (otherwise it’s floating in space) */
+  ast?: DocumentNode;
+  /** (optional) To highlight a specifc part of the code frame, provide line no. (1-based) and col. no. */
+  loc?: SourceLocation['start'];
+  /** (optional) MANUAL codeFrame override (only use for non-JSON errors, like YAML) */
+  code?: string;
+}
+
+export interface DebugEntry {
+  /** `parser` | `plugin` */
+  group: 'parser' | 'plugin';
+  /** Current subtask or submodule */
+  task: string;
+  /** Error message to be logged */
+  message: string;
+  /** (optional) Show code below message */
+  codeFrame?: { code: string; line: number; column: number };
+  /** (optional) Display performance timing */
+  timing?: number;
+}
+
+export default class Logger {
+  constructor(options?: { level?: LogLevel; debugScope: string });
+}
+
+export class TokensJSONError extends Error {
+  level: LogLevel;
+  debugScope: string;
+  node?: AnyNode;
+
+  constructor(message: string, node: AnyNode);
+
+  setLevel(level: LogLevel): void;
+
+  /** Log an error message (always; can’t be silenced) */
+  error(entry: LogEntry): void;
+
+  /** Log an info message (if logging level permits) */
+  info(entry: LogEntry): void;
+
+  /** Log a warning message (if logging level permits) */
+  warn(entry: LogEntry): void;
+
+  /** Log a diagnostics message (if logging level permits) */
+  debug(entry: DebugEntry): void;
+}

package/logger.js

@@ -0,0 +1,121 @@
+import { codeFrameColumns } from '@babel/code-frame';
+import { print } from '@humanwhocodes/momoa';
+import color from 'picocolors';
+import wcmatch from 'wildcard-match';
+
+export const LOG_ORDER = ['error', 'warn', 'info', 'debug'];
+
+const DEBUG_GROUP_COLOR = { core: color.green, plugin: color.magenta };
+
+const MESSAGE_COLOR = { error: color.red, warn: color.yellow };
+
+const timeFormatter = new Intl.DateTimeFormat('en-gb', { timeStyle: 'medium' });
+
+/**
+ * @param {Entry} entry
+ * @param {Severity} severity
+ * @return {string}
+ */
+export function formatMessage(entry, severity) {
+  let message = entry.message;
+  if (entry.label) {
+    message = `${color.bold(`${entry.label}:`)} ${message}`;
+  }
+  if (severity in MESSAGE_COLOR) {
+    message = MESSAGE_COLOR[severity](message);
+  }
+  if (entry.ast || entry.code) {
+    message = `${message}\n\n${codeFrameColumns(entry.ast ? print(entry.ast, { indent: 2 }) : entry.code, {
+      start: entry.loc ?? { line: 1 },
+    })}`;
+  }
+  return message;
+}
+
+export default class Logger {
+  level = 'info';
+  debugScope = '*';
+
+  constructor(options) {
+    if (options?.level) {
+      this.level = options.level;
+    }
+    if (options?.debugScope) {
+      this.debugScope = options.debugScope;
+    }
+  }
+
+  setLevel(level) {
+    this.level = level;
+  }
+
+  /** Log an error message (always; can’t be silenced) */
+  error(entry) {
+    const message = formatMessage(entry, 'error');
+    if (entry.continueOnError) {
+      console.error(message);
+      return;
+    }
+    if (entry.node) {
+      throw new TokensJSONError(message, entry.node);
+    } else {
+      throw new Error(message);
+    }
+  }
+
+  /** Log an info message (if logging level permits) */
+  info(entry) {
+    if (this.level === 'silent' || LOG_ORDER.indexOf(this.level) < LOG_ORDER.indexOf('info')) {
+      return;
+    }
+    // biome-ignore lint/suspicious/noConsoleLog: this is its job
+    console.log(formatMessage(entry, 'info'));
+  }
+
+  /** Log a warning message (if logging level permits) */
+  warn(entry) {
+    if (this.level === 'silent' || LOG_ORDER.indexOf(this.level) < LOG_ORDER.indexOf('warn')) {
+      return;
+    }
+    console.warn(formatMessage(entry, 'warn'));
+  }
+
+  /** Log a diagnostics message (if logging level permits) */
+  debug(entry) {
+    if (this.level === 'silent' || LOG_ORDER.indexOf(this.level) < LOG_ORDER.indexOf('debug')) {
+      return;
+    }
+
+    let message = formatMessage(entry, 'debug');
+
+    const debugPrefix = `${entry.group}:${entry.task}`;
+    if (!wcmatch(this.debugScope)(debugPrefix)) {
+      return;
+    }
+    message = `${DEBUG_GROUP_COLOR[entry.group || 'core'](debugPrefix)} ${color.dim(
+      timeFormatter.format(new Date()),
+    )} ${message}`;
+    if (typeof entry.timing === 'number') {
+      let timing = Math.round(entry.timing);
+      if (timing < 1_000) {
+        timing = `${timing}ms`;
+      } else if (timing < 60_000) {
+        timing = `${Math.round(timing * 100) / 100_000}s`;
+      }
+      message = `${message} ${color.dim(`[${timing}]`)}`;
+    }
+
+    console.debug(message);
+  }
+}
+
+export class TokensJSONError extends Error {
+  /** Erring JSON node */
+  node;
+
+  constructor(message, node) {
+    super(message);
+    this.name = 'TokensJSONError';
+    this.node = node;
+  }
+}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@terrazzo/parser",
+  "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
+  "version": "0.0.0",
+  "author": {
+    "name": "Drew Powers",
+    "email": "drew@pow.rs"
+  },
+  "keywords": [
+    "design tokens",
+    "design tokens community group",
+    "design tokens format module",
+    "dtcg",
+    "cli",
+    "w3c design tokens",
+    "design system",
+    "typescript",
+    "sass",
+    "css",
+    "style tokens",
+    "style system",
+    "linting",
+    "lint"
+  ],
+  "license": "MIT",
+  "type": "module",
+  "main": "./index.js",
+  "dependencies": {
+    "@babel/code-frame": "^7.24.2",
+    "@humanwhocodes/momoa": "^3.0.1",
+    "@types/babel__code-frame": "^7.0.6",
+    "@types/culori": "^2.1.0",
+    "@types/deep-equal": "^1.0.4",
+    "culori": "^4.0.1",
+    "deep-equal": "^2.2.3",
+    "is-what": "^4.1.16",
+    "merge-anything": "^5.1.7",
+    "picocolors": "^1.0.0",
+    "wildcard-match": "^5.1.3",
+    "yaml": "^2.4.1",
+    "@terrazzo/token-tools": "^0.0.0"
+  },
+  "devDependencies": {
+    "strip-ansi": "^7.1.0"
+  },
+  "scripts": {
+    "lint": "biome check .",
+    "test": "pnpm run \"/^test:.*/\"",
+    "test:js": "vitest run",
+    "test:ts": "tsc --noEmit"
+  }
+}

package/parse/index.d.ts

@@ -0,0 +1,32 @@
+import type { DocumentNode } from '@humanwhocodes/momoa';
+import type { ConfigInit } from '../config.js';
+import type { TokenNormalized } from '../types.js';
+import type Logger from '../logger.js';
+
+export * from './validate.js';
+
+export interface ParseOptions {
+  logger?: Logger;
+  /** Skip lint step (default: false) */
+  skipLint?: boolean;
+  config: ConfigInit;
+}
+
+export interface ParseResult {
+  tokens: Record<string, TokenNormalized>;
+  ast: DocumentNode;
+}
+
+/**
+ * Parse and validate Tokens JSON, and lint it
+ */
+export default function parse(input: string | object, options?: ParseOptions): Promise<ParseResult>;
+
+/** Determine if an input is likely a JSON string */
+export function maybeJSONString(input: unknown): boolean;
+
+/** Resolve alias */
+export function resolveAlias(
+  alias: string,
+  options: { tokens: Record<string, TokenNormalized>; logger: Logger; path: string[] },
+): string | undefined;