@terrazzo/parser 0.0.11 → 0.0.13

package/CHANGELOG.md

@@ -0,0 +1,18 @@
+# @terrazzo/parser
+
+## 0.0.13
+
+### Patch Changes
+
+- [#289](https://github.com/terrazzoapp/terrazzo/pull/289) [`0fc9738`](https://github.com/terrazzoapp/terrazzo/commit/0fc9738bb3dfecb680d225e4bd3970f21cfe8079) Thanks [@drwpow](https://github.com/drwpow)! - Add YAML support
+
+- [#291](https://github.com/terrazzoapp/terrazzo/pull/291) [`6a875b1`](https://github.com/terrazzoapp/terrazzo/commit/6a875b163539dba8111911851a7819732056b3aa) Thanks [@drwpow](https://github.com/drwpow)! - Allow negative dimension values
+
+## 0.0.12
+
+### Patch Changes
+
+- [#285](https://github.com/terrazzoapp/terrazzo/pull/285) [`e8a0df1`](https://github.com/terrazzoapp/terrazzo/commit/e8a0df1f3b50cf7cb292bcc475aae271feae4569) Thanks [@drwpow](https://github.com/drwpow)! - Add support for multiple token files
+
+- Updated dependencies [[`e8a0df1`](https://github.com/terrazzoapp/terrazzo/commit/e8a0df1f3b50cf7cb292bcc475aae271feae4569)]:
+  - @terrazzo/token-tools@0.0.6

package/build/index.d.ts

@@ -4,7 +4,7 @@ import type { ConfigInit } from '../config.js';
 import type Logger from '../logger.js';
 
 export interface BuildRunnerOptions {
-  ast: DocumentNode;
+  sources: { filename?: URL; src: string; document: DocumentNode }[];
   config: ConfigInit;
   logger?: Logger;
 }
@@ -73,8 +73,8 @@ export interface TransformHookOptions {
       mode?: string;
     },
   ): void;
-  /** Momoa document */
-  ast: DocumentNode;
+  /** Momoa documents */
+  sources: { filename?: URL; src: string; document: DocumentNode }[];
 }
 
 export interface BuildHookOptions {
@@ -82,8 +82,8 @@ export interface BuildHookOptions {
   tokens: Record<string, TokenNormalized>;
   /** Query transformed values */
   getTransforms(params: TransformParams): TokenTransformed[];
-  /** Momoa document */
-  ast: DocumentNode;
+  /** Momoa documents */
+  sources: { filename?: URL; src: string; document: DocumentNode }[];
   outputFile: (
     /** Filename to output (relative to outDir) */
     filename: string,
@@ -101,8 +101,8 @@ export interface BuildEndHookOptions {
   tokens: Record<string, TokenNormalized>;
   /** Query transformed values */
   getTransforms(params: TransformParams): TokenTransformed[];
-  /** Momoa document */
-  ast: DocumentNode;
+  /** Momoa documents */
+  sources: { filename?: URL; src: string; document: DocumentNode }[];
   /** Final files to be written */
   outputFiles: OutputFileExpanded[];
 }

package/build/index.js

@@ -5,7 +5,7 @@ import Logger from '../logger.js';
 /**
  * @typedef {object} BuildRunnerOptions
  * @typedef {Record<string, TokenNormalized>} BuildRunnerOptions.tokens
- * @typedef {DocumentNode} BuildRunnerOptions.ast
+ * @typedef {Array} BuildRunnerOptions.sources
  * @typedef {ConfigInit} BuildRunnerOptions.config
  * @typedef {Logger} BuildRunnerOptions.logger
  * @typedef {import("@humanwhocodes/momoa").DocumentNode} DocumentNode
@@ -60,7 +60,7 @@ function validateTransformParams({ params, token, logger, pluginName }) {
  * @param {BuildOptions} options
  * @return {Promise<BuildResult>}
  */
-export default async function build(tokens, { ast, logger = new Logger(), config }) {
+export default async function build(tokens, { sources, logger = new Logger(), config }) {
   const formats = {};
   const result = { outputFiles: [] };
 
@@ -95,7 +95,7 @@ export default async function build(tokens, { ast, logger = new Logger(), config
     if (typeof plugin.transform === 'function') {
       await plugin.transform({
         tokens,
-        ast,
+        sources,
         getTransforms,
         setTransform(id, params) {
           if (transformsLocked) {
@@ -165,7 +165,7 @@ export default async function build(tokens, { ast, logger = new Logger(), config
       const pluginBuildStart = performance.now();
       await plugin.build({
         tokens,
-        ast,
+        sources,
         getTransforms,
         outputFile(filename, contents) {
           const resolved = new URL(filename, config.outDir);
@@ -196,7 +196,7 @@ export default async function build(tokens, { ast, logger = new Logger(), config
     if (typeof plugin.buildEnd === 'function') {
       await plugin.buildEnd({
         tokens,
-        ast,
+        sources,
         getTransforms,
         format: (formatID) => createFormatter(formatID),
         outputFiles: structruedClone(result.outputFiles),

package/lib/code-frame.d.ts

@@ -0,0 +1,56 @@
+// MIT License
+//
+// Copyright (c) 2014-present Sebastian McKenzie and other contributors
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+export interface Location {
+  line: number;
+  column: number;
+}
+
+export interface NodeLocation {
+  end?: Location;
+  start: Location;
+}
+
+export interface Options {
+  /** Syntax highlight the code as JavaScript for terminals. default: false */
+  highlightCode?: boolean;
+  /**  The number of lines to show above the error. default: 2 */
+  linesAbove?: number;
+  /**  The number of lines to show below the error. default: 3 */
+  linesBelow?: number;
+  /**
+   * Forcibly syntax highlight the code as JavaScript (for non-terminals);
+   * overrides highlightCode.
+   * default: false
+   */
+  forceColor?: boolean;
+  /**
+   * Pass in a string to be displayed inline (if possible) next to the
+   * highlighted location in the code. If it can't be positioned inline,
+   * it will be placed above the code frame.
+   * default: nothing
+   */
+  message?: string;
+}
+
+export function codeFrameColumns(input: string, location: NodeLocation, options?: Options): string;

package/lib/code-frame.js

@@ -0,0 +1,141 @@
+// This is copied from @babel/code-frame package but without the heavyweight color highlighting
+// (note: Babel loads both chalk AND picocolors, and doesn’t treeshake well)
+// Babel is MIT-licensed and unaffiliated with this project.
+
+// MIT License
+//
+// Copyright (c) 2014-present Sebastian McKenzie and other contributors
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+/**
+ * Extract what lines should be marked and highlighted.
+ */
+
+function getMarkerLines(loc, source, opts = {}) {
+  const startLoc = {
+    column: 0,
+    line: -1,
+    ...loc.start,
+  };
+  const endLoc = {
+    ...startLoc,
+    ...loc.end,
+  };
+  const { linesAbove = 2, linesBelow = 3 } = opts || {};
+  const startLine = startLoc.line;
+  const startColumn = startLoc.column;
+  const endLine = endLoc.line;
+  const endColumn = endLoc.column;
+
+  let start = Math.max(startLine - (linesAbove + 1), 0);
+  let end = Math.min(source.length, endLine + linesBelow);
+
+  if (startLine === -1) {
+    start = 0;
+  }
+
+  if (endLine === -1) {
+    end = source.length;
+  }
+
+  const lineDiff = endLine - startLine;
+  const markerLines = {};
+
+  if (lineDiff) {
+    for (let i = 0; i <= lineDiff; i++) {
+      const lineNumber = i + startLine;
+
+      if (!startColumn) {
+        markerLines[lineNumber] = true;
+      } else if (i === 0) {
+        const sourceLength = source[lineNumber - 1].length;
+
+        markerLines[lineNumber] = [startColumn, sourceLength - startColumn + 1];
+      } else if (i === lineDiff) {
+        markerLines[lineNumber] = [0, endColumn];
+      } else {
+        const sourceLength = source[lineNumber - i].length;
+
+        markerLines[lineNumber] = [0, sourceLength];
+      }
+    }
+  } else {
+    if (startColumn === endColumn) {
+      if (startColumn) {
+        markerLines[startLine] = [startColumn, 0];
+      } else {
+        markerLines[startLine] = true;
+      }
+    } else {
+      markerLines[startLine] = [startColumn, endColumn - startColumn];
+    }
+  }
+
+  return { start, end, markerLines };
+}
+
+/**
+ * RegExp to test for newlines in terminal.
+ */
+
+const NEWLINE = /\r\n|[\n\r\u2028\u2029]/;
+
+export function codeFrameColumns(rawLines, loc, opts = {}) {
+  const lines = rawLines.split(NEWLINE);
+  const { start, end, markerLines } = getMarkerLines(loc, lines, opts);
+  const hasColumns = loc.start && typeof loc.start.column === 'number';
+
+  const numberMaxWidth = String(end).length;
+
+  let frame = rawLines
+    .split(NEWLINE, end)
+    .slice(start, end)
+    .map((line, index) => {
+      const number = start + 1 + index;
+      const paddedNumber = ` ${number}`.slice(-numberMaxWidth);
+      const gutter = ` ${paddedNumber} |`;
+      const hasMarker = markerLines[number];
+      const lastMarkerLine = !markerLines[number + 1];
+      if (hasMarker) {
+        let markerLine = '';
+        if (Array.isArray(hasMarker)) {
+          const markerSpacing = line.slice(0, Math.max(hasMarker[0] - 1, 0)).replace(/[^\t]/g, ' ');
+          const numberOfMarkers = hasMarker[1] || 1;
+
+          markerLine = ['\n ', gutter.replace(/\d/g, ' '), ' ', markerSpacing, '^'.repeat(numberOfMarkers)].join('');
+
+          if (lastMarkerLine && opts.message) {
+            markerLine += ` ${opts.message}`;
+          }
+        }
+        return ['>', gutter, line.length > 0 ? ` ${line}` : '', markerLine].join('');
+      } else {
+        return ` ${gutter}${line.length > 0 ? ` ${line}` : ''}`;
+      }
+    })
+    .join('\n');
+
+  if (opts.message && !hasColumns) {
+    frame = `${' '.repeat(numberMaxWidth + 1)}${opts.message}\n${frame}`;
+  }
+
+  return frame;
+}

package/lint/index.d.ts

@@ -26,7 +26,8 @@ export interface LinterOptions<O = any> {
     id: string;
     severity: LintRuleSeverity;
   };
-  ast: DocumentNode;
+  document: DocumentNode;
+  filename?: URL;
   source?: string;
   /** Any options the user has declared for this plugin */
   options?: O;
@@ -34,7 +35,8 @@ export interface LinterOptions<O = any> {
 export type Linter = (options: LinterOptions) => Promise<LintNotice[] | undefined>;
 
 export interface LintRunnerOptions {
-  ast: DocumentNode;
+  document: DocumentNode;
+  filename?: URL;
   config: ConfigInit;
   logger: Logger;
 }

package/lint/plugin-core/rules/duplicate-values.js

@@ -1,5 +1,4 @@
 import { isAlias, isTokenMatch } from '@terrazzo/token-tools';
-import deepEqual from 'deep-equal';
 
 export default function ruleDuplicateValues({ tokens, rule: { severity }, options }) {
   if (severity === 'off') {
@@ -51,7 +50,7 @@ export default function ruleDuplicateValues({ tokens, rule: { severity }, option
     // everything else: use deepEqual
     let isDuplicate = false;
     for (const v of values[t.$type]?.values() ?? []) {
-      if (deepEqual(t.$value, v)) {
+      if (JSON.stringify(t.$value) === JSON.stringify(v)) {
         notices.push({ message: `Duplicated value (${t.id})`, node: t.sourceNode });
         isDuplicate = true;
         break;

package/logger.d.ts

@@ -13,12 +13,14 @@ export interface LogEntry {
   message: string;
   /** (optional) Prefix message with label */
   label?: string;
+  /** (optional) File in disk */
+  filename?: URL;
   /** Continue on error? (default: false) */
   continueOnError?: boolean;
   /** (optional) Show a code frame for the erring node */
   node?: AnyNode;
   /** (optional) To show a code frame, provide the original source code */
-  source?: string;
+  src?: string;
 }
 
 export interface DebugEntry {
@@ -29,7 +31,7 @@ export interface DebugEntry {
   /** Error message to be logged */
   message: string;
   /** (optional) Show code below message */
-  codeFrame?: { source: string; line: number; column: number };
+  codeFrame?: { src: string; line: number; column: number };
   /** (optional) Display performance timing */
   timing?: number;
 }

package/logger.js

@@ -1,6 +1,6 @@
-import { codeFrameColumns } from '@babel/code-frame';
 import color from 'picocolors';
 import wcmatch from 'wildcard-match';
+import { codeFrameColumns } from './lib/code-frame.js';
 
 export const LOG_ORDER = ['error', 'warn', 'info', 'debug'];
 
@@ -23,8 +23,10 @@ export function formatMessage(entry, severity) {
   if (severity in MESSAGE_COLOR) {
     message = MESSAGE_COLOR[severity](message);
   }
-  if (entry.source) {
-    message = `${message}\n\n${codeFrameColumns(entry.source, { start: entry.node?.loc?.start })}`;
+  if (entry.src) {
+    const start = entry.node?.loc?.start;
+    // note: strip "file://" protocol, but not href
+    message = `${message}\n\n${entry.filename ? `${entry.filename.href.replace(/^file:\/\//, '')}:${start?.line ?? 0}:${start?.column ?? 0}\n\n` : ''}${codeFrameColumns(entry.src, { start }, { highlightCode: false })}`;
   }
   return message;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@terrazzo/parser",
-  "version": "0.0.11",
+  "version": "0.0.13",
   "description": "Parser/validator for the Design Tokens Community Group (DTCG) standard.",
   "type": "module",
   "author": {
@@ -30,19 +30,20 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@babel/code-frame": "^7.24.7",
-    "@humanwhocodes/momoa": "^3.1.1",
+    "@humanwhocodes/momoa": "^3.2.0",
     "@types/babel__code-frame": "^7.0.6",
     "@types/culori": "^2.1.1",
-    "@types/deep-equal": "^1.0.4",
     "culori": "^4.0.1",
-    "deep-equal": "^2.2.3",
     "is-what": "^4.1.16",
     "merge-anything": "^5.1.7",
     "picocolors": "^1.0.1",
     "wildcard-match": "^5.1.3",
-    "yaml": "^2.4.5",
-    "@terrazzo/token-tools": "^0.0.4"
+    "yaml": "^2.5.0",
+    "@terrazzo/token-tools": "^0.0.6"
+  },
+  "devDependencies": {
+    "esbuild": "^0.23.0",
+    "yaml-to-momoa": "^0.0.1"
   },
   "scripts": {
     "lint": "biome check .",

package/parse/index.d.ts

@@ -1,28 +1,39 @@
 import type { DocumentNode } from '@humanwhocodes/momoa';
 import type { TokenNormalized } from '@terrazzo/token-tools';
+import type yamlToMomoa from 'yaml-to-momoa';
 import type { ConfigInit } from '../config.js';
 import type Logger from '../logger.js';
 
 export * from './validate.js';
 
+export interface ParseInput {
+  /** Source filename (if read from disk) */
+  filename?: URL;
+  /** JSON/YAML string, or JSON-serializable object (if already in memory) */
+  src: string | object;
+}
+
 export interface ParseOptions {
   logger?: Logger;
+  config: ConfigInit;
   /** Skip lint step (default: false) */
   skipLint?: boolean;
-  config: ConfigInit;
   /** Continue on error? (Useful for `tz check`) (default: false) */
   continueOnError?: boolean;
+  /** Provide yamlToMomoa module to parse YAML (by default, this isn’t shipped to cut down on package weight) */
+  yamlToMomoa?: typeof yamlToMomoa;
 }
 
 export interface ParseResult {
   tokens: Record<string, TokenNormalized>;
-  ast: DocumentNode;
+  /** ASTs are returned in order of input array */
+  sources: { filename?: URL; src: string; document: DocumentNode }[];
 }
 
 /**
  * Parse and validate Tokens JSON, and lint it
  */
-export default function parse(input: string | object, options?: ParseOptions): Promise<ParseResult>;
+export default function parse(input: ParseInput[], options?: ParseOptions): Promise<ParseResult>;
 
 /** Determine if an input is likely a JSON string */
 export function maybeJSONString(input: unknown): boolean;