@terrazzo/parser 0.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Drew Powers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/build/index.d.ts

@@ -0,0 +1,104 @@
+import type { DocumentNode } from '@humanwhocodes/momoa';
+import type { ConfigInit } from '../config.js';
+import type Logger from '../logger.js';
+import type { TokenNormalized } from '../types.js';
+
+export interface BuildRunnerOptions {
+  ast: DocumentNode;
+  config: ConfigInit;
+  logger?: Logger;
+}
+
+export interface OutputFile {
+  filename: string;
+  contents: string | Buffer;
+}
+
+/** Transformed token with a single value. Note that this may be any type! */
+export interface TokenTransformedSingleValue {
+  /** ID unique to this format. If missing, use `token.id`. */
+  localID?: string;
+  type: 'SINGLE_VALUE';
+  value: string;
+  /** The mode of this value (default: `"."`) */
+  mode: string;
+  /** The original token */
+  token: TokenNormalized;
+}
+
+/** Transformed token with multiple values. Note that this may be any type! */
+export interface TokenTransformedMultiValue {
+  /** ID unique to this format. If missing, use `token.id` */
+  localID?: string;
+  type: 'MULTI_VALUE';
+  value: Record<string, string>;
+  /** The mode of this value (default: `"."`) */
+  mode: string;
+  /** The original token */
+  token: TokenNormalized;
+}
+
+export type TokenTransformed = TokenTransformedSingleValue | TokenTransformedMultiValue;
+
+export interface TransformParams {
+  /** ID of an existing format */
+  format: string;
+  /** Glob of tokens to select (e.g. `"color.*"` to select all tokens starting with `"color."`) */
+  select?: string | string[];
+  /** Mode name, if selecting a mode (default: `"."`) */
+  mode?: string | string[];
+}
+
+export interface TransformHookOptions {
+  /** Map of tokens */
+  tokens: Record<string, TokenNormalized>;
+  /** Query transformed values */
+  getTransforms(params: TransformParams): TokenTransformed;
+  /** Update transformed values */
+  setTransform(
+    id: string,
+    params: {
+      format: string;
+      localID?: string;
+      value: string | Record<string, string>;
+      mode?: string;
+    },
+  ): void;
+  /** Momoa document */
+  ast: DocumentNode;
+}
+
+export interface BuildHookOptions {
+  /** Map of tokens */
+  tokens: Record<string, TokenNormalized>;
+  /** Query transformed values */
+  getTransforms(params: TransformParams): TokenTransformed[];
+  /** Momoa document */
+  ast: DocumentNode;
+  outputFile: (
+    /** Filename to output (relative to outDir) */
+    filename: string,
+    /** Contents to write to file */
+    contents: string | Buffer,
+  ) => void;
+}
+
+export interface BuildRunnerResult {
+  outputFiles: OutputFile[];
+}
+
+export interface BuildEndHookOptions {
+  /** Map of tokens */
+  tokens: Record<string, TokenNormalized>;
+  /** Query transformed values */
+  getTransforms(params: TransformParams): TokenTransformed[];
+  /** Momoa document */
+  ast: DocumentNode;
+  /** Final files to be written */
+  outputFiles: OutputFile[];
+}
+
+export default function build(
+  tokens: Record<string, TokenNormalized>,
+  options: BuildRunnerOptions,
+): Promise<BuildRunnerResult>;

package/build/index.js

@@ -0,0 +1,182 @@
+import { isTokenMatch } from '@terrazzo/token-tools';
+import wcmatch from 'wildcard-match';
+import Logger from '../logger.js';
+
+/**
+ * @typedef {object} BuildRunnerOptions
+ * @typedef {Record<string, TokenNormalized>} BuildRunnerOptions.tokens
+ * @typedef {DocumentNode} BuildRunnerOptions.ast
+ * @typedef {ConfigInit} BuildRunnerOptions.config
+ * @typedef {Logger} BuildRunnerOptions.logger
+ * @typedef {import("@humanwhocodes/momoa").DocumentNode} DocumentNode
+ * @typedef {import("../config.js").ConfigInit} ConfigInit
+ * @typedef {import("../logger.js")} Logger
+ * @typedef {import("../types.js").TokenNormalized} TokenNormalized
+ *
+ * @typedef {object} BuildRunnerResult
+ * @typedef {OutputFile[]} BuildRunnerResult.outputFiles
+ * @typedef {object} OutputFile
+ * @typedef {string} OutputFile.filename
+ * @typedef {string | Buffer} OutputFile.contents
+ */
+
+export const SINGLE_VALUE = 'SINGLE_VALUE';
+export const MULTI_VALUE = 'MULTI_VALUE';
+
+/** Validate plugin setTransform() calls for immediate feedback */
+function validateTransformParams({ params, token, logger, pluginName }) {
+  const baseEntry = { group: 'plugin', task: pluginName };
+
+  // validate ID
+  if (!token) {
+    logger.error({
+      ...baseEntry,
+      message: `setTransform() tried to transform token "${id}" but it doesn’t exist.`,
+    });
+  }
+  // validate value is valid for SINGLE_VALUE or MULTI_VALUE
+  if (
+    !params.value ||
+    (typeof params.value !== 'string' && typeof params.value !== 'object') ||
+    Array.isArray(params.value)
+  ) {
+    logger.error({
+      ...baseEntry,
+      message: `setTransform() value expected string or object of strings, received ${
+        Array.isArray(params.value) ? 'Array' : typeof params.value
+      }`,
+    });
+  }
+  if (typeof params.value === 'object' && Object.values(params.value).some((v) => !v || typeof v !== 'string')) {
+    logger.error({
+      ...baseEntry,
+      message: 'setTransform() value expected object of strings, received some non-string values',
+    });
+  }
+}
+
+/**
+ * Run build stage
+ * @param {BuildOptions} options
+ * @return {Promise<BuildResult>}
+ */
+export default async function build(tokens, { ast, logger = new Logger(), config }) {
+  const formats = {};
+  const result = { outputFiles: [] };
+
+  function getTransforms(params) {
+    return (formats[params.format] ?? []).filter((token) => {
+      if (
+        params.select &&
+        params.select !== '*' &&
+        !isTokenMatch(token.token.id, Array.isArray(params.select) ? params.select : [params.select])
+      ) {
+        return false;
+      }
+      return !params.mode || wcmatch(params.mode)(token.mode);
+    });
+  }
+
+  // transform()
+  let transformsLocked = false; // prevent plugins from transforming after stage has ended
+  const startTransform = performance.now();
+  logger.debug({ group: 'parser', task: 'transform', message: 'Start transform' });
+  for (const plugin of config.plugins) {
+    if (typeof plugin.transform === 'function') {
+      await plugin.transform({
+        tokens,
+        ast,
+        getTransforms,
+        setTransform(id, params) {
+          if (transformsLocked) {
+            logger.warn({
+              message: 'Attempted to call setTransform() after transform step has completed.',
+              group: 'plugin',
+              task: plugin.name,
+            });
+            return;
+          }
+          const token = tokens[id];
+          validateTransformParams({ token, logger, params, pluginName: plugin.name });
+
+          // upsert
+          if (!formats[params.format]) {
+            formats[params.format] = [];
+          }
+          const foundTokenI = formats[params.format].findIndex(
+            (t) =>
+              params.localID === t.localID &&
+              (!params.mode || params.mode === t.mode) &&
+              (!params.variant || params.variant === t.variant),
+          );
+          if (foundTokenI === -1) {
+            formats[params.format].push({
+              ...params,
+              type: typeof params.value === 'string' ? SINGLE_VALUE : MULTI_VALUE,
+              mode: params.mode || '.',
+              token: structuredClone(token),
+            });
+          } else {
+            formats[params.format][foundTokenI].value = params.value;
+            formats[params.format][foundTokenI].type = typeof params.value === 'string' ? SINGLE_VALUE : MULTI_VALUE;
+          }
+        },
+      });
+    }
+  }
+  transformsLocked = true;
+  logger.debug({
+    group: 'parser',
+    task: 'transform',
+    message: 'Finish transform',
+    timing: performance.now() - startTransform,
+  });
+
+  // build()
+  const startBuild = performance.now();
+  logger.debug({ group: 'parser', task: 'build', message: 'Start build' });
+  for (const plugin of config.plugins) {
+    if (typeof plugin.build === 'function') {
+      await plugin.build({
+        tokens,
+        ast,
+        getTransforms,
+        outputFile(filename, contents) {
+          const resolved = new URL(filename, config.outDir);
+          if (result.outputFiles.some((f) => new URL(f.filename, config.outDir).href === resolved.href)) {
+            logger.error({ message: `Can’t overwrite file "${filename}"`, label: plugin.name });
+          }
+          result.outputFiles.push({ filename, contents });
+        },
+      });
+    }
+  }
+  logger.debug({
+    group: 'parser',
+    task: 'build',
+    message: 'Finish build',
+    timing: performance.now() - startBuild,
+  });
+
+  // buildEnd()
+  const startBuildEnd = performance.now();
+  logger.debug({ group: 'parser', task: 'build', message: 'Start buildEnd' });
+  for (const plugin of config.plugins) {
+    if (typeof plugin.buildEnd === 'function') {
+      await plugin.buildEnd({
+        tokens,
+        ast,
+        format: (formatID) => createFormatter(formatID),
+        outputFiles: structruedClone(result.outputFiles),
+      });
+    }
+  }
+  logger.debug({
+    group: 'parser',
+    task: 'build',
+    message: 'Finish buildEnd',
+    timing: performance.now() - startBuildEnd,
+  });
+
+  return result;
+}

package/config.d.ts

@@ -0,0 +1,64 @@
+import type { BuildHookOptions, BuildRunnerResult, TransformHookOptions } from './build/index.js';
+import type { LintRuleShorthand, LintRuleLonghand, Linter } from './lint/index.js';
+import type Logger from './logger.js';
+
+export interface Config {
+  /** Path to tokens.json (default: "./tokens.json") */
+  tokens?: string | string[];
+  /** Output directory (default: "./tokens/") */
+  outDir?: string;
+  /** Specify plugins */
+  plugins?: Plugin[];
+  /** Specify linting settings */
+  lint?: {
+    /** Configure build behavior */
+    build?: {
+      /** Should linters run with `co build`? (default: true) */
+      enabled?: boolean;
+    };
+    /** Configure lint rules */
+    rules?: Record<string, LintRuleShorthand | LintRuleLonghand>;
+  };
+}
+
+export interface ConfigInit {
+  tokens: URL[];
+  outDir: URL;
+  plugins: Plugin[];
+  lint: {
+    build: NonNullable<NonNullable<Config['lint']>['build']>;
+    rules: Record<string, LintRuleLonghand>;
+  };
+}
+
+export interface Plugin {
+  name: string;
+  /** Read config, and optionally modify */
+  // biome-ignore lint/suspicious/noConfusingVoidType format: this helps plugins be a little looser on their typing
+  config?(config: ConfigInit): void | ConfigInit | undefined;
+  /**
+   * Declare:
+   * - `"pre"`: run this plugin BEFORE all others
+   * - `"post"`: run this plugin AFTER all others
+   * - (default) run this plugin in default order (array order)
+   */
+  enforce?: 'pre' | 'post';
+  /** Throw lint errors/warnings */
+  lint?(): Record<string, Linter>;
+  transform?(options: TransformHookOptions): Promise<void>;
+  build?(options: BuildHookOptions): Promise<void>;
+  buildEnd?(result: BuildRunnerResult): Promise<void>;
+}
+
+export interface ConfigOptions {
+  logger?: Logger;
+  /** @terrazzo/parser needs cwd so this can be run without Node.js. Importing defineConfig from @terrazzo/cli doesn’t need this. */
+  cwd: URL;
+}
+
+/**
+ * Validate and normalize a config
+ */
+export default function defineConfig(rawConfig: Config, options: ConfigOptions): ConfigInit;
+
+export function mergeConfigs(a: Config, b: Config): Config;

package/config.js

@@ -0,0 +1,196 @@
+import { merge } from 'merge-anything';
+import coreLintPlugin from './lint/plugin-core/index.js';
+import Logger from './logger.js';
+
+const TRAILING_SLASH_RE = /\/*$/;
+
+/**
+ * Validate and normalize a config
+ * @param {Config} rawConfig
+ * @param {object} options
+ * @param {Logger} options.logger
+ * @param {URL} options.cwd
+ */
+export default function defineConfig(rawConfig, { logger = new Logger(), cwd = import.meta.url } = {}) {
+  const configStart = performance.now();
+
+  logger.debug({ group: 'parser', task: 'config', message: 'Start config validation' });
+
+  const config = { ...rawConfig };
+
+  // config.tokens
+  if (rawConfig.tokens === undefined) {
+    config.tokens = ['./tokens.json']; // will be normalized in next step
+  } else if (typeof rawConfig.tokens === 'string') {
+    config.tokens = [rawConfig.tokens]; // will be normalized in next step
+  } else if (Array.isArray(rawConfig.tokens)) {
+    config.tokens = [];
+    for (const file of rawConfig.tokens) {
+      if (typeof file === 'string') {
+        config.tokens.push(file); // will be normalized in next step
+      } else {
+        logger.error({
+          label: 'config.tokens',
+          message: `Expected array of strings, encountered ${JSON.stringify(file)}`,
+        });
+      }
+    }
+  } else {
+    logger.error({
+      label: 'config.tokens',
+      message: `Expected string or array of strings, received ${typeof rawConfig.tokens}`,
+    });
+  }
+  for (let i = 0; i < config.tokens.length; i++) {
+    const filepath = config.tokens[i];
+    try {
+      config.tokens[i] = new URL(filepath, cwd);
+    } catch {
+      logger.error({ label: 'config.tokens', message: `Invalid URL ${filepath}` });
+    }
+  }
+
+  // config.outDir
+  if (typeof config.outDir === 'undefined') {
+    config.outDir = new URL('./tokens/', cwd);
+  } else if (typeof config.outDir !== 'string') {
+    logger.error({ label: 'config.outDir', message: `Expected string, received ${JSON.stringify(config.outDir)}` });
+  } else {
+    // note: always add trailing slash so URL treats it as a directory
+    config.outDir = new URL(config.outDir.replace(TRAILING_SLASH_RE, '/'), cwd);
+  }
+
+  // config.plugins
+  if (typeof config.plugins === 'undefined') {
+    config.plugins = [];
+  }
+  if (!Array.isArray(config.plugins)) {
+    logger.error({
+      label: 'config.plugins',
+      message: `Expected array of plugins, received ${JSON.stringify(config.plugins)}`,
+    });
+  }
+  config.plugins.push(coreLintPlugin());
+  for (let n = 0; n < config.plugins.length; n++) {
+    const plugin = config.plugins[n];
+    if (typeof plugin !== 'object') {
+      logger.error({ label: `plugin[${n}]`, message: `Expected output plugin, received ${JSON.stringify(plugin)}` });
+    } else if (!plugin.name) {
+      logger.error({ label: `plugin[${n}]`, message: `Missing "name"` });
+    }
+  }
+  // order plugins with "enforce"
+  config.plugins.sort((a, b) => {
+    if (a.enforce === 'pre' && b.enforce !== 'pre') {
+      return -1;
+    } else if (a.enforce === 'post' && b.enforce !== 'post') {
+      return 1;
+    }
+    return 0;
+  });
+
+  // config.lint
+  if (config.lint !== undefined) {
+    if (config.lint === null || typeof config.lint !== 'object' || Array.isArray(config.lint)) {
+      logger.error({ label: 'config.lint', message: 'Must be an object' });
+      return config;
+    }
+
+    if (!config.lint.build) {
+      config.lint.build = { enabled: true };
+    }
+    if (config.lint.build.enabled !== undefined) {
+      if (typeof config.lint.build.enabled !== 'boolean') {
+        logger.error({
+          label: 'config.lint.build.enabled',
+          message: `Expected boolean, received ${JSON.stringify(config.lint.build)}`,
+        });
+      }
+    } else {
+      config.lint.build.enabled = true;
+    }
+
+    if (config.lint.rules !== undefined) {
+      if (config.lint.rules === null || typeof config.lint.rules !== 'object' || Array.isArray(config.lint.rules)) {
+        logger.error({
+          label: 'config.lint.rules',
+          message: `Expected object, received ${JSON.stringify(config.lint.rules)}`,
+        });
+      }
+
+      for (const id in config.lint.rules) {
+        if (!Object.hasOwn(config.lint.rules, id)) {
+          continue;
+        }
+        if (typeof id !== 'string') {
+          logger.error({ label: 'config.lint.rules', message: `Expects string keys, received ${JSON.stringify(id)}` });
+        }
+        const value = config.lint.rules[id];
+        let severity = 'off';
+        let options;
+        if (typeof value === 'number' || typeof value === 'string') {
+          severity = value;
+        } else if (Array.isArray(value)) {
+          severity = value[0];
+          options = value[1];
+        } else if (value !== undefined) {
+          logger.error({
+            label: `config.lint.rule:${id}`,
+            message: `Invalid eyntax. Expected \`string | number | Array\`, received ${JSON.stringify(value)}}`,
+          });
+        }
+        config.lint.rules[id] = { id, severity, options };
+        if (typeof severity === 'number') {
+          if (severity !== 0 && severity !== 1 && severity !== 2) {
+            logger.error({
+              label: `config.lint.rule:${id}`,
+              message: `Invalid number ${severity}. Specify 0 (off), 1 (warn), or 2 (error).`,
+            });
+            return config;
+          }
+          config.lint.rules[id].severity = ['off', 'warn', 'error'][severity];
+        } else if (typeof severity === 'string') {
+          if (severity !== 'off' && severity !== 'warn' && severity !== 'error') {
+            logger.error({
+              label: `config.lint.rule:${id}`,
+              message: `Invalid string ${JSON.stringify(severity)}. Specify "off", "warn", or "error".`,
+            });
+          }
+        } else if (value !== null) {
+          logger.error({
+            label: `config.lint.rule:${id}`,
+            message: `Expected string or number, received ${JSON.stringify(value)}`,
+          });
+        }
+      }
+    }
+  } else {
+    config.lint = {
+      build: { enabled: true },
+      rules: {},
+    };
+  }
+
+  // call plugin.config()
+  for (const plugin of config.plugins) {
+    plugin.config?.({ ...config });
+  }
+
+  logger.debug({
+    group: 'parser',
+    task: 'config',
+    message: 'Finish config validation',
+    timing: performance.now() - configStart,
+  });
+
+  return config;
+}
+
+/**
+ * @param {object} a
+ * @param {object} b
+ * @return {object}
+ */
+export function mergeConfigs(a, b) {
+  return merge(a, b);
+}

package/index.d.ts

@@ -0,0 +1,16 @@
+export { default as build } from './build/index.js';
+export * from './build/index.js';
+
+export { default as defineConfig } from './config.js';
+export * from './config.js';
+
+export { default as lintRunner } from './lint/index.js';
+export * from './lint/index.js';
+
+export { default as Logger } from './logger.js';
+export * from './logger.js';
+
+export { default as parse } from './parse/index.js';
+export * from './parse/index.js';
+
+export * from './types.js';

package/index.js

@@ -0,0 +1,37 @@
+// MIT License
+//
+// Copyright (c) 2021 Drew Powers
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+export { default as build } from './build/index.js';
+export * from './build/index.js';
+
+export { default as defineConfig } from './config.js';
+export * from './config.js';
+
+export { default as lintRunner } from './lint/index.js';
+export * from './lint/index.js';
+
+export { default as Logger } from './logger.js';
+export * from './logger.js';
+
+export { default as parse } from './parse/index.js';
+export * from './parse/index.js';
+
+export * from './types.js';

package/lint/index.d.ts

@@ -0,0 +1,41 @@
+import type { AnyNode, DocumentNode } from '@humanwhocodes/momoa';
+import type { ConfigInit } from '../config.js';
+import type Logger from '../logger.js';
+import type { Group } from '../types.js';
+
+export interface LintNotice {
+  /** Lint message shown to the user */
+  message: string;
+  /** Erring node (used to point to a specific line) */
+  node?: AnyNode;
+}
+
+export type LintRuleSeverity = 'error' | 'warn' | 'off';
+export type LintRuleShorthand = LintRuleSeverity | 0 | 1 | 2;
+export type LintRuleLonghand = [LintRuleSeverity | 0 | 1 | 2, any];
+
+export interface LintRuleNormalized<O = any> {
+  id: string;
+  severity: LintRuleSeverity;
+  options?: O;
+}
+
+export interface LinterOptions<O = any> {
+  tokens: Group;
+  rule: {
+    id: string;
+    severity: LintRuleSeverity;
+  };
+  ast: DocumentNode;
+  /** Any options the user has declared for this plugin */
+  options?: O;
+}
+export type Linter = (options: LinterOptions) => Promise<LintNotice[] | undefined>;
+
+export interface LintRunnerOptions {
+  ast: DocumentNode;
+  config: ConfigInit;
+  logger: Logger;
+}
+
+export default function lintRunner(options: LintRunnerOptions): Promise<void>;