@backtickjs/compiler 0.0.0-stage → 0.1.0

package/src/buildMappings.ts

@@ -0,0 +1,31 @@
+import type { CodeInformation } from "./CodeInformation.js";
+import type { Segment } from "./segmentsToString.js";
+
+export interface SourceMapping {
+  sourceOffsets: number[];
+  generatedOffsets: number[];
+  lengths: number[];
+  generatedLengths: number[];
+  data?: CodeInformation;
+}
+
+export function buildMappings(segments: Segment[]): SourceMapping[] {
+  const mappings: SourceMapping[] = [];
+  let generatedOffset = 0;
+  for (const segment of segments) {
+    if (typeof segment === "string") {
+      generatedOffset += segment.length;
+      continue;
+    }
+    const [text, , sourceOffset, sourceLength, data] = segment;
+    mappings.push({
+      sourceOffsets: [sourceOffset],
+      generatedOffsets: [generatedOffset],
+      lengths: [sourceLength],
+      generatedLengths: [text.length],
+      ...(data && { data }),
+    });
+    generatedOffset += text.length;
+  }
+  return mappings;
+}

package/src/compileModule.ts

@@ -0,0 +1,132 @@
+import type ts from "typescript";
+import type { Plugin } from "./compileScript.js";
+import { transform } from "./transform.js";
+
+// A loader's compile of one file, as a runtime runs it: Node's, Bun's. The
+// loader reads the file and loads what comes back; this does the rest, the
+// same way for every runtime.
+
+/** A compile step's module, as Babel's presets are: its default export makes the step. */
+interface PluginModule {
+  default: () => Plugin;
+}
+
+/**
+ * The framework's compile steps a project names in its `package.json`, made
+ * with the project's own `require`:
+ *
+ * ```json
+ * "backtick": { "plugins": ["@backtickjs/react/plugin"] }
+ * ```
+ */
+export function pluginsFrom(
+  packageJson: string,
+  require: (specifier: string) => unknown,
+): Plugin[] {
+  const { backtick } = JSON.parse(packageJson) as {
+    backtick?: { plugins?: readonly string[] };
+  };
+  return (backtick?.plugins ?? []).map((specifier) =>
+    (require(specifier) as PluginModule).default(),
+  );
+}
+
+export interface CompileModuleOptions {
+  /** The framework's compile steps, as `pluginsFrom` makes them. */
+  readonly plugins: readonly Plugin[];
+  /** Transforms run after Backtick's, as a runtime needs its own. */
+  readonly before?: readonly ts.TransformerFactory<ts.SourceFile>[];
+}
+
+/**
+ * A TypeScript file, its `cs` scripts included, as an ES module with its
+ * source map inline, compiled with the `tsconfig.json` nearest it, as `tsc`
+ * and editors find one, so an app and its server can each have their own.
+ * Throws the compiler's errors: a refused script is emitted with `null` where
+ * its code was, so it must not load.
+ */
+export function compileModule(
+  ts: typeof import("typescript"),
+  fileName: string,
+  source: string,
+  { plugins, before = [] }: CompileModuleOptions,
+): string {
+  const diagnostics: ts.Diagnostic[] = [];
+  const { outputText } = ts.transpileModule(source, {
+    fileName,
+    compilerOptions: compilerOptionsFor(ts, directoryOf(fileName)),
+    transformers: {
+      before: [
+        transform(ts, (diagnostic) => diagnostics.push(diagnostic), {
+          plugins,
+        }),
+        ...before,
+      ],
+    },
+  });
+
+  const errors = diagnostics.filter(
+    (diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error,
+  );
+  if (errors.length > 0) {
+    throw new Error(
+      ts.formatDiagnostics(errors, {
+        getCanonicalFileName: (name) => name,
+        getCurrentDirectory: () => ts.sys.getCurrentDirectory(),
+        getNewLine: () => "\n",
+      }),
+    );
+  }
+  return outputText;
+}
+
+// Read once per config, however many files share it.
+const optionsByConfig = new Map<string, ts.CompilerOptions>();
+
+function compilerOptionsFor(
+  ts: typeof import("typescript"),
+  directory: string,
+): ts.CompilerOptions {
+  const configPath =
+    ts.findConfigFile(directory, ts.sys.fileExists, "tsconfig.json") ?? "";
+  let options = optionsByConfig.get(configPath);
+  if (options === undefined) {
+    let fromConfig: ts.CompilerOptions = {};
+    if (configPath !== "") {
+      const { config } = ts.readConfigFile(configPath, ts.sys.readFile);
+      // Parsed whole, so `extends` is followed, as `tsc` follows it.
+      fromConfig = ts.parseJsonConfigFileContent(
+        config,
+        ts.sys,
+        directoryOf(configPath),
+      ).options;
+    }
+    // Where `tsc` would write its output says nothing about a file compiled
+    // where it loads, but a source map written with it names the file relative
+    // to that output, and a runtime resolves the name relative to the file.
+    const {
+      outDir: _outDir,
+      outFile: _outFile,
+      rootDir: _rootDir,
+      sourceRoot: _sourceRoot,
+      mapRoot: _mapRoot,
+      ...compiling
+    } = fromConfig;
+    options = {
+      ...compiling,
+      module: ts.ModuleKind.ESNext,
+      target: ts.ScriptTarget.ESNext,
+      sourceMap: false,
+      inlineSourceMap: true,
+      inlineSources: true,
+    };
+    optionsByConfig.set(configPath, options);
+  }
+  return options;
+}
+
+// A path's directory, by `/` or `\`, without Node's `path`.
+function directoryOf(path: string): string {
+  const end = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\"));
+  return end === -1 ? "." : path.slice(0, end);
+}

package/src/compileScript.ts

@@ -0,0 +1,97 @@
+import remapping from "@jridgewell/remapping";
+import type ts from "typescript";
+import type { EmittedScript } from "./emitScript.js";
+
+/**
+ * A framework's compile step: a module in, the module it compiles to out, with
+ * its map into the module it was given. Solid's adapter's `solid()` is one.
+ * Synchronous, as a TypeScript transformer is.
+ */
+export type Plugin = (
+  code: string,
+  id: string,
+) => { code: string; map: string };
+
+/** A script as a build writes it, a module table's entry. */
+export interface CompiledScript extends EmittedScript {
+  /** The modules the code requires, by specifier, as Metro records them. */
+  readonly dependencies: readonly string[];
+}
+
+/**
+ * A script as a build writes it: the module whose default export it is,
+ * through each plugin in turn, then a module table's entry,
+ * `(module, exports, require) => { … }`, as webpack and Metro write one: its
+ * body the module as CommonJS, JSX kept where no plugin compiled it, the
+ * modules it requires its dependencies, its map chained onto the script's own
+ * and naming the host file `sourceName`.
+ */
+export function compileScript(
+  ts: typeof import("typescript"),
+  script: EmittedScript,
+  sourceName: string,
+  plugins: readonly Plugin[],
+): CompiledScript {
+  let code = `export default (\n${script.code}\n);`;
+  const maps = [down(renamed(script.map, sourceName))];
+  for (const plugin of plugins) {
+    const compiled = plugin(code, `${sourceName}.jsx`);
+    code = compiled.code;
+    maps.unshift(compiled.map);
+  }
+  // The module as CommonJS, by TypeScript: its imports `require` calls, its
+  // default export `exports.default`, so it runs as a function's body.
+  const dependencies: string[] = [];
+  const output = ts.transpileModule(code, {
+    fileName: "module.jsx",
+    compilerOptions: {
+      jsx: ts.JsxEmit.Preserve,
+      module: ts.ModuleKind.CommonJS,
+      target: ts.ScriptTarget.ESNext,
+      sourceMap: true,
+    },
+    transformers: { before: [specifiers(ts, dependencies)] },
+  });
+  maps.unshift(output.sourceMapText!);
+  const body = output.outputText.replace(/\n\/\/# sourceMappingURL=.*$/, "");
+  // Chained as an array, not by source name: Babel names a map's source by
+  // the id's basename.
+  const map = remapping(maps, () => null, { excludeContent: true }).toString();
+  return {
+    code: `(module, exports, require) => {\n${body}\n}`,
+    map: down(map),
+    dependencies,
+  };
+}
+
+// What a module imports, by specifier, once each, as TypeScript reads it.
+function specifiers(
+  ts: typeof import("typescript"),
+  into: string[],
+): ts.TransformerFactory<ts.SourceFile> {
+  return () => (sourceFile) => {
+    for (const statement of sourceFile.statements) {
+      if (
+        ts.isImportDeclaration(statement) &&
+        ts.isStringLiteral(statement.moduleSpecifier) &&
+        !into.includes(statement.moduleSpecifier.text)
+      ) {
+        into.push(statement.moduleSpecifier.text);
+      }
+    }
+    return sourceFile;
+  };
+}
+
+// A script's map with its one source, the host file, named `sourceName`.
+function renamed(map: string, sourceName: string): string {
+  const parsed = JSON.parse(map) as object;
+  return JSON.stringify({ ...parsed, sources: [sourceName] });
+}
+
+// A map whose code moved down a line, as code does that a wrapper starts on a
+// line of its own.
+function down(map: string): string {
+  const parsed = JSON.parse(map) as { mappings: string };
+  return JSON.stringify({ ...parsed, mappings: `;${parsed.mappings}` });
+}

package/src/diagnostics.ts

@@ -0,0 +1,9 @@
+import type { DiagnosticCategory } from "typescript";
+import type { SourceRange } from "./SourceRange.js";
+
+export interface Diagnostic {
+  range: SourceRange;
+  message: string;
+  category: DiagnosticCategory;
+  code: number;
+}

package/src/emitScript.ts

@@ -0,0 +1,297 @@
+import { addMapping, GenMapping, toEncodedMap } from "@jridgewell/gen-mapping";
+import { eachMapping, TraceMap } from "@jridgewell/trace-mapping";
+import type ts from "typescript";
+import type { ClientScript } from "./parseFile.js";
+import type { BindingResolution, ResolvedParam } from "./resolveBindings.js";
+import { tagRoot } from "./tagRoot.js";
+
+/** A script as the client runs it, and where its code came from. */
+export interface EmittedScript {
+  /**
+   * The script as an expression, `($splice0, …) => body`, a parameter per
+   * entry of a script's `params` (see `paramName`). Types are gone and
+   * everything else, JSX included, is as the script wrote it, for the
+   * framework's own compiler to read when a bundle is built. How it is
+   * delivered is the bundle's to write.
+   */
+  readonly code: string;
+  /**
+   * The code's source map, into the host file. It carries no
+   * `sourcesContent`: the host file is the server's, and a map is shipped.
+   */
+  readonly map: string;
+}
+
+/**
+ * What a name the script wrote becomes: one of its entry's parameters, called
+ * with the bindings a hole hands over where it has `args`, read as it is where
+ * not.
+ */
+export interface Edit {
+  readonly param: string;
+  readonly args?: readonly string[];
+}
+
+/**
+ * A script parameter's name: its kind and its index in a script's `params`,
+ * `$splice0`, `$capture1`. A script cannot bind a name starting with
+ * `$`, so none meets one of its own.
+ */
+export function paramName(param: ResolvedParam, index: number): string {
+  return `$${param.kind}${index}`;
+}
+
+// A binding key as `resolveBindings` writes it, `<name>$<fileHash>$<n>`, back
+// to the name the script wrote.
+function sourceName(key: string): string {
+  return key.replace(/\$[0-9a-z]+\$\d+$/, "");
+}
+
+/**
+ * The edits a script's text needs, by the host-file offset of what each
+ * replaces: every splice it reads, host tags among them, and every binding it
+ * captures. What is a reference, and each parameter's number, was decided by
+ * `resolveBindings`.
+ */
+export function scriptEdits(
+  script: ClientScript,
+  bindings: BindingResolution,
+  params: readonly ResolvedParam[],
+): Map<number, Edit> {
+  const captureParam = new Map(
+    params.flatMap((param, index) =>
+      param.kind === "capture" ? [[param.key, index] as const] : [],
+    ),
+  );
+  const captures = [...captureParam.keys()];
+  const read = (key: string): string => {
+    const index = captureParam.get(key);
+    return index === undefined ? sourceName(key) : `$capture${index}`;
+  };
+
+  const edits = new Map<number, Edit>();
+  const at = (identifier: ts.Identifier) =>
+    script.toSourceRange(identifier).start;
+  params.forEach((param, index) => {
+    if (param.kind === "capture") {
+      return;
+    }
+    const edit: Edit = {
+      param: paramName(param, index),
+      args: [...param.bindings, ...captures].map(read),
+    };
+    for (const ref of param.refs) {
+      edits.set(at(ref), edit);
+    }
+  });
+  for (const [identifier, key] of bindings) {
+    const index = captureParam.get(key);
+    if (
+      index !== undefined &&
+      identifier.getSourceFile() === script.fileWithPlaceholders
+    ) {
+      edits.set(at(identifier), { param: `$capture${index}` });
+    }
+  }
+  return edits;
+}
+
+/**
+ * Emits a script's code by compiling its own text with TypeScript, with
+ * `edits` applied.
+ *
+ * TypeScript is handed the script's text alone, as `parseFile` read it: its
+ * template's escapes decoded, every `${…}` hole its placeholder. An edit is
+ * found where its name stands in the host file, and the source map is moved
+ * there, through the script's own offsets. The script alone, not the host file
+ * blanked around it: that would make each script cost what the whole file
+ * does.
+ */
+export function emitScript(
+  ts: typeof import("typescript"),
+  script: ClientScript,
+  params: readonly string[],
+  edits: ReadonlyMap<number, Edit>,
+): EmittedScript {
+  const sourceFile = script.sourceFile;
+
+  const transformer: ts.TransformerFactory<ts.SourceFile> =
+    (context) => (file) => {
+      const f = context.factory;
+      const editAt = (node: ts.Node) =>
+        edits.get(script.toSourceOffset(node.getStart(file)));
+      const edited = (edit: Edit, from: ts.Node): ts.Expression => {
+        const param = f.createIdentifier(edit.param);
+        const node =
+          edit.args === undefined
+            ? param
+            : f.createCallExpression(
+                param,
+                undefined,
+                edit.args.map((arg) => f.createIdentifier(arg)),
+              );
+        return ts.setOriginalNode(ts.setTextRange(node, from), from);
+      };
+      // A tag's names where the tag reads a splice: they stay as written, the
+      // parameter of the arrow below.
+      const asWritten = new Set<ts.Node>();
+      // A tag that names a splice, `<$Card>`: JSX can't call where a tag goes,
+      // so the element is the body of an arrow its tag is a parameter of, called
+      // with the splice where the element stands, as any splice is read:
+      // `(($Card) => <$Card …/>)($splice0())`. A capture is its value, read as
+      // a tag as it is anywhere.
+      const hostTag = (
+        node: ts.JsxElement | ts.JsxSelfClosingElement,
+      ): ts.Node | undefined => {
+        const opening = ts.isJsxElement(node) ? node.openingElement : node;
+        const root = tagRoot(ts, opening.tagName);
+        const edit = root && editAt(root);
+        if (root === undefined || edit?.args === undefined) {
+          return undefined;
+        }
+        asWritten.add(opening.tagName);
+        if (ts.isJsxElement(node)) {
+          asWritten.add(node.closingElement.tagName);
+        }
+        const call = ts.setTextRange(
+          f.createCallExpression(
+            f.createParenthesizedExpression(
+              f.createArrowFunction(
+                undefined,
+                undefined,
+                [f.createParameterDeclaration(undefined, undefined, root.text)],
+                undefined,
+                f.createToken(ts.SyntaxKind.EqualsGreaterThanToken),
+                ts.visitEachChild(node, visit, context),
+              ),
+            ),
+            undefined,
+            [edited(edit, root)],
+          ),
+          node,
+        );
+        // Among an element's children, an expression is written in braces.
+        return ts.isJsxElement(node.parent) || ts.isJsxFragment(node.parent)
+          ? f.createJsxExpression(undefined, call)
+          : call;
+      };
+      const visit = (node: ts.Node): ts.Node => {
+        // Types are TypeScript's to strip, after this.
+        if (ts.isTypeNode(node) || asWritten.has(node)) {
+          return node;
+        }
+        if (ts.isJsxElement(node) || ts.isJsxSelfClosingElement(node)) {
+          const tagged = hostTag(node);
+          if (tagged !== undefined) {
+            return tagged;
+          }
+        }
+        if (ts.isIdentifier(node)) {
+          const edit = editAt(node);
+          return edit === undefined ? node : edited(edit, node);
+        }
+        // `{ count }` names a key as well as a value.
+        if (ts.isShorthandPropertyAssignment(node)) {
+          const edit = editAt(node.name);
+          if (edit !== undefined) {
+            return f.createPropertyAssignment(
+              node.name.text,
+              edited(edit, node.name),
+            );
+          }
+        }
+        return ts.visitEachChild(node, visit, context);
+      };
+
+      const [statement] = file.statements;
+      if (statement === undefined) {
+        return file;
+      }
+      // Where the script stands, so the entry maps to it.
+      const entry = ts.setTextRange(
+        f.createArrowFunction(
+          undefined,
+          undefined,
+          params.map((name) =>
+            f.createParameterDeclaration(undefined, undefined, name),
+          ),
+          undefined,
+          f.createToken(ts.SyntaxKind.EqualsGreaterThanToken),
+          ts.isExpressionStatement(statement)
+            ? ts.visitNode(statement.expression, visit, ts.isExpression)
+            : ts.visitNode(statement, visit, ts.isBlock),
+        ),
+        statement,
+      );
+      return f.updateSourceFile(file, [
+        ts.setOriginalNode(f.createExpressionStatement(entry), statement),
+      ]);
+    };
+
+  const output = ts.transpileModule(script.textWithPlaceholders, {
+    fileName: sourceFile.fileName,
+    compilerOptions: {
+      target: ts.ScriptTarget.ESNext,
+      module: ts.ModuleKind.ESNext,
+      jsx: ts.JsxEmit.Preserve,
+      sourceMap: true,
+      alwaysStrict: false,
+      removeComments: true,
+    },
+    transformers: { before: [transformer] },
+  });
+  // The file's one expression statement, without its `;` or the comment
+  // naming a map file.
+  const code = output.outputText
+    .replace(/\n\/\/# sourceMappingURL=.*$/, "")
+    .replace(/;\s*$/, "");
+  return {
+    code,
+    map: moved(output.sourceMapText!, script),
+  };
+}
+
+// A map into the script's text, moved to where each position stands in the
+// host file. Named as the host file was, not relative to an output file there
+// is none of, so maps from files in different directories can be combined.
+function moved(map: string, script: ClientScript): string {
+  const sourceFile = script.sourceFile;
+  const lineStarts = [0];
+  for (const line of script.textWithPlaceholders.matchAll(/\r\n|\n|\r/g)) {
+    lineStarts.push(line.index + line[0].length);
+  }
+  // Lines are 1-based here, columns 0-based.
+  const original = (line: number, column: number) => {
+    const at = sourceFile.getLineAndCharacterOfPosition(
+      script.toSourceOffset(lineStarts[line - 1]! + column),
+    );
+    return { line: at.line + 1, column: at.character };
+  };
+  const into = new GenMapping({ file: sourceFile.fileName });
+  eachMapping(new TraceMap(map), (mapping) => {
+    if (mapping.originalLine === null) {
+      addMapping(into, {
+        generated: {
+          line: mapping.generatedLine,
+          column: mapping.generatedColumn,
+        },
+      });
+      return;
+    }
+    // TypeScript writes no names, so there are none to carry.
+    addMapping(into, {
+      generated: {
+        line: mapping.generatedLine,
+        column: mapping.generatedColumn,
+      },
+      source: sourceFile.fileName,
+      original: original(mapping.originalLine, mapping.originalColumn),
+    });
+  });
+  // TypeScript's own map, with only where it points changed.
+  return JSON.stringify({
+    ...(JSON.parse(map) as object),
+    sources: [sourceFile.fileName],
+    mappings: toEncodedMap(into).mappings,
+  });
+}

package/src/emitScripts.ts

@@ -0,0 +1,39 @@
+import {
+  type CompiledScript,
+  compileScript,
+  type Plugin,
+} from "./compileScript.js";
+import { parseSourceText } from "./parseFile.js";
+import { rewriteFile } from "./rewriteFile.js";
+
+/** A script's emitted code, by where the host wrote it. */
+export interface EmittedScriptAt extends CompiledScript {
+  // the offset of its `cs` in the host file
+  readonly start: number;
+}
+
+/**
+ * What each of a file's scripts compiles to for the client, in source order:
+ * nested scripts too, each as its own entry.
+ */
+export function emitScripts(
+  ts: typeof import("typescript"),
+  fileName: string,
+  sourceText: string,
+  // Its framework's compile steps (see `TransformOptions.plugins`).
+  plugins: readonly Plugin[] = [],
+): EmittedScriptAt[] {
+  const file = rewriteFile(ts, parseSourceText(ts, fileName, sourceText));
+  return Array.from(file.scripts)
+    .flatMap(([template, { runtime }]) =>
+      runtime === null
+        ? []
+        : [
+            {
+              start: template.getStart(file.sourceFile),
+              ...compileScript(ts, runtime.emitted, fileName, plugins),
+            },
+          ],
+    )
+    .sort((a, b) => a.start - b.start);
+}

package/src/flattenScripts.ts

@@ -0,0 +1,13 @@
+import type { ClientScript } from "./parseFile.js";
+
+// Yields every client script in the file, descending through splices
+export function* flattenScripts(
+  scripts: ClientScript[],
+): Generator<ClientScript> {
+  for (const script of scripts) {
+    yield script;
+    for (const splice of Object.values(script.splices)) {
+      yield* flattenScripts(splice.scripts);
+    }
+  }
+}

package/src/hashText.ts

@@ -0,0 +1,17 @@
+// 64-bit FNV-1a over the file's text: the per-file content hash that makes
+// compiler output globally distinguishable. The compiler runs one file at a
+// time and cannot hand out coordinated identifiers, and a file's path alone
+// can recur across codebases (two packages each compiled against their own
+// root), so both binding keys and script locations carry this hash. At 64
+// bits an accidental collision between different files is out of reach
+// (the birthday bound sits around four billion files), so a collision
+// implies identical file content, where sharing is correct rather than a
+// mixup.
+export function hashText(input: string): string {
+  let hash = 0xcbf29ce484222325n;
+  for (let i = 0; i < input.length; i++) {
+    hash ^= BigInt(input.charCodeAt(i));
+    hash = (hash * 0x100000001b3n) & 0xffffffffffffffffn;
+  }
+  return hash.toString(36);
+}

package/src/hostElements.ts

@@ -0,0 +1,34 @@
+import type ts from "typescript";
+import type { Diagnostic } from "./diagnostics.js";
+import { isComponentTag } from "./isComponentTag.js";
+
+// Each element written in host code: an intrinsic element is drawn by the
+// client, so it belongs in a script. Every JSX node TypeScript parsed is the
+// host's, a splice's included, as a script's JSX is text in its template.
+export function hostElementDiagnostics(
+  ts: typeof import("typescript"),
+  sourceFile: ts.SourceFile,
+): Diagnostic[] {
+  const diagnostics: Diagnostic[] = [];
+  const visit = (node: ts.Node): void => {
+    if (ts.isJsxOpeningElement(node) || ts.isJsxSelfClosingElement(node)) {
+      const { tagName } = node;
+      const name = tagName.getText(sourceFile);
+      // A member read (`ui.button`) is a value, whatever its case.
+      const named = ts.isIdentifier(tagName) || ts.isJsxNamespacedName(tagName);
+      if (named && !isComponentTag(name)) {
+        diagnostics.push({
+          range: { start: tagName.getStart(sourceFile), end: tagName.getEnd() },
+          message:
+            `\`<${name}>\` is drawn by the client, so it belongs in a ` +
+            `script: cs\`<${name}>…</${name}>\`.`,
+          category: ts.DiagnosticCategory.Error,
+          code: 0,
+        });
+      }
+    }
+    ts.forEachChild(node, visit);
+  };
+  visit(sourceFile);
+  return diagnostics;
+}