@cplieger/deadset-ts 8.0.0-dev.2 → 8.0.0-dev.20

package/src/dependencies.ts

@@ -29,34 +29,55 @@ import {
   type Node,
   type SourceFile,
 } from "@typescript/native/ast";
+import { amdLoadOf, amdPlatform } from "./amd.ts";
 import { automaticTypes, type AutomaticTypes } from "./automatic-types.ts";
 import {
-  configurationFiles,
   declaredIn as declaredBeside,
   dependenciesNamed,
   moduleStrings,
   outsideChain,
+  type ConfigurationFiles,
   type OutsideModule,
   type ParseFile,
-} from "./configuration-files.ts";
+} from "./entries/configuration-files.ts";
 import type { FindingPosition } from "./finding.ts";
 import type { Host } from "./host.ts";
-import { nodeKey, packageScope, type Inventory } from "./inventory.ts";
-import { scriptTokens } from "./manifest.ts";
-import { dirnamePath, isAbsolutePath, joinPath, relativePath, resolvePath } from "./paths.ts";
+import { declarationBindings } from "./import-bindings.ts";
+import { nodeKey, type Inventory } from "./inventory.ts";
+import { packageScope } from "./load/package-scope.ts";
+import {
+  ancestors,
+  cannotStartPackageName,
+  dirnamePath,
+  joinPath,
+  relativePath,
+  resolvePath,
+} from "./paths.ts";
 import type { DependencySection, Module } from "./ref.ts";
 import { UNANSWERED } from "./query.ts";
 import type { Handle, ProjectView } from "./session.ts";
 import { sourceFilesOf, type SourceFiles } from "./source-files.ts";
-import { workflowTokens } from "./workflow-steps.ts";
+import { manifestCommands, nodemonTexts } from "./roots/manifest.ts";
+import { writtenName } from "./platform-calls.ts";
+import { callsPlatform } from "./platform-symbols.ts";
+import { parenthesesRemoved } from "./unwrap.ts";
+import type { Workflows } from "./entries/workflow-steps.ts";
+import {
+  pathsNamed,
+  preloadedPackages,
+  readCommands,
+  type CommandReading,
+} from "./commands/read.ts";
+import { heldNames } from "./text/named-files.ts";
+import { bytewise } from "./text/compare.ts";
+import { isRecord } from "./json-document.ts";
+import { inInstalledPackages, INSTALLED_PACKAGES, MANIFEST, readDocument } from "./fs/manifest.ts";
 
-const MANIFEST = "package.json";
-const MODULES_DIR = "node_modules";
-const MODULES_SEGMENT = `/${MODULES_DIR}/`;
+const MODULES_SEGMENT = `/${INSTALLED_PACKAGES}/`;
 const TYPES_SCOPE = "@types/";
 
-/** A script token naming a JavaScript file by a path no expansion changes. */
-const SCRIPT_FILE = /^[^$`{*?[~]+\.(?:js|mjs|cjs)$/u;
+/** A path a command names that is a JavaScript file's. */
+const SCRIPT_FILE = /\.(?:js|mjs|cjs)$/u;
 
 /** The manifest sections a dependency is declared in, each by the selector its reference spells. */
 const SECTIONS: readonly (readonly [string, DependencySection])[] = [
@@ -160,25 +181,6 @@ export interface Dependencies {
   readonly heldBy: ReadonlyMap<string, readonly string[]>;
 }
 
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-/** One JSON document's decoded value, or undefined where it cannot be read or parsed. */
-function readJSON(host: Host, path: string): { text: string; value: unknown } | undefined {
-  let text: string;
-  try {
-    text = host.readFile(path);
-  } catch {
-    return undefined;
-  }
-  try {
-    return { text, value: JSON.parse(text) };
-  } catch {
-    return undefined;
-  }
-}
-
 /**
  * The package a bare module specifier names: its first segment, or its first two
  * where it starts with a scope. A relative, absolute or scheme-prefixed specifier and
@@ -302,7 +304,7 @@ function configurationPackages(
     }
     seen.add(file);
     for (const entry of extendsOf(host, file, parse)) {
-      if (entry.startsWith(".") || isAbsolutePath(entry)) {
+      if (cannotStartPackageName(entry)) {
         const local = resolvePath(dirnamePath(file), entry);
         pending.push(local.endsWith(".json") ? local : `${local}.json`);
       } else {
@@ -332,34 +334,18 @@ function importOf(specifier: Node): Node | undefined {
 
 /** The local names one import declaration binds. */
 function bindingsOf(node: Node | undefined): readonly Texted[] {
-  if (node === undefined) {
-    return [];
-  }
-  if (isImportEqualsDeclaration(node)) {
-    return [node.name];
-  }
-  if (!isImportDeclaration(node) || node.importClause === undefined) {
-    return [];
-  }
-  const clause = node.importClause;
-  const names: Texted[] = clause.name === undefined ? [] : [clause.name];
-  const bindings = clause.namedBindings;
-  if (bindings !== undefined) {
-    if (bindings.kind === SyntaxKind.NamespaceImport) {
-      names.push(bindings.name);
-    } else {
-      names.push(...bindings.elements.map((element) => element.name));
-    }
-  }
-  return names;
+  return node !== undefined && (isImportDeclaration(node) || isImportEqualsDeclaration(node))
+    ? declarationBindings(node).map((binding) => binding.name)
+    : [];
 }
 
 /**
  * Every package one project needs, and which declaration each use is written in.
  * One local walk and one batched resolution per file: a use of an import binding is
  * an identifier resolving to the binding's own symbol, so a local declaration that
- * shadows the name is not one. A string of a configuration file uses each dependency
- * the manifest beside the file declares that it spells.
+ * shadows the name is not one. A string of a configuration file, as the entry sources
+ * read them in `configuration`, uses each dependency the manifest beside the file
+ * declares that it spells.
  */
 export function projectNeeds<Brand>(
   project: ProjectView<Brand>,
@@ -367,6 +353,8 @@ export function projectNeeds<Brand>(
   host: Host,
   targetRoot: string,
   parse: ParseFile,
+  configuration: ConfigurationFiles,
+  workflows: Workflows,
 ): ProjectNeeds {
   const programPackages = new Set<string>();
   for (const name of project.program.getSourceFileNames()) {
@@ -420,7 +408,6 @@ export function projectNeeds<Brand>(
     return names;
   };
   const files = sourceFilesOf(project, targetRoot);
-  const configuration = configurationFiles(host, targetRoot, files, parse);
   for (const { file, manifestDir } of configuration.modules) {
     const names = declaredAt(joinPath(manifestDir, MANIFEST));
     for (const { node, text } of moduleStrings(file)) {
@@ -429,7 +416,7 @@ export function projectNeeds<Brand>(
   }
   for (const { strings, manifestDirs } of configuration.documents) {
     for (const manifestDir of manifestDirs) {
-      const names = declaredAt(joinPath(manifestDir, "package.json"));
+      const names = declaredAt(joinPath(manifestDir, MANIFEST));
       for (const text of strings) {
         use([PROJECT_OWNER], dependenciesNamed(text, names), joinPath(manifestDir, MANIFEST));
       }
@@ -440,11 +427,11 @@ export function projectNeeds<Brand>(
     targetRoot,
     files,
     parse,
-    scriptRunFiles(host, targetRoot, files),
+    scriptRunFiles(host, targetRoot, files, workflows),
   );
   for (const { file, manifestDirs } of [...configuration.outside, ...ranFiles]) {
     for (const manifestDir of manifestDirs) {
-      const names = declaredAt(joinPath(manifestDir, "package.json"));
+      const names = declaredAt(joinPath(manifestDir, MANIFEST));
       for (const { text } of moduleStrings(file)) {
         use([PROJECT_OWNER], dependenciesNamed(text, names), joinPath(manifestDir, MANIFEST));
       }
@@ -454,44 +441,46 @@ export function projectNeeds<Brand>(
 }
 
 /**
- * The JavaScript files no program holds that a script of a manifest at or above one of the
- * project's files runs, each with the directory of that manifest, and those a workflow
- * step or git hook runs, with the target root's: a token naming such a file below the
- * target root and in no `node_modules`, read against the manifest's directory.
+ * The JavaScript files no program holds that a command string of a manifest at or above one of
+ * the project's files runs or names, each with the directory of that manifest, and those a
+ * workflow step or git hook names, with the target root's: a path `pathsNamed` reads, below the
+ * target root and in no `node_modules`.
  */
 function scriptRunFiles(
   host: Host,
   targetRoot: string,
   files: SourceFiles,
+  workflows: Workflows,
 ): readonly { readonly path: string; readonly manifestDir: string }[] {
   const root = resolvePath(host.workingDirectory(), targetRoot);
   const dirs = new Set<string>([root]);
   for (const name of files.byName.keys()) {
-    for (let dir = dirnamePath(name); !dirs.has(dir); dir = dirnamePath(dir)) {
-      if (relativePath(root, dir) === undefined) {
+    for (const dir of ancestors(dirnamePath(name), root)) {
+      if (dirs.has(dir)) {
         break;
       }
       dirs.add(dir);
     }
   }
-  const tokens = new Map<string, ReadonlySet<string>>([
-    [root, new Set(workflowTokens(host, root))],
+  const named = new Map<string, readonly string[]>([
+    [root, workflows.readings.flatMap(pathsNamed).map((one) => one.path)],
   ]);
   for (const dir of dirs) {
-    const read = readJSON(host, joinPath(dir, MANIFEST));
+    const read = readDocument(host, joinPath(dir, MANIFEST));
     if (read !== undefined && isRecord(read.value)) {
-      const own = scriptTokensOf(read.value);
-      tokens.set(dir, dir === root ? new Set([...own, ...(tokens.get(root) ?? [])]) : own);
+      const own = manifestReadings(host, dir, read.value)
+        .flatMap(pathsNamed)
+        .map((one) => one.path);
+      named.set(dir, [...own, ...(named.get(dir) ?? [])]);
     }
   }
   const found: { path: string; manifestDir: string }[] = [];
-  for (const [manifestDir, held] of tokens) {
-    for (const token of held) {
-      const path = SCRIPT_FILE.test(token) ? resolvePath(manifestDir, token) : undefined;
-      const below = path === undefined ? undefined : relativePath(root, path);
+  for (const [manifestDir, paths] of named) {
+    for (const path of new Set(paths)) {
+      const below = SCRIPT_FILE.test(path) ? relativePath(root, path) : undefined;
       if (
-        path !== undefined &&
-        below?.split("/").includes(MODULES_DIR) === false &&
+        below !== undefined &&
+        !inInstalledPackages(below) &&
         !files.byName.has(path) &&
         host.kindOf(path) === "file"
       ) {
@@ -502,6 +491,54 @@ function scriptRunFiles(
   return found;
 }
 
+/** What the command strings of one manifest, its scripts and nodemon's command, run. */
+function manifestReadings(
+  host: Host,
+  dir: string,
+  manifest: Record<string, unknown>,
+): readonly CommandReading[] {
+  return manifestCommands(host, dir, manifest).map(({ command }) => readCommands(command, dir));
+}
+
+/** What one manifest's command strings use, held with the whole text of nodemon's configuration. */
+function manifestUses(host: Host, dir: string, manifest: Record<string, unknown>): CommandUses {
+  const uses = usesOf(manifestReadings(host, dir, manifest));
+  const texts = nodemonTexts(host, dir, manifest).map(({ text }) => text);
+  return { ...uses, held: [uses.held, ...texts].join("\n") };
+}
+
+/** What the target root's workflow steps, git hooks and recipes use, held with their files' whole text. */
+function workflowUses(workflows: Workflows): CommandUses {
+  const uses = usesOf(workflows.readings);
+  return { ...uses, held: [uses.held, ...workflows.texts].join("\n") };
+}
+
+/** What some command strings say about the packages they use. */
+interface CommandUses {
+  /** Every static word of their commands, which uses a dependency whose command it is. */
+  readonly words: ReadonlySet<string>;
+  /** The package each module a read command preloads by name is in. */
+  readonly preloads: ReadonlySet<string>;
+  /** The whole text of every string, which the dependency rule reads: each package or command name in it is a use. */
+  readonly held: string;
+}
+
+function usesOf(readings: readonly CommandReading[]): CommandUses {
+  return {
+    words: new Set(
+      readings.flatMap((one) =>
+        one.commands.flatMap((command) => command.words.flatMap((word) => word ?? [])),
+      ),
+    ),
+    preloads: new Set(
+      readings
+        .flatMap((one) => one.runs.flatMap(preloadedPackages))
+        .flatMap((specifier) => packageOfSpecifier(specifier) ?? []),
+    ),
+    held: readings.map((one) => one.held).join("\n"),
+  };
+}
+
 /** The declarations of the inventory a use written at one node belongs to. */
 function ownersOf(file: SourceFile, held: Inventory, node: Node): readonly string[] {
   const parent = node.parent as Node | undefined;
@@ -599,8 +636,17 @@ function fileNeeds<Brand>(
   if (names.size > 0 || automatic.globals.size > 0) {
     visit(file);
   }
+  const loads = runtimeLoads(file);
+  const callees = loads.flatMap((load) => load.callees);
+  const amdCallees = new Set(loads.flatMap((load) => (load.amd ? load.callees : [])));
 
-  const asked = [...specifiers, ...bindings.map(({ node }) => node), ...candidates, ...globals];
+  const asked = [
+    ...specifiers,
+    ...bindings.map(({ node }) => node),
+    ...candidates,
+    ...globals,
+    ...callees,
+  ];
   const handles: Handle<Brand>[] = asked.map((node) => project.handle(node));
   const resolved = project.symbolsAt(handles);
 
@@ -676,35 +722,96 @@ function fileNeeds<Brand>(
   for (const reference of file.typeReferenceDirectives) {
     use(ownersOf(file, held, file), typePackages(reference.fileName, programPackages));
   }
-  for (const specifier of resolvedSpecifiers(file)) {
-    const named = packageOfSpecifier(specifier.text);
-    if (named !== undefined) {
+  // A callee the checker leaves unanswered may be a require, so its call holds the
+  // dependency it names by live code, as program.33 holds one.
+  const calleesAt = unresolvedAt + globals.length;
+  const decided = new Map<string, boolean>();
+  const requires = new Set<Identifier>();
+  const unansweredCallees = new Set<Identifier>();
+  callees.forEach((callee, index) => {
+    const symbol = resolved[calleesAt + index];
+    if (symbol === UNANSWERED) {
+      unansweredCallees.add(callee);
+      return;
+    }
+    // The decision reads the callee's symbol, its written name and its platform name, and nothing else.
+    const platform = amdCallees.has(callee) ? amdPlatform(callee) : "require";
+    const key =
+      symbol === undefined ? undefined : `${symbol.id}\u0000${platform}\u0000${callee.text}`;
+    const answer =
+      (key === undefined ? undefined : decided.get(key)) ??
+      callsPlatform(project, platform, callee, symbol);
+    if (key !== undefined) {
+      decided.set(key, answer);
+    }
+    if (answer) {
+      requires.add(callee);
+    }
+  });
+  for (const { specifier, named, callees: through } of loads) {
+    if (through.length === 0 || through.some((one) => requires.has(one))) {
       use(ownersOf(file, held, specifier), [named]);
+    } else if (through.some((one) => unansweredCallees.has(one))) {
+      use([PROJECT_OWNER], [named]);
     }
   }
   return resolved.slice(0, specifiers.length).includes(UNANSWERED);
 }
 
 /**
- * The literal specifiers one file asks the runtime to resolve, through `import.meta.resolve`
- * or `require.resolve`, each of which needs the package it names installed.
+ * A call of one file that may load or resolve a package at run time: the literal specifier
+ * naming it, and the names a `require` would be written with, of which one has to be a
+ * `require` for the call to need the package; none for a use where it stands.
  */
-function resolvedSpecifiers(file: SourceFile): readonly Texted[] {
-  const found: Texted[] = [];
-  if (!file.text.includes(".resolve(")) {
-    return found;
-  }
+interface RuntimeLoad {
+  readonly specifier: Texted;
+  readonly named: string;
+  readonly callees: readonly Identifier[];
+  /** Whether the specifier is an id of an AMD call's dependency list, its callee AMD's `define` or `require`. */
+  readonly amd: boolean;
+}
+
+/**
+ * The calls one file may load or resolve a package with at run time, each of whose first
+ * argument is literal text naming a package: a `require()` call, which the compiler lists among
+ * a file's imports only in a JavaScript file and only through the bare name; a `.resolve` of
+ * another name, which needs it where that name is a `require`; and `import.meta.resolve` or
+ * a `.resolve` of the name `require`, whatever it names, a use where it stands; and each id of
+ * an AMD dependency list (references-ts.37), which needs it where the call is AMD's.
+ */
+function runtimeLoads(file: SourceFile): readonly RuntimeLoad[] {
+  const found: RuntimeLoad[] = [];
   const visit = (node: Node): void => {
-    if (isCallExpression(node) && isPropertyAccessExpression(node.expression)) {
-      const callee = node.expression;
+    const amd = amdLoadOf(node);
+    if (amd !== undefined) {
+      for (const { id } of amd.ids) {
+        const named = packageOfSpecifier(id.text);
+        if (named !== undefined) {
+          found.push({ specifier: id, named, callees: [amd.callee], amd: true });
+        }
+      }
+    } else if (isCallExpression(node)) {
       const [argument] = node.arguments;
-      const receiver = callee.expression;
-      const resolving =
-        callee.name.text === "resolve" &&
-        ((isMetaProperty(receiver) && receiver.name.text === "meta") ||
-          (isIdentifier(receiver) && receiver.text === "require"));
-      if (resolving && argument !== undefined && isLiteralText(argument)) {
-        found.push(argument);
+      if (argument !== undefined && isLiteralText(argument)) {
+        const named = packageOfSpecifier(argument.text);
+        const callee = parenthesesRemoved(node.expression);
+        const receiver =
+          isPropertyAccessExpression(callee) && callee.name.text === "resolve"
+            ? parenthesesRemoved(callee.expression)
+            : undefined;
+        const stands =
+          receiver !== undefined &&
+          ((isMetaProperty(receiver) && receiver.name.text === "meta") ||
+            (isIdentifier(receiver) && receiver.text === "require"));
+        const callees = stands
+          ? []
+          : [
+              writtenName(callee),
+              receiver === undefined ? undefined : writtenName(receiver),
+            ].filter((one) => one !== undefined);
+        if (named !== undefined && (stands || callees.length > 0)) {
+          found.push({ specifier: argument, named, callees, amd: false });
+        }
       }
     }
     node.forEachChild(visit);
@@ -797,8 +904,8 @@ function lineAndColumn(text: string, offset: number): { line: number; column: nu
 
 /** The installed copy of one dependency, read where the package manager puts it for the target. */
 function installedOf(host: Host, root: string, name: string): InstalledDependency | undefined {
-  for (let dir = root; ; dir = dirnamePath(dir)) {
-    const read = readJSON(host, joinPath(dir, MODULES_DIR, name, MANIFEST));
+  for (const dir of ancestors(root)) {
+    const read = readDocument(host, joinPath(dir, INSTALLED_PACKAGES, name, MANIFEST));
     if (read !== undefined && isRecord(read.value)) {
       const bin = read.value["bin"];
       const peers = read.value["peerDependencies"];
@@ -815,10 +922,8 @@ function installedOf(host: Host, root: string, name: string): InstalledDependenc
               : [],
       };
     }
-    if (dirnamePath(dir) === dir) {
-      return undefined;
-    }
   }
+  return undefined;
 }
 
 /** Every dependency the manifest text at `path` declares, in the order the text writes them. */
@@ -857,20 +962,6 @@ export function declarationKey(dependency: DeclaredDependency): string {
   return `${dependency.position.path}\u0000${dependency.name}`;
 }
 
-/** Every token of every value of one manifest's `scripts` member. */
-function scriptTokensOf(manifest: Record<string, unknown>): ReadonlySet<string> {
-  const scripts = manifest["scripts"];
-  const found = new Set<string>();
-  if (isRecord(scripts)) {
-    for (const command of Object.values(scripts)) {
-      if (typeof command === "string") {
-        scriptTokens(command).forEach((token) => found.add(token));
-      }
-    }
-  }
-  return found;
-}
-
 /** Whether a path below the target root lies in the directory `dir` below it, `""` being the root. */
 function isInside(path: string, dir: string): boolean {
   return dir === "" || path.startsWith(`${dir}/`);
@@ -943,6 +1034,7 @@ function addNeededPeers(
 export function dependenciesOf(
   host: Host,
   targetRoot: string,
+  workflows: Workflows,
   perProject: readonly ProjectNeeds[],
   candidates: readonly string[],
   members: readonly string[] = [],
@@ -965,29 +1057,38 @@ export function dependenciesOf(
       unbuilt.add(path);
     }
   }
-  const workflow = new Set(workflowTokens(host, targetRoot));
+  const workflow = workflowUses(workflows);
   const declared: DeclaredDependency[] = [];
   const ran = new Set<string>();
   const peers = new Map<string, readonly string[]>();
   const uninstalledHere = new Set(nested);
   for (const [dir, path] of manifests) {
-    const read = readJSON(host, joinPath(dir, MANIFEST));
+    const read = readDocument(host, joinPath(dir, MANIFEST));
     if (read === undefined || !isRecord(read.value)) {
       continue;
     }
-    const tokens = scriptTokensOf(read.value);
+    const own = manifestUses(host, dir, read.value);
+    const held = `${own.held}\n${workflow.held}`;
     for (const dependency of declaredIn(read.text, read.value, path, scopeOf(path))) {
       const found = installedOf(host, dir, dependency.name);
       if (found === undefined && uninstalledHere.has(dir)) {
         continue;
       }
       declared.push(dependency);
+      if (
+        own.preloads.has(dependency.name) ||
+        workflow.preloads.has(dependency.name) ||
+        heldNames(held, [dependency.name]).size > 0
+      ) {
+        ran.add(declarationKey(dependency));
+      }
       if (found === undefined) {
         continue;
       }
       peers.set(declarationKey(dependency), found.peers);
-      const runs = (command: string): boolean => tokens.has(command) || workflow.has(command);
-      if (found.commands.some(runs)) {
+      const runs = (command: string): boolean =>
+        own.words.has(command) || workflow.words.has(command);
+      if (found.commands.some(runs) || heldNames(held, found.commands).size > 0) {
         ran.add(declarationKey(dependency));
       }
     }
@@ -1045,10 +1146,7 @@ export function dependenciesOf(
   const heldBy = new Map<string, readonly string[]>();
   for (const [key, held] of holders) {
     if (!live.has(key) && !ran.has(key)) {
-      heldBy.set(
-        key,
-        [...held].sort((a, b) => (a < b ? -1 : 1)),
-      );
+      heldBy.set(key, [...held].sort(bytewise));
     }
   }
   const users = new Map<string, number>();
@@ -1060,10 +1158,7 @@ export function dependenciesOf(
   for (const id of candidates) {
     const sole = [...(uses.get(id) ?? [])].filter((one) => users.get(one) === 1 && names.has(one));
     if (sole.length > 0) {
-      lastUses.set(
-        id,
-        sole.sort((a, b) => (a < b ? -1 : 1)),
-      );
+      lastUses.set(id, sole.sort(bytewise));
     }
   }
   const owners = new Map<string, Set<string>>();
@@ -1075,7 +1170,7 @@ export function dependenciesOf(
     }
   }
   const fileLastUses = new Map<string, string[]>();
-  for (const [one, held] of [...owners].sort(([a], [b]) => (a < b ? -1 : 1))) {
+  for (const [one, held] of [...owners].sort(([a], [b]) => bytewise(a, b))) {
     const paths = new Set([...held].map((owner) => owner.replace(/:\d+:\d+$/u, "")));
     const [path] = paths;
     if (paths.size === 1 && path !== undefined && path !== PROJECT_OWNER) {