@atlaspack/core 2.12.1-canary.3354

package/src/requests/ValidationRequest.js

@@ -0,0 +1,79 @@
+// @flow strict-local
+import type {Async} from '@atlaspack/types';
+import type {SharedReference} from '@atlaspack/workers';
+import type {StaticRunOpts} from '../RequestTracker';
+import type {AssetGroup} from '../types';
+import type {ConfigAndCachePath} from './AtlaspackConfigRequest';
+
+import nullthrows from 'nullthrows';
+import AtlaspackConfig from '../AtlaspackConfig';
+import {report} from '../ReporterRunner';
+import Validation from '../Validation';
+import createAtlaspackConfigRequest from './AtlaspackConfigRequest';
+import {requestTypes} from '../RequestTracker';
+
+type ValidationRequest = {|
+  id: string,
+  +type: typeof requestTypes.validation_request,
+  run: (RunOpts<void>) => Async<void>,
+  input: ValidationRequestInput,
+|};
+
+type RunOpts<TResult> = {|
+  input: ValidationRequestInput,
+  ...StaticRunOpts<TResult>,
+|};
+
+type ValidationRequestInput = {|
+  assetRequests: Array<AssetGroup>,
+  optionsRef: SharedReference,
+|};
+
+export default function createValidationRequest(
+  input: ValidationRequestInput,
+): ValidationRequest {
+  return {
+    id: 'validation',
+    type: requestTypes.validation_request,
+    run: async ({input: {assetRequests, optionsRef}, api, options, farm}) => {
+      let {config: processedConfig, cachePath} = nullthrows(
+        await api.runRequest<null, ConfigAndCachePath>(
+          createAtlaspackConfigRequest(),
+        ),
+      );
+
+      let config = new AtlaspackConfig(processedConfig, options);
+      let trackedRequestsDesc = assetRequests.filter(request => {
+        return config.getValidatorNames(request.filePath).length > 0;
+      });
+
+      // Schedule validations on workers for all plugins that implement the one-asset-at-a-time "validate" method.
+      let promises = trackedRequestsDesc.map(
+        async request =>
+          ((await farm.createHandle('runValidate'))({
+            requests: [request],
+            optionsRef: optionsRef,
+            configCachePath: cachePath,
+          }): void),
+      );
+
+      // Skip sending validation requests if no validators were configured
+      if (trackedRequestsDesc.length === 0) {
+        return;
+      }
+
+      // Schedule validations on the main thread for all validation plugins that implement "validateAll".
+      promises.push(
+        new Validation({
+          requests: trackedRequestsDesc,
+          options,
+          config,
+          report,
+          dedicatedThread: true,
+        }).run(),
+      );
+      await Promise.all(promises);
+    },
+    input,
+  };
+}

package/src/requests/WriteBundleRequest.js

@@ -0,0 +1,362 @@
+// @flow strict-local
+
+import type {FileSystem, FileOptions} from '@atlaspack/fs';
+import type {ContentKey} from '@atlaspack/graph';
+import type {Async, FilePath, Compressor} from '@atlaspack/types';
+
+import type {RunAPI, StaticRunOpts} from '../RequestTracker';
+import type {Bundle, PackagedBundleInfo, AtlaspackOptions} from '../types';
+import type BundleGraph from '../BundleGraph';
+import type {BundleInfo} from '../PackagerRunner';
+import type {ConfigAndCachePath} from './AtlaspackConfigRequest';
+import type {LoadedPlugin} from '../AtlaspackConfig';
+import type {ProjectPath} from '../projectPath';
+
+import {HASH_REF_HASH_LEN, HASH_REF_PREFIX} from '../constants';
+import nullthrows from 'nullthrows';
+import path from 'path';
+import {NamedBundle} from '../public/Bundle';
+import {blobToStream, TapStream} from '@atlaspack/utils';
+import {Readable, Transform, pipeline} from 'stream';
+import {
+  fromProjectPath,
+  fromProjectPathRelative,
+  toProjectPath,
+  joinProjectPath,
+  toProjectPathUnsafe,
+} from '../projectPath';
+import createAtlaspackConfigRequest, {
+  getCachedAtlaspackConfig,
+} from './AtlaspackConfigRequest';
+import PluginOptions from '../public/PluginOptions';
+import {PluginLogger} from '@atlaspack/logger';
+import {
+  getDevDepRequests,
+  invalidateDevDeps,
+  createDevDependency,
+  runDevDepRequest,
+} from './DevDepRequest';
+import AtlaspackConfig from '../AtlaspackConfig';
+import ThrowableDiagnostic, {errorToDiagnostic} from '@atlaspack/diagnostic';
+import {PluginTracer, tracer} from '@atlaspack/profiler';
+import {requestTypes} from '../RequestTracker';
+
+const HASH_REF_PREFIX_LEN = HASH_REF_PREFIX.length;
+const BOUNDARY_LENGTH = HASH_REF_PREFIX.length + 32 - 1;
+
+type WriteBundleRequestInput = {|
+  bundleGraph: BundleGraph,
+  bundle: Bundle,
+  info: BundleInfo,
+  hashRefToNameHash: Map<string, string>,
+|};
+
+export type WriteBundleRequestResult = PackagedBundleInfo;
+
+type RunInput<TResult> = {|
+  input: WriteBundleRequestInput,
+  ...StaticRunOpts<TResult>,
+|};
+
+export type WriteBundleRequest = {|
+  id: ContentKey,
+  +type: typeof requestTypes.write_bundle_request,
+  run: (RunInput<PackagedBundleInfo>) => Async<PackagedBundleInfo>,
+  input: WriteBundleRequestInput,
+|};
+
+/**
+ * Writes a bundle to the dist directory, replacing hash references with the final content hashes.
+ */
+export default function createWriteBundleRequest(
+  input: WriteBundleRequestInput,
+): WriteBundleRequest {
+  let name = nullthrows(input.bundle.name);
+  let nameHash = nullthrows(
+    input.hashRefToNameHash.get(input.bundle.hashReference),
+  );
+  return {
+    id: `${input.bundle.id}:${input.info.hash}:${nameHash}:${name}`,
+    type: requestTypes.write_bundle_request,
+    run,
+    input,
+  };
+}
+
+async function run({input, options, api}) {
+  let {bundleGraph, bundle, info, hashRefToNameHash} = input;
+  let {inputFS, outputFS} = options;
+  let name = nullthrows(bundle.name);
+  let thisHashReference = bundle.hashReference;
+
+  if (info.type !== bundle.type) {
+    name = name.slice(0, -path.extname(name).length) + '.' + info.type;
+  }
+
+  if (name.includes(thisHashReference)) {
+    let thisNameHash = nullthrows(hashRefToNameHash.get(thisHashReference));
+    name = name.replace(thisHashReference, thisNameHash);
+  }
+
+  let filePath = joinProjectPath(bundle.target.distDir, name);
+
+  // Watch the bundle and source map for deletion.
+  // Also watch the dist dir because invalidateOnFileDelete does not currently
+  // invalidate when a parent directory is deleted.
+  // TODO: do we want to also watch for file edits?
+  api.invalidateOnFileDelete(bundle.target.distDir);
+  api.invalidateOnFileDelete(filePath);
+
+  let cacheKeys = info.cacheKeys;
+  let mapKey = cacheKeys.map;
+  let fullPath = fromProjectPath(options.projectRoot, filePath);
+  if (mapKey && bundle.env.sourceMap && !bundle.env.sourceMap.inline) {
+    api.invalidateOnFileDelete(
+      toProjectPath(options.projectRoot, fullPath + '.map'),
+    );
+  }
+
+  let dir = path.dirname(fullPath);
+  await outputFS.mkdirp(dir); // ? Got rid of dist exists, is this an expensive operation
+
+  // Use the file mode from the entry asset as the file mode for the bundle.
+  // Don't do this for browser builds, as the executable bit in particular is unnecessary.
+  let publicBundle = NamedBundle.get(bundle, bundleGraph, options);
+  let mainEntry = publicBundle.getMainEntry();
+  let writeOptions =
+    publicBundle.env.isBrowser() || !mainEntry
+      ? undefined
+      : {
+          mode: (await inputFS.stat(mainEntry.filePath)).mode,
+        };
+  let contentStream: Readable;
+  if (info.isLargeBlob) {
+    contentStream = options.cache.getStream(cacheKeys.content);
+  } else {
+    contentStream = blobToStream(
+      await options.cache.getBlob(cacheKeys.content),
+    );
+  }
+  let size = 0;
+  contentStream = contentStream.pipe(
+    new TapStream(buf => {
+      size += buf.length;
+    }),
+  );
+
+  let configResult = nullthrows(
+    await api.runRequest<null, ConfigAndCachePath>(
+      createAtlaspackConfigRequest(),
+    ),
+  );
+  let config = getCachedAtlaspackConfig(configResult, options);
+
+  let {devDeps, invalidDevDeps} = await getDevDepRequests(api);
+  invalidateDevDeps(invalidDevDeps, options, config);
+
+  await writeFiles(
+    contentStream,
+    info,
+    hashRefToNameHash,
+    options,
+    config,
+    outputFS,
+    filePath,
+    writeOptions,
+    devDeps,
+    api,
+  );
+
+  if (
+    mapKey &&
+    bundle.env.sourceMap &&
+    !bundle.env.sourceMap.inline &&
+    (await options.cache.has(mapKey))
+  ) {
+    await writeFiles(
+      blobToStream(await options.cache.getBlob(mapKey)),
+      info,
+      hashRefToNameHash,
+      options,
+      config,
+      outputFS,
+      toProjectPathUnsafe(fromProjectPathRelative(filePath) + '.map'),
+      writeOptions,
+      devDeps,
+      api,
+    );
+  }
+
+  let res = {
+    filePath,
+    type: info.type,
+    stats: {
+      size,
+      time: info.time ?? 0,
+    },
+  };
+
+  api.storeResult(res);
+  return res;
+}
+
+async function writeFiles(
+  inputStream: stream$Readable,
+  info: BundleInfo,
+  hashRefToNameHash: Map<string, string>,
+  options: AtlaspackOptions,
+  config: AtlaspackConfig,
+  outputFS: FileSystem,
+  filePath: ProjectPath,
+  writeOptions: ?FileOptions,
+  devDeps: Map<string, string>,
+  api: RunAPI<PackagedBundleInfo>,
+) {
+  let compressors = await config.getCompressors(
+    fromProjectPathRelative(filePath),
+  );
+  let fullPath = fromProjectPath(options.projectRoot, filePath);
+
+  let stream = info.hashReferences.length
+    ? inputStream.pipe(replaceStream(hashRefToNameHash))
+    : inputStream;
+
+  let promises = [];
+  for (let compressor of compressors) {
+    promises.push(
+      runCompressor(
+        compressor,
+        cloneStream(stream),
+        options,
+        outputFS,
+        fullPath,
+        writeOptions,
+        devDeps,
+        api,
+      ),
+    );
+  }
+
+  await Promise.all(promises);
+}
+
+async function runCompressor(
+  compressor: LoadedPlugin<Compressor>,
+  stream: stream$Readable,
+  options: AtlaspackOptions,
+  outputFS: FileSystem,
+  filePath: FilePath,
+  writeOptions: ?FileOptions,
+  devDeps: Map<string, string>,
+  api: RunAPI<PackagedBundleInfo>,
+) {
+  let measurement;
+  try {
+    measurement = tracer.createMeasurement(
+      compressor.name,
+      'compress',
+      path.relative(options.projectRoot, filePath),
+    );
+    let res = await compressor.plugin.compress({
+      stream,
+      options: new PluginOptions(options),
+      logger: new PluginLogger({origin: compressor.name}),
+      tracer: new PluginTracer({origin: compressor.name, category: 'compress'}),
+    });
+
+    if (res != null) {
+      await new Promise((resolve, reject) =>
+        pipeline(
+          res.stream,
+          outputFS.createWriteStream(
+            filePath + (res.type != null ? '.' + res.type : ''),
+            writeOptions,
+          ),
+          err => {
+            if (err) reject(err);
+            else resolve();
+          },
+        ),
+      );
+    }
+  } catch (err) {
+    throw new ThrowableDiagnostic({
+      diagnostic: errorToDiagnostic(err, {
+        origin: compressor.name,
+      }),
+    });
+  } finally {
+    measurement && measurement.end();
+    // Add dev deps for compressor plugins AFTER running them, to account for lazy require().
+    let devDepRequest = await createDevDependency(
+      {
+        specifier: compressor.name,
+        resolveFrom: compressor.resolveFrom,
+      },
+      devDeps,
+      options,
+    );
+    await runDevDepRequest(api, devDepRequest);
+  }
+}
+
+function replaceStream(hashRefToNameHash) {
+  let boundaryStr = Buffer.alloc(0);
+  let replaced = Buffer.alloc(0);
+  return new Transform({
+    transform(chunk, encoding, cb) {
+      let str = Buffer.concat([boundaryStr, Buffer.from(chunk)]);
+      let lastMatchI = 0;
+      if (replaced.length < str.byteLength) {
+        replaced = Buffer.alloc(str.byteLength);
+      }
+      let replacedLength = 0;
+
+      while (lastMatchI < str.byteLength) {
+        let matchI = str.indexOf(HASH_REF_PREFIX, lastMatchI);
+        if (matchI === -1) {
+          replaced.set(
+            str.subarray(lastMatchI, str.byteLength),
+            replacedLength,
+          );
+          replacedLength += str.byteLength - lastMatchI;
+          break;
+        } else {
+          let match = str
+            .subarray(matchI, matchI + HASH_REF_PREFIX_LEN + HASH_REF_HASH_LEN)
+            .toString();
+          let replacement = Buffer.from(hashRefToNameHash.get(match) ?? match);
+          replaced.set(str.subarray(lastMatchI, matchI), replacedLength);
+          replacedLength += matchI - lastMatchI;
+          replaced.set(replacement, replacedLength);
+          replacedLength += replacement.byteLength;
+          lastMatchI = matchI + HASH_REF_PREFIX_LEN + HASH_REF_HASH_LEN;
+        }
+      }
+
+      boundaryStr = replaced.subarray(
+        replacedLength - BOUNDARY_LENGTH,
+        replacedLength,
+      );
+      let strUpToBoundary = replaced.subarray(
+        0,
+        replacedLength - BOUNDARY_LENGTH,
+      );
+      cb(null, strUpToBoundary);
+    },
+
+    flush(cb) {
+      cb(null, boundaryStr);
+    },
+  });
+}
+
+function cloneStream(readable) {
+  let res = new Readable();
+  // $FlowFixMe
+  res._read = () => {};
+  readable.on('data', chunk => res.push(chunk));
+  readable.on('end', () => res.push(null));
+  readable.on('error', err => res.emit('error', err));
+  return res;
+}

package/src/requests/WriteBundlesRequest.js

@@ -0,0 +1,209 @@
+// @flow strict-local
+
+import type {ContentKey} from '@atlaspack/graph';
+import type {Async} from '@atlaspack/types';
+import type {SharedReference} from '@atlaspack/workers';
+import type {StaticRunOpts} from '../RequestTracker';
+import {requestTypes} from '../RequestTracker';
+import type {PackagedBundleInfo} from '../types';
+import type BundleGraph from '../BundleGraph';
+import type {BundleInfo} from '../PackagerRunner';
+
+import {HASH_REF_PREFIX} from '../constants';
+import {joinProjectPath} from '../projectPath';
+import nullthrows from 'nullthrows';
+import {hashString} from '@atlaspack/rust';
+import {createPackageRequest} from './PackageRequest';
+import createWriteBundleRequest from './WriteBundleRequest';
+
+type WriteBundlesRequestInput = {|
+  bundleGraph: BundleGraph,
+  optionsRef: SharedReference,
+|};
+
+export type WriteBundlesRequestResult = Map<string, PackagedBundleInfo>;
+
+type RunInput<TResult> = {|
+  input: WriteBundlesRequestInput,
+  ...StaticRunOpts<TResult>,
+|};
+
+export type WriteBundlesRequest = {|
+  id: ContentKey,
+  +type: typeof requestTypes.write_bundles_request,
+  run: (
+    RunInput<WriteBundlesRequestResult>,
+  ) => Async<WriteBundlesRequestResult>,
+  input: WriteBundlesRequestInput,
+|};
+
+/**
+ * Packages, optimizes, and writes all bundles to the dist directory.
+ */
+export default function createWriteBundlesRequest(
+  input: WriteBundlesRequestInput,
+): WriteBundlesRequest {
+  return {
+    type: requestTypes.write_bundles_request,
+    id: 'write_bundles:' + input.bundleGraph.getBundleGraphHash(),
+    run,
+    input,
+  };
+}
+
+async function run({input, api, farm, options}) {
+  let {bundleGraph, optionsRef} = input;
+  let {ref, dispose} = await farm.createSharedReference(bundleGraph);
+
+  api.invalidateOnOptionChange('shouldContentHash');
+
+  let res = new Map();
+  let bundleInfoMap: {|
+    [string]: BundleInfo,
+  |} = {};
+  let writeEarlyPromises = {};
+  let hashRefToNameHash = new Map();
+  let bundles = bundleGraph.getBundles().filter(bundle => {
+    // Do not package and write placeholder bundles to disk. We just
+    // need to update the name so other bundles can reference it.
+    if (bundle.isPlaceholder) {
+      let hash = bundle.id.slice(-8);
+      hashRefToNameHash.set(bundle.hashReference, hash);
+      let name = nullthrows(
+        bundle.name,
+        `Expected ${bundle.type} bundle to have a name`,
+      ).replace(bundle.hashReference, hash);
+      res.set(bundle.id, {
+        filePath: joinProjectPath(bundle.target.distDir, name),
+        type: bundle.type, // FIXME: this is wrong if the packager changes the type...
+        stats: {
+          time: 0,
+          size: 0,
+        },
+      });
+      return false;
+    }
+
+    return true;
+  });
+
+  // Package on the main thread if there is only one bundle to package.
+  // This avoids the cost of serializing the bundle graph for single file change builds.
+  let useMainThread =
+    bundles.length === 1 ||
+    bundles.filter(b => !api.canSkipSubrequest(bundleGraph.getHash(b)))
+      .length === 1;
+
+  try {
+    await Promise.all(
+      bundles.map(async bundle => {
+        let request = createPackageRequest({
+          bundle,
+          bundleGraph,
+          bundleGraphReference: ref,
+          optionsRef,
+          useMainThread,
+        });
+
+        let info = await api.runRequest(request);
+
+        if (!useMainThread) {
+          // Force a refresh of the cache to avoid a race condition
+          // between threaded reads and writes that can result in an LMDB cache miss:
+          //   1. The main thread has read some value from cache, necessitating a read transaction.
+          //   2. Concurrently, Thread A finishes a packaging request.
+          //   3. Subsequently, the main thread is tasked with this request, but fails because the read transaction is stale.
+          // This only occurs if the reading thread has a transaction that was created before the writing thread committed,
+          // and the transaction is still live when the reading thread attempts to get the written value.
+          // See https://github.com/parcel-bundler/parcel/issues/9121
+          options.cache.refresh();
+        }
+
+        bundleInfoMap[bundle.id] = info;
+        if (!info.hashReferences.length) {
+          hashRefToNameHash.set(
+            bundle.hashReference,
+            options.shouldContentHash
+              ? info.hash.slice(-8)
+              : bundle.id.slice(-8),
+          );
+          let writeBundleRequest = createWriteBundleRequest({
+            bundle,
+            info,
+            hashRefToNameHash,
+            bundleGraph,
+          });
+          let promise = api.runRequest(writeBundleRequest);
+          // If the promise rejects before we await it (below), we don't want to crash the build.
+          promise.catch(() => {});
+          writeEarlyPromises[bundle.id] = promise;
+        }
+      }),
+    );
+    assignComplexNameHashes(hashRefToNameHash, bundles, bundleInfoMap, options);
+    await Promise.all(
+      bundles.map(bundle => {
+        let promise =
+          writeEarlyPromises[bundle.id] ??
+          api.runRequest(
+            createWriteBundleRequest({
+              bundle,
+              info: bundleInfoMap[bundle.id],
+              hashRefToNameHash,
+              bundleGraph,
+            }),
+          );
+
+        return promise.then(r => res.set(bundle.id, r));
+      }),
+    );
+
+    api.storeResult(res);
+    return res;
+  } finally {
+    await dispose();
+  }
+}
+
+function assignComplexNameHashes(
+  hashRefToNameHash,
+  bundles,
+  bundleInfoMap,
+  options,
+) {
+  for (let bundle of bundles) {
+    if (hashRefToNameHash.get(bundle.hashReference) != null) {
+      continue;
+    }
+    hashRefToNameHash.set(
+      bundle.hashReference,
+      options.shouldContentHash
+        ? hashString(
+            [...getBundlesIncludedInHash(bundle.id, bundleInfoMap)]
+              .map(bundleId => bundleInfoMap[bundleId].hash)
+              .join(':'),
+          ).slice(-8)
+        : bundle.id.slice(-8),
+    );
+  }
+}
+
+function getBundlesIncludedInHash(
+  bundleId,
+  bundleInfoMap,
+  included = new Set(),
+) {
+  included.add(bundleId);
+  for (let hashRef of bundleInfoMap[bundleId].hashReferences) {
+    let referencedId = getIdFromHashRef(hashRef);
+    if (!included.has(referencedId)) {
+      getBundlesIncludedInHash(referencedId, bundleInfoMap, included);
+    }
+  }
+
+  return included;
+}
+
+function getIdFromHashRef(hashRef: string) {
+  return hashRef.slice(HASH_REF_PREFIX.length);
+}