@atlaspack/core 2.12.1-canary.3354

package/src/UncommittedAsset.js

@@ -0,0 +1,426 @@
+// @flow strict-local
+
+import type {
+  AST,
+  Blob,
+  DependencyOptions,
+  FileCreateInvalidation,
+  GenerateOutput,
+  PackageName,
+  TransformerResult,
+} from '@atlaspack/types';
+import type {Asset, Dependency, AtlaspackOptions, Invalidations} from './types';
+
+import invariant from 'assert';
+import {Readable} from 'stream';
+import SourceMap from '@parcel/source-map';
+import {
+  blobToStream,
+  bufferStream,
+  streamFromPromise,
+  TapStream,
+  loadSourceMap,
+  SOURCEMAP_RE,
+} from '@atlaspack/utils';
+import {hashString, hashBuffer, Hash} from '@atlaspack/rust';
+import {serializeRaw} from './serializer';
+import {createDependency, mergeDependencies} from './Dependency';
+import {mergeEnvironments} from './Environment';
+import {ATLASPACK_VERSION} from './constants';
+import {createAsset, createAssetIdFromOptions} from './assetUtils';
+import {BundleBehaviorNames} from './types';
+import {invalidateOnFileCreateToInternal, createInvalidations} from './utils';
+import {type ProjectPath, fromProjectPath} from './projectPath';
+
+type UncommittedAssetOptions = {|
+  value: Asset,
+  options: AtlaspackOptions,
+  content?: ?Blob,
+  mapBuffer?: ?Buffer,
+  ast?: ?AST,
+  isASTDirty?: ?boolean,
+  idBase?: ?string,
+  invalidations?: Invalidations,
+|};
+
+export default class UncommittedAsset {
+  value: Asset;
+  options: AtlaspackOptions;
+  content: ?(Blob | Promise<Buffer>);
+  mapBuffer: ?Buffer;
+  sourceContent: ?string;
+  map: ?SourceMap;
+  ast: ?AST;
+  isASTDirty: boolean;
+  idBase: ?string;
+  invalidations: Invalidations;
+  generate: ?() => Promise<GenerateOutput>;
+
+  constructor({
+    value,
+    options,
+    content,
+    mapBuffer,
+    ast,
+    isASTDirty,
+    idBase,
+    invalidations,
+  }: UncommittedAssetOptions) {
+    this.value = value;
+    this.options = options;
+    this.content = content;
+    this.mapBuffer = mapBuffer;
+    this.ast = ast;
+    this.isASTDirty = isASTDirty || false;
+    this.idBase = idBase;
+    this.invalidations = invalidations || createInvalidations();
+  }
+
+  /*
+   * Prepares the asset for being serialized to the cache by committing its
+   * content and map of the asset to the cache.
+   */
+  async commit(): Promise<void> {
+    // If there is a dirty AST, clear out any old content and map as these
+    // must be regenerated later and shouldn't be committed.
+    if (this.ast != null && this.isASTDirty) {
+      this.content = null;
+      this.mapBuffer = null;
+    }
+
+    let size = 0;
+    let outputHash = '';
+    let contentKey = this.content == null ? null : this.getCacheKey('content');
+    let mapKey = this.mapBuffer == null ? null : this.getCacheKey('map');
+    let astKey = this.ast == null ? null : this.getCacheKey('ast');
+
+    // Since we can only read from the stream once, compute the content length
+    // and hash while it's being written to the cache.
+    await Promise.all([
+      contentKey != null &&
+        this.commitContent(contentKey).then(
+          s => ((size = s.size), (outputHash = s.hash)),
+        ),
+      this.mapBuffer != null &&
+        mapKey != null &&
+        this.options.cache.setBlob(mapKey, this.mapBuffer),
+      astKey != null &&
+        this.options.cache.setBlob(astKey, serializeRaw(this.ast)),
+    ]);
+    this.value.contentKey = contentKey;
+    this.value.mapKey = mapKey;
+    this.value.astKey = astKey;
+    this.value.outputHash = outputHash;
+
+    if (this.content != null) {
+      this.value.stats.size = size;
+    }
+
+    this.value.isLargeBlob = this.content instanceof Readable;
+    this.value.committed = true;
+  }
+
+  async commitContent(
+    contentKey: string,
+  ): Promise<{|size: number, hash: string|}> {
+    let content = await this.content;
+    if (content == null) {
+      return {size: 0, hash: ''};
+    }
+
+    let size = 0;
+    if (content instanceof Readable) {
+      let hash = new Hash();
+      await this.options.cache.setStream(
+        contentKey,
+        content.pipe(
+          new TapStream(buf => {
+            hash.writeBuffer(buf);
+            size += buf.length;
+          }),
+        ),
+      );
+
+      return {size, hash: hash.finish()};
+    }
+
+    let hash;
+    if (typeof content === 'string') {
+      hash = hashString(content);
+      size = Buffer.byteLength(content);
+    } else {
+      hash = hashBuffer(content);
+      size = content.length;
+    }
+
+    await this.options.cache.setBlob(contentKey, content);
+    return {size, hash};
+  }
+
+  async getCode(): Promise<string> {
+    if (this.ast != null && this.isASTDirty) {
+      throw new Error(
+        'Cannot call getCode() on an asset with a dirty AST. For transformers, implement canReuseAST() and check asset.isASTDirty.',
+      );
+    }
+
+    let content = await this.content;
+    if (typeof content === 'string' || content instanceof Buffer) {
+      return content.toString();
+    } else if (content != null) {
+      this.content = bufferStream(content);
+      return (await this.content).toString();
+    }
+
+    invariant(false, 'Internal error: missing content');
+  }
+
+  async getBuffer(): Promise<Buffer> {
+    let content = await this.content;
+    if (content == null) {
+      return Buffer.alloc(0);
+    } else if (content instanceof Buffer) {
+      return content;
+    } else if (typeof content === 'string') {
+      return Buffer.from(content);
+    }
+
+    this.content = bufferStream(content);
+    return this.content;
+  }
+
+  getStream(): Readable {
+    if (this.content instanceof Readable) {
+      // Remove content if it's a stream, as it should not be reused.
+      let content = this.content;
+      this.content = null;
+      return content;
+    }
+
+    if (this.content instanceof Promise) {
+      return streamFromPromise(this.content);
+    }
+
+    return blobToStream(this.content ?? Buffer.alloc(0));
+  }
+
+  setCode(code: string) {
+    this.content = code;
+    this.clearAST();
+  }
+
+  setBuffer(buffer: Buffer) {
+    this.content = buffer;
+    this.clearAST();
+  }
+
+  setStream(stream: Readable) {
+    this.content = stream;
+    this.clearAST();
+  }
+
+  async loadExistingSourcemap(): Promise<?SourceMap> {
+    if (this.map) {
+      return this.map;
+    }
+
+    let code = await this.getCode();
+    let map = await loadSourceMap(
+      fromProjectPath(this.options.projectRoot, this.value.filePath),
+      code,
+      {
+        fs: this.options.inputFS,
+        projectRoot: this.options.projectRoot,
+      },
+    );
+
+    if (map) {
+      this.map = map;
+      this.mapBuffer = map.toBuffer();
+      this.setCode(code.replace(SOURCEMAP_RE, ''));
+    }
+
+    return this.map;
+  }
+
+  getMapBuffer(): Promise<?Buffer> {
+    return Promise.resolve(this.mapBuffer);
+  }
+
+  async getMap(): Promise<?SourceMap> {
+    if (this.map == null) {
+      let mapBuffer = this.mapBuffer ?? (await this.getMapBuffer());
+      if (mapBuffer) {
+        // Get sourcemap from flatbuffer
+        this.map = new SourceMap(this.options.projectRoot, mapBuffer);
+      }
+    }
+
+    return this.map;
+  }
+
+  setMap(map: ?SourceMap): void {
+    // If we have sourceContent available, it means this asset is source code without
+    // a previous source map. Ensure that the map set by the transformer has the original
+    // source content available.
+    if (map != null && this.sourceContent != null) {
+      map.setSourceContent(
+        fromProjectPath(this.options.projectRoot, this.value.filePath),
+        // $FlowFixMe
+        this.sourceContent,
+      );
+      this.sourceContent = null;
+    }
+
+    this.map = map;
+    this.mapBuffer = this.map?.toBuffer();
+  }
+
+  getAST(): Promise<?AST> {
+    return Promise.resolve(this.ast);
+  }
+
+  setAST(ast: AST): void {
+    this.ast = ast;
+    this.isASTDirty = true;
+    this.value.astGenerator = {
+      type: ast.type,
+      version: ast.version,
+    };
+  }
+
+  clearAST() {
+    this.ast = null;
+    this.isASTDirty = false;
+    this.value.astGenerator = null;
+  }
+
+  getCacheKey(key: string): string {
+    return hashString(ATLASPACK_VERSION + key + this.value.id);
+  }
+
+  addDependency(opts: DependencyOptions): string {
+    // eslint-disable-next-line no-unused-vars
+    let {env, symbols, ...rest} = opts;
+    let dep = createDependency(this.options.projectRoot, {
+      ...rest,
+      // $FlowFixMe "convert" the $ReadOnlyMaps to the interal mutable one
+      symbols,
+      env: mergeEnvironments(this.options.projectRoot, this.value.env, env),
+      sourceAssetId: this.value.id,
+      sourcePath: fromProjectPath(
+        this.options.projectRoot,
+        this.value.filePath,
+      ),
+    });
+    let existing = this.value.dependencies.get(dep.id);
+    if (existing) {
+      mergeDependencies(existing, dep);
+    } else {
+      this.value.dependencies.set(dep.id, dep);
+    }
+    return dep.id;
+  }
+
+  invalidateOnFileChange(filePath: ProjectPath) {
+    this.invalidations.invalidateOnFileChange.add(filePath);
+  }
+
+  invalidateOnFileCreate(invalidation: FileCreateInvalidation) {
+    this.invalidations.invalidateOnFileCreate.push(
+      invalidateOnFileCreateToInternal(this.options.projectRoot, invalidation),
+    );
+  }
+
+  invalidateOnEnvChange(key: string) {
+    this.invalidations.invalidateOnEnvChange.add(key);
+  }
+
+  invalidateOnBuild() {
+    this.invalidations.invalidateOnBuild = true;
+  }
+
+  invalidateOnStartup() {
+    this.invalidations.invalidateOnStartup = true;
+  }
+
+  getDependencies(): Array<Dependency> {
+    return Array.from(this.value.dependencies.values());
+  }
+
+  createChildAsset(
+    result: TransformerResult,
+    plugin: PackageName,
+    configPath: ProjectPath,
+    configKeyPath?: string,
+  ): UncommittedAsset {
+    let content = result.content ?? null;
+
+    let asset = new UncommittedAsset({
+      value: createAsset(this.options.projectRoot, {
+        idBase: this.idBase,
+        filePath: this.value.filePath,
+        type: result.type,
+        bundleBehavior:
+          result.bundleBehavior ??
+          (this.value.bundleBehavior == null
+            ? null
+            : BundleBehaviorNames[this.value.bundleBehavior]),
+        isBundleSplittable:
+          result.isBundleSplittable ?? this.value.isBundleSplittable,
+        isSource: this.value.isSource,
+        env: mergeEnvironments(
+          this.options.projectRoot,
+          this.value.env,
+          result.env,
+        ),
+        dependencies:
+          this.value.type === result.type
+            ? new Map(this.value.dependencies)
+            : new Map(),
+        meta: {
+          ...this.value.meta,
+          ...result.meta,
+        },
+        pipeline:
+          result.pipeline ??
+          (this.value.type === result.type ? this.value.pipeline : null),
+        stats: {
+          time: 0,
+          size: this.value.stats.size,
+        },
+        // $FlowFixMe
+        symbols: result.symbols,
+        sideEffects: result.sideEffects ?? this.value.sideEffects,
+        uniqueKey: result.uniqueKey,
+        astGenerator: result.ast
+          ? {type: result.ast.type, version: result.ast.version}
+          : null,
+        plugin,
+        configPath,
+        configKeyPath,
+      }),
+      options: this.options,
+      content,
+      ast: result.ast,
+      isASTDirty: result.ast === this.ast ? this.isASTDirty : true,
+      mapBuffer: result.map ? result.map.toBuffer() : null,
+      idBase: this.idBase,
+      invalidations: this.invalidations,
+    });
+
+    let dependencies = result.dependencies;
+    if (dependencies) {
+      for (let dep of dependencies) {
+        asset.addDependency(dep);
+      }
+    }
+
+    return asset;
+  }
+
+  updateId() {
+    // $FlowFixMe - this is fine
+    this.value.id = createAssetIdFromOptions(this.value);
+  }
+}

package/src/Validation.js

@@ -0,0 +1,223 @@
+// @flow strict-local
+
+import type {WorkerApi} from '@atlaspack/workers';
+import type {AssetGroup, AtlaspackOptions, ReportFn} from './types';
+import type {Validator, ValidateResult} from '@atlaspack/types';
+import type {Diagnostic} from '@atlaspack/diagnostic';
+
+import path from 'path';
+import {resolveConfig} from '@atlaspack/utils';
+import logger, {PluginLogger} from '@atlaspack/logger';
+import ThrowableDiagnostic, {errorToDiagnostic} from '@atlaspack/diagnostic';
+import AtlaspackConfig from './AtlaspackConfig';
+import UncommittedAsset from './UncommittedAsset';
+import {createAsset} from './assetUtils';
+import {Asset} from './public/Asset';
+import PluginOptions from './public/PluginOptions';
+import summarizeRequest from './summarizeRequest';
+import {fromProjectPath, fromProjectPathRelative} from './projectPath';
+import {PluginTracer} from '@atlaspack/profiler';
+import {hashString} from '@atlaspack/rust';
+
+export type ValidationOpts = {|
+  config: AtlaspackConfig,
+  /**
+   * If true, this Validation instance will run all validators that implement the single-threaded "validateAll" method.
+   * If falsy, it will run validators that implement the one-asset-at-a-time "validate" method.
+   */
+  dedicatedThread?: boolean,
+  options: AtlaspackOptions,
+  requests: AssetGroup[],
+  report: ReportFn,
+  workerApi?: WorkerApi,
+|};
+
+export default class Validation {
+  allAssets: {[validatorName: string]: UncommittedAsset[], ...} = {};
+  allValidators: {[validatorName: string]: Validator, ...} = {};
+  dedicatedThread: boolean;
+  impactfulOptions: $Shape<AtlaspackOptions>;
+  options: AtlaspackOptions;
+  atlaspackConfig: AtlaspackConfig;
+  report: ReportFn;
+  requests: AssetGroup[];
+  workerApi: ?WorkerApi;
+
+  constructor({
+    config,
+    dedicatedThread,
+    options,
+    requests,
+    report,
+    workerApi,
+  }: ValidationOpts) {
+    this.dedicatedThread = dedicatedThread ?? false;
+    this.options = options;
+    this.atlaspackConfig = config;
+    this.report = report;
+    this.requests = requests;
+    this.workerApi = workerApi;
+  }
+
+  async run(): Promise<void> {
+    let pluginOptions = new PluginOptions(this.options);
+    await this.buildAssetsAndValidators();
+    await Promise.all(
+      Object.keys(this.allValidators).map(async validatorName => {
+        let assets = this.allAssets[validatorName];
+        if (assets) {
+          let plugin = this.allValidators[validatorName];
+          let validatorLogger = new PluginLogger({origin: validatorName});
+          let validatorTracer = new PluginTracer({
+            origin: validatorName,
+            category: 'validator',
+          });
+          let validatorResults: Array<?ValidateResult> = [];
+          try {
+            // If the plugin supports the single-threading validateAll method, pass all assets to it.
+            if (plugin.validateAll && this.dedicatedThread) {
+              validatorResults = await plugin.validateAll({
+                assets: assets.map(asset => new Asset(asset)),
+                options: pluginOptions,
+                logger: validatorLogger,
+                tracer: validatorTracer,
+                resolveConfigWithPath: (
+                  configNames: Array<string>,
+                  assetFilePath: string,
+                ) =>
+                  resolveConfig(
+                    this.options.inputFS,
+                    assetFilePath,
+                    configNames,
+                    this.options.projectRoot,
+                  ),
+              });
+            }
+
+            // Otherwise, pass the assets one-at-a-time
+            else if (plugin.validate && !this.dedicatedThread) {
+              await Promise.all(
+                assets.map(async input => {
+                  let config = null;
+                  let publicAsset = new Asset(input);
+                  if (plugin.getConfig) {
+                    config = await plugin.getConfig({
+                      asset: publicAsset,
+                      options: pluginOptions,
+                      logger: validatorLogger,
+                      tracer: validatorTracer,
+                      resolveConfig: (configNames: Array<string>) =>
+                        resolveConfig(
+                          this.options.inputFS,
+                          publicAsset.filePath,
+                          configNames,
+                          this.options.projectRoot,
+                        ),
+                    });
+                  }
+
+                  let validatorResult = await plugin.validate({
+                    asset: publicAsset,
+                    options: pluginOptions,
+                    config,
+                    logger: validatorLogger,
+                    tracer: validatorTracer,
+                  });
+                  validatorResults.push(validatorResult);
+                }),
+              );
+            }
+            this.handleResults(validatorResults);
+          } catch (e) {
+            throw new ThrowableDiagnostic({
+              diagnostic: errorToDiagnostic(e, {
+                origin: validatorName,
+              }),
+            });
+          }
+        }
+      }),
+    );
+  }
+
+  async buildAssetsAndValidators() {
+    // Figure out what validators need to be run, and group the assets by the relevant validators.
+    await Promise.all(
+      this.requests.map(async request => {
+        this.report({
+          type: 'validation',
+          filePath: fromProjectPath(this.options.projectRoot, request.filePath),
+        });
+
+        let asset = await this.loadAsset(request);
+
+        let validators = await this.atlaspackConfig.getValidators(
+          request.filePath,
+        );
+
+        for (let validator of validators) {
+          this.allValidators[validator.name] = validator.plugin;
+          if (this.allAssets[validator.name]) {
+            this.allAssets[validator.name].push(asset);
+          } else {
+            this.allAssets[validator.name] = [asset];
+          }
+        }
+      }),
+    );
+  }
+
+  handleResults(validatorResults: Array<?ValidateResult>) {
+    let warnings: Array<Diagnostic> = [];
+    let errors: Array<Diagnostic> = [];
+    validatorResults.forEach(result => {
+      if (result) {
+        warnings.push(...result.warnings);
+        errors.push(...result.errors);
+      }
+    });
+
+    if (errors.length > 0) {
+      throw new ThrowableDiagnostic({
+        diagnostic: errors,
+      });
+    }
+
+    if (warnings.length > 0) {
+      logger.warn(warnings);
+    }
+  }
+
+  async loadAsset(request: AssetGroup): Promise<UncommittedAsset> {
+    let {filePath, env, code, sideEffects, query} = request;
+    let {content, size, isSource} = await summarizeRequest(
+      this.options.inputFS,
+      {
+        filePath: fromProjectPath(this.options.projectRoot, request.filePath),
+      },
+    );
+
+    // If the transformer request passed code rather than a filename,
+    // use a hash as the base for the id to ensure it is unique.
+    let idBase =
+      code != null ? hashString(code) : fromProjectPathRelative(filePath);
+    return new UncommittedAsset({
+      idBase,
+      value: createAsset(this.options.projectRoot, {
+        idBase,
+        filePath: filePath,
+        isSource,
+        type: path.extname(fromProjectPathRelative(filePath)).slice(1),
+        query,
+        env: env,
+        stats: {
+          time: 0,
+          size,
+        },
+        sideEffects: sideEffects,
+      }),
+      options: this.options,
+      content,
+    });
+  }
+}