@atlaspack/core 2.16.2-canary.5 → 2.16.2-canary.500

package/src/{Atlaspack.js → Atlaspack.ts}

@@ -1,4 +1,3 @@
-// @flow strict-local
 import type {
   Asset,
   AsyncSubscription,
@@ -44,64 +43,84 @@ import createPathRequest from './requests/PathRequest';
 import {createEnvironment} from './Environment';
 import {createDependency} from './Dependency';
 import {Disposable} from '@atlaspack/events';
-import {init as initSourcemaps} from '@parcel/source-map';
+import {init as initSourcemaps} from '@atlaspack/source-map';
 import {LMDBLiteCache} from '@atlaspack/cache';
 import {
+  // @ts-expect-error TS2305
   init as initRust,
   initializeMonitoring,
   closeMonitoring,
-  type Lmdb,
+  Lmdb,
 } from '@atlaspack/rust';
 import {
   fromProjectPath,
   toProjectPath,
   fromProjectPathRelative,
 } from './projectPath';
-import {tracer} from '@atlaspack/profiler';
-import {setFeatureFlags, DEFAULT_FEATURE_FLAGS} from '@atlaspack/feature-flags';
+import {tracer, NativeProfiler} from '@atlaspack/profiler';
+import {
+  setFeatureFlags,
+  DEFAULT_FEATURE_FLAGS,
+  getFeatureFlag,
+} from '@atlaspack/feature-flags';
 import {AtlaspackV3, FileSystemV3} from './atlaspack-v3';
 import createAssetGraphRequestJS from './requests/AssetGraphRequest';
 import {createAssetGraphRequestRust} from './requests/AssetGraphRequestRust';
 import type {AssetGraphRequestResult} from './requests/AssetGraphRequest';
+import {getBundleGraph} from './requests/BundleGraphRequestRust';
 import {loadRustWorkerThreadDylibHack} from './rustWorkerThreadDylibHack';
 
 registerCoreWithSerializer();
 
 export const INTERNAL_TRANSFORM: symbol = Symbol('internal_transform');
 export const INTERNAL_RESOLVE: symbol = Symbol('internal_resolve');
-export const WORKER_PATH: string = path.join(__dirname, 'worker.js');
+export let WORKER_PATH: string = path.join(__dirname, 'worker.js');
+if (__filename.endsWith('.ts')) {
+  WORKER_PATH = path.join(__dirname, 'worker.ts');
+}
 
 export default class Atlaspack {
-  #requestTracker /*: RequestTracker*/;
-  #config /*: AtlaspackConfig*/;
-  #farm /*: WorkerFarm*/;
-  #initialized /*: boolean*/ = false;
-  #disposable /*: Disposable */;
-  #initialOptions /*: InitialAtlaspackOptions */;
-  #reporterRunner /*: ReporterRunner*/;
-  #resolvedOptions /*: ?AtlaspackOptions*/ = null;
-  #optionsRef /*: SharedReference */;
-  #watchAbortController /*: AbortController*/;
-  #watchQueue /*: PromiseQueue<?BuildEvent>*/ = new PromiseQueue<?BuildEvent>({
+  // @ts-expect-error TS2564
+  #requestTracker: RequestTracker;
+  // @ts-expect-error TS2564
+  #config: AtlaspackConfig;
+  // @ts-expect-error TS2564
+  #farm: WorkerFarm;
+  #initialized: boolean = false;
+  // @ts-expect-error TS2564
+  #disposable: Disposable;
+  #initialOptions: InitialAtlaspackOptions;
+  // @ts-expect-error TS2564
+  #reporterRunner: ReporterRunner;
+  #resolvedOptions: AtlaspackOptions | null | undefined = null;
+  // @ts-expect-error TS2564
+  #optionsRef: SharedReference;
+  // @ts-expect-error TS2564
+  #watchAbortController: AbortController;
+  #watchQueue: PromiseQueue<BuildEvent | null | undefined> = new PromiseQueue<
+    BuildEvent | null | undefined
+  >({
     maxConcurrent: 1,
   });
-  #watchEvents /*: ValueEmitter<
-    | {|
-        +error: Error,
-        +buildEvent?: void,
-      |}
-    | {|
-        +buildEvent: BuildEvent,
-        +error?: void,
-      |},
-  > */;
-  #watcherSubscription /*: ?AsyncSubscription*/;
-  #watcherCount /*: number*/ = 0;
-  #requestedAssetIds /*: Set<string>*/ = new Set();
-
-  rustAtlaspack: AtlaspackV3 | null | void;
-
-  isProfiling /*: boolean */;
+  // @ts-expect-error TS2564
+  #watchEvents: ValueEmitter<
+    | {
+        readonly error: Error;
+        readonly buildEvent?: undefined;
+      }
+    | {
+        readonly buildEvent: BuildEvent;
+        readonly error?: undefined;
+      }
+  >;
+  #watcherSubscription: AsyncSubscription | null | undefined;
+  #watcherCount: number = 0;
+  #requestedAssetIds: Set<string> = new Set();
+
+  rustAtlaspack: AtlaspackV3 | null | undefined;
+
+  // @ts-expect-error TS2564
+  isProfiling: boolean;
 
   constructor(options: InitialAtlaspackOptions) {
     this.#initialOptions = options;
@@ -115,7 +134,7 @@ export default class Atlaspack {
     const featureFlags = {
       ...DEFAULT_FEATURE_FLAGS,
       ...this.#initialOptions.featureFlags,
-    };
+    } as const;
     setFeatureFlags(featureFlags);
 
     loadRustWorkerThreadDylibHack();
@@ -138,7 +157,7 @@ export default class Atlaspack {
         process.off('exit', onExit);
         onExit();
       });
-    } catch (e) {
+    } catch (e: any) {
       // Fallthrough
       // eslint-disable-next-line no-console
       console.warn(e);
@@ -151,7 +170,11 @@ export default class Atlaspack {
     this.#resolvedOptions = resolvedOptions;
 
     let rustAtlaspack: AtlaspackV3;
-    if (resolvedOptions.featureFlags.atlaspackV3) {
+    if (
+      resolvedOptions.featureFlags.atlaspackV3 ||
+      resolvedOptions.featureFlags.fullNative ||
+      resolvedOptions.featureFlags.nativePackager
+    ) {
       // eslint-disable-next-line no-unused-vars
       let {entries, inputFS, outputFS, ...options} = this.#initialOptions;
 
@@ -161,26 +184,38 @@ export default class Atlaspack {
 
       const lmdb: Lmdb = resolvedOptions.cache.getNativeRef();
 
-      // $FlowFixMe
       const version = require('../package.json').version;
       await lmdb.put('current_session_version', Buffer.from(version));
 
+      let threads = undefined;
+      if (process.env.ATLASPACK_NATIVE_THREADS !== undefined) {
+        threads = parseInt(process.env.ATLASPACK_NATIVE_THREADS, 10);
+      } else if (process.env.NODE_ENV === 'test') {
+        threads = 2;
+      }
+
       rustAtlaspack = await AtlaspackV3.create({
         ...options,
+        // @ts-expect-error TS2353
         corePath: path.join(__dirname, '..'),
-        threads: process.env.NODE_ENV === 'test' ? 2 : undefined,
+        threads,
         entries: Array.isArray(entries)
           ? entries
           : entries == null
-          ? undefined
-          : [entries],
+            ? undefined
+            : [entries],
         env: resolvedOptions.env,
         fs: inputFS && new FileSystemV3(inputFS),
-        // $FlowFixMe ProjectPath is a string
         defaultTargetOptions: resolvedOptions.defaultTargetOptions,
+        serveOptions: resolvedOptions.serveOptions,
         lmdb,
+        featureFlags: resolvedOptions.featureFlags,
+      });
+      this.#disposable.add(() => {
+        rustAtlaspack.end();
       });
     }
+    // @ts-expect-error TS2454
     this.rustAtlaspack = rustAtlaspack;
 
     let {config} = await loadAtlaspackConfig(resolvedOptions);
@@ -231,6 +266,7 @@ export default class Atlaspack {
     this.#requestTracker = await RequestTracker.init({
       farm: this.#farm,
       options: resolvedOptions,
+      // @ts-expect-error TS2454
       rustAtlaspack,
     });
 
@@ -272,7 +308,7 @@ export default class Atlaspack {
 
       try {
         await this.#requestTracker.writeToCache(abortController.signal);
-      } catch (err) {
+      } catch (err: any) {
         if (!abortController.signal.aborted) {
           // We expect abort errors if we interrupt the cache write
           throw err;
@@ -283,7 +319,7 @@ export default class Atlaspack {
     }
   }
 
-  async _startNextBuild(): Promise<?BuildEvent> {
+  async _startNextBuild(): Promise<BuildEvent | null | undefined> {
     this.#watchAbortController = new AbortController();
     await this.clearBuildCaches();
 
@@ -297,7 +333,7 @@ export default class Atlaspack {
       });
 
       return buildEvent;
-    } catch (err) {
+    } catch (err: any) {
       // Ignore BuildAbortErrors and only emit critical errors.
       if (!(err instanceof BuildAbortError)) {
         throw err;
@@ -309,12 +345,13 @@ export default class Atlaspack {
   }
 
   async watch(
-    cb?: (err: ?Error, buildEvent?: BuildEvent) => mixed,
+    cb?: (err?: Error | null | undefined, buildEvent?: BuildEvent) => unknown,
   ): Promise<AsyncSubscription> {
     if (!this.#initialized) {
       await this._init();
     }
 
+    // @ts-expect-error TS7034
     let watchEventsDisposable;
     if (cb) {
       watchEventsDisposable = this.#watchEvents.addListener(
@@ -334,8 +371,10 @@ export default class Atlaspack {
 
     this.#watcherCount++;
 
+    // @ts-expect-error TS7034
     let unsubscribePromise;
     const unsubscribe = async () => {
+      // @ts-expect-error TS7005
       if (watchEventsDisposable) {
         watchEventsDisposable.dispose();
       }
@@ -353,10 +392,12 @@ export default class Atlaspack {
 
     return {
       unsubscribe() {
+        // @ts-expect-error TS7005
         if (unsubscribePromise == null) {
           unsubscribePromise = unsubscribe();
         }
 
+        // @ts-expect-error TS7005
         return unsubscribePromise;
       },
     };
@@ -365,20 +406,28 @@ export default class Atlaspack {
   async _build({
     signal,
     startTime = Date.now(),
-  }: {|
-    signal?: AbortSignal,
-    startTime?: number,
-  |} = {
+  }: {
+    signal?: AbortSignal;
+    startTime?: number;
+  } = {
     /*::...null*/
   }): Promise<BuildEvent> {
-    this.#requestTracker.setSignal(signal);
     let options = nullthrows(this.#resolvedOptions);
     try {
       if (options.shouldProfile) {
         await this.startProfiling();
       }
+      if (options.nativeProfiler) {
+        const nativeProfiler = new NativeProfiler();
+        await nativeProfiler.startProfiling(options.nativeProfiler);
+      }
       if (options.shouldTrace) {
         tracer.enable();
+        // We need to ensure the tracer is disabled when Atlaspack is disposed as it is a module level object.
+        // While rare (except for tests), if another instance is created later it should not have tracing enabled.
+        this.#disposable.add(() => {
+          tracer.disable();
+        });
       }
       await this.#reporterRunner.report({
         type: 'buildStart',
@@ -386,19 +435,64 @@ export default class Atlaspack {
 
       this.#requestTracker.graph.invalidateOnBuildNodes();
 
-      let request = createAtlaspackBuildRequest({
-        optionsRef: this.#optionsRef,
-        requestedAssetIds: this.#requestedAssetIds,
-        signal,
-      });
+      let bundleGraph: any;
+      let bundleInfo: Map<any, any>;
+      let changedAssets: Map<any, any>;
+      let assetRequests: Array<any>;
+      let scopeHoistingStats: any;
+
+      if (getFeatureFlag('fullNative') && this.rustAtlaspack) {
+        let [result, error] = await this.rustAtlaspack.build(
+          (eventJson: string) => {
+            let event = JSON.parse(eventJson);
+            this.#reporterRunner.report(event);
+          },
+        );
 
-      let {bundleGraph, bundleInfo, changedAssets, assetRequests} =
-        await this.#requestTracker.runRequest(request, {force: true});
+        if (error) {
+          throw new ThrowableDiagnostic({diagnostic: error});
+        }
+        ({bundleGraph, changedAssets} = getBundleGraph(result));
+        bundleInfo = new Map(
+          (result.bundleInfo ?? []).map(
+            (info: {
+              bundleId: string;
+              filePath: string;
+              type: string;
+              size: number;
+              time: number;
+            }) => [
+              info.bundleId,
+              {
+                filePath: toProjectPath(options.projectRoot, info.filePath),
+                bundleId: info.bundleId,
+                type: info.type,
+                stats: {size: info.size, time: info.time},
+              },
+            ],
+          ),
+        );
+        assetRequests = result.assetRequests ?? [];
+      } else {
+        let request = createAtlaspackBuildRequest({
+          optionsRef: this.#optionsRef,
+          requestedAssetIds: this.#requestedAssetIds,
+          signal,
+        });
+
+        ({
+          bundleGraph,
+          bundleInfo,
+          changedAssets,
+          assetRequests,
+          scopeHoistingStats,
+        } = await this.#requestTracker.runRequest(request, {force: true}));
+      }
 
       this.#requestedAssetIds.clear();
 
       await dumpGraphToGraphViz(
-        // $FlowFixMe
+        // @ts-expect-error TS2345
         this.#requestTracker.graph,
         'RequestGraph',
         requestGraphEdgeTypes,
@@ -407,14 +501,20 @@ export default class Atlaspack {
       let event = {
         type: 'buildSuccess',
         changedAssets: new Map(
-          Array.from(changedAssets).map(([id, asset]) => [
+          Array.from(changedAssets).map(([id, asset]: [any, any]) => [
             id,
             assetFromValue(asset, options),
           ]),
         ),
         bundleGraph: new BundleGraph<IPackagedBundle>(
           bundleGraph,
-          (bundle, bundleGraph, options) =>
+          (
+            // @ts-expect-error TS2304
+            bundle: Bundle,
+            // @ts-expect-error TS2314
+            bundleGraph: BundleGraph,
+            options: AtlaspackOptions,
+          ) =>
             PackagedBundle.getWithInfo(
               bundle,
               bundleGraph,
@@ -424,6 +524,7 @@ export default class Atlaspack {
           options,
         ),
         buildTime: Date.now() - startTime,
+        // @ts-expect-error TS7006
         requestBundle: async (bundle) => {
           let bundleNode = bundleGraph._graph.getNodeByContentKey(bundle.id);
           invariant(bundleNode?.type === 'bundle', 'Bundle does not exist');
@@ -437,6 +538,7 @@ export default class Atlaspack {
               buildTime: 0,
               requestBundle: event.requestBundle,
               unstable_requestStats: {},
+              scopeHoistingStats: event.scopeHoistingStats,
             };
           }
 
@@ -454,17 +556,23 @@ export default class Atlaspack {
 
           let results = await this.#watchQueue.run();
           let result = results.filter(Boolean).pop();
+          // @ts-expect-error TS18049
           if (result.type === 'buildFailure') {
+            // @ts-expect-error TS18049
             throw new BuildError(result.diagnostics);
           }
 
           return result;
         },
         unstable_requestStats: this.#requestTracker.flushStats(),
+        nativeCacheStats: await this.rustAtlaspack?.completeCacheSession(),
+        scopeHoistingStats,
       };
 
+      // @ts-expect-error TS2345
       await this.#reporterRunner.report(event);
       await this.#requestTracker.runRequest(
+        // @ts-expect-error TS2345
         createValidationRequest({optionsRef: this.#optionsRef, assetRequests}),
         {force: assetRequests.length > 0},
       );
@@ -473,8 +581,9 @@ export default class Atlaspack {
         throw this.#reporterRunner.errors;
       }
 
+      // @ts-expect-error TS2322
       return event;
-    } catch (e) {
+    } catch (e: any) {
       if (e instanceof BuildAbortError) {
         throw e;
       }
@@ -484,9 +593,12 @@ export default class Atlaspack {
         type: 'buildFailure',
         diagnostics: Array.isArray(diagnostic) ? diagnostic : [diagnostic],
         unstable_requestStats: this.#requestTracker.flushStats(),
+        nativeCacheStats: await this.rustAtlaspack?.completeCacheSession(),
       };
 
+      // @ts-expect-error TS2345
       await this.#reporterRunner.report(event);
+      // @ts-expect-error TS2322
       return event;
     } finally {
       if (this.isProfiling) {
@@ -522,14 +634,15 @@ export default class Atlaspack {
         });
 
         let nativeInvalid = false;
-        if (this.rustAtlaspack) {
+        if (getFeatureFlag('atlaspackV3') && this.rustAtlaspack) {
           nativeInvalid = await this.rustAtlaspack.respondToFsEvents(events);
         }
 
-        let isInvalid = await this.#requestTracker.respondToFSEvents(
-          events,
-          Number.POSITIVE_INFINITY,
-        );
+        let {didInvalidate: isInvalid} =
+          await this.#requestTracker.respondToFSEvents(
+            events,
+            Number.POSITIVE_INFINITY,
+          );
 
         if (
           (nativeInvalid || isInvalid) &&
@@ -613,13 +726,15 @@ export default class Atlaspack {
       lazyIncludes: [],
       lazyExcludes: [],
       requestedAssetIds: this.#requestedAssetIds,
-    };
+    } as const;
 
     const start = Date.now();
     const result = await this.#requestTracker.runRequest(
-      this.rustAtlaspack != null
-        ? createAssetGraphRequestRust(this.rustAtlaspack)(input)
-        : createAssetGraphRequestJS(input),
+      getFeatureFlag('atlaspackV3') && this.rustAtlaspack != null
+        ? // @ts-expect-error TS2345
+          createAssetGraphRequestRust(this.rustAtlaspack)(input)
+        : // @ts-expect-error TS2345
+          createAssetGraphRequestJS(input),
       {force: true},
     );
 
@@ -639,6 +754,20 @@ export default class Atlaspack {
     return result;
   }
 
+  /**
+   * Copy the cache to a new directory and compact it.
+   */
+  async unstable_compactCache(): Promise<void> {
+    await this._init();
+
+    const cache = nullthrows(this.#resolvedOptions).cache;
+    if (cache instanceof LMDBLiteCache) {
+      await cache.compact('parcel-cache-compacted');
+    } else {
+      throw new Error('Cache is not an LMDBLiteCache');
+    }
+  }
+
   async unstable_transform(
     options: AtlaspackTransformOptions,
   ): Promise<Array<Asset>> {
@@ -673,7 +802,7 @@ export default class Atlaspack {
 
   async unstable_resolve(
     request: AtlaspackResolveOptions,
-  ): Promise<?AtlaspackResolveResult> {
+  ): Promise<AtlaspackResolveResult | null | undefined> {
     if (!this.#initialized) {
       await this._init();
     }
@@ -712,6 +841,7 @@ export default class Atlaspack {
     }
 
     return {
+      // @ts-expect-error TS2322
       filePath: fromProjectPath(projectRoot, res.filePath),
       code: res.code,
       query: res.query,
@@ -728,7 +858,7 @@ export class BuildError extends ThrowableDiagnostic {
 }
 
 export function createWorkerFarm(
-  options: $Shape<FarmOptions> = {},
+  options: Partial<FarmOptions> = {},
 ): WorkerFarm {
   return new WorkerFarm({
     ...options,

package/src/{AtlaspackConfig.schema.js → AtlaspackConfig.schema.ts}

@@ -1,19 +1,14 @@
-// @flow strict-local
 import type {PackageName} from '@atlaspack/types';
 import type {SchemaEntity} from '@atlaspack/utils';
 
 // Parcel validates plugin package names due to:
-//
 // * https://github.com/parcel-bundler/parcel/issues/3397#issuecomment-521353931
-//
 // Ultimately:
-//
 // * We do not care about package names
 // * Validation makes interop between parcel/atlaspack confusing.
-//
 export function validatePackageName(
   // eslint-disable-next-line no-unused-vars
-  pkg: ?PackageName,
+  pkg: PackageName | null | undefined,
   // eslint-disable-next-line no-unused-vars
   pluginType: string,
   // eslint-disable-next-line no-unused-vars
@@ -27,7 +22,7 @@ const validatePluginName = (pluginType: string, key: string) => {
 
     try {
       validatePackageName(val, pluginType, key);
-    } catch (e) {
+    } catch (e: any) {
       return e.message;
     }
   };
@@ -39,7 +34,7 @@ const validateExtends = (val: string): void => {
 
   try {
     validatePackageName(val, 'config', 'extends');
-  } catch (e) {
+  } catch (e: any) {
     return e.message;
   }
 };
@@ -96,20 +91,31 @@ export default {
     },
     bundler: {
       type: 'string',
-      __validate: (validatePluginName('bundler', 'bundler'): (string) => void),
+      __validate: validatePluginName('bundler', 'bundler') as (
+        arg1: string,
+      ) => void,
     },
-    resolvers: (pipelineSchema('resolver', 'resolvers'): SchemaEntity),
-    transformers: (mapPipelineSchema(
+    resolvers: pipelineSchema('resolver', 'resolvers') as SchemaEntity,
+    transformers: mapPipelineSchema(
       'transformer',
       'transformers',
-    ): SchemaEntity),
-    validators: (mapPipelineSchema('validator', 'validators'): SchemaEntity),
-    namers: (pipelineSchema('namer', 'namers'): SchemaEntity),
-    packagers: (mapStringSchema('packager', 'packagers'): SchemaEntity),
-    optimizers: (mapPipelineSchema('optimizer', 'optimizers'): SchemaEntity),
-    compressors: (mapPipelineSchema('compressor', 'compressors'): SchemaEntity),
-    reporters: (pipelineSchema('reporter', 'reporters'): SchemaEntity),
-    runtimes: (pipelineSchema('runtime', 'runtimes'): SchemaEntity),
+    ) as SchemaEntity,
+    validators: mapPipelineSchema('validator', 'validators') as SchemaEntity,
+    namers: pipelineSchema('namer', 'namers') as SchemaEntity,
+    packagers: mapStringSchema('packager', 'packagers') as SchemaEntity,
+    optimizers: mapPipelineSchema('optimizer', 'optimizers') as SchemaEntity,
+    compressors: mapPipelineSchema('compressor', 'compressors') as SchemaEntity,
+    reporters: pipelineSchema('reporter', 'reporters') as SchemaEntity,
+    runtimes: pipelineSchema('runtime', 'runtimes') as SchemaEntity,
+    // This isn't currently supported in V2, however we need to allow it to
+    // parse as even V3 builds need the AtlaspackConfigRequest
+    unstable_alias: {
+      type: 'object',
+      properties: {},
+      additionalProperties: {
+        type: 'string',
+      },
+    },
     filePath: {
       type: 'string',
     },