@atlaspack/core 2.16.2-canary.48 → 2.16.2-canary.481

package/src/{CommittedAsset.js → CommittedAsset.ts}

@@ -1,24 +1,22 @@
-// @flow strict-local
-
 import {Readable} from 'stream';
 
 import {deserializeRaw} from '@atlaspack/build-cache';
 import type {AST, Blob} from '@atlaspack/types';
 import {bufferStream, blobToStream, streamFromPromise} from '@atlaspack/utils';
-import SourceMap from '@parcel/source-map';
+import SourceMap from '@atlaspack/source-map';
 
 import {generateFromAST} from './assetUtils';
 import type {Asset, Dependency, AtlaspackOptions} from './types';
 
 export default class CommittedAsset {
-  key: ?string;
+  key: string | null | undefined;
   value: Asset;
   options: AtlaspackOptions;
-  content: ?Promise<Buffer | string>;
-  mapBuffer: ?Promise<?Buffer>;
-  map: ?Promise<?SourceMap>;
-  ast: ?Promise<AST>;
-  code: ?string;
+  content: Promise<Buffer | string> | null | undefined;
+  mapBuffer: Promise<Buffer | null | undefined> | null | undefined;
+  map: Promise<SourceMap | null | undefined> | null | undefined;
+  ast: Promise<AST> | null | undefined;
+  code: string | null | undefined;
 
   constructor(value: Asset, options: AtlaspackOptions) {
     this.value = value;
@@ -63,6 +61,7 @@ export default class CommittedAsset {
     if (typeof content === 'string' || content instanceof Buffer) {
       return content.toString();
     } else if (content != null) {
+      // @ts-expect-error TS2345
       this.content = bufferStream(content);
       return (await this.content).toString();
     }
@@ -79,7 +78,9 @@ export default class CommittedAsset {
       return Buffer.from(content);
     }
 
+    // @ts-expect-error TS2345
     this.content = bufferStream(content);
+    // @ts-expect-error TS2322
     return this.content;
   }
 
@@ -90,7 +91,7 @@ export default class CommittedAsset {
       : blobToStream(content);
   }
 
-  getMapBuffer(): Promise<?Buffer> {
+  getMapBuffer(): Promise<Buffer | null | undefined> {
     let mapKey = this.value.mapKey;
     if (mapKey != null && this.mapBuffer == null) {
       this.mapBuffer = (async () => {
@@ -116,7 +117,7 @@ export default class CommittedAsset {
           }
 
           return await this.options.cache.getBlob(mapKey);
-        } catch (err) {
+        } catch (err: any) {
           if (err.code === 'ENOENT' && this.value.astKey != null) {
             return (await generateFromAST(this)).map?.toBuffer();
           } else {
@@ -126,10 +127,11 @@ export default class CommittedAsset {
       })();
     }
 
+    // @ts-expect-error TS2322
     return this.mapBuffer ?? Promise.resolve();
   }
 
-  getMap(): Promise<?SourceMap> {
+  getMap(): Promise<SourceMap | null | undefined> {
     if (this.map == null) {
       this.map = (async () => {
         let mapBuffer = await this.getMapBuffer();
@@ -143,7 +145,7 @@ export default class CommittedAsset {
     return this.map;
   }
 
-  getAST(): Promise<?AST> {
+  getAST(): Promise<AST | null | undefined> {
     if (this.value.astKey == null) {
       return Promise.resolve(null);
     }

package/src/{Dependency.js → Dependency.ts}

@@ -1,4 +1,3 @@
-// @flow
 import type {
   FilePath,
   Meta,
@@ -8,7 +7,7 @@ import type {
   BundleBehavior as IBundleBehavior,
   SemverRange,
 } from '@atlaspack/types';
-import type {Dependency, Environment, Target} from './types';
+import type {Dependency, Target} from './types';
 import {createDependencyId as createDependencyIdRust} from '@atlaspack/rust';
 import {
   SpecifierType,
@@ -21,31 +20,41 @@ import {toInternalSourceLocation} from './utils';
 import {toProjectPath} from './projectPath';
 import assert from 'assert';
 import {identifierRegistry} from './IdentifierRegistry';
+import {fromEnvironmentId, toEnvironmentId} from './EnvironmentManager';
+import type {EnvironmentRef} from './EnvironmentManager';
 
-type DependencyOpts = {|
-  id?: string,
-  sourcePath?: FilePath,
-  sourceAssetId?: string,
-  specifier: DependencySpecifier,
-  specifierType: $Keys<typeof SpecifierType>,
-  priority?: $Keys<typeof Priority>,
-  needsStableName?: boolean,
-  bundleBehavior?: ?IBundleBehavior,
-  isEntry?: boolean,
-  isOptional?: boolean,
-  loc?: SourceLocation,
-  env: Environment,
-  packageConditions?: Array<string>,
-  meta?: Meta,
-  resolveFrom?: FilePath,
-  range?: SemverRange,
-  target?: Target,
-  symbols?: ?Map<
-    Symbol,
-    {|local: Symbol, loc: ?SourceLocation, isWeak: boolean, meta?: ?Meta|},
-  >,
-  pipeline?: ?string,
-|};
+type DependencyOpts = {
+  id?: string;
+  sourcePath?: FilePath;
+  sourceAssetId?: string;
+  specifier: DependencySpecifier;
+  specifierType: keyof typeof SpecifierType;
+  priority?: keyof typeof Priority;
+  needsStableName?: boolean;
+  bundleBehavior?: IBundleBehavior | null | undefined;
+  isEntry?: boolean;
+  isOptional?: boolean;
+  loc?: SourceLocation;
+  env: EnvironmentRef;
+  packageConditions?: Array<string>;
+  meta?: Meta;
+  resolveFrom?: FilePath;
+  range?: SemverRange;
+  target?: Target;
+  symbols?:
+    | Map<
+        symbol,
+        {
+          local: symbol;
+          loc: SourceLocation | null | undefined;
+          isWeak: boolean;
+          meta?: Meta | null | undefined;
+        }
+      >
+    | null
+    | undefined;
+  pipeline?: string | null | undefined;
+};
 
 export function createDependencyId({
   sourceAssetId,
@@ -57,30 +66,36 @@ export function createDependencyId({
   bundleBehavior,
   priority,
   packageConditions,
-}: {|
-  sourceAssetId?: string | void,
-  specifier: DependencySpecifier,
-  env: Environment,
-  target?: Target | void,
-  pipeline?: ?string,
-  specifierType: $Keys<typeof SpecifierType>,
-  bundleBehavior?: ?IBundleBehavior,
-  priority?: $Keys<typeof Priority> | void,
-  packageConditions?: Array<string> | void,
-|}): string {
+}: {
+  sourceAssetId?: string | undefined;
+  specifier: DependencySpecifier;
+  env: EnvironmentRef;
+  target?: Target | undefined;
+  pipeline?: string | null | undefined;
+  specifierType: keyof typeof SpecifierType;
+  bundleBehavior?: IBundleBehavior | null | undefined;
+  priority?: keyof typeof Priority | undefined;
+  packageConditions?: Array<string> | undefined;
+}): string {
   assert(typeof specifierType === 'string');
   assert(typeof priority === 'string' || priority == null);
   const params = {
     sourceAssetId,
     specifier,
-    environmentId: env.id,
-    target,
+    environmentId: toEnvironmentId(env),
+    target:
+      target != null
+        ? {
+            ...target,
+            env: fromEnvironmentId(target.env),
+          }
+        : null,
     pipeline,
     specifierType: SpecifierType[specifierType],
     bundleBehavior,
     priority: priority ? Priority[priority] : Priority.sync,
     packageConditions,
-  };
+  } as const;
   const id = createDependencyIdRust(params);
   identifierRegistry.addIdentifier('dependency', id, params);
   return id;
@@ -126,7 +141,7 @@ export function createDependency(
     symbols:
       opts.symbols &&
       new Map(
-        [...opts.symbols].map(([k, v]) => [
+        [...opts.symbols].map(([k, v]: [any, any]) => [
           k,
           {
             local: v.local,
@@ -164,9 +179,11 @@ function convertConditions(conditions: Array<string>, dep: Dependency) {
   // Store common package conditions as bit flags to reduce size.
   // Custom conditions are stored as strings.
   let packageConditions = 0;
-  let customConditions = [];
+  let customConditions: Array<string> = [];
   for (let condition of conditions) {
+    // @ts-expect-error TS7053
     if (ExportsCondition[condition]) {
+      // @ts-expect-error TS7053
       packageConditions |= ExportsCondition[condition];
     } else {
       customConditions.push(condition);

package/src/{Environment.js → Environment.ts}

@@ -1,8 +1,8 @@
-// @flow
 import type {
   EnvironmentOptions,
   Environment as IEnvironment,
   FilePath,
+  EnvMap,
 } from '@atlaspack/types';
 import type {Environment, InternalSourceLocation} from './types';
 import {createEnvironmentId} from '@atlaspack/rust';
@@ -10,16 +10,18 @@ import {toInternalSourceLocation} from './utils';
 import PublicEnvironment from './public/Environment';
 import {environmentToInternalEnvironment} from './public/Environment';
 import {identifierRegistry} from './IdentifierRegistry';
+import {toEnvironmentRef} from './EnvironmentManager';
+import type {EnvironmentRef} from './EnvironmentManager';
 
 const DEFAULT_ENGINES = {
   browsers: ['> 0.25%'],
   node: '>= 8.0.0',
-};
+} as const;
 
-type EnvironmentOpts = {|
-  ...EnvironmentOptions,
-  loc?: ?InternalSourceLocation,
-|};
+type EnvironmentOpts = EnvironmentOptions & {
+  loc?: InternalSourceLocation | null | undefined;
+  customEnv?: EnvMap | null | undefined;
+};
 
 export function createEnvironment({
   context,
@@ -33,9 +35,10 @@ export function createEnvironment({
   sourceMap,
   unstableSingleFileOutput = false,
   loc,
+  customEnv,
 }: EnvironmentOpts = {
   /*::...null*/
-}): Environment {
+}): EnvironmentRef {
   if (context == null) {
     if (engines?.node) {
       context = 'node';
@@ -57,8 +60,10 @@ export function createEnvironment({
       case 'browser':
       case 'web-worker':
       case 'service-worker':
+      case 'tesseract':
       case 'electron-renderer':
         engines = {
+          // @ts-expect-error TS2322
           browsers: DEFAULT_ENGINES.browsers,
         };
         break;
@@ -77,6 +82,7 @@ export function createEnvironment({
       case 'browser':
       case 'web-worker':
       case 'service-worker':
+      case 'tesseract':
       default:
         includeNodeModules = true;
         break;
@@ -99,6 +105,7 @@ export function createEnvironment({
   let res: Environment = {
     id: '',
     context,
+    // @ts-expect-error TS2322
     engines,
     includeNodeModules,
     outputFormat,
@@ -109,27 +116,28 @@ export function createEnvironment({
     sourceMap,
     unstableSingleFileOutput,
     loc,
+    customEnv,
   };
 
   res.id = getEnvironmentHash(res);
-  return Object.freeze(res);
+
+  return toEnvironmentRef(Object.freeze(res));
 }
 
 export function mergeEnvironments(
   projectRoot: FilePath,
   a: Environment,
-  b: ?(EnvironmentOptions | IEnvironment),
-): Environment {
+  b?: EnvironmentOptions | IEnvironment | null,
+): EnvironmentRef {
   // If merging the same object, avoid copying.
   if (a === b || !b) {
-    return a;
+    return toEnvironmentRef(a);
   }
 
   if (b instanceof PublicEnvironment) {
-    return environmentToInternalEnvironment(b);
+    return toEnvironmentRef(environmentToInternalEnvironment(b));
   }
 
-  // $FlowFixMe - ignore the `id` that is already on a
   return createEnvironment({
     ...a,
     ...b,
@@ -137,7 +145,7 @@ export function mergeEnvironments(
   });
 }
 
-function getEnvironmentHash(env: Environment): string {
+export function getEnvironmentHash(env: Environment): string {
   const data = {
     context: env.context,
     engines: env.engines,
@@ -148,7 +156,8 @@ function getEnvironmentHash(env: Environment): string {
     shouldOptimize: env.shouldOptimize,
     shouldScopeHoist: env.shouldScopeHoist,
     sourceMap: env.sourceMap,
-  };
+    customEnv: env.customEnv,
+  } as const;
   const id = createEnvironmentId(data);
   identifierRegistry.addIdentifier('environment', id, data);
   return id;

package/src/EnvironmentManager.ts

@@ -0,0 +1,154 @@
+/*!
+ * At the moment we're doing this change for `CoreEnvironment`,
+ * but the same change must be made for `TypesEnvironment` in @atlaspack/types.
+ */
+import type {Environment as CoreEnvironment} from './types';
+import {Cache} from '@atlaspack/cache';
+import {
+  addEnvironment,
+  getEnvironment,
+  getAllEnvironments,
+  setAllEnvironments,
+} from '@atlaspack/rust';
+import {getFeatureFlag} from '@atlaspack/feature-flags';
+import {instrument} from '@atlaspack/logger';
+import {ATLASPACK_VERSION} from './constants';
+
+const localEnvironmentCache = new Map<string, CoreEnvironment>();
+
+export type EnvironmentId = string;
+/**
+ * When deduplication is cleaned-up this will always be a string.
+ */
+export type EnvironmentRef = EnvironmentId | CoreEnvironment;
+
+/**
+ * Convert environment to a ref.
+ * This is what we should be using to store environments.
+ */
+export function toEnvironmentRef(env: CoreEnvironment): EnvironmentRef {
+  if (!getFeatureFlag('environmentDeduplication')) {
+    return env;
+  }
+
+  const id = toEnvironmentId(env);
+  return id;
+}
+
+/**
+ * Convert environment to a string ID
+ */
+export function toEnvironmentId(
+  /**
+   * Redundant type during roll-out
+   */
+  env: CoreEnvironment | EnvironmentRef,
+): string {
+  if (!getFeatureFlag('environmentDeduplication')) {
+    return typeof env === 'string' ? env : env.id;
+  }
+
+  if (typeof env === 'string') {
+    return env;
+  }
+
+  addEnvironment(env);
+  return env.id;
+}
+
+export function fromEnvironmentId(id: EnvironmentRef): CoreEnvironment {
+  if (!getFeatureFlag('environmentDeduplication')) {
+    if (typeof id === 'string') {
+      throw new Error(
+        'This should never happen when environmentDeduplication feature-flag is off',
+      );
+    } else {
+      return id;
+    }
+  }
+
+  if (typeof id !== 'string') {
+    return id;
+  }
+
+  const localEnv = localEnvironmentCache.get(id);
+
+  if (localEnv) {
+    return localEnv;
+  }
+
+  const env = Object.freeze(getEnvironment(id));
+  // @ts-expect-error TS2345
+  localEnvironmentCache.set(id, env);
+  // @ts-expect-error TS2322
+  return env;
+}
+
+/**
+ * Writes all environments and their IDs to the cache
+ * @param {Cache} cache
+ * @returns {Promise<void>}
+ */
+export async function writeEnvironmentsToCache(cache: Cache): Promise<void> {
+  const environments = getAllEnvironments();
+  const environmentIds = new Set<string>();
+
+  // Store each environment individually
+  for (const env of environments) {
+    // @ts-expect-error TS18046
+    environmentIds.add(env.id);
+    // @ts-expect-error TS18046
+    const envKey = `Environment/${ATLASPACK_VERSION}/${env.id}`;
+
+    await instrument(
+      `RequestTracker::writeToCache::cache.put(${envKey})`,
+      async () => {
+        await cache.set(envKey, env);
+      },
+    );
+  }
+
+  // Store the list of environment IDs
+  await instrument(
+    `RequestTracker::writeToCache::cache.put(${`EnvironmentManager/${ATLASPACK_VERSION}`})`,
+    async () => {
+      await cache.set(
+        `EnvironmentManager/${ATLASPACK_VERSION}`,
+        Array.from(environmentIds),
+      );
+    },
+  );
+}
+
+/**
+ * Loads all environments and their IDs from the cache
+ * @param {Cache} cache
+ * @returns {Promise<void>}
+ */
+export async function loadEnvironmentsFromCache(cache: Cache): Promise<void> {
+  const cachedEnvIds = await cache.get(
+    `EnvironmentManager/${ATLASPACK_VERSION}`,
+  );
+
+  if (cachedEnvIds == null) {
+    return;
+  }
+
+  const environments: Array<
+    // @ts-expect-error TS2552
+    Environment & {
+      id: string;
+    }
+  > = [];
+  // @ts-expect-error TS2488
+  for (const envId of cachedEnvIds) {
+    const envKey = `Environment/${ATLASPACK_VERSION}/${envId}`;
+    const cachedEnv = await cache.get(envKey);
+    if (cachedEnv != null) {
+      environments.push(cachedEnv);
+    }
+  }
+  if (environments.length > 0) {
+    setAllEnvironments(environments);
+  }
+}

package/src/{IdentifierRegistry.js → IdentifierRegistry.ts}

@@ -1,5 +1,3 @@
-// @flow strict-local
-
 import fs from 'fs';
 
 export class IdentifierRegistry {
@@ -10,7 +8,7 @@ export class IdentifierRegistry {
     this.#enabled = enabled;
   }
 
-  addIdentifier(type: string, identifier: string, data: mixed) {
+  addIdentifier(type: string, identifier: string, data: unknown) {
     if (this.#enabled) {
       if (!this.#createdDirectory) {
         fs.mkdirSync('./.atlaspack', {recursive: true});
@@ -19,7 +17,6 @@ export class IdentifierRegistry {
 
       fs.appendFile(
         './.atlaspack/atlaspack-identifiers.txt',
-        // $FlowFixMe
         `${type} ${identifier} ${JSON.stringify(data)}\n`,
         () => {},
       );

package/src/{InternalConfig.js → InternalConfig.ts}

@@ -1,9 +1,6 @@
-// @flow strict-local
-
 import type {PackageName, ConfigResult} from '@atlaspack/types';
 import type {
   Config,
-  Environment,
   InternalFileCreateInvalidation,
   InternalDevDepOptions,
 } from './types';
@@ -13,25 +10,27 @@ import {fromProjectPathRelative} from './projectPath';
 import {createEnvironment} from './Environment';
 import {hashString} from '@atlaspack/rust';
 import {identifierRegistry} from './IdentifierRegistry';
+import type {EnvironmentRef} from './EnvironmentManager';
+import {toEnvironmentId} from './EnvironmentManager';
 
-type ConfigOpts = {|
-  plugin: PackageName,
-  searchPath: ProjectPath,
-  isSource?: boolean,
-  env?: Environment,
-  result?: ConfigResult,
-  invalidateOnFileChange?: Set<ProjectPath>,
-  invalidateOnConfigKeyChange?: Array<{|
-    filePath: ProjectPath,
-    configKey: string,
-  |}>,
-  invalidateOnFileCreate?: Array<InternalFileCreateInvalidation>,
-  invalidateOnEnvChange?: Set<string>,
-  invalidateOnOptionChange?: Set<string>,
-  devDeps?: Array<InternalDevDepOptions>,
-  invalidateOnStartup?: boolean,
-  invalidateOnBuild?: boolean,
-|};
+export type ConfigOpts = {
+  plugin: PackageName;
+  searchPath: ProjectPath;
+  isSource?: boolean;
+  env?: EnvironmentRef;
+  result?: ConfigResult;
+  invalidateOnFileChange?: Set<ProjectPath>;
+  invalidateOnConfigKeyChange?: Array<{
+    filePath: ProjectPath;
+    configKey: string[];
+  }>;
+  invalidateOnFileCreate?: Array<InternalFileCreateInvalidation>;
+  invalidateOnEnvChange?: Set<string>;
+  invalidateOnOptionChange?: Set<string>;
+  devDeps?: Array<InternalDevDepOptions>;
+  invalidateOnStartup?: boolean;
+  invalidateOnBuild?: boolean;
+};
 
 export function createConfig({
   plugin,
@@ -52,13 +51,13 @@ export function createConfig({
   const configId = hashString(
     plugin +
       fromProjectPathRelative(searchPath) +
-      environment.id +
+      toEnvironmentId(environment) +
       String(isSource),
   );
   identifierRegistry.addIdentifier('config_request', configId, {
     plugin,
     searchPath,
-    environmentId: environment.id,
+    environmentId: toEnvironmentId(environment),
     isSource,
   });
   return {