@expo/fingerprint 0.0.1

package/src/hash/Hash.ts

@@ -0,0 +1,204 @@
+import { createHash } from 'crypto';
+import { createReadStream } from 'fs';
+import fs from 'fs/promises';
+import minimatch from 'minimatch';
+import pLimit from 'p-limit';
+import path from 'path';
+
+import type {
+  Fingerprint,
+  FingerprintSource,
+  HashResult,
+  HashSource,
+  HashSourceContents,
+  NormalizedOptions,
+} from '../Fingerprint.types';
+import { profile } from '../utils/Profile';
+
+/**
+ * Create a `Fingerprint` from `HashSources` array
+ */
+export async function createFingerprintFromSourcesAsync(
+  sources: HashSource[],
+  projectRoot: string,
+  options: NormalizedOptions
+): Promise<Fingerprint> {
+  const limiter = pLimit(options.concurrentIoLimit);
+  const fingerprintSources = await Promise.all(
+    sources.map((source) => createFingerprintSourceAsync(source, limiter, projectRoot, options))
+  );
+
+  const hasher = createHash(options.hashAlgorithm);
+  for (const source of fingerprintSources) {
+    if (source.hash != null) {
+      hasher.update(createSourceId(source));
+      hasher.update(source.hash);
+    }
+  }
+  const hash = hasher.digest('hex');
+
+  return {
+    sources: fingerprintSources,
+    hash,
+  };
+}
+
+/**
+ * Create a `FingerprintSource` from a `HashSource`
+ * This function will get a hash value and merge back to original source
+ */
+export async function createFingerprintSourceAsync(
+  source: HashSource,
+  limiter: pLimit.Limit,
+  projectRoot: string,
+  options: NormalizedOptions
+): Promise<FingerprintSource> {
+  let result: HashResult | null = null;
+  switch (source.type) {
+    case 'contents':
+      result = await createContentsHashResultsAsync(source, options);
+      break;
+    case 'file':
+      result = await createFileHashResultsAsync(source.filePath, limiter, projectRoot, options);
+      break;
+    case 'dir':
+      result = await profile(
+        createDirHashResultsAsync,
+        `createDirHashResultsAsync(${source.filePath})`
+      )(source.filePath, limiter, projectRoot, options);
+      break;
+    default:
+      throw new Error('Unsupported source type');
+  }
+
+  return { ...source, hash: result?.hex ?? null };
+}
+
+/**
+ * Create a `HashResult` from a file
+ */
+export async function createFileHashResultsAsync(
+  filePath: string,
+  limiter: pLimit.Limit,
+  projectRoot: string,
+  options: NormalizedOptions
+): Promise<HashResult> {
+  // Backup code for faster hashing
+  /*
+  return limiter(async () => {
+    const hasher = createHash(options.hashAlgorithm);
+
+    const stat = await fs.stat(filePath);
+    hasher.update(`${stat.size}`);
+
+    const buffer = Buffer.alloc(4096);
+    const fd = await fs.open(filePath, 'r');
+    await fd.read(buffer, 0, buffer.length, 0);
+    await fd.close();
+    hasher.update(buffer);
+    console.log('stat', filePath, stat.size);
+    return { id: path.relative(projectRoot, filePath), hex: hasher.digest('hex') };
+  });
+  */
+
+  return limiter(() => {
+    return new Promise<HashResult>((resolve, reject) => {
+      let resolved = false;
+      const hasher = createHash(options.hashAlgorithm);
+      const stream = createReadStream(path.join(projectRoot, filePath));
+      stream.on('close', () => {
+        if (!resolved) {
+          const hex = hasher.digest('hex');
+          resolve({ id: filePath, hex });
+          resolved = true;
+        }
+      });
+      stream.on('error', (e) => {
+        reject(e);
+      });
+      stream.on('data', (chunk) => {
+        hasher.update(chunk);
+      });
+    });
+  });
+}
+
+/**
+ * Indicate the given `dirPath` should be excluded by `dirExcludes`
+ */
+function isExcludedDir(dirPath: string, dirExcludes: string[]): boolean {
+  for (const exclude of dirExcludes) {
+    if (minimatch(dirPath, exclude)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Create `HashResult` for a dir.
+ * If the dir is excluded, returns null rather than a HashResult
+ */
+export async function createDirHashResultsAsync(
+  dirPath: string,
+  limiter: pLimit.Limit,
+  projectRoot: string,
+  options: NormalizedOptions,
+  depth: number = 0
+): Promise<HashResult | null> {
+  if (isExcludedDir(dirPath, options.dirExcludes)) {
+    return null;
+  }
+  const dirents = (await fs.readdir(path.join(projectRoot, dirPath), { withFileTypes: true })).sort(
+    (a, b) => a.name.localeCompare(b.name)
+  );
+  const promises: Promise<HashResult | null>[] = [];
+  for (const dirent of dirents) {
+    if (dirent.isDirectory()) {
+      const filePath = path.join(dirPath, dirent.name);
+      promises.push(createDirHashResultsAsync(filePath, limiter, projectRoot, options, depth + 1));
+    } else if (dirent.isFile()) {
+      const filePath = path.join(dirPath, dirent.name);
+      promises.push(createFileHashResultsAsync(filePath, limiter, projectRoot, options));
+    }
+  }
+
+  const hasher = createHash(options.hashAlgorithm);
+  const results = await Promise.all(promises);
+  for (const result of results) {
+    if (result != null) {
+      hasher.update(result.id);
+      hasher.update(result.hex);
+    }
+  }
+  const hex = hasher.digest('hex');
+
+  return { id: dirPath, hex };
+}
+
+/**
+ * Create `HashResult` for a `HashSourceContents`
+ */
+export async function createContentsHashResultsAsync(
+  source: HashSourceContents,
+  options: NormalizedOptions
+): Promise<HashResult> {
+  const hex = createHash(options.hashAlgorithm).update(source.contents).digest('hex');
+  return { id: source.id, hex };
+}
+
+/**
+ * Create id from given source
+ */
+export function createSourceId(source: HashSource): string {
+  switch (source.type) {
+    case 'contents':
+      return source.id;
+    case 'file':
+      return source.filePath;
+    case 'dir':
+      return source.filePath;
+    default:
+      throw new Error('Unsupported source type');
+  }
+}

package/src/hash/__tests__/Hash-test.ts

@@ -0,0 +1,192 @@
+import { createHash } from 'crypto';
+import { vol } from 'memfs';
+import pLimit from 'p-limit';
+import path from 'path';
+
+import { HashSource } from '../../Fingerprint.types';
+import { normalizeOptions } from '../../Options';
+import {
+  createContentsHashResultsAsync,
+  createDirHashResultsAsync,
+  createFileHashResultsAsync,
+  createFingerprintFromSourcesAsync,
+  createFingerprintSourceAsync,
+  createSourceId,
+} from '../Hash';
+
+jest.mock('fs');
+jest.mock('fs/promises');
+
+describe(createFingerprintFromSourcesAsync, () => {
+  afterEach(() => {
+    vol.reset();
+  });
+
+  it('snapshot', async () => {
+    vol.mkdirSync('/app');
+    vol.writeFileSync(path.join('/app', 'app.json'), '{}');
+
+    const sources: HashSource[] = [
+      { type: 'contents', id: 'foo', contents: 'HelloWorld', reasons: ['foo'] },
+      { type: 'file', filePath: 'app.json', reasons: ['expoConfig'] },
+    ];
+
+    expect(await createFingerprintFromSourcesAsync(sources, '/app', normalizeOptions()))
+      .toMatchInlineSnapshot(`
+      Object {
+        "hash": "ec7d81780f735d5e289b27cdcc04a6c99d2621dc",
+        "sources": Array [
+          Object {
+            "contents": "HelloWorld",
+            "hash": "db8ac1c259eb89d4a131b253bacfca5f319d54f2",
+            "id": "foo",
+            "reasons": Array [
+              "foo",
+            ],
+            "type": "contents",
+          },
+          Object {
+            "filePath": "app.json",
+            "hash": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
+            "reasons": Array [
+              "expoConfig",
+            ],
+            "type": "file",
+          },
+        ],
+      }
+    `);
+  });
+});
+
+describe(createFingerprintSourceAsync, () => {
+  it('should merge hash value to original source', async () => {
+    const source: HashSource = {
+      type: 'contents',
+      id: 'foo',
+      contents: 'HelloWorld',
+      reasons: ['foo'],
+    };
+    const expectedResult = {
+      ...source,
+      hash: 'db8ac1c259eb89d4a131b253bacfca5f319d54f2',
+    };
+    expect(
+      await createFingerprintSourceAsync(source, pLimit(1), '/app', normalizeOptions())
+    ).toEqual(expectedResult);
+  });
+});
+
+describe(createContentsHashResultsAsync, () => {
+  it('should return {id, hex} result', async () => {
+    const id = 'foo';
+    const contents = '{}';
+    const options = normalizeOptions();
+    const result = await createContentsHashResultsAsync(
+      {
+        type: 'contents',
+        id,
+        contents,
+        reasons: [id],
+      },
+      options
+    );
+
+    const expectHex = createHash(options.hashAlgorithm).update(contents).digest('hex');
+    expect(result.id).toEqual(id);
+    expect(result.hex).toEqual(expectHex);
+  });
+});
+
+describe(createFileHashResultsAsync, () => {
+  afterEach(() => {
+    vol.reset();
+  });
+
+  it('should return {id, hex} result', async () => {
+    const filePath = 'app.json';
+    const contents = '{}';
+    const limiter = pLimit(1);
+    const options = normalizeOptions();
+    vol.mkdirSync('/app');
+    vol.writeFileSync(path.join('/app', filePath), contents);
+
+    const result = await createFileHashResultsAsync(filePath, limiter, '/app', options);
+
+    const expectHex = createHash(options.hashAlgorithm).update(contents).digest('hex');
+    expect(result.id).toEqual(filePath);
+    expect(result.hex).toEqual(expectHex);
+  });
+});
+
+describe(createDirHashResultsAsync, () => {
+  afterEach(() => {
+    vol.reset();
+  });
+
+  it('should return {id, hex} result', async () => {
+    const limiter = pLimit(3);
+    const options = normalizeOptions();
+    const volJSON = {
+      '/app/ios/Podfile': '...',
+      '/app/eas.json': '{}',
+      '/app/app.json': '{}',
+      '/app/android/build.gradle': '...',
+    };
+    vol.fromJSON(volJSON);
+    const result = await createDirHashResultsAsync('.', limiter, '/app', options);
+
+    expect(result?.id).toEqual('.');
+    expect(result?.hex).not.toBe('');
+  });
+
+  it('should return stable result from sorted files', async () => {
+    const limiter = pLimit(3);
+    const options = normalizeOptions();
+    const volJSON = {
+      '/app/ios/Podfile': '...',
+      '/app/eas.json': '{}',
+      '/app/app.json': '{}',
+      '/app/android/build.gradle': '...',
+    };
+    vol.fromJSON(volJSON);
+    const result = await createDirHashResultsAsync('.', limiter, '/app', options);
+
+    vol.reset();
+    const sortedVolJSON = {
+      '/app/app.json': '{}',
+      '/app/eas.json': '{}',
+      '/app/android/build.gradle': '...',
+      '/app/ios/Podfile': '...',
+    };
+    vol.fromJSON(sortedVolJSON);
+    const sortedResult = await createDirHashResultsAsync('.', limiter, '/app', options);
+
+    expect(result?.id).toEqual(sortedResult?.id);
+    expect(result?.hex).toEqual(sortedResult?.hex);
+  });
+});
+
+describe(createSourceId, () => {
+  it(`should use filePath as id for file or dir`, () => {
+    const fileSource: HashSource = {
+      type: 'file',
+      filePath: '/app/app.json',
+      reasons: ['expoConfig'],
+    };
+    expect(createSourceId(fileSource)).toBe('/app/app.json');
+
+    const dirSource: HashSource = { type: 'dir', filePath: '/app/ios', reasons: ['bareNativeDir'] };
+    expect(createSourceId(dirSource)).toBe('/app/ios');
+  });
+
+  it(`should use given id for contents`, () => {
+    const source: HashSource = {
+      type: 'contents',
+      id: 'foo',
+      contents: 'HelloWorld',
+      reasons: ['foo'],
+    };
+    expect(createSourceId(source)).toBe('foo');
+  });
+});

package/src/index.ts

@@ -0,0 +1,3 @@
+export { createFingerprintAsync, createProjectHashAsync } from './Fingerprint';
+
+export * from './Fingerprint.types';

package/src/sourcer/Bare.ts

@@ -0,0 +1,114 @@
+import spawnAsync from '@expo/spawn-async';
+import chalk from 'chalk';
+import path from 'path';
+
+import type { HashSource, NormalizedOptions, Platform } from '../Fingerprint.types';
+import { getFileBasedHashSourceAsync } from './Utils';
+
+const debug = require('debug')('expo:fingerprint:sourcer:Bare');
+
+export async function getBareAndroidSourcesAsync(
+  projectRoot: string,
+  options: NormalizedOptions
+): Promise<HashSource[]> {
+  if (options.platforms.includes('android')) {
+    const result = await getFileBasedHashSourceAsync(projectRoot, 'android', 'bareNativeDir');
+    if (result != null) {
+      debug(`Adding bare native dir - ${chalk.dim('android')}`);
+      return [result];
+    }
+  }
+  return [];
+}
+
+export async function getBareIosSourcesAsync(
+  projectRoot: string,
+  options: NormalizedOptions
+): Promise<HashSource[]> {
+  if (options.platforms.includes('ios')) {
+    const result = await getFileBasedHashSourceAsync(projectRoot, 'ios', 'bareNativeDir');
+    if (result != null) {
+      debug(`Adding bare native dir - ${chalk.dim('ios')}`);
+      return [result];
+    }
+  }
+  return [];
+}
+
+export async function getPackageJsonScriptSourcesAsync(
+  projectRoot: string,
+  options: NormalizedOptions
+) {
+  let packageJson;
+  try {
+    packageJson = require(`${projectRoot}/package.json`);
+  } catch (e: unknown) {
+    debug(`Unable to read package.json from ${projectRoot}/package.json: ` + e);
+    return [];
+  }
+  const results: HashSource[] = [];
+  if (packageJson.scripts) {
+    debug(`Adding package.json contents - ${chalk.dim('scripts')}`);
+    const id = 'packageJson:scripts';
+    results.push({
+      type: 'contents',
+      id,
+      contents: JSON.stringify(packageJson.scripts),
+      reasons: [id],
+    });
+  }
+  return results;
+}
+
+export async function getGitIgnoreSourcesAsync(projectRoot: string, options: NormalizedOptions) {
+  const result = await getFileBasedHashSourceAsync(projectRoot, '.gitignore', 'bareGitIgnore');
+  if (result != null) {
+    debug(`Adding file - ${chalk.dim('.gitignore')}`);
+    return [result];
+  }
+  return [];
+}
+
+export async function getRncliAutolinkingSourcesAsync(
+  projectRoot: string,
+  options: NormalizedOptions
+): Promise<HashSource[]> {
+  try {
+    const results: HashSource[] = [];
+    const { stdout } = await spawnAsync('npx', ['react-native', 'config'], { cwd: projectRoot });
+    const config = JSON.parse(stdout);
+    const { root } = config;
+    const reasons = ['bareRncliAutolinking'];
+    for (const depData of Object.values<any>(config.dependencies)) {
+      const filePath = path.relative(root, depData.root);
+      results.push({ type: 'dir', filePath, reasons });
+      debug(`Adding react-native-cli autolinking dir - ${chalk.dim(filePath)}`);
+      for (const platform of options.platforms) {
+        const platformData = getRncliPlatformData(depData, root, platform);
+        if (platformData) {
+          results.push({
+            type: 'contents',
+            id: `rncliAutolinkingConfig:${depData.name}:${platform}`,
+            contents: platformData,
+            reasons,
+          });
+        }
+      }
+    }
+    return results;
+  } catch {
+    return [];
+  }
+}
+
+function getRncliPlatformData(dependency: any, root: string, platform: Platform): string {
+  const platformData = dependency.platforms[platform];
+  if (!platformData) {
+    return '';
+  }
+  const json: Record<string, string> = {};
+  for (const [key, value] of Object.entries<any>(platformData)) {
+    json[key] = value?.startsWith?.(root) ? path.relative(root, value) : value;
+  }
+  return JSON.stringify(json);
+}